From c375936e81af8ee2a3c881c77004f0e73755fa41 Mon Sep 17 00:00:00 2001 From: John Chase <68833933+joohhnnn@users.noreply.github.com> Date: Fri, 16 Jun 2023 20:21:43 +0800 Subject: [PATCH 01/90] rlp: fix typos (#27484) fix typos Co-authored-by: john --- rlp/encode.go | 2 +- rlp/encode_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rlp/encode.go b/rlp/encode.go index 3fac0bd2d3..ffb42b2997 100644 --- a/rlp/encode.go +++ b/rlp/encode.go @@ -419,7 +419,7 @@ func makeEncoderWriter(typ reflect.Type) writer { // package json simply doesn't call MarshalJSON for this case, but encodes the // value as if it didn't implement the interface. We don't want to handle it that // way. - return fmt.Errorf("rlp: unadressable value of type %v, EncodeRLP is pointer method", val.Type()) + return fmt.Errorf("rlp: unaddressable value of type %v, EncodeRLP is pointer method", val.Type()) } return val.Addr().Interface().(Encoder).EncodeRLP(w) } diff --git a/rlp/encode_test.go b/rlp/encode_test.go index 02be47d0ef..314958eb56 100644 --- a/rlp/encode_test.go +++ b/rlp/encode_test.go @@ -396,7 +396,7 @@ var encTests = []encTest{ {val: &struct{ TE testEncoder }{testEncoder{errors.New("test error")}}, error: "test error"}, // Verify the error for non-addressable non-pointer Encoder. - {val: testEncoder{}, error: "rlp: unadressable value of type rlp.testEncoder, EncodeRLP is pointer method"}, + {val: testEncoder{}, error: "rlp: unaddressable value of type rlp.testEncoder, EncodeRLP is pointer method"}, // Verify Encoder takes precedence over []byte. {val: []byteEncoder{0, 1, 2, 3, 4}, output: "C5C0C0C0C0C0"}, From d40a255e973775575d8d16456252f93ac75c09f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 16 Jun 2023 15:29:40 +0300 Subject: [PATCH 02/90] all: move main transaction pool into a subpool (#27463) * all: move main transaction pool into a subpool * go.mod: remove superfluous updates * core/txpool: review fixes, handle txs rejected by all subpools * core/txpool: typos --- cmd/utils/flags.go | 11 +- core/txpool/errors.go | 53 + core/txpool/{ => legacypool}/journal.go | 2 +- core/txpool/legacypool/legacypool.go | 1847 ++++++++++++++++ .../legacypool2_test.go} | 56 +- .../legacypool_test.go} | 433 ++-- core/txpool/{ => legacypool}/list.go | 2 +- core/txpool/{ => legacypool}/list_test.go | 2 +- core/txpool/{ => legacypool}/noncer.go | 2 +- core/txpool/subpool.go | 111 + core/txpool/txpool.go | 1961 ++--------------- core/types/transaction.go | 10 +- core/types/transaction_signing.go | 6 +- core/types/transaction_test.go | 4 +- eth/api_backend.go | 21 +- eth/backend.go | 12 +- eth/catalyst/api_test.go | 25 +- eth/ethconfig/config.go | 6 +- eth/ethconfig/gen_config.go | 6 +- eth/fetcher/tx_fetcher.go | 17 +- eth/fetcher/tx_fetcher_test.go | 30 +- eth/handler.go | 14 +- eth/handler_eth_test.go | 19 +- eth/handler_test.go | 27 +- eth/protocols/eth/broadcast.go | 8 +- eth/protocols/eth/handler.go | 4 +- eth/protocols/eth/handler_test.go | 10 +- eth/protocols/eth/handlers.go | 2 +- event/multisub.go | 50 + event/multisub_test.go | 175 ++ internal/ethapi/api_test.go | 4 +- internal/ethapi/backend.go | 4 +- internal/ethapi/transaction_args_test.go | 4 +- internal/flags/categories.go | 3 +- les/api_backend.go | 4 +- les/server_requests.go | 9 +- les/test_helper.go | 7 +- light/txpool.go | 12 +- miner/miner_test.go | 17 +- miner/stress/clique/main.go | 5 +- miner/worker.go | 10 +- miner/worker_test.go | 20 +- params/protocol_params.go | 2 + tests/fuzzers/les/les-fuzzer.go | 6 +- tests/fuzzers/txfetcher/txfetcher_fuzzer.go | 3 +- 45 files changed, 2931 insertions(+), 2105 deletions(-) create mode 100644 core/txpool/errors.go rename core/txpool/{ => legacypool}/journal.go (99%) create mode 100644 core/txpool/legacypool/legacypool.go rename core/txpool/{txpool2_test.go => legacypool/legacypool2_test.go} (84%) rename core/txpool/{txpool_test.go => legacypool/legacypool_test.go} (87%) rename core/txpool/{ => legacypool}/list.go (99%) rename core/txpool/{ => legacypool}/list_test.go (99%) rename core/txpool/{ => legacypool}/noncer.go (99%) create mode 100644 core/txpool/subpool.go create mode 100644 event/multisub.go create mode 100644 event/multisub_test.go diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 692970477d..de4e58a3b1 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -41,7 +41,7 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/txpool" + "github.com/ethereum/go-ethereum/core/txpool/legacypool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" @@ -334,18 +334,18 @@ var ( TxPoolJournalFlag = &cli.StringFlag{ Name: "txpool.journal", Usage: "Disk journal for local transaction to survive node restarts", - Value: txpool.DefaultConfig.Journal, + Value: ethconfig.Defaults.TxPool.Journal, Category: flags.TxPoolCategory, } TxPoolRejournalFlag = &cli.DurationFlag{ Name: "txpool.rejournal", Usage: "Time interval to regenerate the local transaction journal", - Value: txpool.DefaultConfig.Rejournal, + Value: ethconfig.Defaults.TxPool.Rejournal, Category: flags.TxPoolCategory, } TxPoolPriceLimitFlag = &cli.Uint64Flag{ Name: "txpool.pricelimit", - Usage: "Minimum gas price limit to enforce for acceptance into the pool", + Usage: "Minimum gas price tip to enforce for acceptance into the pool", Value: ethconfig.Defaults.TxPool.PriceLimit, Category: flags.TxPoolCategory, } @@ -385,7 +385,6 @@ var ( Value: ethconfig.Defaults.TxPool.Lifetime, Category: flags.TxPoolCategory, } - // Performance tuning settings CacheFlag = &cli.IntFlag{ Name: "cache", @@ -1500,7 +1499,7 @@ func setGPO(ctx *cli.Context, cfg *gasprice.Config, light bool) { } } -func setTxPool(ctx *cli.Context, cfg *txpool.Config) { +func setTxPool(ctx *cli.Context, cfg *legacypool.Config) { if ctx.IsSet(TxPoolLocalsFlag.Name) { locals := strings.Split(ctx.String(TxPoolLocalsFlag.Name), ",") for _, account := range locals { diff --git a/core/txpool/errors.go b/core/txpool/errors.go new file mode 100644 index 0000000000..b8c1c914a3 --- /dev/null +++ b/core/txpool/errors.go @@ -0,0 +1,53 @@ +// Copyright 2014 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package txpool + +import "errors" + +var ( + // ErrAlreadyKnown is returned if the transactions is already contained + // within the pool. + ErrAlreadyKnown = errors.New("already known") + + // ErrInvalidSender is returned if the transaction contains an invalid signature. + ErrInvalidSender = errors.New("invalid sender") + + // ErrUnderpriced is returned if a transaction's gas price is below the minimum + // configured for the transaction pool. + ErrUnderpriced = errors.New("transaction underpriced") + + // ErrReplaceUnderpriced is returned if a transaction is attempted to be replaced + // with a different one without the required price bump. + ErrReplaceUnderpriced = errors.New("replacement transaction underpriced") + + // ErrGasLimit is returned if a transaction's requested gas limit exceeds the + // maximum allowance of the current block. + ErrGasLimit = errors.New("exceeds block gas limit") + + // ErrNegativeValue is a sanity error to ensure no one is able to specify a + // transaction with a negative value. + ErrNegativeValue = errors.New("negative value") + + // ErrOversizedData is returned if the input data of a transaction is greater + // than some meaningful limit a user might use. This is not a consensus error + // making the transaction invalid, rather a DOS protection. + ErrOversizedData = errors.New("oversized data") + + // ErrFutureReplacePending is returned if a future transaction replaces a pending + // transaction. Future transactions should only be able to replace other future transactions. + ErrFutureReplacePending = errors.New("future transaction tries to replace pending") +) diff --git a/core/txpool/journal.go b/core/txpool/legacypool/journal.go similarity index 99% rename from core/txpool/journal.go rename to core/txpool/legacypool/journal.go index 1b330b0c3c..f04ab8fc14 100644 --- a/core/txpool/journal.go +++ b/core/txpool/legacypool/journal.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package txpool +package legacypool import ( "errors" diff --git a/core/txpool/legacypool/legacypool.go b/core/txpool/legacypool/legacypool.go new file mode 100644 index 0000000000..3525747c72 --- /dev/null +++ b/core/txpool/legacypool/legacypool.go @@ -0,0 +1,1847 @@ +// Copyright 2014 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package legacypool implements the normal EVM execution transaction pool. +package legacypool + +import ( + "errors" + "math" + "math/big" + "sort" + "sync" + "sync/atomic" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/prque" + "github.com/ethereum/go-ethereum/consensus/misc" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/txpool" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/params" +) + +const ( + // txSlotSize is used to calculate how many data slots a single transaction + // takes up based on its size. The slots are used as DoS protection, ensuring + // that validating a new transaction remains a constant operation (in reality + // O(maxslots), where max slots are 4 currently). + txSlotSize = 32 * 1024 + + // txMaxSize is the maximum size a single transaction can have. This field has + // non-trivial consequences: larger transactions are significantly harder and + // more expensive to propagate; larger transactions also take more resources + // to validate whether they fit into the pool or not. + txMaxSize = 4 * txSlotSize // 128KB +) + +var ( + // ErrAlreadyKnown is returned if the transactions is already contained + // within the pool. + ErrAlreadyKnown = errors.New("already known") + + // ErrTxPoolOverflow is returned if the transaction pool is full and can't accept + // another remote transaction. + ErrTxPoolOverflow = errors.New("txpool is full") +) + +var ( + evictionInterval = time.Minute // Time interval to check for evictable transactions + statsReportInterval = 8 * time.Second // Time interval to report transaction pool stats +) + +var ( + // Metrics for the pending pool + pendingDiscardMeter = metrics.NewRegisteredMeter("txpool/pending/discard", nil) + pendingReplaceMeter = metrics.NewRegisteredMeter("txpool/pending/replace", nil) + pendingRateLimitMeter = metrics.NewRegisteredMeter("txpool/pending/ratelimit", nil) // Dropped due to rate limiting + pendingNofundsMeter = metrics.NewRegisteredMeter("txpool/pending/nofunds", nil) // Dropped due to out-of-funds + + // Metrics for the queued pool + queuedDiscardMeter = metrics.NewRegisteredMeter("txpool/queued/discard", nil) + queuedReplaceMeter = metrics.NewRegisteredMeter("txpool/queued/replace", nil) + queuedRateLimitMeter = metrics.NewRegisteredMeter("txpool/queued/ratelimit", nil) // Dropped due to rate limiting + queuedNofundsMeter = metrics.NewRegisteredMeter("txpool/queued/nofunds", nil) // Dropped due to out-of-funds + queuedEvictionMeter = metrics.NewRegisteredMeter("txpool/queued/eviction", nil) // Dropped due to lifetime + + // General tx metrics + knownTxMeter = metrics.NewRegisteredMeter("txpool/known", nil) + validTxMeter = metrics.NewRegisteredMeter("txpool/valid", nil) + invalidTxMeter = metrics.NewRegisteredMeter("txpool/invalid", nil) + underpricedTxMeter = metrics.NewRegisteredMeter("txpool/underpriced", nil) + overflowedTxMeter = metrics.NewRegisteredMeter("txpool/overflowed", nil) + + // throttleTxMeter counts how many transactions are rejected due to too-many-changes between + // txpool reorgs. + throttleTxMeter = metrics.NewRegisteredMeter("txpool/throttle", nil) + // reorgDurationTimer measures how long time a txpool reorg takes. + reorgDurationTimer = metrics.NewRegisteredTimer("txpool/reorgtime", nil) + // dropBetweenReorgHistogram counts how many drops we experience between two reorg runs. It is expected + // that this number is pretty low, since txpool reorgs happen very frequently. + dropBetweenReorgHistogram = metrics.NewRegisteredHistogram("txpool/dropbetweenreorg", nil, metrics.NewExpDecaySample(1028, 0.015)) + + pendingGauge = metrics.NewRegisteredGauge("txpool/pending", nil) + queuedGauge = metrics.NewRegisteredGauge("txpool/queued", nil) + localGauge = metrics.NewRegisteredGauge("txpool/local", nil) + slotsGauge = metrics.NewRegisteredGauge("txpool/slots", nil) + + reheapTimer = metrics.NewRegisteredTimer("txpool/reheap", nil) +) + +// BlockChain defines the minimal set of methods needed to back a tx pool with +// a chain. Exists to allow mocking the live chain out of tests. +type BlockChain interface { + // Config retrieves the chain's fork configuration. + Config() *params.ChainConfig + + // CurrentBlock returns the current head of the chain. + CurrentBlock() *types.Header + + // GetBlock retrieves a specific block, used during pool resets. + GetBlock(hash common.Hash, number uint64) *types.Block + + // StateAt returns a state database for a given root hash (generally the head). + StateAt(root common.Hash) (*state.StateDB, error) +} + +// Config are the configuration parameters of the transaction pool. +type Config struct { + Locals []common.Address // Addresses that should be treated by default as local + NoLocals bool // Whether local transaction handling should be disabled + Journal string // Journal of local transactions to survive node restarts + Rejournal time.Duration // Time interval to regenerate the local transaction journal + + PriceLimit uint64 // Minimum gas price to enforce for acceptance into the pool + PriceBump uint64 // Minimum price bump percentage to replace an already existing transaction (nonce) + + AccountSlots uint64 // Number of executable transaction slots guaranteed per account + GlobalSlots uint64 // Maximum number of executable transaction slots for all accounts + AccountQueue uint64 // Maximum number of non-executable transaction slots permitted per account + GlobalQueue uint64 // Maximum number of non-executable transaction slots for all accounts + + Lifetime time.Duration // Maximum amount of time non-executable transaction are queued +} + +// DefaultConfig contains the default configurations for the transaction pool. +var DefaultConfig = Config{ + Journal: "transactions.rlp", + Rejournal: time.Hour, + + PriceLimit: 1, + PriceBump: 10, + + AccountSlots: 16, + GlobalSlots: 4096 + 1024, // urgent + floating queue capacity with 4:1 ratio + AccountQueue: 64, + GlobalQueue: 1024, + + Lifetime: 3 * time.Hour, +} + +// sanitize checks the provided user configurations and changes anything that's +// unreasonable or unworkable. +func (config *Config) sanitize() Config { + conf := *config + if conf.Rejournal < time.Second { + log.Warn("Sanitizing invalid txpool journal time", "provided", conf.Rejournal, "updated", time.Second) + conf.Rejournal = time.Second + } + if conf.PriceLimit < 1 { + log.Warn("Sanitizing invalid txpool price limit", "provided", conf.PriceLimit, "updated", DefaultConfig.PriceLimit) + conf.PriceLimit = DefaultConfig.PriceLimit + } + if conf.PriceBump < 1 { + log.Warn("Sanitizing invalid txpool price bump", "provided", conf.PriceBump, "updated", DefaultConfig.PriceBump) + conf.PriceBump = DefaultConfig.PriceBump + } + if conf.AccountSlots < 1 { + log.Warn("Sanitizing invalid txpool account slots", "provided", conf.AccountSlots, "updated", DefaultConfig.AccountSlots) + conf.AccountSlots = DefaultConfig.AccountSlots + } + if conf.GlobalSlots < 1 { + log.Warn("Sanitizing invalid txpool global slots", "provided", conf.GlobalSlots, "updated", DefaultConfig.GlobalSlots) + conf.GlobalSlots = DefaultConfig.GlobalSlots + } + if conf.AccountQueue < 1 { + log.Warn("Sanitizing invalid txpool account queue", "provided", conf.AccountQueue, "updated", DefaultConfig.AccountQueue) + conf.AccountQueue = DefaultConfig.AccountQueue + } + if conf.GlobalQueue < 1 { + log.Warn("Sanitizing invalid txpool global queue", "provided", conf.GlobalQueue, "updated", DefaultConfig.GlobalQueue) + conf.GlobalQueue = DefaultConfig.GlobalQueue + } + if conf.Lifetime < 1 { + log.Warn("Sanitizing invalid txpool lifetime", "provided", conf.Lifetime, "updated", DefaultConfig.Lifetime) + conf.Lifetime = DefaultConfig.Lifetime + } + return conf +} + +// LegacyPool contains all currently known transactions. Transactions +// enter the pool when they are received from the network or submitted +// locally. They exit the pool when they are included in the blockchain. +// +// The pool separates processable transactions (which can be applied to the +// current state) and future transactions. Transactions move between those +// two states over time as they are received and processed. +type LegacyPool struct { + config Config + chainconfig *params.ChainConfig + chain BlockChain + gasTip atomic.Pointer[big.Int] + txFeed event.Feed + scope event.SubscriptionScope + signer types.Signer + mu sync.RWMutex + + currentHead atomic.Pointer[types.Header] // Current head of the blockchain + currentState *state.StateDB // Current state in the blockchain head + pendingNonces *noncer // Pending state tracking virtual nonces + + locals *accountSet // Set of local transaction to exempt from eviction rules + journal *journal // Journal of local transaction to back up to disk + + pending map[common.Address]*list // All currently processable transactions + queue map[common.Address]*list // Queued but non-processable transactions + beats map[common.Address]time.Time // Last heartbeat from each known account + all *lookup // All transactions to allow lookups + priced *pricedList // All transactions sorted by price + + reqResetCh chan *txpoolResetRequest + reqPromoteCh chan *accountSet + queueTxEventCh chan *types.Transaction + reorgDoneCh chan chan struct{} + reorgShutdownCh chan struct{} // requests shutdown of scheduleReorgLoop + wg sync.WaitGroup // tracks loop, scheduleReorgLoop + initDoneCh chan struct{} // is closed once the pool is initialized (for tests) + + changesSinceReorg int // A counter for how many drops we've performed in-between reorg. +} + +type txpoolResetRequest struct { + oldHead, newHead *types.Header +} + +// New creates a new transaction pool to gather, sort and filter inbound +// transactions from the network. +func New(config Config, chain BlockChain) *LegacyPool { + // Sanitize the input to ensure no vulnerable gas prices are set + config = (&config).sanitize() + + // Create the transaction pool with its initial settings + pool := &LegacyPool{ + config: config, + chain: chain, + chainconfig: chain.Config(), + signer: types.LatestSigner(chain.Config()), + pending: make(map[common.Address]*list), + queue: make(map[common.Address]*list), + beats: make(map[common.Address]time.Time), + all: newLookup(), + reqResetCh: make(chan *txpoolResetRequest), + reqPromoteCh: make(chan *accountSet), + queueTxEventCh: make(chan *types.Transaction), + reorgDoneCh: make(chan chan struct{}), + reorgShutdownCh: make(chan struct{}), + initDoneCh: make(chan struct{}), + } + pool.locals = newAccountSet(pool.signer) + for _, addr := range config.Locals { + log.Info("Setting new local account", "address", addr) + pool.locals.add(addr) + } + pool.priced = newPricedList(pool.all) + + if !config.NoLocals && config.Journal != "" { + pool.journal = newTxJournal(config.Journal) + } + return pool +} + +// Filter returns whether the given transaction can be consumed by the legacy +// pool, specifically, whether it is a Legacy, AccessList or Dynamic transaction. +func (pool *LegacyPool) Filter(tx *types.Transaction) bool { + switch tx.Type() { + case types.LegacyTxType, types.AccessListTxType, types.DynamicFeeTxType: + return true + default: + return false + } +} + +// Init sets the gas price needed to keep a transaction in the pool and the chain +// head to allow balance / nonce checks. The transaction journal will be loaded +// from disk and filtered based on the provided starting settings. The internal +// goroutines will be spun up and the pool deemed operational afterwards. +func (pool *LegacyPool) Init(gasTip *big.Int, head *types.Header) error { + // Set the basic pool parameters + pool.gasTip.Store(gasTip) + pool.reset(nil, head) + + // Start the reorg loop early, so it can handle requests generated during + // journal loading. + pool.wg.Add(1) + go pool.scheduleReorgLoop() + + // If local transactions and journaling is enabled, load from disk + if pool.journal != nil { + if err := pool.journal.load(pool.addLocals); err != nil { + log.Warn("Failed to load transaction journal", "err", err) + } + if err := pool.journal.rotate(pool.local()); err != nil { + log.Warn("Failed to rotate transaction journal", "err", err) + } + } + pool.wg.Add(1) + go pool.loop() + return nil +} + +// loop is the transaction pool's main event loop, waiting for and reacting to +// outside blockchain events as well as for various reporting and transaction +// eviction events. +func (pool *LegacyPool) loop() { + defer pool.wg.Done() + + var ( + prevPending, prevQueued, prevStales int + + // Start the stats reporting and transaction eviction tickers + report = time.NewTicker(statsReportInterval) + evict = time.NewTicker(evictionInterval) + journal = time.NewTicker(pool.config.Rejournal) + ) + defer report.Stop() + defer evict.Stop() + defer journal.Stop() + + // Notify tests that the init phase is done + close(pool.initDoneCh) + for { + select { + // Handle pool shutdown + case <-pool.reorgShutdownCh: + return + + // Handle stats reporting ticks + case <-report.C: + pool.mu.RLock() + pending, queued := pool.stats() + pool.mu.RUnlock() + stales := int(pool.priced.stales.Load()) + + if pending != prevPending || queued != prevQueued || stales != prevStales { + log.Debug("Transaction pool status report", "executable", pending, "queued", queued, "stales", stales) + prevPending, prevQueued, prevStales = pending, queued, stales + } + + // Handle inactive account transaction eviction + case <-evict.C: + pool.mu.Lock() + for addr := range pool.queue { + // Skip local transactions from the eviction mechanism + if pool.locals.contains(addr) { + continue + } + // Any non-locals old enough should be removed + if time.Since(pool.beats[addr]) > pool.config.Lifetime { + list := pool.queue[addr].Flatten() + for _, tx := range list { + pool.removeTx(tx.Hash(), true) + } + queuedEvictionMeter.Mark(int64(len(list))) + } + } + pool.mu.Unlock() + + // Handle local transaction journal rotation + case <-journal.C: + if pool.journal != nil { + pool.mu.Lock() + if err := pool.journal.rotate(pool.local()); err != nil { + log.Warn("Failed to rotate local tx journal", "err", err) + } + pool.mu.Unlock() + } + } + } +} + +// Close terminates the transaction pool. +func (pool *LegacyPool) Close() error { + // Unsubscribe all subscriptions registered from txpool + pool.scope.Close() + + // Terminate the pool reorger and return + close(pool.reorgShutdownCh) + pool.wg.Wait() + + if pool.journal != nil { + pool.journal.close() + } + log.Info("Transaction pool stopped") + return nil +} + +// Reset implements txpool.SubPool, allowing the legacy pool's internal state to be +// kept in sync with the main transacion pool's internal state. +func (pool *LegacyPool) Reset(oldHead, newHead *types.Header) { + wait := pool.requestReset(oldHead, newHead) + <-wait +} + +// SubscribeTransactions registers a subscription of NewTxsEvent and +// starts sending event to the given channel. +func (pool *LegacyPool) SubscribeTransactions(ch chan<- core.NewTxsEvent) event.Subscription { + return pool.scope.Track(pool.txFeed.Subscribe(ch)) +} + +// SetGasTip updates the minimum gas tip required by the transaction pool for a +// new transaction, and drops all transactions below this threshold. +func (pool *LegacyPool) SetGasTip(tip *big.Int) { + pool.mu.Lock() + defer pool.mu.Unlock() + + old := pool.gasTip.Load() + pool.gasTip.Store(new(big.Int).Set(tip)) + + // If the min miner fee increased, remove transactions below the new threshold + if tip.Cmp(old) > 0 { + // pool.priced is sorted by GasFeeCap, so we have to iterate through pool.all instead + drop := pool.all.RemotesBelowTip(tip) + for _, tx := range drop { + pool.removeTx(tx.Hash(), false) + } + pool.priced.Removed(len(drop)) + } + log.Info("LEgacy pool tip threshold updated", "tip", tip) +} + +// Nonce returns the next nonce of an account, with all transactions executable +// by the pool already applied on top. +func (pool *LegacyPool) Nonce(addr common.Address) uint64 { + pool.mu.RLock() + defer pool.mu.RUnlock() + + return pool.pendingNonces.get(addr) +} + +// Stats retrieves the current pool stats, namely the number of pending and the +// number of queued (non-executable) transactions. +func (pool *LegacyPool) Stats() (int, int) { + pool.mu.RLock() + defer pool.mu.RUnlock() + + return pool.stats() +} + +// stats retrieves the current pool stats, namely the number of pending and the +// number of queued (non-executable) transactions. +func (pool *LegacyPool) stats() (int, int) { + pending := 0 + for _, list := range pool.pending { + pending += list.Len() + } + queued := 0 + for _, list := range pool.queue { + queued += list.Len() + } + return pending, queued +} + +// Content retrieves the data content of the transaction pool, returning all the +// pending as well as queued transactions, grouped by account and sorted by nonce. +func (pool *LegacyPool) Content() (map[common.Address][]*types.Transaction, map[common.Address][]*types.Transaction) { + pool.mu.Lock() + defer pool.mu.Unlock() + + pending := make(map[common.Address][]*types.Transaction, len(pool.pending)) + for addr, list := range pool.pending { + pending[addr] = list.Flatten() + } + queued := make(map[common.Address][]*types.Transaction, len(pool.queue)) + for addr, list := range pool.queue { + queued[addr] = list.Flatten() + } + return pending, queued +} + +// ContentFrom retrieves the data content of the transaction pool, returning the +// pending as well as queued transactions of this address, grouped by nonce. +func (pool *LegacyPool) ContentFrom(addr common.Address) ([]*types.Transaction, []*types.Transaction) { + pool.mu.RLock() + defer pool.mu.RUnlock() + + var pending []*types.Transaction + if list, ok := pool.pending[addr]; ok { + pending = list.Flatten() + } + var queued []*types.Transaction + if list, ok := pool.queue[addr]; ok { + queued = list.Flatten() + } + return pending, queued +} + +// Pending retrieves all currently processable transactions, grouped by origin +// account and sorted by nonce. The returned transaction set is a copy and can be +// freely modified by calling code. +// +// The enforceTips parameter can be used to do an extra filtering on the pending +// transactions and only return those whose **effective** tip is large enough in +// the next pending execution environment. +func (pool *LegacyPool) Pending(enforceTips bool) map[common.Address][]*types.Transaction { + pool.mu.Lock() + defer pool.mu.Unlock() + + pending := make(map[common.Address][]*types.Transaction, len(pool.pending)) + for addr, list := range pool.pending { + txs := list.Flatten() + + // If the miner requests tip enforcement, cap the lists now + if enforceTips && !pool.locals.contains(addr) { + for i, tx := range txs { + if tx.EffectiveGasTipIntCmp(pool.gasTip.Load(), pool.priced.urgent.baseFee) < 0 { + txs = txs[:i] + break + } + } + } + if len(txs) > 0 { + pending[addr] = txs + } + } + return pending +} + +// Locals retrieves the accounts currently considered local by the pool. +func (pool *LegacyPool) Locals() []common.Address { + pool.mu.Lock() + defer pool.mu.Unlock() + + return pool.locals.flatten() +} + +// local retrieves all currently known local transactions, grouped by origin +// account and sorted by nonce. The returned transaction set is a copy and can be +// freely modified by calling code. +func (pool *LegacyPool) local() map[common.Address]types.Transactions { + txs := make(map[common.Address]types.Transactions) + for addr := range pool.locals.accounts { + if pending := pool.pending[addr]; pending != nil { + txs[addr] = append(txs[addr], pending.Flatten()...) + } + if queued := pool.queue[addr]; queued != nil { + txs[addr] = append(txs[addr], queued.Flatten()...) + } + } + return txs +} + +// validateTxBasics checks whether a transaction is valid according to the consensus +// rules, but does not check state-dependent validation such as sufficient balance. +// This check is meant as an early check which only needs to be performed once, +// and does not require the pool mutex to be held. +func (pool *LegacyPool) validateTxBasics(tx *types.Transaction, local bool) error { + opts := &txpool.ValidationOptions{ + Config: pool.chainconfig, + Accept: 0 | + 1< pool.config.GlobalSlots+pool.config.GlobalQueue { + // If the new transaction is underpriced, don't accept it + if !isLocal && pool.priced.Underpriced(tx) { + log.Trace("Discarding underpriced transaction", "hash", hash, "gasTipCap", tx.GasTipCap(), "gasFeeCap", tx.GasFeeCap()) + underpricedTxMeter.Mark(1) + return false, txpool.ErrUnderpriced + } + + // We're about to replace a transaction. The reorg does a more thorough + // analysis of what to remove and how, but it runs async. We don't want to + // do too many replacements between reorg-runs, so we cap the number of + // replacements to 25% of the slots + if pool.changesSinceReorg > int(pool.config.GlobalSlots/4) { + throttleTxMeter.Mark(1) + return false, ErrTxPoolOverflow + } + + // New transaction is better than our worse ones, make room for it. + // If it's a local transaction, forcibly discard all available transactions. + // Otherwise if we can't make enough room for new one, abort the operation. + drop, success := pool.priced.Discard(pool.all.Slots()-int(pool.config.GlobalSlots+pool.config.GlobalQueue)+numSlots(tx), isLocal) + + // Special case, we still can't make the room for the new remote one. + if !isLocal && !success { + log.Trace("Discarding overflown transaction", "hash", hash) + overflowedTxMeter.Mark(1) + return false, ErrTxPoolOverflow + } + + // If the new transaction is a future transaction it should never churn pending transactions + if !isLocal && pool.isGapped(from, tx) { + var replacesPending bool + for _, dropTx := range drop { + dropSender, _ := types.Sender(pool.signer, dropTx) + if list := pool.pending[dropSender]; list != nil && list.Contains(dropTx.Nonce()) { + replacesPending = true + break + } + } + // Add all transactions back to the priced queue + if replacesPending { + for _, dropTx := range drop { + pool.priced.Put(dropTx, false) + } + log.Trace("Discarding future transaction replacing pending tx", "hash", hash) + return false, txpool.ErrFutureReplacePending + } + } + + // Kick out the underpriced remote transactions. + for _, tx := range drop { + log.Trace("Discarding freshly underpriced transaction", "hash", tx.Hash(), "gasTipCap", tx.GasTipCap(), "gasFeeCap", tx.GasFeeCap()) + underpricedTxMeter.Mark(1) + dropped := pool.removeTx(tx.Hash(), false) + pool.changesSinceReorg += dropped + } + } + + // Try to replace an existing transaction in the pending pool + if list := pool.pending[from]; list != nil && list.Contains(tx.Nonce()) { + // Nonce already pending, check if required price bump is met + inserted, old := list.Add(tx, pool.config.PriceBump) + if !inserted { + pendingDiscardMeter.Mark(1) + return false, txpool.ErrReplaceUnderpriced + } + // New transaction is better, replace old one + if old != nil { + pool.all.Remove(old.Hash()) + pool.priced.Removed(1) + pendingReplaceMeter.Mark(1) + } + pool.all.Add(tx, isLocal) + pool.priced.Put(tx, isLocal) + pool.journalTx(from, tx) + pool.queueTxEvent(tx) + log.Trace("Pooled new executable transaction", "hash", hash, "from", from, "to", tx.To()) + + // Successful promotion, bump the heartbeat + pool.beats[from] = time.Now() + return old != nil, nil + } + // New transaction isn't replacing a pending one, push into queue + replaced, err = pool.enqueueTx(hash, tx, isLocal, true) + if err != nil { + return false, err + } + // Mark local addresses and journal local transactions + if local && !pool.locals.contains(from) { + log.Info("Setting new local account", "address", from) + pool.locals.add(from) + pool.priced.Removed(pool.all.RemoteToLocals(pool.locals)) // Migrate the remotes if it's marked as local first time. + } + if isLocal { + localGauge.Inc(1) + } + pool.journalTx(from, tx) + + log.Trace("Pooled new future transaction", "hash", hash, "from", from, "to", tx.To()) + return replaced, nil +} + +// isGapped reports whether the given transaction is immediately executable. +func (pool *LegacyPool) isGapped(from common.Address, tx *types.Transaction) bool { + // Short circuit if transaction falls within the scope of the pending list + // or matches the next pending nonce which can be promoted as an executable + // transaction afterwards. Note, the tx staleness is already checked in + // 'validateTx' function previously. + next := pool.pendingNonces.get(from) + if tx.Nonce() <= next { + return false + } + // The transaction has a nonce gap with pending list, it's only considered + // as executable if transactions in queue can fill up the nonce gap. + queue, ok := pool.queue[from] + if !ok { + return true + } + for nonce := next; nonce < tx.Nonce(); nonce++ { + if !queue.Contains(nonce) { + return true // txs in queue can't fill up the nonce gap + } + } + return false +} + +// enqueueTx inserts a new transaction into the non-executable transaction queue. +// +// Note, this method assumes the pool lock is held! +func (pool *LegacyPool) enqueueTx(hash common.Hash, tx *types.Transaction, local bool, addAll bool) (bool, error) { + // Try to insert the transaction into the future queue + from, _ := types.Sender(pool.signer, tx) // already validated + if pool.queue[from] == nil { + pool.queue[from] = newList(false) + } + inserted, old := pool.queue[from].Add(tx, pool.config.PriceBump) + if !inserted { + // An older transaction was better, discard this + queuedDiscardMeter.Mark(1) + return false, txpool.ErrReplaceUnderpriced + } + // Discard any previous transaction and mark this + if old != nil { + pool.all.Remove(old.Hash()) + pool.priced.Removed(1) + queuedReplaceMeter.Mark(1) + } else { + // Nothing was replaced, bump the queued counter + queuedGauge.Inc(1) + } + // If the transaction isn't in lookup set but it's expected to be there, + // show the error log. + if pool.all.Get(hash) == nil && !addAll { + log.Error("Missing transaction in lookup set, please report the issue", "hash", hash) + } + if addAll { + pool.all.Add(tx, local) + pool.priced.Put(tx, local) + } + // If we never record the heartbeat, do it right now. + if _, exist := pool.beats[from]; !exist { + pool.beats[from] = time.Now() + } + return old != nil, nil +} + +// journalTx adds the specified transaction to the local disk journal if it is +// deemed to have been sent from a local account. +func (pool *LegacyPool) journalTx(from common.Address, tx *types.Transaction) { + // Only journal if it's enabled and the transaction is local + if pool.journal == nil || !pool.locals.contains(from) { + return + } + if err := pool.journal.insert(tx); err != nil { + log.Warn("Failed to journal local transaction", "err", err) + } +} + +// promoteTx adds a transaction to the pending (processable) list of transactions +// and returns whether it was inserted or an older was better. +// +// Note, this method assumes the pool lock is held! +func (pool *LegacyPool) promoteTx(addr common.Address, hash common.Hash, tx *types.Transaction) bool { + // Try to insert the transaction into the pending queue + if pool.pending[addr] == nil { + pool.pending[addr] = newList(true) + } + list := pool.pending[addr] + + inserted, old := list.Add(tx, pool.config.PriceBump) + if !inserted { + // An older transaction was better, discard this + pool.all.Remove(hash) + pool.priced.Removed(1) + pendingDiscardMeter.Mark(1) + return false + } + // Otherwise discard any previous transaction and mark this + if old != nil { + pool.all.Remove(old.Hash()) + pool.priced.Removed(1) + pendingReplaceMeter.Mark(1) + } else { + // Nothing was replaced, bump the pending counter + pendingGauge.Inc(1) + } + // Set the potentially new pending nonce and notify any subsystems of the new tx + pool.pendingNonces.set(addr, tx.Nonce()+1) + + // Successful promotion, bump the heartbeat + pool.beats[addr] = time.Now() + return true +} + +// Add enqueues a batch of transactions into the pool if they are valid. Depending +// on the local flag, full pricing contraints will or will not be applied. +// +// If sync is set, the method will block until all internal maintenance related +// to the add is finished. Only use this during tests for determinism! +func (pool *LegacyPool) Add(txs []*txpool.Transaction, local bool, sync bool) []error { + unwrapped := make([]*types.Transaction, len(txs)) + for i, tx := range txs { + unwrapped[i] = tx.Tx + } + return pool.addTxs(unwrapped, local, sync) +} + +// addLocals enqueues a batch of transactions into the pool if they are valid, marking the +// senders as a local ones, ensuring they go around the local pricing constraints. +// +// This method is used to add transactions from the RPC API and performs synchronous pool +// reorganization and event propagation. +func (pool *LegacyPool) addLocals(txs []*types.Transaction) []error { + return pool.addTxs(txs, !pool.config.NoLocals, true) +} + +// addLocal enqueues a single local transaction into the pool if it is valid. This is +// a convenience wrapper around addLocals. +func (pool *LegacyPool) addLocal(tx *types.Transaction) error { + errs := pool.addLocals([]*types.Transaction{tx}) + return errs[0] +} + +// addRemotes enqueues a batch of transactions into the pool if they are valid. If the +// senders are not among the locally tracked ones, full pricing constraints will apply. +// +// This method is used to add transactions from the p2p network and does not wait for pool +// reorganization and internal event propagation. +func (pool *LegacyPool) addRemotes(txs []*types.Transaction) []error { + return pool.addTxs(txs, false, false) +} + +// addRemote enqueues a single transaction into the pool if it is valid. This is a convenience +// wrapper around addRemotes. +func (pool *LegacyPool) addRemote(tx *types.Transaction) error { + errs := pool.addRemotes([]*types.Transaction{tx}) + return errs[0] +} + +// addRemotesSync is like addRemotes, but waits for pool reorganization. Tests use this method. +func (pool *LegacyPool) addRemotesSync(txs []*types.Transaction) []error { + return pool.addTxs(txs, false, true) +} + +// This is like addRemotes with a single transaction, but waits for pool reorganization. Tests use this method. +func (pool *LegacyPool) addRemoteSync(tx *types.Transaction) error { + return pool.addTxs([]*types.Transaction{tx}, false, true)[0] +} + +// addTxs attempts to queue a batch of transactions if they are valid. +func (pool *LegacyPool) addTxs(txs []*types.Transaction, local, sync bool) []error { + // Filter out known ones without obtaining the pool lock or recovering signatures + var ( + errs = make([]error, len(txs)) + news = make([]*types.Transaction, 0, len(txs)) + ) + for i, tx := range txs { + // If the transaction is known, pre-set the error slot + if pool.all.Get(tx.Hash()) != nil { + errs[i] = ErrAlreadyKnown + knownTxMeter.Mark(1) + continue + } + // Exclude transactions with basic errors, e.g invalid signatures and + // insufficient intrinsic gas as soon as possible and cache senders + // in transactions before obtaining lock + if err := pool.validateTxBasics(tx, local); err != nil { + errs[i] = err + invalidTxMeter.Mark(1) + continue + } + // Accumulate all unknown transactions for deeper processing + news = append(news, tx) + } + if len(news) == 0 { + return errs + } + + // Process all the new transaction and merge any errors into the original slice + pool.mu.Lock() + newErrs, dirtyAddrs := pool.addTxsLocked(news, local) + pool.mu.Unlock() + + var nilSlot = 0 + for _, err := range newErrs { + for errs[nilSlot] != nil { + nilSlot++ + } + errs[nilSlot] = err + nilSlot++ + } + // Reorg the pool internals if needed and return + done := pool.requestPromoteExecutables(dirtyAddrs) + if sync { + <-done + } + return errs +} + +// addTxsLocked attempts to queue a batch of transactions if they are valid. +// The transaction pool lock must be held. +func (pool *LegacyPool) addTxsLocked(txs []*types.Transaction, local bool) ([]error, *accountSet) { + dirty := newAccountSet(pool.signer) + errs := make([]error, len(txs)) + for i, tx := range txs { + replaced, err := pool.add(tx, local) + errs[i] = err + if err == nil && !replaced { + dirty.addTx(tx) + } + } + validTxMeter.Mark(int64(len(dirty.accounts))) + return errs, dirty +} + +// Status returns the status (unknown/pending/queued) of a batch of transactions +// identified by their hashes. +func (pool *LegacyPool) Status(hash common.Hash) txpool.TxStatus { + tx := pool.get(hash) + if tx == nil { + return txpool.TxStatusUnknown + } + from, _ := types.Sender(pool.signer, tx) // already validated + + pool.mu.RLock() + defer pool.mu.RUnlock() + + if txList := pool.pending[from]; txList != nil && txList.txs.items[tx.Nonce()] != nil { + return txpool.TxStatusPending + } else if txList := pool.queue[from]; txList != nil && txList.txs.items[tx.Nonce()] != nil { + return txpool.TxStatusQueued + } + return txpool.TxStatusUnknown +} + +// Get returns a transaction if it is contained in the pool and nil otherwise. +func (pool *LegacyPool) Get(hash common.Hash) *txpool.Transaction { + tx := pool.get(hash) + if tx == nil { + return nil + } + return &txpool.Transaction{Tx: tx} +} + +// get returns a transaction if it is contained in the pool and nil otherwise. +func (pool *LegacyPool) get(hash common.Hash) *types.Transaction { + return pool.all.Get(hash) +} + +// Has returns an indicator whether txpool has a transaction cached with the +// given hash. +func (pool *LegacyPool) Has(hash common.Hash) bool { + return pool.all.Get(hash) != nil +} + +// removeTx removes a single transaction from the queue, moving all subsequent +// transactions back to the future queue. +// Returns the number of transactions removed from the pending queue. +func (pool *LegacyPool) removeTx(hash common.Hash, outofbound bool) int { + // Fetch the transaction we wish to delete + tx := pool.all.Get(hash) + if tx == nil { + return 0 + } + addr, _ := types.Sender(pool.signer, tx) // already validated during insertion + + // Remove it from the list of known transactions + pool.all.Remove(hash) + if outofbound { + pool.priced.Removed(1) + } + if pool.locals.contains(addr) { + localGauge.Dec(1) + } + // Remove the transaction from the pending lists and reset the account nonce + if pending := pool.pending[addr]; pending != nil { + if removed, invalids := pending.Remove(tx); removed { + // If no more pending transactions are left, remove the list + if pending.Empty() { + delete(pool.pending, addr) + } + // Postpone any invalidated transactions + for _, tx := range invalids { + // Internal shuffle shouldn't touch the lookup set. + pool.enqueueTx(tx.Hash(), tx, false, false) + } + // Update the account nonce if needed + pool.pendingNonces.setIfLower(addr, tx.Nonce()) + // Reduce the pending counter + pendingGauge.Dec(int64(1 + len(invalids))) + return 1 + len(invalids) + } + } + // Transaction is in the future queue + if future := pool.queue[addr]; future != nil { + if removed, _ := future.Remove(tx); removed { + // Reduce the queued counter + queuedGauge.Dec(1) + } + if future.Empty() { + delete(pool.queue, addr) + delete(pool.beats, addr) + } + } + return 0 +} + +// requestReset requests a pool reset to the new head block. +// The returned channel is closed when the reset has occurred. +func (pool *LegacyPool) requestReset(oldHead *types.Header, newHead *types.Header) chan struct{} { + select { + case pool.reqResetCh <- &txpoolResetRequest{oldHead, newHead}: + return <-pool.reorgDoneCh + case <-pool.reorgShutdownCh: + return pool.reorgShutdownCh + } +} + +// requestPromoteExecutables requests transaction promotion checks for the given addresses. +// The returned channel is closed when the promotion checks have occurred. +func (pool *LegacyPool) requestPromoteExecutables(set *accountSet) chan struct{} { + select { + case pool.reqPromoteCh <- set: + return <-pool.reorgDoneCh + case <-pool.reorgShutdownCh: + return pool.reorgShutdownCh + } +} + +// queueTxEvent enqueues a transaction event to be sent in the next reorg run. +func (pool *LegacyPool) queueTxEvent(tx *types.Transaction) { + select { + case pool.queueTxEventCh <- tx: + case <-pool.reorgShutdownCh: + } +} + +// scheduleReorgLoop schedules runs of reset and promoteExecutables. Code above should not +// call those methods directly, but request them being run using requestReset and +// requestPromoteExecutables instead. +func (pool *LegacyPool) scheduleReorgLoop() { + defer pool.wg.Done() + + var ( + curDone chan struct{} // non-nil while runReorg is active + nextDone = make(chan struct{}) + launchNextRun bool + reset *txpoolResetRequest + dirtyAccounts *accountSet + queuedEvents = make(map[common.Address]*sortedMap) + ) + for { + // Launch next background reorg if needed + if curDone == nil && launchNextRun { + // Run the background reorg and announcements + go pool.runReorg(nextDone, reset, dirtyAccounts, queuedEvents) + + // Prepare everything for the next round of reorg + curDone, nextDone = nextDone, make(chan struct{}) + launchNextRun = false + + reset, dirtyAccounts = nil, nil + queuedEvents = make(map[common.Address]*sortedMap) + } + + select { + case req := <-pool.reqResetCh: + // Reset request: update head if request is already pending. + if reset == nil { + reset = req + } else { + reset.newHead = req.newHead + } + launchNextRun = true + pool.reorgDoneCh <- nextDone + + case req := <-pool.reqPromoteCh: + // Promote request: update address set if request is already pending. + if dirtyAccounts == nil { + dirtyAccounts = req + } else { + dirtyAccounts.merge(req) + } + launchNextRun = true + pool.reorgDoneCh <- nextDone + + case tx := <-pool.queueTxEventCh: + // Queue up the event, but don't schedule a reorg. It's up to the caller to + // request one later if they want the events sent. + addr, _ := types.Sender(pool.signer, tx) + if _, ok := queuedEvents[addr]; !ok { + queuedEvents[addr] = newSortedMap() + } + queuedEvents[addr].Put(tx) + + case <-curDone: + curDone = nil + + case <-pool.reorgShutdownCh: + // Wait for current run to finish. + if curDone != nil { + <-curDone + } + close(nextDone) + return + } + } +} + +// runReorg runs reset and promoteExecutables on behalf of scheduleReorgLoop. +func (pool *LegacyPool) runReorg(done chan struct{}, reset *txpoolResetRequest, dirtyAccounts *accountSet, events map[common.Address]*sortedMap) { + defer func(t0 time.Time) { + reorgDurationTimer.Update(time.Since(t0)) + }(time.Now()) + defer close(done) + + var promoteAddrs []common.Address + if dirtyAccounts != nil && reset == nil { + // Only dirty accounts need to be promoted, unless we're resetting. + // For resets, all addresses in the tx queue will be promoted and + // the flatten operation can be avoided. + promoteAddrs = dirtyAccounts.flatten() + } + pool.mu.Lock() + if reset != nil { + // Reset from the old head to the new, rescheduling any reorged transactions + pool.reset(reset.oldHead, reset.newHead) + + // Nonces were reset, discard any events that became stale + for addr := range events { + events[addr].Forward(pool.pendingNonces.get(addr)) + if events[addr].Len() == 0 { + delete(events, addr) + } + } + // Reset needs promote for all addresses + promoteAddrs = make([]common.Address, 0, len(pool.queue)) + for addr := range pool.queue { + promoteAddrs = append(promoteAddrs, addr) + } + } + // Check for pending transactions for every account that sent new ones + promoted := pool.promoteExecutables(promoteAddrs) + + // If a new block appeared, validate the pool of pending transactions. This will + // remove any transaction that has been included in the block or was invalidated + // because of another transaction (e.g. higher gas price). + if reset != nil { + pool.demoteUnexecutables() + if reset.newHead != nil && pool.chainconfig.IsLondon(new(big.Int).Add(reset.newHead.Number, big.NewInt(1))) { + pendingBaseFee := misc.CalcBaseFee(pool.chainconfig, reset.newHead) + pool.priced.SetBaseFee(pendingBaseFee) + } + // Update all accounts to the latest known pending nonce + nonces := make(map[common.Address]uint64, len(pool.pending)) + for addr, list := range pool.pending { + highestPending := list.LastElement() + nonces[addr] = highestPending.Nonce() + 1 + } + pool.pendingNonces.setAll(nonces) + } + // Ensure pool.queue and pool.pending sizes stay within the configured limits. + pool.truncatePending() + pool.truncateQueue() + + dropBetweenReorgHistogram.Update(int64(pool.changesSinceReorg)) + pool.changesSinceReorg = 0 // Reset change counter + pool.mu.Unlock() + + // Notify subsystems for newly added transactions + for _, tx := range promoted { + addr, _ := types.Sender(pool.signer, tx) + if _, ok := events[addr]; !ok { + events[addr] = newSortedMap() + } + events[addr].Put(tx) + } + if len(events) > 0 { + var txs []*types.Transaction + for _, set := range events { + txs = append(txs, set.Flatten()...) + } + pool.txFeed.Send(core.NewTxsEvent{Txs: txs}) + } +} + +// reset retrieves the current state of the blockchain and ensures the content +// of the transaction pool is valid with regard to the chain state. +func (pool *LegacyPool) reset(oldHead, newHead *types.Header) { + // If we're reorging an old state, reinject all dropped transactions + var reinject types.Transactions + + if oldHead != nil && oldHead.Hash() != newHead.ParentHash { + // If the reorg is too deep, avoid doing it (will happen during fast sync) + oldNum := oldHead.Number.Uint64() + newNum := newHead.Number.Uint64() + + if depth := uint64(math.Abs(float64(oldNum) - float64(newNum))); depth > 64 { + log.Debug("Skipping deep transaction reorg", "depth", depth) + } else { + // Reorg seems shallow enough to pull in all transactions into memory + var discarded, included types.Transactions + var ( + rem = pool.chain.GetBlock(oldHead.Hash(), oldHead.Number.Uint64()) + add = pool.chain.GetBlock(newHead.Hash(), newHead.Number.Uint64()) + ) + if rem == nil { + // This can happen if a setHead is performed, where we simply discard the old + // head from the chain. + // If that is the case, we don't have the lost transactions anymore, and + // there's nothing to add + if newNum >= oldNum { + // If we reorged to a same or higher number, then it's not a case of setHead + log.Warn("Transaction pool reset with missing oldhead", + "old", oldHead.Hash(), "oldnum", oldNum, "new", newHead.Hash(), "newnum", newNum) + return + } + // If the reorg ended up on a lower number, it's indicative of setHead being the cause + log.Debug("Skipping transaction reset caused by setHead", + "old", oldHead.Hash(), "oldnum", oldNum, "new", newHead.Hash(), "newnum", newNum) + // We still need to update the current state s.th. the lost transactions can be readded by the user + } else { + for rem.NumberU64() > add.NumberU64() { + discarded = append(discarded, rem.Transactions()...) + if rem = pool.chain.GetBlock(rem.ParentHash(), rem.NumberU64()-1); rem == nil { + log.Error("Unrooted old chain seen by tx pool", "block", oldHead.Number, "hash", oldHead.Hash()) + return + } + } + for add.NumberU64() > rem.NumberU64() { + included = append(included, add.Transactions()...) + if add = pool.chain.GetBlock(add.ParentHash(), add.NumberU64()-1); add == nil { + log.Error("Unrooted new chain seen by tx pool", "block", newHead.Number, "hash", newHead.Hash()) + return + } + } + for rem.Hash() != add.Hash() { + discarded = append(discarded, rem.Transactions()...) + if rem = pool.chain.GetBlock(rem.ParentHash(), rem.NumberU64()-1); rem == nil { + log.Error("Unrooted old chain seen by tx pool", "block", oldHead.Number, "hash", oldHead.Hash()) + return + } + included = append(included, add.Transactions()...) + if add = pool.chain.GetBlock(add.ParentHash(), add.NumberU64()-1); add == nil { + log.Error("Unrooted new chain seen by tx pool", "block", newHead.Number, "hash", newHead.Hash()) + return + } + } + reinject = types.TxDifference(discarded, included) + } + } + } + // Initialize the internal state to the current head + if newHead == nil { + newHead = pool.chain.CurrentBlock() // Special case during testing + } + statedb, err := pool.chain.StateAt(newHead.Root) + if err != nil { + log.Error("Failed to reset txpool state", "err", err) + return + } + pool.currentHead.Store(newHead) + pool.currentState = statedb + pool.pendingNonces = newNoncer(statedb) + + // Inject any transactions discarded due to reorgs + log.Debug("Reinjecting stale transactions", "count", len(reinject)) + core.SenderCacher.Recover(pool.signer, reinject) + pool.addTxsLocked(reinject, false) +} + +// promoteExecutables moves transactions that have become processable from the +// future queue to the set of pending transactions. During this process, all +// invalidated transactions (low nonce, low balance) are deleted. +func (pool *LegacyPool) promoteExecutables(accounts []common.Address) []*types.Transaction { + // Track the promoted transactions to broadcast them at once + var promoted []*types.Transaction + + // Iterate over all accounts and promote any executable transactions + gasLimit := pool.currentHead.Load().GasLimit + for _, addr := range accounts { + list := pool.queue[addr] + if list == nil { + continue // Just in case someone calls with a non existing account + } + // Drop all transactions that are deemed too old (low nonce) + forwards := list.Forward(pool.currentState.GetNonce(addr)) + for _, tx := range forwards { + hash := tx.Hash() + pool.all.Remove(hash) + } + log.Trace("Removed old queued transactions", "count", len(forwards)) + // Drop all transactions that are too costly (low balance or out of gas) + drops, _ := list.Filter(pool.currentState.GetBalance(addr), gasLimit) + for _, tx := range drops { + hash := tx.Hash() + pool.all.Remove(hash) + } + log.Trace("Removed unpayable queued transactions", "count", len(drops)) + queuedNofundsMeter.Mark(int64(len(drops))) + + // Gather all executable transactions and promote them + readies := list.Ready(pool.pendingNonces.get(addr)) + for _, tx := range readies { + hash := tx.Hash() + if pool.promoteTx(addr, hash, tx) { + promoted = append(promoted, tx) + } + } + log.Trace("Promoted queued transactions", "count", len(promoted)) + queuedGauge.Dec(int64(len(readies))) + + // Drop all transactions over the allowed limit + var caps types.Transactions + if !pool.locals.contains(addr) { + caps = list.Cap(int(pool.config.AccountQueue)) + for _, tx := range caps { + hash := tx.Hash() + pool.all.Remove(hash) + log.Trace("Removed cap-exceeding queued transaction", "hash", hash) + } + queuedRateLimitMeter.Mark(int64(len(caps))) + } + // Mark all the items dropped as removed + pool.priced.Removed(len(forwards) + len(drops) + len(caps)) + queuedGauge.Dec(int64(len(forwards) + len(drops) + len(caps))) + if pool.locals.contains(addr) { + localGauge.Dec(int64(len(forwards) + len(drops) + len(caps))) + } + // Delete the entire queue entry if it became empty. + if list.Empty() { + delete(pool.queue, addr) + delete(pool.beats, addr) + } + } + return promoted +} + +// truncatePending removes transactions from the pending queue if the pool is above the +// pending limit. The algorithm tries to reduce transaction counts by an approximately +// equal number for all for accounts with many pending transactions. +func (pool *LegacyPool) truncatePending() { + pending := uint64(0) + for _, list := range pool.pending { + pending += uint64(list.Len()) + } + if pending <= pool.config.GlobalSlots { + return + } + + pendingBeforeCap := pending + // Assemble a spam order to penalize large transactors first + spammers := prque.New[int64, common.Address](nil) + for addr, list := range pool.pending { + // Only evict transactions from high rollers + if !pool.locals.contains(addr) && uint64(list.Len()) > pool.config.AccountSlots { + spammers.Push(addr, int64(list.Len())) + } + } + // Gradually drop transactions from offenders + offenders := []common.Address{} + for pending > pool.config.GlobalSlots && !spammers.Empty() { + // Retrieve the next offender if not local address + offender, _ := spammers.Pop() + offenders = append(offenders, offender) + + // Equalize balances until all the same or below threshold + if len(offenders) > 1 { + // Calculate the equalization threshold for all current offenders + threshold := pool.pending[offender].Len() + + // Iteratively reduce all offenders until below limit or threshold reached + for pending > pool.config.GlobalSlots && pool.pending[offenders[len(offenders)-2]].Len() > threshold { + for i := 0; i < len(offenders)-1; i++ { + list := pool.pending[offenders[i]] + + caps := list.Cap(list.Len() - 1) + for _, tx := range caps { + // Drop the transaction from the global pools too + hash := tx.Hash() + pool.all.Remove(hash) + + // Update the account nonce to the dropped transaction + pool.pendingNonces.setIfLower(offenders[i], tx.Nonce()) + log.Trace("Removed fairness-exceeding pending transaction", "hash", hash) + } + pool.priced.Removed(len(caps)) + pendingGauge.Dec(int64(len(caps))) + if pool.locals.contains(offenders[i]) { + localGauge.Dec(int64(len(caps))) + } + pending-- + } + } + } + } + + // If still above threshold, reduce to limit or min allowance + if pending > pool.config.GlobalSlots && len(offenders) > 0 { + for pending > pool.config.GlobalSlots && uint64(pool.pending[offenders[len(offenders)-1]].Len()) > pool.config.AccountSlots { + for _, addr := range offenders { + list := pool.pending[addr] + + caps := list.Cap(list.Len() - 1) + for _, tx := range caps { + // Drop the transaction from the global pools too + hash := tx.Hash() + pool.all.Remove(hash) + + // Update the account nonce to the dropped transaction + pool.pendingNonces.setIfLower(addr, tx.Nonce()) + log.Trace("Removed fairness-exceeding pending transaction", "hash", hash) + } + pool.priced.Removed(len(caps)) + pendingGauge.Dec(int64(len(caps))) + if pool.locals.contains(addr) { + localGauge.Dec(int64(len(caps))) + } + pending-- + } + } + } + pendingRateLimitMeter.Mark(int64(pendingBeforeCap - pending)) +} + +// truncateQueue drops the oldest transactions in the queue if the pool is above the global queue limit. +func (pool *LegacyPool) truncateQueue() { + queued := uint64(0) + for _, list := range pool.queue { + queued += uint64(list.Len()) + } + if queued <= pool.config.GlobalQueue { + return + } + + // Sort all accounts with queued transactions by heartbeat + addresses := make(addressesByHeartbeat, 0, len(pool.queue)) + for addr := range pool.queue { + if !pool.locals.contains(addr) { // don't drop locals + addresses = append(addresses, addressByHeartbeat{addr, pool.beats[addr]}) + } + } + sort.Sort(sort.Reverse(addresses)) + + // Drop transactions until the total is below the limit or only locals remain + for drop := queued - pool.config.GlobalQueue; drop > 0 && len(addresses) > 0; { + addr := addresses[len(addresses)-1] + list := pool.queue[addr.address] + + addresses = addresses[:len(addresses)-1] + + // Drop all transactions if they are less than the overflow + if size := uint64(list.Len()); size <= drop { + for _, tx := range list.Flatten() { + pool.removeTx(tx.Hash(), true) + } + drop -= size + queuedRateLimitMeter.Mark(int64(size)) + continue + } + // Otherwise drop only last few transactions + txs := list.Flatten() + for i := len(txs) - 1; i >= 0 && drop > 0; i-- { + pool.removeTx(txs[i].Hash(), true) + drop-- + queuedRateLimitMeter.Mark(1) + } + } +} + +// demoteUnexecutables removes invalid and processed transactions from the pools +// executable/pending queue and any subsequent transactions that become unexecutable +// are moved back into the future queue. +// +// Note: transactions are not marked as removed in the priced list because re-heaping +// is always explicitly triggered by SetBaseFee and it would be unnecessary and wasteful +// to trigger a re-heap is this function +func (pool *LegacyPool) demoteUnexecutables() { + // Iterate over all accounts and demote any non-executable transactions + gasLimit := pool.currentHead.Load().GasLimit + for addr, list := range pool.pending { + nonce := pool.currentState.GetNonce(addr) + + // Drop all transactions that are deemed too old (low nonce) + olds := list.Forward(nonce) + for _, tx := range olds { + hash := tx.Hash() + pool.all.Remove(hash) + log.Trace("Removed old pending transaction", "hash", hash) + } + // Drop all transactions that are too costly (low balance or out of gas), and queue any invalids back for later + drops, invalids := list.Filter(pool.currentState.GetBalance(addr), gasLimit) + for _, tx := range drops { + hash := tx.Hash() + log.Trace("Removed unpayable pending transaction", "hash", hash) + pool.all.Remove(hash) + } + pendingNofundsMeter.Mark(int64(len(drops))) + + for _, tx := range invalids { + hash := tx.Hash() + log.Trace("Demoting pending transaction", "hash", hash) + + // Internal shuffle shouldn't touch the lookup set. + pool.enqueueTx(hash, tx, false, false) + } + pendingGauge.Dec(int64(len(olds) + len(drops) + len(invalids))) + if pool.locals.contains(addr) { + localGauge.Dec(int64(len(olds) + len(drops) + len(invalids))) + } + // If there's a gap in front, alert (should never happen) and postpone all transactions + if list.Len() > 0 && list.txs.Get(nonce) == nil { + gapped := list.Cap(0) + for _, tx := range gapped { + hash := tx.Hash() + log.Error("Demoting invalidated transaction", "hash", hash) + + // Internal shuffle shouldn't touch the lookup set. + pool.enqueueTx(hash, tx, false, false) + } + pendingGauge.Dec(int64(len(gapped))) + } + // Delete the entire pending entry if it became empty. + if list.Empty() { + delete(pool.pending, addr) + } + } +} + +// addressByHeartbeat is an account address tagged with its last activity timestamp. +type addressByHeartbeat struct { + address common.Address + heartbeat time.Time +} + +type addressesByHeartbeat []addressByHeartbeat + +func (a addressesByHeartbeat) Len() int { return len(a) } +func (a addressesByHeartbeat) Less(i, j int) bool { return a[i].heartbeat.Before(a[j].heartbeat) } +func (a addressesByHeartbeat) Swap(i, j int) { a[i], a[j] = a[j], a[i] } + +// accountSet is simply a set of addresses to check for existence, and a signer +// capable of deriving addresses from transactions. +type accountSet struct { + accounts map[common.Address]struct{} + signer types.Signer + cache *[]common.Address +} + +// newAccountSet creates a new address set with an associated signer for sender +// derivations. +func newAccountSet(signer types.Signer, addrs ...common.Address) *accountSet { + as := &accountSet{ + accounts: make(map[common.Address]struct{}, len(addrs)), + signer: signer, + } + for _, addr := range addrs { + as.add(addr) + } + return as +} + +// contains checks if a given address is contained within the set. +func (as *accountSet) contains(addr common.Address) bool { + _, exist := as.accounts[addr] + return exist +} + +// containsTx checks if the sender of a given tx is within the set. If the sender +// cannot be derived, this method returns false. +func (as *accountSet) containsTx(tx *types.Transaction) bool { + if addr, err := types.Sender(as.signer, tx); err == nil { + return as.contains(addr) + } + return false +} + +// add inserts a new address into the set to track. +func (as *accountSet) add(addr common.Address) { + as.accounts[addr] = struct{}{} + as.cache = nil +} + +// addTx adds the sender of tx into the set. +func (as *accountSet) addTx(tx *types.Transaction) { + if addr, err := types.Sender(as.signer, tx); err == nil { + as.add(addr) + } +} + +// flatten returns the list of addresses within this set, also caching it for later +// reuse. The returned slice should not be changed! +func (as *accountSet) flatten() []common.Address { + if as.cache == nil { + accounts := make([]common.Address, 0, len(as.accounts)) + for account := range as.accounts { + accounts = append(accounts, account) + } + as.cache = &accounts + } + return *as.cache +} + +// merge adds all addresses from the 'other' set into 'as'. +func (as *accountSet) merge(other *accountSet) { + for addr := range other.accounts { + as.accounts[addr] = struct{}{} + } + as.cache = nil +} + +// lookup is used internally by LegacyPool to track transactions while allowing +// lookup without mutex contention. +// +// Note, although this type is properly protected against concurrent access, it +// is **not** a type that should ever be mutated or even exposed outside of the +// transaction pool, since its internal state is tightly coupled with the pools +// internal mechanisms. The sole purpose of the type is to permit out-of-bound +// peeking into the pool in LegacyPool.Get without having to acquire the widely scoped +// LegacyPool.mu mutex. +// +// This lookup set combines the notion of "local transactions", which is useful +// to build upper-level structure. +type lookup struct { + slots int + lock sync.RWMutex + locals map[common.Hash]*types.Transaction + remotes map[common.Hash]*types.Transaction +} + +// newLookup returns a new lookup structure. +func newLookup() *lookup { + return &lookup{ + locals: make(map[common.Hash]*types.Transaction), + remotes: make(map[common.Hash]*types.Transaction), + } +} + +// Range calls f on each key and value present in the map. The callback passed +// should return the indicator whether the iteration needs to be continued. +// Callers need to specify which set (or both) to be iterated. +func (t *lookup) Range(f func(hash common.Hash, tx *types.Transaction, local bool) bool, local bool, remote bool) { + t.lock.RLock() + defer t.lock.RUnlock() + + if local { + for key, value := range t.locals { + if !f(key, value, true) { + return + } + } + } + if remote { + for key, value := range t.remotes { + if !f(key, value, false) { + return + } + } + } +} + +// Get returns a transaction if it exists in the lookup, or nil if not found. +func (t *lookup) Get(hash common.Hash) *types.Transaction { + t.lock.RLock() + defer t.lock.RUnlock() + + if tx := t.locals[hash]; tx != nil { + return tx + } + return t.remotes[hash] +} + +// GetLocal returns a transaction if it exists in the lookup, or nil if not found. +func (t *lookup) GetLocal(hash common.Hash) *types.Transaction { + t.lock.RLock() + defer t.lock.RUnlock() + + return t.locals[hash] +} + +// GetRemote returns a transaction if it exists in the lookup, or nil if not found. +func (t *lookup) GetRemote(hash common.Hash) *types.Transaction { + t.lock.RLock() + defer t.lock.RUnlock() + + return t.remotes[hash] +} + +// Count returns the current number of transactions in the lookup. +func (t *lookup) Count() int { + t.lock.RLock() + defer t.lock.RUnlock() + + return len(t.locals) + len(t.remotes) +} + +// LocalCount returns the current number of local transactions in the lookup. +func (t *lookup) LocalCount() int { + t.lock.RLock() + defer t.lock.RUnlock() + + return len(t.locals) +} + +// RemoteCount returns the current number of remote transactions in the lookup. +func (t *lookup) RemoteCount() int { + t.lock.RLock() + defer t.lock.RUnlock() + + return len(t.remotes) +} + +// Slots returns the current number of slots used in the lookup. +func (t *lookup) Slots() int { + t.lock.RLock() + defer t.lock.RUnlock() + + return t.slots +} + +// Add adds a transaction to the lookup. +func (t *lookup) Add(tx *types.Transaction, local bool) { + t.lock.Lock() + defer t.lock.Unlock() + + t.slots += numSlots(tx) + slotsGauge.Update(int64(t.slots)) + + if local { + t.locals[tx.Hash()] = tx + } else { + t.remotes[tx.Hash()] = tx + } +} + +// Remove removes a transaction from the lookup. +func (t *lookup) Remove(hash common.Hash) { + t.lock.Lock() + defer t.lock.Unlock() + + tx, ok := t.locals[hash] + if !ok { + tx, ok = t.remotes[hash] + } + if !ok { + log.Error("No transaction found to be deleted", "hash", hash) + return + } + t.slots -= numSlots(tx) + slotsGauge.Update(int64(t.slots)) + + delete(t.locals, hash) + delete(t.remotes, hash) +} + +// RemoteToLocals migrates the transactions belongs to the given locals to locals +// set. The assumption is held the locals set is thread-safe to be used. +func (t *lookup) RemoteToLocals(locals *accountSet) int { + t.lock.Lock() + defer t.lock.Unlock() + + var migrated int + for hash, tx := range t.remotes { + if locals.containsTx(tx) { + t.locals[hash] = tx + delete(t.remotes, hash) + migrated += 1 + } + } + return migrated +} + +// RemotesBelowTip finds all remote transactions below the given tip threshold. +func (t *lookup) RemotesBelowTip(threshold *big.Int) types.Transactions { + found := make(types.Transactions, 0, 128) + t.Range(func(hash common.Hash, tx *types.Transaction, local bool) bool { + if tx.GasTipCapIntCmp(threshold) < 0 { + found = append(found, tx) + } + return true + }, false, true) // Only iterate remotes + return found +} + +// numSlots calculates the number of slots needed for a single transaction. +func numSlots(tx *types.Transaction) int { + return int((tx.Size() + txSlotSize - 1) / txSlotSize) +} diff --git a/core/txpool/txpool2_test.go b/core/txpool/legacypool/legacypool2_test.go similarity index 84% rename from core/txpool/txpool2_test.go rename to core/txpool/legacypool/legacypool2_test.go index b679050562..5de34588af 100644 --- a/core/txpool/txpool2_test.go +++ b/core/txpool/legacypool/legacypool2_test.go @@ -13,7 +13,7 @@ // // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package txpool +package legacypool import ( "crypto/ecdsa" @@ -33,7 +33,7 @@ func pricedValuedTransaction(nonce uint64, value int64, gaslimit uint64, gaspric return tx } -func count(t *testing.T, pool *TxPool) (pending int, queued int) { +func count(t *testing.T, pool *LegacyPool) (pending int, queued int) { t.Helper() pending, queued = pool.stats() if err := validatePoolInternals(pool); err != nil { @@ -42,7 +42,7 @@ func count(t *testing.T, pool *TxPool) (pending int, queued int) { return pending, queued } -func fillPool(t testing.TB, pool *TxPool) { +func fillPool(t testing.TB, pool *LegacyPool) { t.Helper() // Create a number of test accounts, fund them and make transactions executableTxs := types.Transactions{} @@ -56,8 +56,8 @@ func fillPool(t testing.TB, pool *TxPool) { } } // Import the batch and verify that limits have been enforced - pool.AddRemotesSync(executableTxs) - pool.AddRemotesSync(nonExecutableTxs) + pool.addRemotesSync(executableTxs) + pool.addRemotesSync(nonExecutableTxs) pending, queued := pool.Stats() slots := pool.all.Slots() // sanity-check that the test prerequisites are ok (pending full) @@ -79,12 +79,13 @@ func TestTransactionFutureAttack(t *testing.T) { // Create the pool to test the limit enforcement with statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - blockchain := newTestBlockChain(1000000, statedb, new(event.Feed)) + blockchain := newTestBlockChain(eip1559Config, 1000000, statedb, new(event.Feed)) config := testTxPoolConfig config.GlobalQueue = 100 config.GlobalSlots = 100 - pool := New(config, eip1559Config, blockchain) - defer pool.Stop() + pool := New(config, blockchain) + pool.Init(new(big.Int).SetUint64(config.PriceLimit), blockchain.CurrentBlock()) + defer pool.Close() fillPool(t, pool) pending, _ := pool.Stats() // Now, future transaction attack starts, let's add a bunch of expensive non-executables, and see if the pending-count drops @@ -96,7 +97,7 @@ func TestTransactionFutureAttack(t *testing.T) { futureTxs = append(futureTxs, pricedTransaction(1000+uint64(j), 100000, big.NewInt(500), key)) } for i := 0; i < 5; i++ { - pool.AddRemotesSync(futureTxs) + pool.addRemotesSync(futureTxs) newPending, newQueued := count(t, pool) t.Logf("pending: %d queued: %d, all: %d\n", newPending, newQueued, pool.all.Slots()) } @@ -115,9 +116,10 @@ func TestTransactionFuture1559(t *testing.T) { t.Parallel() // Create the pool to test the pricing enforcement with statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - blockchain := newTestBlockChain(1000000, statedb, new(event.Feed)) - pool := New(testTxPoolConfig, eip1559Config, blockchain) - defer pool.Stop() + blockchain := newTestBlockChain(eip1559Config, 1000000, statedb, new(event.Feed)) + pool := New(testTxPoolConfig, blockchain) + pool.Init(new(big.Int).SetUint64(testTxPoolConfig.PriceLimit), blockchain.CurrentBlock()) + defer pool.Close() // Create a number of test accounts, fund them and make transactions fillPool(t, pool) @@ -131,7 +133,7 @@ func TestTransactionFuture1559(t *testing.T) { for j := 0; j < int(pool.config.GlobalSlots+pool.config.GlobalQueue); j++ { futureTxs = append(futureTxs, dynamicFeeTx(1000+uint64(j), 100000, big.NewInt(200), big.NewInt(101), key)) } - pool.AddRemotesSync(futureTxs) + pool.addRemotesSync(futureTxs) } newPending, _ := pool.Stats() // Pending should not have been touched @@ -147,9 +149,10 @@ func TestTransactionZAttack(t *testing.T) { t.Parallel() // Create the pool to test the pricing enforcement with statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - blockchain := newTestBlockChain(1000000, statedb, new(event.Feed)) - pool := New(testTxPoolConfig, eip1559Config, blockchain) - defer pool.Stop() + blockchain := newTestBlockChain(eip1559Config, 1000000, statedb, new(event.Feed)) + pool := New(testTxPoolConfig, blockchain) + pool.Init(new(big.Int).SetUint64(testTxPoolConfig.PriceLimit), blockchain.CurrentBlock()) + defer pool.Close() // Create a number of test accounts, fund them and make transactions fillPool(t, pool) @@ -181,7 +184,7 @@ func TestTransactionZAttack(t *testing.T) { key, _ := crypto.GenerateKey() pool.currentState.AddBalance(crypto.PubkeyToAddress(key.PublicKey), big.NewInt(100000000000)) futureTxs = append(futureTxs, pricedTransaction(1000+uint64(j), 21000, big.NewInt(500), key)) - pool.AddRemotesSync(futureTxs) + pool.addRemotesSync(futureTxs) } overDraftTxs := types.Transactions{} @@ -192,11 +195,11 @@ func TestTransactionZAttack(t *testing.T) { overDraftTxs = append(overDraftTxs, pricedValuedTransaction(uint64(j), 600000000000, 21000, big.NewInt(500), key)) } } - pool.AddRemotesSync(overDraftTxs) - pool.AddRemotesSync(overDraftTxs) - pool.AddRemotesSync(overDraftTxs) - pool.AddRemotesSync(overDraftTxs) - pool.AddRemotesSync(overDraftTxs) + pool.addRemotesSync(overDraftTxs) + pool.addRemotesSync(overDraftTxs) + pool.addRemotesSync(overDraftTxs) + pool.addRemotesSync(overDraftTxs) + pool.addRemotesSync(overDraftTxs) newPending, newQueued := count(t, pool) newIvPending := countInvalidPending() @@ -214,12 +217,13 @@ func TestTransactionZAttack(t *testing.T) { func BenchmarkFutureAttack(b *testing.B) { // Create the pool to test the limit enforcement with statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - blockchain := newTestBlockChain(1000000, statedb, new(event.Feed)) + blockchain := newTestBlockChain(eip1559Config, 1000000, statedb, new(event.Feed)) config := testTxPoolConfig config.GlobalQueue = 100 config.GlobalSlots = 100 - pool := New(config, eip1559Config, blockchain) - defer pool.Stop() + pool := New(config, blockchain) + pool.Init(new(big.Int).SetUint64(testTxPoolConfig.PriceLimit), blockchain.CurrentBlock()) + defer pool.Close() fillPool(b, pool) key, _ := crypto.GenerateKey() @@ -231,6 +235,6 @@ func BenchmarkFutureAttack(b *testing.B) { } b.ResetTimer() for i := 0; i < 5; i++ { - pool.AddRemotesSync(futureTxs) + pool.addRemotesSync(futureTxs) } } diff --git a/core/txpool/txpool_test.go b/core/txpool/legacypool/legacypool_test.go similarity index 87% rename from core/txpool/txpool_test.go rename to core/txpool/legacypool/legacypool_test.go index 319e25bead..e3302487b3 100644 --- a/core/txpool/txpool_test.go +++ b/core/txpool/legacypool/legacypool_test.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package txpool +package legacypool import ( "crypto/ecdsa" @@ -32,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/event" @@ -59,17 +60,22 @@ func init() { } type testBlockChain struct { + config *params.ChainConfig gasLimit atomic.Uint64 statedb *state.StateDB chainHeadFeed *event.Feed } -func newTestBlockChain(gasLimit uint64, statedb *state.StateDB, chainHeadFeed *event.Feed) *testBlockChain { - bc := testBlockChain{statedb: statedb, chainHeadFeed: new(event.Feed)} +func newTestBlockChain(config *params.ChainConfig, gasLimit uint64, statedb *state.StateDB, chainHeadFeed *event.Feed) *testBlockChain { + bc := testBlockChain{config: config, statedb: statedb, chainHeadFeed: new(event.Feed)} bc.gasLimit.Store(gasLimit) return &bc } +func (bc *testBlockChain) Config() *params.ChainConfig { + return bc.config +} + func (bc *testBlockChain) CurrentBlock() *types.Header { return &types.Header{ Number: new(big.Int), @@ -121,24 +127,26 @@ func dynamicFeeTx(nonce uint64, gaslimit uint64, gasFee *big.Int, tip *big.Int, return tx } -func setupPool() (*TxPool, *ecdsa.PrivateKey) { +func setupPool() (*LegacyPool, *ecdsa.PrivateKey) { return setupPoolWithConfig(params.TestChainConfig) } -func setupPoolWithConfig(config *params.ChainConfig) (*TxPool, *ecdsa.PrivateKey) { +func setupPoolWithConfig(config *params.ChainConfig) (*LegacyPool, *ecdsa.PrivateKey) { statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - blockchain := newTestBlockChain(10000000, statedb, new(event.Feed)) + blockchain := newTestBlockChain(config, 10000000, statedb, new(event.Feed)) key, _ := crypto.GenerateKey() - pool := New(testTxPoolConfig, config, blockchain) - + pool := New(testTxPoolConfig, blockchain) + if err := pool.Init(new(big.Int).SetUint64(testTxPoolConfig.PriceLimit), blockchain.CurrentBlock()); err != nil { + panic(err) + } // wait for the pool to initialize <-pool.initDoneCh return pool, key } // validatePoolInternals checks various consistency invariants within the pool. -func validatePoolInternals(pool *TxPool) error { +func validatePoolInternals(pool *LegacyPool) error { pool.mu.RLock() defer pool.mu.RUnlock() @@ -242,20 +250,21 @@ func TestStateChangeDuringReset(t *testing.T) { // setup pool with 2 transaction in it statedb.SetBalance(address, new(big.Int).SetUint64(params.Ether)) - blockchain := &testChain{newTestBlockChain(1000000000, statedb, new(event.Feed)), address, &trigger} + blockchain := &testChain{newTestBlockChain(params.TestChainConfig, 1000000000, statedb, new(event.Feed)), address, &trigger} tx0 := transaction(0, 100000, key) tx1 := transaction(1, 100000, key) - pool := New(testTxPoolConfig, params.TestChainConfig, blockchain) - defer pool.Stop() + pool := New(testTxPoolConfig, blockchain) + pool.Init(new(big.Int).SetUint64(testTxPoolConfig.PriceLimit), blockchain.CurrentBlock()) + defer pool.Close() nonce := pool.Nonce(address) if nonce != 0 { t.Fatalf("Invalid nonce, want 0, got %d", nonce) } - pool.AddRemotesSync([]*types.Transaction{tx0, tx1}) + pool.addRemotesSync([]*types.Transaction{tx0, tx1}) nonce = pool.Nonce(address) if nonce != 2 { @@ -272,13 +281,13 @@ func TestStateChangeDuringReset(t *testing.T) { } } -func testAddBalance(pool *TxPool, addr common.Address, amount *big.Int) { +func testAddBalance(pool *LegacyPool, addr common.Address, amount *big.Int) { pool.mu.Lock() pool.currentState.AddBalance(addr, amount) pool.mu.Unlock() } -func testSetNonce(pool *TxPool, addr common.Address, nonce uint64) { +func testSetNonce(pool *LegacyPool, addr common.Address, nonce uint64) { pool.mu.Lock() pool.currentState.SetNonce(addr, nonce) pool.mu.Unlock() @@ -288,36 +297,36 @@ func TestInvalidTransactions(t *testing.T) { t.Parallel() pool, key := setupPool() - defer pool.Stop() + defer pool.Close() tx := transaction(0, 100, key) from, _ := deriveSender(tx) // Intrinsic gas too low testAddBalance(pool, from, big.NewInt(1)) - if err, want := pool.AddRemote(tx), core.ErrIntrinsicGas; !errors.Is(err, want) { + if err, want := pool.addRemote(tx), core.ErrIntrinsicGas; !errors.Is(err, want) { t.Errorf("want %v have %v", want, err) } // Insufficient funds tx = transaction(0, 100000, key) - if err, want := pool.AddRemote(tx), core.ErrInsufficientFunds; !errors.Is(err, want) { + if err, want := pool.addRemote(tx), core.ErrInsufficientFunds; !errors.Is(err, want) { t.Errorf("want %v have %v", want, err) } testSetNonce(pool, from, 1) testAddBalance(pool, from, big.NewInt(0xffffffffffffff)) tx = transaction(0, 100000, key) - if err, want := pool.AddRemote(tx), core.ErrNonceTooLow; !errors.Is(err, want) { + if err, want := pool.addRemote(tx), core.ErrNonceTooLow; !errors.Is(err, want) { t.Errorf("want %v have %v", want, err) } tx = transaction(1, 100000, key) pool.gasTip.Store(big.NewInt(1000)) - if err, want := pool.AddRemote(tx), ErrUnderpriced; !errors.Is(err, want) { + if err, want := pool.addRemote(tx), txpool.ErrUnderpriced; !errors.Is(err, want) { t.Errorf("want %v have %v", want, err) } - if err := pool.AddLocal(tx); err != nil { + if err := pool.addLocal(tx); err != nil { t.Error("expected", nil, "got", err) } } @@ -326,7 +335,7 @@ func TestQueue(t *testing.T) { t.Parallel() pool, key := setupPool() - defer pool.Stop() + defer pool.Close() tx := transaction(0, 100, key) from, _ := deriveSender(tx) @@ -357,7 +366,7 @@ func TestQueue2(t *testing.T) { t.Parallel() pool, key := setupPool() - defer pool.Stop() + defer pool.Close() tx1 := transaction(0, 100, key) tx2 := transaction(10, 100, key) @@ -383,13 +392,13 @@ func TestNegativeValue(t *testing.T) { t.Parallel() pool, key := setupPool() - defer pool.Stop() + defer pool.Close() tx, _ := types.SignTx(types.NewTransaction(0, common.Address{}, big.NewInt(-1), 100, big.NewInt(1), nil), types.HomesteadSigner{}, key) from, _ := deriveSender(tx) testAddBalance(pool, from, big.NewInt(1)) - if err := pool.AddRemote(tx); err != ErrNegativeValue { - t.Error("expected", ErrNegativeValue, "got", err) + if err := pool.addRemote(tx); err != txpool.ErrNegativeValue { + t.Error("expected", txpool.ErrNegativeValue, "got", err) } } @@ -397,11 +406,11 @@ func TestTipAboveFeeCap(t *testing.T) { t.Parallel() pool, key := setupPoolWithConfig(eip1559Config) - defer pool.Stop() + defer pool.Close() tx := dynamicFeeTx(0, 100, big.NewInt(1), big.NewInt(2), key) - if err := pool.AddRemote(tx); err != core.ErrTipAboveFeeCap { + if err := pool.addRemote(tx); err != core.ErrTipAboveFeeCap { t.Error("expected", core.ErrTipAboveFeeCap, "got", err) } } @@ -410,18 +419,18 @@ func TestVeryHighValues(t *testing.T) { t.Parallel() pool, key := setupPoolWithConfig(eip1559Config) - defer pool.Stop() + defer pool.Close() veryBigNumber := big.NewInt(1) veryBigNumber.Lsh(veryBigNumber, 300) tx := dynamicFeeTx(0, 100, big.NewInt(1), veryBigNumber, key) - if err := pool.AddRemote(tx); err != core.ErrTipVeryHigh { + if err := pool.addRemote(tx); err != core.ErrTipVeryHigh { t.Error("expected", core.ErrTipVeryHigh, "got", err) } tx2 := dynamicFeeTx(0, 100, veryBigNumber, big.NewInt(1), key) - if err := pool.AddRemote(tx2); err != core.ErrFeeCapVeryHigh { + if err := pool.addRemote(tx2); err != core.ErrFeeCapVeryHigh { t.Error("expected", core.ErrFeeCapVeryHigh, "got", err) } } @@ -430,14 +439,14 @@ func TestChainFork(t *testing.T) { t.Parallel() pool, key := setupPool() - defer pool.Stop() + defer pool.Close() addr := crypto.PubkeyToAddress(key.PublicKey) resetState := func() { statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) statedb.AddBalance(addr, big.NewInt(100000000000000)) - pool.chain = newTestBlockChain(1000000, statedb, new(event.Feed)) + pool.chain = newTestBlockChain(pool.chainconfig, 1000000, statedb, new(event.Feed)) <-pool.requestReset(nil, nil) } resetState() @@ -459,14 +468,14 @@ func TestDoubleNonce(t *testing.T) { t.Parallel() pool, key := setupPool() - defer pool.Stop() + defer pool.Close() addr := crypto.PubkeyToAddress(key.PublicKey) resetState := func() { statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) statedb.AddBalance(addr, big.NewInt(100000000000000)) - pool.chain = newTestBlockChain(1000000, statedb, new(event.Feed)) + pool.chain = newTestBlockChain(pool.chainconfig, 1000000, statedb, new(event.Feed)) <-pool.requestReset(nil, nil) } resetState() @@ -510,7 +519,7 @@ func TestMissingNonce(t *testing.T) { t.Parallel() pool, key := setupPool() - defer pool.Stop() + defer pool.Close() addr := crypto.PubkeyToAddress(key.PublicKey) testAddBalance(pool, addr, big.NewInt(100000000000000)) @@ -534,7 +543,7 @@ func TestNonceRecovery(t *testing.T) { const n = 10 pool, key := setupPool() - defer pool.Stop() + defer pool.Close() addr := crypto.PubkeyToAddress(key.PublicKey) testSetNonce(pool, addr, n) @@ -542,7 +551,7 @@ func TestNonceRecovery(t *testing.T) { <-pool.requestReset(nil, nil) tx := transaction(n, 100000, key) - if err := pool.AddRemote(tx); err != nil { + if err := pool.addRemote(tx); err != nil { t.Error(err) } // simulate some weird re-order of transactions and missing nonce(s) @@ -560,7 +569,7 @@ func TestDropping(t *testing.T) { // Create a test account and fund it pool, key := setupPool() - defer pool.Stop() + defer pool.Close() account := crypto.PubkeyToAddress(key.PublicKey) testAddBalance(pool, account, big.NewInt(1000)) @@ -664,10 +673,11 @@ func TestPostponing(t *testing.T) { // Create the pool to test the postponing with statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - blockchain := newTestBlockChain(1000000, statedb, new(event.Feed)) + blockchain := newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) - pool := New(testTxPoolConfig, params.TestChainConfig, blockchain) - defer pool.Stop() + pool := New(testTxPoolConfig, blockchain) + pool.Init(new(big.Int).SetUint64(testTxPoolConfig.PriceLimit), blockchain.CurrentBlock()) + defer pool.Close() // Create two test accounts to produce different gap profiles with keys := make([]*ecdsa.PrivateKey, 2) @@ -692,7 +702,7 @@ func TestPostponing(t *testing.T) { txs = append(txs, tx) } } - for i, err := range pool.AddRemotesSync(txs) { + for i, err := range pool.addRemotesSync(txs) { if err != nil { t.Fatalf("tx %d: failed to add transactions: %v", i, err) } @@ -777,7 +787,7 @@ func TestGapFilling(t *testing.T) { // Create a test account and fund it pool, key := setupPool() - defer pool.Stop() + defer pool.Close() account := crypto.PubkeyToAddress(key.PublicKey) testAddBalance(pool, account, big.NewInt(1000000)) @@ -788,7 +798,7 @@ func TestGapFilling(t *testing.T) { defer sub.Unsubscribe() // Create a pending and a queued transaction with a nonce-gap in between - pool.AddRemotesSync([]*types.Transaction{ + pool.addRemotesSync([]*types.Transaction{ transaction(0, 100000, key), transaction(2, 100000, key), }) @@ -831,7 +841,7 @@ func TestQueueAccountLimiting(t *testing.T) { // Create a test account and fund it pool, key := setupPool() - defer pool.Stop() + defer pool.Close() account := crypto.PubkeyToAddress(key.PublicKey) testAddBalance(pool, account, big.NewInt(1000000)) @@ -876,14 +886,15 @@ func testQueueGlobalLimiting(t *testing.T, nolocals bool) { // Create the pool to test the limit enforcement with statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - blockchain := newTestBlockChain(1000000, statedb, new(event.Feed)) + blockchain := newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) config := testTxPoolConfig config.NoLocals = nolocals config.GlobalQueue = config.AccountQueue*3 - 1 // reduce the queue limits to shorten test time (-1 to make it non divisible) - pool := New(config, params.TestChainConfig, blockchain) - defer pool.Stop() + pool := New(config, blockchain) + pool.Init(new(big.Int).SetUint64(testTxPoolConfig.PriceLimit), blockchain.CurrentBlock()) + defer pool.Close() // Create a number of test accounts and fund them (last one will be the local) keys := make([]*ecdsa.PrivateKey, 5) @@ -905,7 +916,7 @@ func testQueueGlobalLimiting(t *testing.T, nolocals bool) { nonces[addr]++ } // Import the batch and verify that limits have been enforced - pool.AddRemotesSync(txs) + pool.addRemotesSync(txs) queued := 0 for addr, list := range pool.queue { @@ -922,7 +933,7 @@ func testQueueGlobalLimiting(t *testing.T, nolocals bool) { for i := uint64(0); i < 3*config.GlobalQueue; i++ { txs = append(txs, transaction(i+1, 100000, local)) } - pool.AddLocals(txs) + pool.addLocals(txs) // If locals are disabled, the previous eviction algorithm should apply here too if nolocals { @@ -968,14 +979,15 @@ func testQueueTimeLimiting(t *testing.T, nolocals bool) { // Create the pool to test the non-expiration enforcement statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - blockchain := newTestBlockChain(1000000, statedb, new(event.Feed)) + blockchain := newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) config := testTxPoolConfig config.Lifetime = time.Second config.NoLocals = nolocals - pool := New(config, params.TestChainConfig, blockchain) - defer pool.Stop() + pool := New(config, blockchain) + pool.Init(new(big.Int).SetUint64(config.PriceLimit), blockchain.CurrentBlock()) + defer pool.Close() // Create two test accounts to ensure remotes expire but locals do not local, _ := crypto.GenerateKey() @@ -985,10 +997,10 @@ func testQueueTimeLimiting(t *testing.T, nolocals bool) { testAddBalance(pool, crypto.PubkeyToAddress(remote.PublicKey), big.NewInt(1000000000)) // Add the two transactions and ensure they both are queued up - if err := pool.AddLocal(pricedTransaction(1, 100000, big.NewInt(1), local)); err != nil { + if err := pool.addLocal(pricedTransaction(1, 100000, big.NewInt(1), local)); err != nil { t.Fatalf("failed to add local transaction: %v", err) } - if err := pool.AddRemote(pricedTransaction(1, 100000, big.NewInt(1), remote)); err != nil { + if err := pool.addRemote(pricedTransaction(1, 100000, big.NewInt(1), remote)); err != nil { t.Fatalf("failed to add remote transaction: %v", err) } pending, queued := pool.Stats() @@ -1055,7 +1067,7 @@ func testQueueTimeLimiting(t *testing.T, nolocals bool) { } // Queue gapped transactions - if err := pool.AddLocal(pricedTransaction(4, 100000, big.NewInt(1), local)); err != nil { + if err := pool.addLocal(pricedTransaction(4, 100000, big.NewInt(1), local)); err != nil { t.Fatalf("failed to add remote transaction: %v", err) } if err := pool.addRemoteSync(pricedTransaction(4, 100000, big.NewInt(1), remote)); err != nil { @@ -1064,7 +1076,7 @@ func testQueueTimeLimiting(t *testing.T, nolocals bool) { time.Sleep(5 * evictionInterval) // A half lifetime pass // Queue executable transactions, the life cycle should be restarted. - if err := pool.AddLocal(pricedTransaction(2, 100000, big.NewInt(1), local)); err != nil { + if err := pool.addLocal(pricedTransaction(2, 100000, big.NewInt(1), local)); err != nil { t.Fatalf("failed to add remote transaction: %v", err) } if err := pool.addRemoteSync(pricedTransaction(2, 100000, big.NewInt(1), remote)); err != nil { @@ -1112,7 +1124,7 @@ func TestPendingLimiting(t *testing.T) { // Create a test account and fund it pool, key := setupPool() - defer pool.Stop() + defer pool.Close() account := crypto.PubkeyToAddress(key.PublicKey) testAddBalance(pool, account, big.NewInt(1000000000000)) @@ -1153,13 +1165,14 @@ func TestPendingGlobalLimiting(t *testing.T) { // Create the pool to test the limit enforcement with statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - blockchain := newTestBlockChain(1000000, statedb, new(event.Feed)) + blockchain := newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) config := testTxPoolConfig config.GlobalSlots = config.AccountSlots * 10 - pool := New(config, params.TestChainConfig, blockchain) - defer pool.Stop() + pool := New(config, blockchain) + pool.Init(new(big.Int).SetUint64(config.PriceLimit), blockchain.CurrentBlock()) + defer pool.Close() // Create a number of test accounts and fund them keys := make([]*ecdsa.PrivateKey, 5) @@ -1179,7 +1192,7 @@ func TestPendingGlobalLimiting(t *testing.T) { } } // Import the batch and verify that limits have been enforced - pool.AddRemotesSync(txs) + pool.addRemotesSync(txs) pending := 0 for _, list := range pool.pending { @@ -1201,7 +1214,7 @@ func TestAllowedTxSize(t *testing.T) { // Create a test account and fund it pool, key := setupPool() - defer pool.Stop() + defer pool.Close() account := crypto.PubkeyToAddress(key.PublicKey) testAddBalance(pool, account, big.NewInt(1000000000)) @@ -1254,15 +1267,16 @@ func TestCapClearsFromAll(t *testing.T) { // Create the pool to test the limit enforcement with statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - blockchain := newTestBlockChain(1000000, statedb, new(event.Feed)) + blockchain := newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) config := testTxPoolConfig config.AccountSlots = 2 config.AccountQueue = 2 config.GlobalSlots = 8 - pool := New(config, params.TestChainConfig, blockchain) - defer pool.Stop() + pool := New(config, blockchain) + pool.Init(new(big.Int).SetUint64(config.PriceLimit), blockchain.CurrentBlock()) + defer pool.Close() // Create a number of test accounts and fund them key, _ := crypto.GenerateKey() @@ -1274,7 +1288,7 @@ func TestCapClearsFromAll(t *testing.T) { txs = append(txs, transaction(uint64(j), 100000, key)) } // Import the batch and verify that limits have been enforced - pool.AddRemotes(txs) + pool.addRemotes(txs) if err := validatePoolInternals(pool); err != nil { t.Fatalf("pool internal state corrupted: %v", err) } @@ -1288,13 +1302,14 @@ func TestPendingMinimumAllowance(t *testing.T) { // Create the pool to test the limit enforcement with statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - blockchain := newTestBlockChain(1000000, statedb, new(event.Feed)) + blockchain := newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) config := testTxPoolConfig config.GlobalSlots = 1 - pool := New(config, params.TestChainConfig, blockchain) - defer pool.Stop() + pool := New(config, blockchain) + pool.Init(new(big.Int).SetUint64(config.PriceLimit), blockchain.CurrentBlock()) + defer pool.Close() // Create a number of test accounts and fund them keys := make([]*ecdsa.PrivateKey, 5) @@ -1314,7 +1329,7 @@ func TestPendingMinimumAllowance(t *testing.T) { } } // Import the batch and verify that limits have been enforced - pool.AddRemotesSync(txs) + pool.addRemotesSync(txs) for addr, list := range pool.pending { if list.Len() != int(config.AccountSlots) { @@ -1336,10 +1351,11 @@ func TestRepricing(t *testing.T) { // Create the pool to test the pricing enforcement with statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - blockchain := newTestBlockChain(1000000, statedb, new(event.Feed)) + blockchain := newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) - pool := New(testTxPoolConfig, params.TestChainConfig, blockchain) - defer pool.Stop() + pool := New(testTxPoolConfig, blockchain) + pool.Init(new(big.Int).SetUint64(testTxPoolConfig.PriceLimit), blockchain.CurrentBlock()) + defer pool.Close() // Keep track of transaction events to ensure all executables get announced events := make(chan core.NewTxsEvent, 32) @@ -1370,8 +1386,8 @@ func TestRepricing(t *testing.T) { ltx := pricedTransaction(0, 100000, big.NewInt(1), keys[3]) // Import the batch and that both pending and queued transactions match up - pool.AddRemotesSync(txs) - pool.AddLocal(ltx) + pool.addRemotesSync(txs) + pool.addLocal(ltx) pending, queued := pool.Stats() if pending != 7 { @@ -1403,14 +1419,14 @@ func TestRepricing(t *testing.T) { t.Fatalf("pool internal state corrupted: %v", err) } // Check that we can't add the old transactions back - if err := pool.AddRemote(pricedTransaction(1, 100000, big.NewInt(1), keys[0])); !errors.Is(err, ErrUnderpriced) { - t.Fatalf("adding underpriced pending transaction error mismatch: have %v, want %v", err, ErrUnderpriced) + if err := pool.addRemote(pricedTransaction(1, 100000, big.NewInt(1), keys[0])); !errors.Is(err, txpool.ErrUnderpriced) { + t.Fatalf("adding underpriced pending transaction error mismatch: have %v, want %v", err, txpool.ErrUnderpriced) } - if err := pool.AddRemote(pricedTransaction(0, 100000, big.NewInt(1), keys[1])); !errors.Is(err, ErrUnderpriced) { - t.Fatalf("adding underpriced pending transaction error mismatch: have %v, want %v", err, ErrUnderpriced) + if err := pool.addRemote(pricedTransaction(0, 100000, big.NewInt(1), keys[1])); !errors.Is(err, txpool.ErrUnderpriced) { + t.Fatalf("adding underpriced pending transaction error mismatch: have %v, want %v", err, txpool.ErrUnderpriced) } - if err := pool.AddRemote(pricedTransaction(2, 100000, big.NewInt(1), keys[2])); !errors.Is(err, ErrUnderpriced) { - t.Fatalf("adding underpriced queued transaction error mismatch: have %v, want %v", err, ErrUnderpriced) + if err := pool.addRemote(pricedTransaction(2, 100000, big.NewInt(1), keys[2])); !errors.Is(err, txpool.ErrUnderpriced) { + t.Fatalf("adding underpriced queued transaction error mismatch: have %v, want %v", err, txpool.ErrUnderpriced) } if err := validateEvents(events, 0); err != nil { t.Fatalf("post-reprice event firing failed: %v", err) @@ -1420,7 +1436,7 @@ func TestRepricing(t *testing.T) { } // However we can add local underpriced transactions tx := pricedTransaction(1, 100000, big.NewInt(1), keys[3]) - if err := pool.AddLocal(tx); err != nil { + if err := pool.addLocal(tx); err != nil { t.Fatalf("failed to add underpriced local transaction: %v", err) } if pending, _ = pool.Stats(); pending != 3 { @@ -1433,13 +1449,13 @@ func TestRepricing(t *testing.T) { t.Fatalf("pool internal state corrupted: %v", err) } // And we can fill gaps with properly priced transactions - if err := pool.AddRemote(pricedTransaction(1, 100000, big.NewInt(2), keys[0])); err != nil { + if err := pool.addRemote(pricedTransaction(1, 100000, big.NewInt(2), keys[0])); err != nil { t.Fatalf("failed to add pending transaction: %v", err) } - if err := pool.AddRemote(pricedTransaction(0, 100000, big.NewInt(2), keys[1])); err != nil { + if err := pool.addRemote(pricedTransaction(0, 100000, big.NewInt(2), keys[1])); err != nil { t.Fatalf("failed to add pending transaction: %v", err) } - if err := pool.AddRemote(pricedTransaction(2, 100000, big.NewInt(2), keys[2])); err != nil { + if err := pool.addRemote(pricedTransaction(2, 100000, big.NewInt(2), keys[2])); err != nil { t.Fatalf("failed to add queued transaction: %v", err) } if err := validateEvents(events, 5); err != nil { @@ -1460,7 +1476,7 @@ func TestRepricingDynamicFee(t *testing.T) { // Create the pool to test the pricing enforcement with pool, _ := setupPoolWithConfig(eip1559Config) - defer pool.Stop() + defer pool.Close() // Keep track of transaction events to ensure all executables get announced events := make(chan core.NewTxsEvent, 32) @@ -1491,8 +1507,8 @@ func TestRepricingDynamicFee(t *testing.T) { ltx := dynamicFeeTx(0, 100000, big.NewInt(2), big.NewInt(1), keys[3]) // Import the batch and that both pending and queued transactions match up - pool.AddRemotesSync(txs) - pool.AddLocal(ltx) + pool.addRemotesSync(txs) + pool.addLocal(ltx) pending, queued := pool.Stats() if pending != 7 { @@ -1525,16 +1541,16 @@ func TestRepricingDynamicFee(t *testing.T) { } // Check that we can't add the old transactions back tx := pricedTransaction(1, 100000, big.NewInt(1), keys[0]) - if err := pool.AddRemote(tx); !errors.Is(err, ErrUnderpriced) { - t.Fatalf("adding underpriced pending transaction error mismatch: have %v, want %v", err, ErrUnderpriced) + if err := pool.addRemote(tx); !errors.Is(err, txpool.ErrUnderpriced) { + t.Fatalf("adding underpriced pending transaction error mismatch: have %v, want %v", err, txpool.ErrUnderpriced) } tx = dynamicFeeTx(0, 100000, big.NewInt(2), big.NewInt(1), keys[1]) - if err := pool.AddRemote(tx); !errors.Is(err, ErrUnderpriced) { - t.Fatalf("adding underpriced pending transaction error mismatch: have %v, want %v", err, ErrUnderpriced) + if err := pool.addRemote(tx); !errors.Is(err, txpool.ErrUnderpriced) { + t.Fatalf("adding underpriced pending transaction error mismatch: have %v, want %v", err, txpool.ErrUnderpriced) } tx = dynamicFeeTx(2, 100000, big.NewInt(1), big.NewInt(1), keys[2]) - if err := pool.AddRemote(tx); !errors.Is(err, ErrUnderpriced) { - t.Fatalf("adding underpriced queued transaction error mismatch: have %v, want %v", err, ErrUnderpriced) + if err := pool.addRemote(tx); !errors.Is(err, txpool.ErrUnderpriced) { + t.Fatalf("adding underpriced queued transaction error mismatch: have %v, want %v", err, txpool.ErrUnderpriced) } if err := validateEvents(events, 0); err != nil { t.Fatalf("post-reprice event firing failed: %v", err) @@ -1544,7 +1560,7 @@ func TestRepricingDynamicFee(t *testing.T) { } // However we can add local underpriced transactions tx = dynamicFeeTx(1, 100000, big.NewInt(1), big.NewInt(1), keys[3]) - if err := pool.AddLocal(tx); err != nil { + if err := pool.addLocal(tx); err != nil { t.Fatalf("failed to add underpriced local transaction: %v", err) } if pending, _ = pool.Stats(); pending != 3 { @@ -1558,15 +1574,15 @@ func TestRepricingDynamicFee(t *testing.T) { } // And we can fill gaps with properly priced transactions tx = pricedTransaction(1, 100000, big.NewInt(2), keys[0]) - if err := pool.AddRemote(tx); err != nil { + if err := pool.addRemote(tx); err != nil { t.Fatalf("failed to add pending transaction: %v", err) } tx = dynamicFeeTx(0, 100000, big.NewInt(3), big.NewInt(2), keys[1]) - if err := pool.AddRemote(tx); err != nil { + if err := pool.addRemote(tx); err != nil { t.Fatalf("failed to add pending transaction: %v", err) } tx = dynamicFeeTx(2, 100000, big.NewInt(2), big.NewInt(2), keys[2]) - if err := pool.AddRemote(tx); err != nil { + if err := pool.addRemote(tx); err != nil { t.Fatalf("failed to add queued transaction: %v", err) } if err := validateEvents(events, 5); err != nil { @@ -1584,10 +1600,11 @@ func TestRepricingKeepsLocals(t *testing.T) { // Create the pool to test the pricing enforcement with statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - blockchain := newTestBlockChain(1000000, statedb, new(event.Feed)) + blockchain := newTestBlockChain(eip1559Config, 1000000, statedb, new(event.Feed)) - pool := New(testTxPoolConfig, eip1559Config, blockchain) - defer pool.Stop() + pool := New(testTxPoolConfig, blockchain) + pool.Init(new(big.Int).SetUint64(testTxPoolConfig.PriceLimit), blockchain.CurrentBlock()) + defer pool.Close() // Create a number of test accounts and fund them keys := make([]*ecdsa.PrivateKey, 3) @@ -1599,23 +1616,23 @@ func TestRepricingKeepsLocals(t *testing.T) { for i := uint64(0); i < 500; i++ { // Add pending transaction. pendingTx := pricedTransaction(i, 100000, big.NewInt(int64(i)), keys[2]) - if err := pool.AddLocal(pendingTx); err != nil { + if err := pool.addLocal(pendingTx); err != nil { t.Fatal(err) } // Add queued transaction. queuedTx := pricedTransaction(i+501, 100000, big.NewInt(int64(i)), keys[2]) - if err := pool.AddLocal(queuedTx); err != nil { + if err := pool.addLocal(queuedTx); err != nil { t.Fatal(err) } // Add pending dynamic fee transaction. pendingTx = dynamicFeeTx(i, 100000, big.NewInt(int64(i)+1), big.NewInt(int64(i)), keys[1]) - if err := pool.AddLocal(pendingTx); err != nil { + if err := pool.addLocal(pendingTx); err != nil { t.Fatal(err) } // Add queued dynamic fee transaction. queuedTx = dynamicFeeTx(i+501, 100000, big.NewInt(int64(i)+1), big.NewInt(int64(i)), keys[1]) - if err := pool.AddLocal(queuedTx); err != nil { + if err := pool.addLocal(queuedTx); err != nil { t.Fatal(err) } } @@ -1657,14 +1674,15 @@ func TestUnderpricing(t *testing.T) { // Create the pool to test the pricing enforcement with statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - blockchain := newTestBlockChain(1000000, statedb, new(event.Feed)) + blockchain := newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) config := testTxPoolConfig config.GlobalSlots = 2 config.GlobalQueue = 2 - pool := New(config, params.TestChainConfig, blockchain) - defer pool.Stop() + pool := New(config, blockchain) + pool.Init(new(big.Int).SetUint64(config.PriceLimit), blockchain.CurrentBlock()) + defer pool.Close() // Keep track of transaction events to ensure all executables get announced events := make(chan core.NewTxsEvent, 32) @@ -1688,8 +1706,8 @@ func TestUnderpricing(t *testing.T) { ltx := pricedTransaction(0, 100000, big.NewInt(1), keys[2]) // Import the batch and that both pending and queued transactions match up - pool.AddRemotes(txs) - pool.AddLocal(ltx) + pool.addRemotes(txs) + pool.addLocal(ltx) pending, queued := pool.Stats() if pending != 3 { @@ -1705,26 +1723,26 @@ func TestUnderpricing(t *testing.T) { t.Fatalf("pool internal state corrupted: %v", err) } // Ensure that adding an underpriced transaction on block limit fails - if err := pool.AddRemote(pricedTransaction(0, 100000, big.NewInt(1), keys[1])); !errors.Is(err, ErrUnderpriced) { - t.Fatalf("adding underpriced pending transaction error mismatch: have %v, want %v", err, ErrUnderpriced) + if err := pool.addRemote(pricedTransaction(0, 100000, big.NewInt(1), keys[1])); !errors.Is(err, txpool.ErrUnderpriced) { + t.Fatalf("adding underpriced pending transaction error mismatch: have %v, want %v", err, txpool.ErrUnderpriced) } // Replace a future transaction with a future transaction - if err := pool.AddRemote(pricedTransaction(1, 100000, big.NewInt(2), keys[1])); err != nil { // +K1:1 => -K1:1 => Pend K0:0, K0:1, K2:0; Que K1:1 + if err := pool.addRemote(pricedTransaction(1, 100000, big.NewInt(2), keys[1])); err != nil { // +K1:1 => -K1:1 => Pend K0:0, K0:1, K2:0; Que K1:1 t.Fatalf("failed to add well priced transaction: %v", err) } // Ensure that adding high priced transactions drops cheap ones, but not own - if err := pool.AddRemote(pricedTransaction(0, 100000, big.NewInt(3), keys[1])); err != nil { // +K1:0 => -K1:1 => Pend K0:0, K0:1, K1:0, K2:0; Que - + if err := pool.addRemote(pricedTransaction(0, 100000, big.NewInt(3), keys[1])); err != nil { // +K1:0 => -K1:1 => Pend K0:0, K0:1, K1:0, K2:0; Que - t.Fatalf("failed to add well priced transaction: %v", err) } - if err := pool.AddRemote(pricedTransaction(2, 100000, big.NewInt(4), keys[1])); err != nil { // +K1:2 => -K0:0 => Pend K1:0, K2:0; Que K0:1 K1:2 + if err := pool.addRemote(pricedTransaction(2, 100000, big.NewInt(4), keys[1])); err != nil { // +K1:2 => -K0:0 => Pend K1:0, K2:0; Que K0:1 K1:2 t.Fatalf("failed to add well priced transaction: %v", err) } - if err := pool.AddRemote(pricedTransaction(3, 100000, big.NewInt(5), keys[1])); err != nil { // +K1:3 => -K0:1 => Pend K1:0, K2:0; Que K1:2 K1:3 + if err := pool.addRemote(pricedTransaction(3, 100000, big.NewInt(5), keys[1])); err != nil { // +K1:3 => -K0:1 => Pend K1:0, K2:0; Que K1:2 K1:3 t.Fatalf("failed to add well priced transaction: %v", err) } // Ensure that replacing a pending transaction with a future transaction fails - if err := pool.AddRemote(pricedTransaction(5, 100000, big.NewInt(6), keys[1])); err != ErrFutureReplacePending { - t.Fatalf("adding future replace transaction error mismatch: have %v, want %v", err, ErrFutureReplacePending) + if err := pool.addRemote(pricedTransaction(5, 100000, big.NewInt(6), keys[1])); err != txpool.ErrFutureReplacePending { + t.Fatalf("adding future replace transaction error mismatch: have %v, want %v", err, txpool.ErrFutureReplacePending) } pending, queued = pool.Stats() if pending != 2 { @@ -1741,11 +1759,11 @@ func TestUnderpricing(t *testing.T) { } // Ensure that adding local transactions can push out even higher priced ones ltx = pricedTransaction(1, 100000, big.NewInt(0), keys[2]) - if err := pool.AddLocal(ltx); err != nil { + if err := pool.addLocal(ltx); err != nil { t.Fatalf("failed to append underpriced local transaction: %v", err) } ltx = pricedTransaction(0, 100000, big.NewInt(0), keys[3]) - if err := pool.AddLocal(ltx); err != nil { + if err := pool.addLocal(ltx); err != nil { t.Fatalf("failed to add new underpriced local transaction: %v", err) } pending, queued = pool.Stats() @@ -1771,14 +1789,15 @@ func TestStableUnderpricing(t *testing.T) { // Create the pool to test the pricing enforcement with statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - blockchain := newTestBlockChain(1000000, statedb, new(event.Feed)) + blockchain := newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) config := testTxPoolConfig config.GlobalSlots = 128 config.GlobalQueue = 0 - pool := New(config, params.TestChainConfig, blockchain) - defer pool.Stop() + pool := New(config, blockchain) + pool.Init(new(big.Int).SetUint64(config.PriceLimit), blockchain.CurrentBlock()) + defer pool.Close() // Keep track of transaction events to ensure all executables get announced events := make(chan core.NewTxsEvent, 32) @@ -1796,7 +1815,7 @@ func TestStableUnderpricing(t *testing.T) { for i := uint64(0); i < config.GlobalSlots; i++ { txs = append(txs, pricedTransaction(i, 100000, big.NewInt(1), keys[0])) } - pool.AddRemotesSync(txs) + pool.addRemotesSync(txs) pending, queued := pool.Stats() if pending != int(config.GlobalSlots) { @@ -1839,7 +1858,7 @@ func TestUnderpricingDynamicFee(t *testing.T) { t.Parallel() pool, _ := setupPoolWithConfig(eip1559Config) - defer pool.Stop() + defer pool.Close() pool.config.GlobalSlots = 2 pool.config.GlobalQueue = 2 @@ -1866,8 +1885,8 @@ func TestUnderpricingDynamicFee(t *testing.T) { ltx := dynamicFeeTx(0, 100000, big.NewInt(2), big.NewInt(1), keys[2]) // Import the batch and that both pending and queued transactions match up - pool.AddRemotes(txs) // Pend K0:0, K0:1; Que K1:1 - pool.AddLocal(ltx) // +K2:0 => Pend K0:0, K0:1, K2:0; Que K1:1 + pool.addRemotes(txs) // Pend K0:0, K0:1; Que K1:1 + pool.addLocal(ltx) // +K2:0 => Pend K0:0, K0:1, K2:0; Que K1:1 pending, queued := pool.Stats() if pending != 3 { @@ -1885,22 +1904,22 @@ func TestUnderpricingDynamicFee(t *testing.T) { // Ensure that adding an underpriced transaction fails tx := dynamicFeeTx(0, 100000, big.NewInt(2), big.NewInt(1), keys[1]) - if err := pool.AddRemote(tx); !errors.Is(err, ErrUnderpriced) { // Pend K0:0, K0:1, K2:0; Que K1:1 - t.Fatalf("adding underpriced pending transaction error mismatch: have %v, want %v", err, ErrUnderpriced) + if err := pool.addRemote(tx); !errors.Is(err, txpool.ErrUnderpriced) { // Pend K0:0, K0:1, K2:0; Que K1:1 + t.Fatalf("adding underpriced pending transaction error mismatch: have %v, want %v", err, txpool.ErrUnderpriced) } // Ensure that adding high priced transactions drops cheap ones, but not own tx = pricedTransaction(0, 100000, big.NewInt(2), keys[1]) - if err := pool.AddRemote(tx); err != nil { // +K1:0, -K1:1 => Pend K0:0, K0:1, K1:0, K2:0; Que - + if err := pool.addRemote(tx); err != nil { // +K1:0, -K1:1 => Pend K0:0, K0:1, K1:0, K2:0; Que - t.Fatalf("failed to add well priced transaction: %v", err) } tx = pricedTransaction(1, 100000, big.NewInt(3), keys[1]) - if err := pool.AddRemote(tx); err != nil { // +K1:2, -K0:1 => Pend K0:0 K1:0, K2:0; Que K1:2 + if err := pool.addRemote(tx); err != nil { // +K1:2, -K0:1 => Pend K0:0 K1:0, K2:0; Que K1:2 t.Fatalf("failed to add well priced transaction: %v", err) } tx = dynamicFeeTx(2, 100000, big.NewInt(4), big.NewInt(1), keys[1]) - if err := pool.AddRemote(tx); err != nil { // +K1:3, -K1:0 => Pend K0:0 K2:0; Que K1:2 K1:3 + if err := pool.addRemote(tx); err != nil { // +K1:3, -K1:0 => Pend K0:0 K2:0; Que K1:2 K1:3 t.Fatalf("failed to add well priced transaction: %v", err) } pending, queued = pool.Stats() @@ -1918,11 +1937,11 @@ func TestUnderpricingDynamicFee(t *testing.T) { } // Ensure that adding local transactions can push out even higher priced ones ltx = dynamicFeeTx(1, 100000, big.NewInt(0), big.NewInt(0), keys[2]) - if err := pool.AddLocal(ltx); err != nil { + if err := pool.addLocal(ltx); err != nil { t.Fatalf("failed to append underpriced local transaction: %v", err) } ltx = dynamicFeeTx(0, 100000, big.NewInt(0), big.NewInt(0), keys[3]) - if err := pool.AddLocal(ltx); err != nil { + if err := pool.addLocal(ltx); err != nil { t.Fatalf("failed to add new underpriced local transaction: %v", err) } pending, queued = pool.Stats() @@ -1946,7 +1965,7 @@ func TestDualHeapEviction(t *testing.T) { t.Parallel() pool, _ := setupPoolWithConfig(eip1559Config) - defer pool.Stop() + defer pool.Close() pool.config.GlobalSlots = 10 pool.config.GlobalQueue = 10 @@ -1975,7 +1994,7 @@ func TestDualHeapEviction(t *testing.T) { tx = dynamicFeeTx(0, 100000, big.NewInt(int64(baseFee+200+i)), big.NewInt(1), key) highCap = tx } - pool.AddRemotesSync([]*types.Transaction{tx}) + pool.addRemotesSync([]*types.Transaction{tx}) } pending, queued := pool.Stats() if pending+queued != 20 { @@ -2003,10 +2022,11 @@ func TestDeduplication(t *testing.T) { // Create the pool to test the pricing enforcement with statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - blockchain := newTestBlockChain(1000000, statedb, new(event.Feed)) + blockchain := newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) - pool := New(testTxPoolConfig, params.TestChainConfig, blockchain) - defer pool.Stop() + pool := New(testTxPoolConfig, blockchain) + pool.Init(new(big.Int).SetUint64(testTxPoolConfig.PriceLimit), blockchain.CurrentBlock()) + defer pool.Close() // Create a test account to add transactions with key, _ := crypto.GenerateKey() @@ -2021,7 +2041,7 @@ func TestDeduplication(t *testing.T) { for i := 0; i < len(txs); i += 2 { firsts = append(firsts, txs[i]) } - errs := pool.AddRemotesSync(firsts) + errs := pool.addRemotesSync(firsts) if len(errs) != len(firsts) { t.Fatalf("first add mismatching result count: have %d, want %d", len(errs), len(firsts)) } @@ -2038,7 +2058,7 @@ func TestDeduplication(t *testing.T) { t.Fatalf("queued transactions mismatched: have %d, want %d", queued, len(txs)/2-1) } // Try to add all of them now and ensure previous ones error out as knowns - errs = pool.AddRemotesSync(txs) + errs = pool.addRemotesSync(txs) if len(errs) != len(txs) { t.Fatalf("all add mismatching result count: have %d, want %d", len(errs), len(txs)) } @@ -2069,10 +2089,11 @@ func TestReplacement(t *testing.T) { // Create the pool to test the pricing enforcement with statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - blockchain := newTestBlockChain(1000000, statedb, new(event.Feed)) + blockchain := newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) - pool := New(testTxPoolConfig, params.TestChainConfig, blockchain) - defer pool.Stop() + pool := New(testTxPoolConfig, blockchain) + pool.Init(new(big.Int).SetUint64(testTxPoolConfig.PriceLimit), blockchain.CurrentBlock()) + defer pool.Close() // Keep track of transaction events to ensure all executables get announced events := make(chan core.NewTxsEvent, 32) @@ -2090,10 +2111,10 @@ func TestReplacement(t *testing.T) { if err := pool.addRemoteSync(pricedTransaction(0, 100000, big.NewInt(1), key)); err != nil { t.Fatalf("failed to add original cheap pending transaction: %v", err) } - if err := pool.AddRemote(pricedTransaction(0, 100001, big.NewInt(1), key)); err != ErrReplaceUnderpriced { - t.Fatalf("original cheap pending transaction replacement error mismatch: have %v, want %v", err, ErrReplaceUnderpriced) + if err := pool.addRemote(pricedTransaction(0, 100001, big.NewInt(1), key)); err != txpool.ErrReplaceUnderpriced { + t.Fatalf("original cheap pending transaction replacement error mismatch: have %v, want %v", err, txpool.ErrReplaceUnderpriced) } - if err := pool.AddRemote(pricedTransaction(0, 100000, big.NewInt(2), key)); err != nil { + if err := pool.addRemote(pricedTransaction(0, 100000, big.NewInt(2), key)); err != nil { t.Fatalf("failed to replace original cheap pending transaction: %v", err) } if err := validateEvents(events, 2); err != nil { @@ -2103,10 +2124,10 @@ func TestReplacement(t *testing.T) { if err := pool.addRemoteSync(pricedTransaction(0, 100000, big.NewInt(price), key)); err != nil { t.Fatalf("failed to add original proper pending transaction: %v", err) } - if err := pool.AddRemote(pricedTransaction(0, 100001, big.NewInt(threshold-1), key)); err != ErrReplaceUnderpriced { - t.Fatalf("original proper pending transaction replacement error mismatch: have %v, want %v", err, ErrReplaceUnderpriced) + if err := pool.addRemote(pricedTransaction(0, 100001, big.NewInt(threshold-1), key)); err != txpool.ErrReplaceUnderpriced { + t.Fatalf("original proper pending transaction replacement error mismatch: have %v, want %v", err, txpool.ErrReplaceUnderpriced) } - if err := pool.AddRemote(pricedTransaction(0, 100000, big.NewInt(threshold), key)); err != nil { + if err := pool.addRemote(pricedTransaction(0, 100000, big.NewInt(threshold), key)); err != nil { t.Fatalf("failed to replace original proper pending transaction: %v", err) } if err := validateEvents(events, 2); err != nil { @@ -2114,23 +2135,23 @@ func TestReplacement(t *testing.T) { } // Add queued transactions, ensuring the minimum price bump is enforced for replacement (for ultra low prices too) - if err := pool.AddRemote(pricedTransaction(2, 100000, big.NewInt(1), key)); err != nil { + if err := pool.addRemote(pricedTransaction(2, 100000, big.NewInt(1), key)); err != nil { t.Fatalf("failed to add original cheap queued transaction: %v", err) } - if err := pool.AddRemote(pricedTransaction(2, 100001, big.NewInt(1), key)); err != ErrReplaceUnderpriced { - t.Fatalf("original cheap queued transaction replacement error mismatch: have %v, want %v", err, ErrReplaceUnderpriced) + if err := pool.addRemote(pricedTransaction(2, 100001, big.NewInt(1), key)); err != txpool.ErrReplaceUnderpriced { + t.Fatalf("original cheap queued transaction replacement error mismatch: have %v, want %v", err, txpool.ErrReplaceUnderpriced) } - if err := pool.AddRemote(pricedTransaction(2, 100000, big.NewInt(2), key)); err != nil { + if err := pool.addRemote(pricedTransaction(2, 100000, big.NewInt(2), key)); err != nil { t.Fatalf("failed to replace original cheap queued transaction: %v", err) } - if err := pool.AddRemote(pricedTransaction(2, 100000, big.NewInt(price), key)); err != nil { + if err := pool.addRemote(pricedTransaction(2, 100000, big.NewInt(price), key)); err != nil { t.Fatalf("failed to add original proper queued transaction: %v", err) } - if err := pool.AddRemote(pricedTransaction(2, 100001, big.NewInt(threshold-1), key)); err != ErrReplaceUnderpriced { - t.Fatalf("original proper queued transaction replacement error mismatch: have %v, want %v", err, ErrReplaceUnderpriced) + if err := pool.addRemote(pricedTransaction(2, 100001, big.NewInt(threshold-1), key)); err != txpool.ErrReplaceUnderpriced { + t.Fatalf("original proper queued transaction replacement error mismatch: have %v, want %v", err, txpool.ErrReplaceUnderpriced) } - if err := pool.AddRemote(pricedTransaction(2, 100000, big.NewInt(threshold), key)); err != nil { + if err := pool.addRemote(pricedTransaction(2, 100000, big.NewInt(threshold), key)); err != nil { t.Fatalf("failed to replace original proper queued transaction: %v", err) } @@ -2149,7 +2170,7 @@ func TestReplacementDynamicFee(t *testing.T) { // Create the pool to test the pricing enforcement with pool, key := setupPoolWithConfig(eip1559Config) - defer pool.Stop() + defer pool.Close() testAddBalance(pool, crypto.PubkeyToAddress(key.PublicKey), big.NewInt(1000000000)) // Keep track of transaction events to ensure all executables get announced @@ -2191,12 +2212,12 @@ func TestReplacementDynamicFee(t *testing.T) { } // 2. Don't bump tip or feecap => discard tx = dynamicFeeTx(nonce, 100001, big.NewInt(2), big.NewInt(1), key) - if err := pool.AddRemote(tx); err != ErrReplaceUnderpriced { - t.Fatalf("original cheap %s transaction replacement error mismatch: have %v, want %v", stage, err, ErrReplaceUnderpriced) + if err := pool.addRemote(tx); err != txpool.ErrReplaceUnderpriced { + t.Fatalf("original cheap %s transaction replacement error mismatch: have %v, want %v", stage, err, txpool.ErrReplaceUnderpriced) } // 3. Bump both more than min => accept tx = dynamicFeeTx(nonce, 100000, big.NewInt(3), big.NewInt(2), key) - if err := pool.AddRemote(tx); err != nil { + if err := pool.addRemote(tx); err != nil { t.Fatalf("failed to replace original cheap %s transaction: %v", stage, err) } // 4. Check events match expected (2 new executable txs during pending, 0 during queue) @@ -2214,27 +2235,27 @@ func TestReplacementDynamicFee(t *testing.T) { } // 6. Bump tip max allowed so it's still underpriced => discard tx = dynamicFeeTx(nonce, 100000, big.NewInt(gasFeeCap), big.NewInt(tipThreshold-1), key) - if err := pool.AddRemote(tx); err != ErrReplaceUnderpriced { - t.Fatalf("original proper %s transaction replacement error mismatch: have %v, want %v", stage, err, ErrReplaceUnderpriced) + if err := pool.addRemote(tx); err != txpool.ErrReplaceUnderpriced { + t.Fatalf("original proper %s transaction replacement error mismatch: have %v, want %v", stage, err, txpool.ErrReplaceUnderpriced) } // 7. Bump fee cap max allowed so it's still underpriced => discard tx = dynamicFeeTx(nonce, 100000, big.NewInt(feeCapThreshold-1), big.NewInt(gasTipCap), key) - if err := pool.AddRemote(tx); err != ErrReplaceUnderpriced { - t.Fatalf("original proper %s transaction replacement error mismatch: have %v, want %v", stage, err, ErrReplaceUnderpriced) + if err := pool.addRemote(tx); err != txpool.ErrReplaceUnderpriced { + t.Fatalf("original proper %s transaction replacement error mismatch: have %v, want %v", stage, err, txpool.ErrReplaceUnderpriced) } // 8. Bump tip min for acceptance => accept tx = dynamicFeeTx(nonce, 100000, big.NewInt(gasFeeCap), big.NewInt(tipThreshold), key) - if err := pool.AddRemote(tx); err != ErrReplaceUnderpriced { - t.Fatalf("original proper %s transaction replacement error mismatch: have %v, want %v", stage, err, ErrReplaceUnderpriced) + if err := pool.addRemote(tx); err != txpool.ErrReplaceUnderpriced { + t.Fatalf("original proper %s transaction replacement error mismatch: have %v, want %v", stage, err, txpool.ErrReplaceUnderpriced) } // 9. Bump fee cap min for acceptance => accept tx = dynamicFeeTx(nonce, 100000, big.NewInt(feeCapThreshold), big.NewInt(gasTipCap), key) - if err := pool.AddRemote(tx); err != ErrReplaceUnderpriced { - t.Fatalf("original proper %s transaction replacement error mismatch: have %v, want %v", stage, err, ErrReplaceUnderpriced) + if err := pool.addRemote(tx); err != txpool.ErrReplaceUnderpriced { + t.Fatalf("original proper %s transaction replacement error mismatch: have %v, want %v", stage, err, txpool.ErrReplaceUnderpriced) } // 10. Check events match expected (3 new executable txs during pending, 0 during queue) tx = dynamicFeeTx(nonce, 100000, big.NewInt(feeCapThreshold), big.NewInt(tipThreshold), key) - if err := pool.AddRemote(tx); err != nil { + if err := pool.addRemote(tx); err != nil { t.Fatalf("failed to replace original cheap %s transaction: %v", stage, err) } // 11. Check events match expected (3 new executable txs during pending, 0 during queue) @@ -2274,14 +2295,15 @@ func testJournaling(t *testing.T, nolocals bool) { // Create the original pool to inject transaction into the journal statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - blockchain := newTestBlockChain(1000000, statedb, new(event.Feed)) + blockchain := newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) config := testTxPoolConfig config.NoLocals = nolocals config.Journal = journal config.Rejournal = time.Second - pool := New(config, params.TestChainConfig, blockchain) + pool := New(config, blockchain) + pool.Init(new(big.Int).SetUint64(config.PriceLimit), blockchain.CurrentBlock()) // Create two test accounts to ensure remotes expire but locals do not local, _ := crypto.GenerateKey() @@ -2291,13 +2313,13 @@ func testJournaling(t *testing.T, nolocals bool) { testAddBalance(pool, crypto.PubkeyToAddress(remote.PublicKey), big.NewInt(1000000000)) // Add three local and a remote transactions and ensure they are queued up - if err := pool.AddLocal(pricedTransaction(0, 100000, big.NewInt(1), local)); err != nil { + if err := pool.addLocal(pricedTransaction(0, 100000, big.NewInt(1), local)); err != nil { t.Fatalf("failed to add local transaction: %v", err) } - if err := pool.AddLocal(pricedTransaction(1, 100000, big.NewInt(1), local)); err != nil { + if err := pool.addLocal(pricedTransaction(1, 100000, big.NewInt(1), local)); err != nil { t.Fatalf("failed to add local transaction: %v", err) } - if err := pool.AddLocal(pricedTransaction(2, 100000, big.NewInt(1), local)); err != nil { + if err := pool.addLocal(pricedTransaction(2, 100000, big.NewInt(1), local)); err != nil { t.Fatalf("failed to add local transaction: %v", err) } if err := pool.addRemoteSync(pricedTransaction(0, 100000, big.NewInt(1), remote)); err != nil { @@ -2314,11 +2336,12 @@ func testJournaling(t *testing.T, nolocals bool) { t.Fatalf("pool internal state corrupted: %v", err) } // Terminate the old pool, bump the local nonce, create a new pool and ensure relevant transaction survive - pool.Stop() + pool.Close() statedb.SetNonce(crypto.PubkeyToAddress(local.PublicKey), 1) - blockchain = newTestBlockChain(1000000, statedb, new(event.Feed)) + blockchain = newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) - pool = New(config, params.TestChainConfig, blockchain) + pool = New(config, blockchain) + pool.Init(new(big.Int).SetUint64(config.PriceLimit), blockchain.CurrentBlock()) pending, queued = pool.Stats() if queued != 0 { @@ -2340,11 +2363,12 @@ func testJournaling(t *testing.T, nolocals bool) { statedb.SetNonce(crypto.PubkeyToAddress(local.PublicKey), 2) <-pool.requestReset(nil, nil) time.Sleep(2 * config.Rejournal) - pool.Stop() + pool.Close() statedb.SetNonce(crypto.PubkeyToAddress(local.PublicKey), 1) - blockchain = newTestBlockChain(1000000, statedb, new(event.Feed)) - pool = New(config, params.TestChainConfig, blockchain) + blockchain = newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) + pool = New(config, blockchain) + pool.Init(new(big.Int).SetUint64(config.PriceLimit), blockchain.CurrentBlock()) pending, queued = pool.Stats() if pending != 0 { @@ -2362,7 +2386,7 @@ func testJournaling(t *testing.T, nolocals bool) { if err := validatePoolInternals(pool); err != nil { t.Fatalf("pool internal state corrupted: %v", err) } - pool.Stop() + pool.Close() } // TestStatusCheck tests that the pool can correctly retrieve the @@ -2372,10 +2396,11 @@ func TestStatusCheck(t *testing.T) { // Create the pool to test the status retrievals with statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - blockchain := newTestBlockChain(1000000, statedb, new(event.Feed)) + blockchain := newTestBlockChain(params.TestChainConfig, 1000000, statedb, new(event.Feed)) - pool := New(testTxPoolConfig, params.TestChainConfig, blockchain) - defer pool.Stop() + pool := New(testTxPoolConfig, blockchain) + pool.Init(new(big.Int).SetUint64(testTxPoolConfig.PriceLimit), blockchain.CurrentBlock()) + defer pool.Close() // Create the test accounts to check various transaction statuses with keys := make([]*ecdsa.PrivateKey, 3) @@ -2392,7 +2417,7 @@ func TestStatusCheck(t *testing.T) { txs = append(txs, pricedTransaction(2, 100000, big.NewInt(1), keys[2])) // Queued only // Import the transaction and ensure they are correctly added - pool.AddRemotesSync(txs) + pool.addRemotesSync(txs) pending, queued := pool.Stats() if pending != 2 { @@ -2410,13 +2435,11 @@ func TestStatusCheck(t *testing.T) { hashes[i] = tx.Hash() } hashes = append(hashes, common.Hash{}) + expect := []txpool.TxStatus{txpool.TxStatusPending, txpool.TxStatusPending, txpool.TxStatusQueued, txpool.TxStatusQueued, txpool.TxStatusUnknown} - statuses := pool.Status(hashes) - expect := []TxStatus{TxStatusPending, TxStatusPending, TxStatusQueued, TxStatusQueued, TxStatusUnknown} - - for i := 0; i < len(statuses); i++ { - if statuses[i] != expect[i] { - t.Errorf("transaction %d: status mismatch: have %v, want %v", i, statuses[i], expect[i]) + for i := 0; i < len(hashes); i++ { + if status := pool.Status(hashes[i]); status != expect[i] { + t.Errorf("transaction %d: status mismatch: have %v, want %v", i, status, expect[i]) } } } @@ -2448,7 +2471,7 @@ func BenchmarkPendingDemotion10000(b *testing.B) { benchmarkPendingDemotion(b, 1 func benchmarkPendingDemotion(b *testing.B, size int) { // Add a batch of transactions to a pool one by one pool, key := setupPool() - defer pool.Stop() + defer pool.Close() account := crypto.PubkeyToAddress(key.PublicKey) testAddBalance(pool, account, big.NewInt(1000000)) @@ -2473,7 +2496,7 @@ func BenchmarkFuturePromotion10000(b *testing.B) { benchmarkFuturePromotion(b, 1 func benchmarkFuturePromotion(b *testing.B, size int) { // Add a batch of transactions to a pool one by one pool, key := setupPool() - defer pool.Stop() + defer pool.Close() account := crypto.PubkeyToAddress(key.PublicKey) testAddBalance(pool, account, big.NewInt(1000000)) @@ -2501,7 +2524,7 @@ func BenchmarkBatchLocalInsert10000(b *testing.B) { benchmarkBatchInsert(b, 1000 func benchmarkBatchInsert(b *testing.B, size int, local bool) { // Generate a batch of transactions to enqueue into the pool pool, key := setupPool() - defer pool.Stop() + defer pool.Close() account := crypto.PubkeyToAddress(key.PublicKey) testAddBalance(pool, account, big.NewInt(1000000000000000000)) @@ -2517,9 +2540,9 @@ func benchmarkBatchInsert(b *testing.B, size int, local bool) { b.ResetTimer() for _, batch := range batches { if local { - pool.AddLocals(batch) + pool.addLocals(batch) } else { - pool.AddRemotes(batch) + pool.addRemotes(batch) } } } @@ -2547,15 +2570,15 @@ func BenchmarkInsertRemoteWithAllLocals(b *testing.B) { pool, _ := setupPool() testAddBalance(pool, account, big.NewInt(100000000)) for _, local := range locals { - pool.AddLocal(local) + pool.addLocal(local) } b.StartTimer() // Assign a high enough balance for testing testAddBalance(pool, remoteAddr, big.NewInt(100000000)) for i := 0; i < len(remotes); i++ { - pool.AddRemotes([]*types.Transaction{remotes[i]}) + pool.addRemotes([]*types.Transaction{remotes[i]}) } - pool.Stop() + pool.Close() } } @@ -2563,7 +2586,7 @@ func BenchmarkInsertRemoteWithAllLocals(b *testing.B) { func BenchmarkMultiAccountBatchInsert(b *testing.B) { // Generate a batch of transactions to enqueue into the pool pool, _ := setupPool() - defer pool.Stop() + defer pool.Close() b.ReportAllocs() batches := make(types.Transactions, b.N) for i := 0; i < b.N; i++ { @@ -2576,6 +2599,6 @@ func BenchmarkMultiAccountBatchInsert(b *testing.B) { // Benchmark importing the transactions into the queue b.ResetTimer() for _, tx := range batches { - pool.AddRemotesSync([]*types.Transaction{tx}) + pool.addRemotesSync([]*types.Transaction{tx}) } } diff --git a/core/txpool/list.go b/core/txpool/legacypool/list.go similarity index 99% rename from core/txpool/list.go rename to core/txpool/legacypool/list.go index fae7c2fcac..d3b6768f81 100644 --- a/core/txpool/list.go +++ b/core/txpool/legacypool/list.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package txpool +package legacypool import ( "container/heap" diff --git a/core/txpool/list_test.go b/core/txpool/legacypool/list_test.go similarity index 99% rename from core/txpool/list_test.go rename to core/txpool/legacypool/list_test.go index 4e1a5d5e83..b5cd34b23b 100644 --- a/core/txpool/list_test.go +++ b/core/txpool/legacypool/list_test.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package txpool +package legacypool import ( "math/big" diff --git a/core/txpool/noncer.go b/core/txpool/legacypool/noncer.go similarity index 99% rename from core/txpool/noncer.go rename to core/txpool/legacypool/noncer.go index ba7fbedad5..2c65dd2cae 100644 --- a/core/txpool/noncer.go +++ b/core/txpool/legacypool/noncer.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package txpool +package legacypool import ( "sync" diff --git a/core/txpool/subpool.go b/core/txpool/subpool.go new file mode 100644 index 0000000000..835f0de97a --- /dev/null +++ b/core/txpool/subpool.go @@ -0,0 +1,111 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package txpool + +import ( + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto/kzg4844" + "github.com/ethereum/go-ethereum/event" +) + +// Transaction is a helper struct to group together a canonical transaction with +// satellite data items that are needed by the pool but are not part of the chain. +type Transaction struct { + Tx *types.Transaction // Canonical transaction + + BlobTxBlobs []kzg4844.Blob // Blobs needed by the blob pool + BlobTxCommits []kzg4844.Commitment // Commitments needed by the blob pool + BlobTxProofs []kzg4844.Proof // Proofs needed by the blob pool +} + +// SubPool represents a specialized transaction pool that lives on its own (e.g. +// blob pool). Since independent of how many specialized pools we have, they do +// need to be updated in lockstep and assemble into one coherent view for block +// production, this interface defines the common methods that allow the primary +// transaction pool to manage the subpools. +type SubPool interface { + // Filter is a selector used to decide whether a transaction whould be added + // to this particular subpool. + Filter(tx *types.Transaction) bool + + // Init sets the base parameters of the subpool, allowing it to load any saved + // transactions from disk and also permitting internal maintenance routines to + // start up. + // + // These should not be passed as a constructor argument - nor should the pools + // start by themselves - in order to keep multiple subpools in lockstep with + // one another. + Init(gasTip *big.Int, head *types.Header) error + + // Close terminates any background processing threads and releases any held + // resources. + Close() error + + // Reset retrieves the current state of the blockchain and ensures the content + // of the transaction pool is valid with regard to the chain state. + Reset(oldHead, newHead *types.Header) + + // SetGasTip updates the minimum price required by the subpool for a new + // transaction, and drops all transactions below this threshold. + SetGasTip(tip *big.Int) + + // Has returns an indicator whether subpool has a transaction cached with the + // given hash. + Has(hash common.Hash) bool + + // Get returns a transaction if it is contained in the pool, or nil otherwise. + Get(hash common.Hash) *Transaction + + // Add enqueues a batch of transactions into the pool if they are valid. Due + // to the large transaction churn, add may postpone fully integrating the tx + // to a later point to batch multiple ones together. + Add(txs []*Transaction, local bool, sync bool) []error + + // Pending retrieves all currently processable transactions, grouped by origin + // account and sorted by nonce. + Pending(enforceTips bool) map[common.Address][]*types.Transaction + + // SubscribeTransactions subscribes to new transaction events. + SubscribeTransactions(ch chan<- core.NewTxsEvent) event.Subscription + + // Nonce returns the next nonce of an account, with all transactions executable + // by the pool already applied on top. + Nonce(addr common.Address) uint64 + + // Stats retrieves the current pool stats, namely the number of pending and the + // number of queued (non-executable) transactions. + Stats() (int, int) + + // Content retrieves the data content of the transaction pool, returning all the + // pending as well as queued transactions, grouped by account and sorted by nonce. + Content() (map[common.Address][]*types.Transaction, map[common.Address][]*types.Transaction) + + // ContentFrom retrieves the data content of the transaction pool, returning the + // pending as well as queued transactions of this address, grouped by nonce. + ContentFrom(addr common.Address) ([]*types.Transaction, []*types.Transaction) + + // Locals retrieves the accounts currently considered local by the pool. + Locals() []common.Address + + // Status returns the known status (unknown/pending/queued) of a transaction + // identified by their hashes. + Status(hash common.Hash) TxStatus +} diff --git a/core/txpool/txpool.go b/core/txpool/txpool.go index 1395725402..f2c94563bc 100644 --- a/core/txpool/txpool.go +++ b/core/txpool/txpool.go @@ -1,4 +1,4 @@ -// Copyright 2014 The go-ethereum Authors +// Copyright 2023 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify @@ -18,121 +18,12 @@ package txpool import ( "errors" - "math" "math/big" - "sort" - "sync" - "sync/atomic" - "time" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/prque" - "github.com/ethereum/go-ethereum/consensus/misc" "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/metrics" - "github.com/ethereum/go-ethereum/params" -) - -const ( - // chainHeadChanSize is the size of channel listening to ChainHeadEvent. - chainHeadChanSize = 10 - - // txSlotSize is used to calculate how many data slots a single transaction - // takes up based on its size. The slots are used as DoS protection, ensuring - // that validating a new transaction remains a constant operation (in reality - // O(maxslots), where max slots are 4 currently). - txSlotSize = 32 * 1024 - - // txMaxSize is the maximum size a single transaction can have. This field has - // non-trivial consequences: larger transactions are significantly harder and - // more expensive to propagate; larger transactions also take more resources - // to validate whether they fit into the pool or not. - txMaxSize = 4 * txSlotSize // 128KB -) - -var ( - // ErrAlreadyKnown is returned if the transactions is already contained - // within the pool. - ErrAlreadyKnown = errors.New("already known") - - // ErrInvalidSender is returned if the transaction contains an invalid signature. - ErrInvalidSender = errors.New("invalid sender") - - // ErrUnderpriced is returned if a transaction's gas price is below the minimum - // configured for the transaction pool. - ErrUnderpriced = errors.New("transaction underpriced") - - // ErrTxPoolOverflow is returned if the transaction pool is full and can't accept - // another remote transaction. - ErrTxPoolOverflow = errors.New("txpool is full") - - // ErrReplaceUnderpriced is returned if a transaction is attempted to be replaced - // with a different one without the required price bump. - ErrReplaceUnderpriced = errors.New("replacement transaction underpriced") - - // ErrGasLimit is returned if a transaction's requested gas limit exceeds the - // maximum allowance of the current block. - ErrGasLimit = errors.New("exceeds block gas limit") - - // ErrNegativeValue is a sanity error to ensure no one is able to specify a - // transaction with a negative value. - ErrNegativeValue = errors.New("negative value") - - // ErrOversizedData is returned if the input data of a transaction is greater - // than some meaningful limit a user might use. This is not a consensus error - // making the transaction invalid, rather a DOS protection. - ErrOversizedData = errors.New("oversized data") - - // ErrFutureReplacePending is returned if a future transaction replaces a pending - // transaction. Future transactions should only be able to replace other future transactions. - ErrFutureReplacePending = errors.New("future transaction tries to replace pending") -) - -var ( - evictionInterval = time.Minute // Time interval to check for evictable transactions - statsReportInterval = 8 * time.Second // Time interval to report transaction pool stats -) - -var ( - // Metrics for the pending pool - pendingDiscardMeter = metrics.NewRegisteredMeter("txpool/pending/discard", nil) - pendingReplaceMeter = metrics.NewRegisteredMeter("txpool/pending/replace", nil) - pendingRateLimitMeter = metrics.NewRegisteredMeter("txpool/pending/ratelimit", nil) // Dropped due to rate limiting - pendingNofundsMeter = metrics.NewRegisteredMeter("txpool/pending/nofunds", nil) // Dropped due to out-of-funds - - // Metrics for the queued pool - queuedDiscardMeter = metrics.NewRegisteredMeter("txpool/queued/discard", nil) - queuedReplaceMeter = metrics.NewRegisteredMeter("txpool/queued/replace", nil) - queuedRateLimitMeter = metrics.NewRegisteredMeter("txpool/queued/ratelimit", nil) // Dropped due to rate limiting - queuedNofundsMeter = metrics.NewRegisteredMeter("txpool/queued/nofunds", nil) // Dropped due to out-of-funds - queuedEvictionMeter = metrics.NewRegisteredMeter("txpool/queued/eviction", nil) // Dropped due to lifetime - - // General tx metrics - knownTxMeter = metrics.NewRegisteredMeter("txpool/known", nil) - validTxMeter = metrics.NewRegisteredMeter("txpool/valid", nil) - invalidTxMeter = metrics.NewRegisteredMeter("txpool/invalid", nil) - underpricedTxMeter = metrics.NewRegisteredMeter("txpool/underpriced", nil) - overflowedTxMeter = metrics.NewRegisteredMeter("txpool/overflowed", nil) - - // throttleTxMeter counts how many transactions are rejected due to too-many-changes between - // txpool reorgs. - throttleTxMeter = metrics.NewRegisteredMeter("txpool/throttle", nil) - // reorgDurationTimer measures how long time a txpool reorg takes. - reorgDurationTimer = metrics.NewRegisteredTimer("txpool/reorgtime", nil) - // dropBetweenReorgHistogram counts how many drops we experience between two reorg runs. It is expected - // that this number is pretty low, since txpool reorgs happen very frequently. - dropBetweenReorgHistogram = metrics.NewRegisteredHistogram("txpool/dropbetweenreorg", nil, metrics.NewExpDecaySample(1028, 0.015)) - - pendingGauge = metrics.NewRegisteredGauge("txpool/pending", nil) - queuedGauge = metrics.NewRegisteredGauge("txpool/queued", nil) - localGauge = metrics.NewRegisteredGauge("txpool/local", nil) - slotsGauge = metrics.NewRegisteredGauge("txpool/slots", nil) - - reheapTimer = metrics.NewRegisteredTimer("txpool/reheap", nil) ) // TxStatus is the current status of a transaction as seen by the pool. @@ -145,1705 +36,309 @@ const ( TxStatusIncluded ) -// blockChain provides the state of blockchain and current gas limit to do -// some pre checks in tx pool and event subscribers. -type blockChain interface { +// BlockChain defines the minimal set of methods needed to back a tx pool with +// a chain. Exists to allow mocking the live chain out of tests. +type BlockChain interface { + // CurrentBlock returns the current head of the chain. CurrentBlock() *types.Header - GetBlock(hash common.Hash, number uint64) *types.Block - StateAt(root common.Hash) (*state.StateDB, error) + // SubscribeChainHeadEvent subscribes to new blocks being added to the chain. SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription } -// Config are the configuration parameters of the transaction pool. -type Config struct { - Locals []common.Address // Addresses that should be treated by default as local - NoLocals bool // Whether local transaction handling should be disabled - Journal string // Journal of local transactions to survive node restarts - Rejournal time.Duration // Time interval to regenerate the local transaction journal - - PriceLimit uint64 // Minimum gas price to enforce for acceptance into the pool - PriceBump uint64 // Minimum price bump percentage to replace an already existing transaction (nonce) - - AccountSlots uint64 // Number of executable transaction slots guaranteed per account - GlobalSlots uint64 // Maximum number of executable transaction slots for all accounts - AccountQueue uint64 // Maximum number of non-executable transaction slots permitted per account - GlobalQueue uint64 // Maximum number of non-executable transaction slots for all accounts - - Lifetime time.Duration // Maximum amount of time non-executable transaction are queued -} - -// DefaultConfig contains the default configurations for the transaction pool. -var DefaultConfig = Config{ - Journal: "transactions.rlp", - Rejournal: time.Hour, - - PriceLimit: 1, - PriceBump: 10, - - AccountSlots: 16, - GlobalSlots: 4096 + 1024, // urgent + floating queue capacity with 4:1 ratio - AccountQueue: 64, - GlobalQueue: 1024, - - Lifetime: 3 * time.Hour, -} - -// sanitize checks the provided user configurations and changes anything that's -// unreasonable or unworkable. -func (config *Config) sanitize() Config { - conf := *config - if conf.Rejournal < time.Second { - log.Warn("Sanitizing invalid txpool journal time", "provided", conf.Rejournal, "updated", time.Second) - conf.Rejournal = time.Second - } - if conf.PriceLimit < 1 { - log.Warn("Sanitizing invalid txpool price limit", "provided", conf.PriceLimit, "updated", DefaultConfig.PriceLimit) - conf.PriceLimit = DefaultConfig.PriceLimit - } - if conf.PriceBump < 1 { - log.Warn("Sanitizing invalid txpool price bump", "provided", conf.PriceBump, "updated", DefaultConfig.PriceBump) - conf.PriceBump = DefaultConfig.PriceBump - } - if conf.AccountSlots < 1 { - log.Warn("Sanitizing invalid txpool account slots", "provided", conf.AccountSlots, "updated", DefaultConfig.AccountSlots) - conf.AccountSlots = DefaultConfig.AccountSlots - } - if conf.GlobalSlots < 1 { - log.Warn("Sanitizing invalid txpool global slots", "provided", conf.GlobalSlots, "updated", DefaultConfig.GlobalSlots) - conf.GlobalSlots = DefaultConfig.GlobalSlots - } - if conf.AccountQueue < 1 { - log.Warn("Sanitizing invalid txpool account queue", "provided", conf.AccountQueue, "updated", DefaultConfig.AccountQueue) - conf.AccountQueue = DefaultConfig.AccountQueue - } - if conf.GlobalQueue < 1 { - log.Warn("Sanitizing invalid txpool global queue", "provided", conf.GlobalQueue, "updated", DefaultConfig.GlobalQueue) - conf.GlobalQueue = DefaultConfig.GlobalQueue - } - if conf.Lifetime < 1 { - log.Warn("Sanitizing invalid txpool lifetime", "provided", conf.Lifetime, "updated", DefaultConfig.Lifetime) - conf.Lifetime = DefaultConfig.Lifetime - } - return conf -} - -// TxPool contains all currently known transactions. Transactions -// enter the pool when they are received from the network or submitted -// locally. They exit the pool when they are included in the blockchain. -// -// The pool separates processable transactions (which can be applied to the -// current state) and future transactions. Transactions move between those -// two states over time as they are received and processed. +// TxPool is an aggregator for various transaction specific pools, collectively +// tracking all the transactions deemed interesting by the node. Transactions +// enter the pool when they are received from the network or submitted locally. +// They exit the pool when they are included in the blockchain or evicted due to +// resource constraints. type TxPool struct { - config Config - chainconfig *params.ChainConfig - chain blockChain - gasTip atomic.Pointer[big.Int] - txFeed event.Feed - scope event.SubscriptionScope - signer types.Signer - mu sync.RWMutex - - currentHead atomic.Pointer[types.Header] // Current head of the blockchain - currentState *state.StateDB // Current state in the blockchain head - pendingNonces *noncer // Pending state tracking virtual nonces - - locals *accountSet // Set of local transaction to exempt from eviction rules - journal *journal // Journal of local transaction to back up to disk - - pending map[common.Address]*list // All currently processable transactions - queue map[common.Address]*list // Queued but non-processable transactions - beats map[common.Address]time.Time // Last heartbeat from each known account - all *lookup // All transactions to allow lookups - priced *pricedList // All transactions sorted by price - - chainHeadCh chan core.ChainHeadEvent - chainHeadSub event.Subscription - reqResetCh chan *txpoolResetRequest - reqPromoteCh chan *accountSet - queueTxEventCh chan *types.Transaction - reorgDoneCh chan chan struct{} - reorgShutdownCh chan struct{} // requests shutdown of scheduleReorgLoop - wg sync.WaitGroup // tracks loop, scheduleReorgLoop - initDoneCh chan struct{} // is closed once the pool is initialized (for tests) - - changesSinceReorg int // A counter for how many drops we've performed in-between reorg. -} - -type txpoolResetRequest struct { - oldHead, newHead *types.Header + subpools []SubPool // List of subpools for specialized transaction handling + subs event.SubscriptionScope // Subscription scope to unscubscribe all on shutdown + quit chan chan error // Quit channel to tear down the head updater } // New creates a new transaction pool to gather, sort and filter inbound // transactions from the network. -func New(config Config, chainconfig *params.ChainConfig, chain blockChain) *TxPool { - // Sanitize the input to ensure no vulnerable gas prices are set - config = (&config).sanitize() +func New(gasTip *big.Int, chain BlockChain, subpools []SubPool) (*TxPool, error) { + // Retrieve the current head so that all subpools and this main coordinator + // pool will have the same starting state, even if the chain moves forward + // during initialization. + head := chain.CurrentBlock() - // Create the transaction pool with its initial settings pool := &TxPool{ - config: config, - chainconfig: chainconfig, - chain: chain, - signer: types.LatestSigner(chainconfig), - pending: make(map[common.Address]*list), - queue: make(map[common.Address]*list), - beats: make(map[common.Address]time.Time), - all: newLookup(), - chainHeadCh: make(chan core.ChainHeadEvent, chainHeadChanSize), - reqResetCh: make(chan *txpoolResetRequest), - reqPromoteCh: make(chan *accountSet), - queueTxEventCh: make(chan *types.Transaction), - reorgDoneCh: make(chan chan struct{}), - reorgShutdownCh: make(chan struct{}), - initDoneCh: make(chan struct{}), + subpools: subpools, + quit: make(chan chan error), } - pool.gasTip.Store(new(big.Int).SetUint64(config.PriceLimit)) - pool.locals = newAccountSet(pool.signer) - for _, addr := range config.Locals { - log.Info("Setting new local account", "address", addr) - pool.locals.add(addr) - } - pool.priced = newPricedList(pool.all) - pool.reset(nil, chain.CurrentBlock()) - - // Start the reorg loop early so it can handle requests generated during journal loading. - pool.wg.Add(1) - go pool.scheduleReorgLoop() - - // If local transactions and journaling is enabled, load from disk - if !config.NoLocals && config.Journal != "" { - pool.journal = newTxJournal(config.Journal) - - if err := pool.journal.load(pool.AddLocals); err != nil { - log.Warn("Failed to load transaction journal", "err", err) - } - if err := pool.journal.rotate(pool.local()); err != nil { - log.Warn("Failed to rotate transaction journal", "err", err) + for i, subpool := range subpools { + if err := subpool.Init(gasTip, head); err != nil { + for j := i - 1; j >= 0; j-- { + subpools[j].Close() + } + return nil, err } } + go pool.loop(head, chain) + return pool, nil +} - // Subscribe events from blockchain and start the main event loop. - pool.chainHeadSub = pool.chain.SubscribeChainHeadEvent(pool.chainHeadCh) - pool.wg.Add(1) - go pool.loop() +// Close terminates the transaction pool and all its subpools. +func (p *TxPool) Close() error { + var errs []error - return pool + // Terminate the reset loop and wait for it to finish + errc := make(chan error) + p.quit <- errc + errs = append(errs, <-errc) + + // Terminate each subpool + for _, subpool := range p.subpools { + errs = append(errs, subpool.Close()) + } + return errors.Join(errs...) } // loop is the transaction pool's main event loop, waiting for and reacting to // outside blockchain events as well as for various reporting and transaction // eviction events. -func (pool *TxPool) loop() { - defer pool.wg.Done() - +func (p *TxPool) loop(head *types.Header, chain BlockChain) { + // Subscribe to chain head events to trigger subpool resets var ( - prevPending, prevQueued, prevStales int - // Start the stats reporting and transaction eviction tickers - report = time.NewTicker(statsReportInterval) - evict = time.NewTicker(evictionInterval) - journal = time.NewTicker(pool.config.Rejournal) - // Track the previous head headers for transaction reorgs - head = pool.chain.CurrentBlock() + newHeadCh = make(chan core.ChainHeadEvent) + newHeadSub = chain.SubscribeChainHeadEvent(newHeadCh) ) - defer report.Stop() - defer evict.Stop() - defer journal.Stop() + defer newHeadSub.Unsubscribe() - // Notify tests that the init phase is done - close(pool.initDoneCh) - for { - select { - // Handle ChainHeadEvent - case ev := <-pool.chainHeadCh: - if ev.Block != nil { - pool.requestReset(head, ev.Block.Header()) - head = ev.Block.Header() - } - - // System shutdown. - case <-pool.chainHeadSub.Err(): - close(pool.reorgShutdownCh) - return - - // Handle stats reporting ticks - case <-report.C: - pool.mu.RLock() - pending, queued := pool.stats() - pool.mu.RUnlock() - stales := int(pool.priced.stales.Load()) - - if pending != prevPending || queued != prevQueued || stales != prevStales { - log.Debug("Transaction pool status report", "executable", pending, "queued", queued, "stales", stales) - prevPending, prevQueued, prevStales = pending, queued, stales - } - - // Handle inactive account transaction eviction - case <-evict.C: - pool.mu.Lock() - for addr := range pool.queue { - // Skip local transactions from the eviction mechanism - if pool.locals.contains(addr) { - continue - } - // Any non-locals old enough should be removed - if time.Since(pool.beats[addr]) > pool.config.Lifetime { - list := pool.queue[addr].Flatten() - for _, tx := range list { - pool.removeTx(tx.Hash(), true) + // Track the previous and current head to feed to an idle reset + var ( + oldHead = head + newHead = oldHead + ) + // Consume chain head events and start resets when none is running + var ( + resetBusy = make(chan struct{}, 1) // Allow 1 reset to run concurrently + resetDone = make(chan *types.Header) + ) + var errc chan error + for errc == nil { + // Something interesting might have happened, run a reset if there is + // one needed but none is running. The resetter will run on its own + // goroutine to allow chain head events to be consumed contiguously. + if newHead != oldHead { + // Try to inject a busy marker and start a reset if successful + select { + case resetBusy <- struct{}{}: + // Busy marker injected, start a new subpool reset + go func(oldHead, newHead *types.Header) { + for _, subpool := range p.subpools { + subpool.Reset(oldHead, newHead) } - queuedEvictionMeter.Mark(int64(len(list))) - } - } - pool.mu.Unlock() + resetDone <- newHead + }(oldHead, newHead) - // Handle local transaction journal rotation - case <-journal.C: - if pool.journal != nil { - pool.mu.Lock() - if err := pool.journal.rotate(pool.local()); err != nil { - log.Warn("Failed to rotate local tx journal", "err", err) - } - pool.mu.Unlock() + default: + // Reset already running, wait until it finishes } } + // Wait for the next chain head event or a previous reset finish + select { + case event := <-newHeadCh: + // Chain moved forward, store the head for later consumption + newHead = event.Block.Header() + + case head := <-resetDone: + // Previous reset finished, update the old head and allow a new reset + oldHead = head + <-resetBusy + + case errc = <-p.quit: + // Termination requested, break out on the next loop round + } } -} - -// Stop terminates the transaction pool. -func (pool *TxPool) Stop() { - // Unsubscribe all subscriptions registered from txpool - pool.scope.Close() - - // Unsubscribe subscriptions registered from blockchain - pool.chainHeadSub.Unsubscribe() - pool.wg.Wait() - - if pool.journal != nil { - pool.journal.close() - } - log.Info("Transaction pool stopped") -} - -// SubscribeNewTxsEvent registers a subscription of NewTxsEvent and -// starts sending event to the given channel. -func (pool *TxPool) SubscribeNewTxsEvent(ch chan<- core.NewTxsEvent) event.Subscription { - return pool.scope.Track(pool.txFeed.Subscribe(ch)) + // Notify the closer of termination (no error possible for now) + errc <- nil } // SetGasTip updates the minimum gas tip required by the transaction pool for a // new transaction, and drops all transactions below this threshold. -func (pool *TxPool) SetGasTip(tip *big.Int) { - pool.mu.Lock() - defer pool.mu.Unlock() +func (p *TxPool) SetGasTip(tip *big.Int) { + for _, subpool := range p.subpools { + subpool.SetGasTip(tip) + } +} - old := pool.gasTip.Load() - pool.gasTip.Store(new(big.Int).Set(tip)) - - // If the min miner fee increased, remove transactions below the new threshold - if tip.Cmp(old) > 0 { - // pool.priced is sorted by GasFeeCap, so we have to iterate through pool.all instead - drop := pool.all.RemotesBelowTip(tip) - for _, tx := range drop { - pool.removeTx(tx.Hash(), false) +// Has returns an indicator whether the pool has a transaction cached with the +// given hash. +func (p *TxPool) Has(hash common.Hash) bool { + for _, subpool := range p.subpools { + if subpool.Has(hash) { + return true } - pool.priced.Removed(len(drop)) } - log.Info("Transaction pool tip threshold updated", "tip", tip) + return false } -// Nonce returns the next nonce of an account, with all transactions executable -// by the pool already applied on top. -func (pool *TxPool) Nonce(addr common.Address) uint64 { - pool.mu.RLock() - defer pool.mu.RUnlock() - - return pool.pendingNonces.get(addr) +// Get returns a transaction if it is contained in the pool, or nil otherwise. +func (p *TxPool) Get(hash common.Hash) *Transaction { + for _, subpool := range p.subpools { + if tx := subpool.Get(hash); tx != nil { + return tx + } + } + return nil } -// Stats retrieves the current pool stats, namely the number of pending and the -// number of queued (non-executable) transactions. -func (pool *TxPool) Stats() (int, int) { - pool.mu.RLock() - defer pool.mu.RUnlock() +// Add enqueues a batch of transactions into the pool if they are valid. Due +// to the large transaction churn, add may postpone fully integrating the tx +// to a later point to batch multiple ones together. +func (p *TxPool) Add(txs []*Transaction, local bool, sync bool) []error { + // Split the input transactions between the subpools. It shouldn't really + // happen that we receive merged batches, but better graceful than strange + // errors. + // + // We also need to track how the transactions were split across the subpools, + // so we can piece back the returned errors into the original order. + txsets := make([][]*Transaction, len(p.subpools)) + splits := make([]int, len(txs)) - return pool.stats() -} + for i, tx := range txs { + // Mark this transaction belonging to no-subpool + splits[i] = -1 -// stats retrieves the current pool stats, namely the number of pending and the -// number of queued (non-executable) transactions. -func (pool *TxPool) stats() (int, int) { - pending := 0 - for _, list := range pool.pending { - pending += list.Len() + // Try to find a subpool that accepts the transaction + for j, subpool := range p.subpools { + if subpool.Filter(tx.Tx) { + txsets[j] = append(txsets[j], tx) + splits[i] = j + break + } + } } - queued := 0 - for _, list := range pool.queue { - queued += list.Len() + // Add the transactions split apart to the individual subpools and piece + // back the errors into the original sort order. + errsets := make([][]error, len(p.subpools)) + for i := 0; i < len(p.subpools); i++ { + errsets[i] = p.subpools[i].Add(txsets[i], local, sync) } - return pending, queued -} - -// Content retrieves the data content of the transaction pool, returning all the -// pending as well as queued transactions, grouped by account and sorted by nonce. -func (pool *TxPool) Content() (map[common.Address]types.Transactions, map[common.Address]types.Transactions) { - pool.mu.Lock() - defer pool.mu.Unlock() - - pending := make(map[common.Address]types.Transactions, len(pool.pending)) - for addr, list := range pool.pending { - pending[addr] = list.Flatten() + errs := make([]error, len(txs)) + for i, split := range splits { + // If the transaction was rejected by all subpools, mark it unsupported + if split == -1 { + errs[i] = core.ErrTxTypeNotSupported + continue + } + // Find which subpool handled it and pull in the corresponding error + errs[i] = errsets[split][0] + errsets[split] = errsets[split][1:] } - queued := make(map[common.Address]types.Transactions, len(pool.queue)) - for addr, list := range pool.queue { - queued[addr] = list.Flatten() - } - return pending, queued -} - -// ContentFrom retrieves the data content of the transaction pool, returning the -// pending as well as queued transactions of this address, grouped by nonce. -func (pool *TxPool) ContentFrom(addr common.Address) (types.Transactions, types.Transactions) { - pool.mu.RLock() - defer pool.mu.RUnlock() - - var pending types.Transactions - if list, ok := pool.pending[addr]; ok { - pending = list.Flatten() - } - var queued types.Transactions - if list, ok := pool.queue[addr]; ok { - queued = list.Flatten() - } - return pending, queued + return errs } // Pending retrieves all currently processable transactions, grouped by origin -// account and sorted by nonce. The returned transaction set is a copy and can be -// freely modified by calling code. -// -// The enforceTips parameter can be used to do an extra filtering on the pending -// transactions and only return those whose **effective** tip is large enough in -// the next pending execution environment. -func (pool *TxPool) Pending(enforceTips bool) map[common.Address]types.Transactions { - pool.mu.Lock() - defer pool.mu.Unlock() - - pending := make(map[common.Address]types.Transactions, len(pool.pending)) - for addr, list := range pool.pending { - txs := list.Flatten() - - // If the miner requests tip enforcement, cap the lists now - if enforceTips && !pool.locals.contains(addr) { - for i, tx := range txs { - if tx.EffectiveGasTipIntCmp(pool.gasTip.Load(), pool.priced.urgent.baseFee) < 0 { - txs = txs[:i] - break - } - } - } - if len(txs) > 0 { - pending[addr] = txs - } - } - return pending -} - -// Locals retrieves the accounts currently considered local by the pool. -func (pool *TxPool) Locals() []common.Address { - pool.mu.Lock() - defer pool.mu.Unlock() - - return pool.locals.flatten() -} - -// local retrieves all currently known local transactions, grouped by origin -// account and sorted by nonce. The returned transaction set is a copy and can be -// freely modified by calling code. -func (pool *TxPool) local() map[common.Address]types.Transactions { - txs := make(map[common.Address]types.Transactions) - for addr := range pool.locals.accounts { - if pending := pool.pending[addr]; pending != nil { - txs[addr] = append(txs[addr], pending.Flatten()...) - } - if queued := pool.queue[addr]; queued != nil { - txs[addr] = append(txs[addr], queued.Flatten()...) +// account and sorted by nonce. +func (p *TxPool) Pending(enforceTips bool) map[common.Address][]*types.Transaction { + txs := make(map[common.Address][]*types.Transaction) + for _, subpool := range p.subpools { + for addr, set := range subpool.Pending(enforceTips) { + txs[addr] = set } } return txs } -// validateTxBasics checks whether a transaction is valid according to the consensus -// rules, but does not check state-dependent validation such as sufficient balance. -// This check is meant as an early check which only needs to be performed once, -// and does not require the pool mutex to be held. -func (pool *TxPool) validateTxBasics(tx *types.Transaction, local bool) error { - opts := &ValidationOptions{ - Config: pool.chainconfig, - Accept: 0 | - 1< pool.config.GlobalSlots+pool.config.GlobalQueue { - // If the new transaction is underpriced, don't accept it - if !isLocal && pool.priced.Underpriced(tx) { - log.Trace("Discarding underpriced transaction", "hash", hash, "gasTipCap", tx.GasTipCap(), "gasFeeCap", tx.GasFeeCap()) - underpricedTxMeter.Mark(1) - return false, ErrUnderpriced - } - - // We're about to replace a transaction. The reorg does a more thorough - // analysis of what to remove and how, but it runs async. We don't want to - // do too many replacements between reorg-runs, so we cap the number of - // replacements to 25% of the slots - if pool.changesSinceReorg > int(pool.config.GlobalSlots/4) { - throttleTxMeter.Mark(1) - return false, ErrTxPoolOverflow - } - - // New transaction is better than our worse ones, make room for it. - // If it's a local transaction, forcibly discard all available transactions. - // Otherwise if we can't make enough room for new one, abort the operation. - drop, success := pool.priced.Discard(pool.all.Slots()-int(pool.config.GlobalSlots+pool.config.GlobalQueue)+numSlots(tx), isLocal) - - // Special case, we still can't make the room for the new remote one. - if !isLocal && !success { - log.Trace("Discarding overflown transaction", "hash", hash) - overflowedTxMeter.Mark(1) - return false, ErrTxPoolOverflow - } - - // If the new transaction is a future transaction it should never churn pending transactions - if !isLocal && pool.isGapped(from, tx) { - var replacesPending bool - for _, dropTx := range drop { - dropSender, _ := types.Sender(pool.signer, dropTx) - if list := pool.pending[dropSender]; list != nil && list.Contains(dropTx.Nonce()) { - replacesPending = true - break - } - } - // Add all transactions back to the priced queue - if replacesPending { - for _, dropTx := range drop { - pool.priced.Put(dropTx, false) - } - log.Trace("Discarding future transaction replacing pending tx", "hash", hash) - return false, ErrFutureReplacePending - } - } - - // Kick out the underpriced remote transactions. - for _, tx := range drop { - log.Trace("Discarding freshly underpriced transaction", "hash", tx.Hash(), "gasTipCap", tx.GasTipCap(), "gasFeeCap", tx.GasFeeCap()) - underpricedTxMeter.Mark(1) - dropped := pool.removeTx(tx.Hash(), false) - pool.changesSinceReorg += dropped +// Nonce returns the next nonce of an account, with all transactions executable +// by the pool already applied on top. +func (p *TxPool) Nonce(addr common.Address) uint64 { + // Since (for now) accounts are unique to subpools, only one pool will have + // (at max) a non-state nonce. To avoid stateful lookups, just return the + // highest nonce for now. + var nonce uint64 + for _, subpool := range p.subpools { + if next := subpool.Nonce(addr); nonce < next { + nonce = next } } - - // Try to replace an existing transaction in the pending pool - if list := pool.pending[from]; list != nil && list.Contains(tx.Nonce()) { - // Nonce already pending, check if required price bump is met - inserted, old := list.Add(tx, pool.config.PriceBump) - if !inserted { - pendingDiscardMeter.Mark(1) - return false, ErrReplaceUnderpriced - } - // New transaction is better, replace old one - if old != nil { - pool.all.Remove(old.Hash()) - pool.priced.Removed(1) - pendingReplaceMeter.Mark(1) - } - pool.all.Add(tx, isLocal) - pool.priced.Put(tx, isLocal) - pool.journalTx(from, tx) - pool.queueTxEvent(tx) - log.Trace("Pooled new executable transaction", "hash", hash, "from", from, "to", tx.To()) - - // Successful promotion, bump the heartbeat - pool.beats[from] = time.Now() - return old != nil, nil - } - // New transaction isn't replacing a pending one, push into queue - replaced, err = pool.enqueueTx(hash, tx, isLocal, true) - if err != nil { - return false, err - } - // Mark local addresses and journal local transactions - if local && !pool.locals.contains(from) { - log.Info("Setting new local account", "address", from) - pool.locals.add(from) - pool.priced.Removed(pool.all.RemoteToLocals(pool.locals)) // Migrate the remotes if it's marked as local first time. - } - if isLocal { - localGauge.Inc(1) - } - pool.journalTx(from, tx) - - log.Trace("Pooled new future transaction", "hash", hash, "from", from, "to", tx.To()) - return replaced, nil + return nonce } -// isGapped reports whether the given transaction is immediately executable. -func (pool *TxPool) isGapped(from common.Address, tx *types.Transaction) bool { - // Short circuit if transaction falls within the scope of the pending list - // or matches the next pending nonce which can be promoted as an executable - // transaction afterwards. Note, the tx staleness is already checked in - // 'validateTx' function previously. - next := pool.pendingNonces.get(from) - if tx.Nonce() <= next { - return false +// Stats retrieves the current pool stats, namely the number of pending and the +// number of queued (non-executable) transactions. +func (p *TxPool) Stats() (int, int) { + var runnable, blocked int + for _, subpool := range p.subpools { + run, block := subpool.Stats() + + runnable += run + blocked += block } - // The transaction has a nonce gap with pending list, it's only considered - // as executable if transactions in queue can fill up the nonce gap. - queue, ok := pool.queue[from] - if !ok { - return true - } - for nonce := next; nonce < tx.Nonce(); nonce++ { - if !queue.Contains(nonce) { - return true // txs in queue can't fill up the nonce gap - } - } - return false + return runnable, blocked } -// enqueueTx inserts a new transaction into the non-executable transaction queue. -// -// Note, this method assumes the pool lock is held! -func (pool *TxPool) enqueueTx(hash common.Hash, tx *types.Transaction, local bool, addAll bool) (bool, error) { - // Try to insert the transaction into the future queue - from, _ := types.Sender(pool.signer, tx) // already validated - if pool.queue[from] == nil { - pool.queue[from] = newList(false) - } - inserted, old := pool.queue[from].Add(tx, pool.config.PriceBump) - if !inserted { - // An older transaction was better, discard this - queuedDiscardMeter.Mark(1) - return false, ErrReplaceUnderpriced - } - // Discard any previous transaction and mark this - if old != nil { - pool.all.Remove(old.Hash()) - pool.priced.Removed(1) - queuedReplaceMeter.Mark(1) - } else { - // Nothing was replaced, bump the queued counter - queuedGauge.Inc(1) - } - // If the transaction isn't in lookup set but it's expected to be there, - // show the error log. - if pool.all.Get(hash) == nil && !addAll { - log.Error("Missing transaction in lookup set, please report the issue", "hash", hash) - } - if addAll { - pool.all.Add(tx, local) - pool.priced.Put(tx, local) - } - // If we never record the heartbeat, do it right now. - if _, exist := pool.beats[from]; !exist { - pool.beats[from] = time.Now() - } - return old != nil, nil -} - -// journalTx adds the specified transaction to the local disk journal if it is -// deemed to have been sent from a local account. -func (pool *TxPool) journalTx(from common.Address, tx *types.Transaction) { - // Only journal if it's enabled and the transaction is local - if pool.journal == nil || !pool.locals.contains(from) { - return - } - if err := pool.journal.insert(tx); err != nil { - log.Warn("Failed to journal local transaction", "err", err) - } -} - -// promoteTx adds a transaction to the pending (processable) list of transactions -// and returns whether it was inserted or an older was better. -// -// Note, this method assumes the pool lock is held! -func (pool *TxPool) promoteTx(addr common.Address, hash common.Hash, tx *types.Transaction) bool { - // Try to insert the transaction into the pending queue - if pool.pending[addr] == nil { - pool.pending[addr] = newList(true) - } - list := pool.pending[addr] - - inserted, old := list.Add(tx, pool.config.PriceBump) - if !inserted { - // An older transaction was better, discard this - pool.all.Remove(hash) - pool.priced.Removed(1) - pendingDiscardMeter.Mark(1) - return false - } - // Otherwise discard any previous transaction and mark this - if old != nil { - pool.all.Remove(old.Hash()) - pool.priced.Removed(1) - pendingReplaceMeter.Mark(1) - } else { - // Nothing was replaced, bump the pending counter - pendingGauge.Inc(1) - } - // Set the potentially new pending nonce and notify any subsystems of the new tx - pool.pendingNonces.set(addr, tx.Nonce()+1) - - // Successful promotion, bump the heartbeat - pool.beats[addr] = time.Now() - return true -} - -// AddLocals enqueues a batch of transactions into the pool if they are valid, marking the -// senders as a local ones, ensuring they go around the local pricing constraints. -// -// This method is used to add transactions from the RPC API and performs synchronous pool -// reorganization and event propagation. -func (pool *TxPool) AddLocals(txs []*types.Transaction) []error { - return pool.addTxs(txs, !pool.config.NoLocals, true) -} - -// AddLocal enqueues a single local transaction into the pool if it is valid. This is -// a convenience wrapper around AddLocals. -func (pool *TxPool) AddLocal(tx *types.Transaction) error { - errs := pool.AddLocals([]*types.Transaction{tx}) - return errs[0] -} - -// AddRemotes enqueues a batch of transactions into the pool if they are valid. If the -// senders are not among the locally tracked ones, full pricing constraints will apply. -// -// This method is used to add transactions from the p2p network and does not wait for pool -// reorganization and internal event propagation. -func (pool *TxPool) AddRemotes(txs []*types.Transaction) []error { - return pool.addTxs(txs, false, false) -} - -// AddRemotesSync is like AddRemotes, but waits for pool reorganization. Tests use this method. -func (pool *TxPool) AddRemotesSync(txs []*types.Transaction) []error { - return pool.addTxs(txs, false, true) -} - -// This is like AddRemotes with a single transaction, but waits for pool reorganization. Tests use this method. -func (pool *TxPool) addRemoteSync(tx *types.Transaction) error { - errs := pool.AddRemotesSync([]*types.Transaction{tx}) - return errs[0] -} - -// AddRemote enqueues a single transaction into the pool if it is valid. This is a convenience -// wrapper around AddRemotes. -// -// Deprecated: use AddRemotes -func (pool *TxPool) AddRemote(tx *types.Transaction) error { - errs := pool.AddRemotes([]*types.Transaction{tx}) - return errs[0] -} - -// addTxs attempts to queue a batch of transactions if they are valid. -func (pool *TxPool) addTxs(txs []*types.Transaction, local, sync bool) []error { - // Filter out known ones without obtaining the pool lock or recovering signatures +// Content retrieves the data content of the transaction pool, returning all the +// pending as well as queued transactions, grouped by account and sorted by nonce. +func (p *TxPool) Content() (map[common.Address][]*types.Transaction, map[common.Address][]*types.Transaction) { var ( - errs = make([]error, len(txs)) - news = make([]*types.Transaction, 0, len(txs)) + runnable = make(map[common.Address][]*types.Transaction) + blocked = make(map[common.Address][]*types.Transaction) ) - for i, tx := range txs { - // If the transaction is known, pre-set the error slot - if pool.all.Get(tx.Hash()) != nil { - errs[i] = ErrAlreadyKnown - knownTxMeter.Mark(1) - continue - } - // Exclude transactions with basic errors, e.g invalid signatures and - // insufficient intrinsic gas as soon as possible and cache senders - // in transactions before obtaining lock - if err := pool.validateTxBasics(tx, local); err != nil { - errs[i] = err - invalidTxMeter.Mark(1) - continue - } - // Accumulate all unknown transactions for deeper processing - news = append(news, tx) - } - if len(news) == 0 { - return errs - } + for _, subpool := range p.subpools { + run, block := subpool.Content() - // Process all the new transaction and merge any errors into the original slice - pool.mu.Lock() - newErrs, dirtyAddrs := pool.addTxsLocked(news, local) - pool.mu.Unlock() - - var nilSlot = 0 - for _, err := range newErrs { - for errs[nilSlot] != nil { - nilSlot++ + for addr, txs := range run { + runnable[addr] = txs + } + for addr, txs := range block { + blocked[addr] = txs } - errs[nilSlot] = err - nilSlot++ } - // Reorg the pool internals if needed and return - done := pool.requestPromoteExecutables(dirtyAddrs) - if sync { - <-done - } - return errs + return runnable, blocked } -// addTxsLocked attempts to queue a batch of transactions if they are valid. -// The transaction pool lock must be held. -func (pool *TxPool) addTxsLocked(txs []*types.Transaction, local bool) ([]error, *accountSet) { - dirty := newAccountSet(pool.signer) - errs := make([]error, len(txs)) - for i, tx := range txs { - replaced, err := pool.add(tx, local) - errs[i] = err - if err == nil && !replaced { - dirty.addTx(tx) +// ContentFrom retrieves the data content of the transaction pool, returning the +// pending as well as queued transactions of this address, grouped by nonce. +func (p *TxPool) ContentFrom(addr common.Address) ([]*types.Transaction, []*types.Transaction) { + for _, subpool := range p.subpools { + run, block := subpool.ContentFrom(addr) + if len(run) != 0 || len(block) != 0 { + return run, block } } - validTxMeter.Mark(int64(len(dirty.accounts))) - return errs, dirty + return []*types.Transaction{}, []*types.Transaction{} } -// Status returns the status (unknown/pending/queued) of a batch of transactions +// Locals retrieves the accounts currently considered local by the pool. +func (p *TxPool) Locals() []common.Address { + // Retrieve the locals from each subpool and deduplicate them + locals := make(map[common.Address]struct{}) + for _, subpool := range p.subpools { + for _, local := range subpool.Locals() { + locals[local] = struct{}{} + } + } + // Flatten and return the deduplicated local set + flat := make([]common.Address, 0, len(locals)) + for local := range locals { + flat = append(flat, local) + } + return flat +} + +// Status returns the known status (unknown/pending/queued) of a transaction // identified by their hashes. -func (pool *TxPool) Status(hashes []common.Hash) []TxStatus { - status := make([]TxStatus, len(hashes)) - for i, hash := range hashes { - tx := pool.Get(hash) - if tx == nil { - continue - } - from, _ := types.Sender(pool.signer, tx) // already validated - pool.mu.RLock() - if txList := pool.pending[from]; txList != nil && txList.txs.items[tx.Nonce()] != nil { - status[i] = TxStatusPending - } else if txList := pool.queue[from]; txList != nil && txList.txs.items[tx.Nonce()] != nil { - status[i] = TxStatusQueued - } - // implicit else: the tx may have been included into a block between - // checking pool.Get and obtaining the lock. In that case, TxStatusUnknown is correct - pool.mu.RUnlock() - } - return status -} - -// Get returns a transaction if it is contained in the pool and nil otherwise. -func (pool *TxPool) Get(hash common.Hash) *types.Transaction { - return pool.all.Get(hash) -} - -// Has returns an indicator whether txpool has a transaction cached with the -// given hash. -func (pool *TxPool) Has(hash common.Hash) bool { - return pool.all.Get(hash) != nil -} - -// removeTx removes a single transaction from the queue, moving all subsequent -// transactions back to the future queue. -// Returns the number of transactions removed from the pending queue. -func (pool *TxPool) removeTx(hash common.Hash, outofbound bool) int { - // Fetch the transaction we wish to delete - tx := pool.all.Get(hash) - if tx == nil { - return 0 - } - addr, _ := types.Sender(pool.signer, tx) // already validated during insertion - - // Remove it from the list of known transactions - pool.all.Remove(hash) - if outofbound { - pool.priced.Removed(1) - } - if pool.locals.contains(addr) { - localGauge.Dec(1) - } - // Remove the transaction from the pending lists and reset the account nonce - if pending := pool.pending[addr]; pending != nil { - if removed, invalids := pending.Remove(tx); removed { - // If no more pending transactions are left, remove the list - if pending.Empty() { - delete(pool.pending, addr) - } - // Postpone any invalidated transactions - for _, tx := range invalids { - // Internal shuffle shouldn't touch the lookup set. - pool.enqueueTx(tx.Hash(), tx, false, false) - } - // Update the account nonce if needed - pool.pendingNonces.setIfLower(addr, tx.Nonce()) - // Reduce the pending counter - pendingGauge.Dec(int64(1 + len(invalids))) - return 1 + len(invalids) +func (p *TxPool) Status(hash common.Hash) TxStatus { + for _, subpool := range p.subpools { + if status := subpool.Status(hash); status != TxStatusUnknown { + return status } } - // Transaction is in the future queue - if future := pool.queue[addr]; future != nil { - if removed, _ := future.Remove(tx); removed { - // Reduce the queued counter - queuedGauge.Dec(1) - } - if future.Empty() { - delete(pool.queue, addr) - delete(pool.beats, addr) - } - } - return 0 -} - -// requestReset requests a pool reset to the new head block. -// The returned channel is closed when the reset has occurred. -func (pool *TxPool) requestReset(oldHead *types.Header, newHead *types.Header) chan struct{} { - select { - case pool.reqResetCh <- &txpoolResetRequest{oldHead, newHead}: - return <-pool.reorgDoneCh - case <-pool.reorgShutdownCh: - return pool.reorgShutdownCh - } -} - -// requestPromoteExecutables requests transaction promotion checks for the given addresses. -// The returned channel is closed when the promotion checks have occurred. -func (pool *TxPool) requestPromoteExecutables(set *accountSet) chan struct{} { - select { - case pool.reqPromoteCh <- set: - return <-pool.reorgDoneCh - case <-pool.reorgShutdownCh: - return pool.reorgShutdownCh - } -} - -// queueTxEvent enqueues a transaction event to be sent in the next reorg run. -func (pool *TxPool) queueTxEvent(tx *types.Transaction) { - select { - case pool.queueTxEventCh <- tx: - case <-pool.reorgShutdownCh: - } -} - -// scheduleReorgLoop schedules runs of reset and promoteExecutables. Code above should not -// call those methods directly, but request them being run using requestReset and -// requestPromoteExecutables instead. -func (pool *TxPool) scheduleReorgLoop() { - defer pool.wg.Done() - - var ( - curDone chan struct{} // non-nil while runReorg is active - nextDone = make(chan struct{}) - launchNextRun bool - reset *txpoolResetRequest - dirtyAccounts *accountSet - queuedEvents = make(map[common.Address]*sortedMap) - ) - for { - // Launch next background reorg if needed - if curDone == nil && launchNextRun { - // Run the background reorg and announcements - go pool.runReorg(nextDone, reset, dirtyAccounts, queuedEvents) - - // Prepare everything for the next round of reorg - curDone, nextDone = nextDone, make(chan struct{}) - launchNextRun = false - - reset, dirtyAccounts = nil, nil - queuedEvents = make(map[common.Address]*sortedMap) - } - - select { - case req := <-pool.reqResetCh: - // Reset request: update head if request is already pending. - if reset == nil { - reset = req - } else { - reset.newHead = req.newHead - } - launchNextRun = true - pool.reorgDoneCh <- nextDone - - case req := <-pool.reqPromoteCh: - // Promote request: update address set if request is already pending. - if dirtyAccounts == nil { - dirtyAccounts = req - } else { - dirtyAccounts.merge(req) - } - launchNextRun = true - pool.reorgDoneCh <- nextDone - - case tx := <-pool.queueTxEventCh: - // Queue up the event, but don't schedule a reorg. It's up to the caller to - // request one later if they want the events sent. - addr, _ := types.Sender(pool.signer, tx) - if _, ok := queuedEvents[addr]; !ok { - queuedEvents[addr] = newSortedMap() - } - queuedEvents[addr].Put(tx) - - case <-curDone: - curDone = nil - - case <-pool.reorgShutdownCh: - // Wait for current run to finish. - if curDone != nil { - <-curDone - } - close(nextDone) - return - } - } -} - -// runReorg runs reset and promoteExecutables on behalf of scheduleReorgLoop. -func (pool *TxPool) runReorg(done chan struct{}, reset *txpoolResetRequest, dirtyAccounts *accountSet, events map[common.Address]*sortedMap) { - defer func(t0 time.Time) { - reorgDurationTimer.Update(time.Since(t0)) - }(time.Now()) - defer close(done) - - var promoteAddrs []common.Address - if dirtyAccounts != nil && reset == nil { - // Only dirty accounts need to be promoted, unless we're resetting. - // For resets, all addresses in the tx queue will be promoted and - // the flatten operation can be avoided. - promoteAddrs = dirtyAccounts.flatten() - } - pool.mu.Lock() - if reset != nil { - // Reset from the old head to the new, rescheduling any reorged transactions - pool.reset(reset.oldHead, reset.newHead) - - // Nonces were reset, discard any events that became stale - for addr := range events { - events[addr].Forward(pool.pendingNonces.get(addr)) - if events[addr].Len() == 0 { - delete(events, addr) - } - } - // Reset needs promote for all addresses - promoteAddrs = make([]common.Address, 0, len(pool.queue)) - for addr := range pool.queue { - promoteAddrs = append(promoteAddrs, addr) - } - } - // Check for pending transactions for every account that sent new ones - promoted := pool.promoteExecutables(promoteAddrs) - - // If a new block appeared, validate the pool of pending transactions. This will - // remove any transaction that has been included in the block or was invalidated - // because of another transaction (e.g. higher gas price). - if reset != nil { - pool.demoteUnexecutables() - if reset.newHead != nil && pool.chainconfig.IsLondon(new(big.Int).Add(reset.newHead.Number, big.NewInt(1))) { - pendingBaseFee := misc.CalcBaseFee(pool.chainconfig, reset.newHead) - pool.priced.SetBaseFee(pendingBaseFee) - } - // Update all accounts to the latest known pending nonce - nonces := make(map[common.Address]uint64, len(pool.pending)) - for addr, list := range pool.pending { - highestPending := list.LastElement() - nonces[addr] = highestPending.Nonce() + 1 - } - pool.pendingNonces.setAll(nonces) - } - // Ensure pool.queue and pool.pending sizes stay within the configured limits. - pool.truncatePending() - pool.truncateQueue() - - dropBetweenReorgHistogram.Update(int64(pool.changesSinceReorg)) - pool.changesSinceReorg = 0 // Reset change counter - pool.mu.Unlock() - - // Notify subsystems for newly added transactions - for _, tx := range promoted { - addr, _ := types.Sender(pool.signer, tx) - if _, ok := events[addr]; !ok { - events[addr] = newSortedMap() - } - events[addr].Put(tx) - } - if len(events) > 0 { - var txs []*types.Transaction - for _, set := range events { - txs = append(txs, set.Flatten()...) - } - pool.txFeed.Send(core.NewTxsEvent{Txs: txs}) - } -} - -// reset retrieves the current state of the blockchain and ensures the content -// of the transaction pool is valid with regard to the chain state. -func (pool *TxPool) reset(oldHead, newHead *types.Header) { - // If we're reorging an old state, reinject all dropped transactions - var reinject types.Transactions - - if oldHead != nil && oldHead.Hash() != newHead.ParentHash { - // If the reorg is too deep, avoid doing it (will happen during fast sync) - oldNum := oldHead.Number.Uint64() - newNum := newHead.Number.Uint64() - - if depth := uint64(math.Abs(float64(oldNum) - float64(newNum))); depth > 64 { - log.Debug("Skipping deep transaction reorg", "depth", depth) - } else { - // Reorg seems shallow enough to pull in all transactions into memory - var discarded, included types.Transactions - var ( - rem = pool.chain.GetBlock(oldHead.Hash(), oldHead.Number.Uint64()) - add = pool.chain.GetBlock(newHead.Hash(), newHead.Number.Uint64()) - ) - if rem == nil { - // This can happen if a setHead is performed, where we simply discard the old - // head from the chain. - // If that is the case, we don't have the lost transactions anymore, and - // there's nothing to add - if newNum >= oldNum { - // If we reorged to a same or higher number, then it's not a case of setHead - log.Warn("Transaction pool reset with missing oldhead", - "old", oldHead.Hash(), "oldnum", oldNum, "new", newHead.Hash(), "newnum", newNum) - return - } - // If the reorg ended up on a lower number, it's indicative of setHead being the cause - log.Debug("Skipping transaction reset caused by setHead", - "old", oldHead.Hash(), "oldnum", oldNum, "new", newHead.Hash(), "newnum", newNum) - // We still need to update the current state s.th. the lost transactions can be readded by the user - } else { - for rem.NumberU64() > add.NumberU64() { - discarded = append(discarded, rem.Transactions()...) - if rem = pool.chain.GetBlock(rem.ParentHash(), rem.NumberU64()-1); rem == nil { - log.Error("Unrooted old chain seen by tx pool", "block", oldHead.Number, "hash", oldHead.Hash()) - return - } - } - for add.NumberU64() > rem.NumberU64() { - included = append(included, add.Transactions()...) - if add = pool.chain.GetBlock(add.ParentHash(), add.NumberU64()-1); add == nil { - log.Error("Unrooted new chain seen by tx pool", "block", newHead.Number, "hash", newHead.Hash()) - return - } - } - for rem.Hash() != add.Hash() { - discarded = append(discarded, rem.Transactions()...) - if rem = pool.chain.GetBlock(rem.ParentHash(), rem.NumberU64()-1); rem == nil { - log.Error("Unrooted old chain seen by tx pool", "block", oldHead.Number, "hash", oldHead.Hash()) - return - } - included = append(included, add.Transactions()...) - if add = pool.chain.GetBlock(add.ParentHash(), add.NumberU64()-1); add == nil { - log.Error("Unrooted new chain seen by tx pool", "block", newHead.Number, "hash", newHead.Hash()) - return - } - } - reinject = types.TxDifference(discarded, included) - } - } - } - // Initialize the internal state to the current head - if newHead == nil { - newHead = pool.chain.CurrentBlock() // Special case during testing - } - statedb, err := pool.chain.StateAt(newHead.Root) - if err != nil { - log.Error("Failed to reset txpool state", "err", err) - return - } - pool.currentHead.Store(newHead) - pool.currentState = statedb - pool.pendingNonces = newNoncer(statedb) - - // Inject any transactions discarded due to reorgs - log.Debug("Reinjecting stale transactions", "count", len(reinject)) - core.SenderCacher.Recover(pool.signer, reinject) - pool.addTxsLocked(reinject, false) -} - -// promoteExecutables moves transactions that have become processable from the -// future queue to the set of pending transactions. During this process, all -// invalidated transactions (low nonce, low balance) are deleted. -func (pool *TxPool) promoteExecutables(accounts []common.Address) []*types.Transaction { - // Track the promoted transactions to broadcast them at once - var promoted []*types.Transaction - - // Iterate over all accounts and promote any executable transactions - gasLimit := pool.currentHead.Load().GasLimit - for _, addr := range accounts { - list := pool.queue[addr] - if list == nil { - continue // Just in case someone calls with a non existing account - } - // Drop all transactions that are deemed too old (low nonce) - forwards := list.Forward(pool.currentState.GetNonce(addr)) - for _, tx := range forwards { - hash := tx.Hash() - pool.all.Remove(hash) - } - log.Trace("Removed old queued transactions", "count", len(forwards)) - // Drop all transactions that are too costly (low balance or out of gas) - drops, _ := list.Filter(pool.currentState.GetBalance(addr), gasLimit) - for _, tx := range drops { - hash := tx.Hash() - pool.all.Remove(hash) - } - log.Trace("Removed unpayable queued transactions", "count", len(drops)) - queuedNofundsMeter.Mark(int64(len(drops))) - - // Gather all executable transactions and promote them - readies := list.Ready(pool.pendingNonces.get(addr)) - for _, tx := range readies { - hash := tx.Hash() - if pool.promoteTx(addr, hash, tx) { - promoted = append(promoted, tx) - } - } - log.Trace("Promoted queued transactions", "count", len(promoted)) - queuedGauge.Dec(int64(len(readies))) - - // Drop all transactions over the allowed limit - var caps types.Transactions - if !pool.locals.contains(addr) { - caps = list.Cap(int(pool.config.AccountQueue)) - for _, tx := range caps { - hash := tx.Hash() - pool.all.Remove(hash) - log.Trace("Removed cap-exceeding queued transaction", "hash", hash) - } - queuedRateLimitMeter.Mark(int64(len(caps))) - } - // Mark all the items dropped as removed - pool.priced.Removed(len(forwards) + len(drops) + len(caps)) - queuedGauge.Dec(int64(len(forwards) + len(drops) + len(caps))) - if pool.locals.contains(addr) { - localGauge.Dec(int64(len(forwards) + len(drops) + len(caps))) - } - // Delete the entire queue entry if it became empty. - if list.Empty() { - delete(pool.queue, addr) - delete(pool.beats, addr) - } - } - return promoted -} - -// truncatePending removes transactions from the pending queue if the pool is above the -// pending limit. The algorithm tries to reduce transaction counts by an approximately -// equal number for all for accounts with many pending transactions. -func (pool *TxPool) truncatePending() { - pending := uint64(0) - for _, list := range pool.pending { - pending += uint64(list.Len()) - } - if pending <= pool.config.GlobalSlots { - return - } - - pendingBeforeCap := pending - // Assemble a spam order to penalize large transactors first - spammers := prque.New[int64, common.Address](nil) - for addr, list := range pool.pending { - // Only evict transactions from high rollers - if !pool.locals.contains(addr) && uint64(list.Len()) > pool.config.AccountSlots { - spammers.Push(addr, int64(list.Len())) - } - } - // Gradually drop transactions from offenders - offenders := []common.Address{} - for pending > pool.config.GlobalSlots && !spammers.Empty() { - // Retrieve the next offender if not local address - offender, _ := spammers.Pop() - offenders = append(offenders, offender) - - // Equalize balances until all the same or below threshold - if len(offenders) > 1 { - // Calculate the equalization threshold for all current offenders - threshold := pool.pending[offender].Len() - - // Iteratively reduce all offenders until below limit or threshold reached - for pending > pool.config.GlobalSlots && pool.pending[offenders[len(offenders)-2]].Len() > threshold { - for i := 0; i < len(offenders)-1; i++ { - list := pool.pending[offenders[i]] - - caps := list.Cap(list.Len() - 1) - for _, tx := range caps { - // Drop the transaction from the global pools too - hash := tx.Hash() - pool.all.Remove(hash) - - // Update the account nonce to the dropped transaction - pool.pendingNonces.setIfLower(offenders[i], tx.Nonce()) - log.Trace("Removed fairness-exceeding pending transaction", "hash", hash) - } - pool.priced.Removed(len(caps)) - pendingGauge.Dec(int64(len(caps))) - if pool.locals.contains(offenders[i]) { - localGauge.Dec(int64(len(caps))) - } - pending-- - } - } - } - } - - // If still above threshold, reduce to limit or min allowance - if pending > pool.config.GlobalSlots && len(offenders) > 0 { - for pending > pool.config.GlobalSlots && uint64(pool.pending[offenders[len(offenders)-1]].Len()) > pool.config.AccountSlots { - for _, addr := range offenders { - list := pool.pending[addr] - - caps := list.Cap(list.Len() - 1) - for _, tx := range caps { - // Drop the transaction from the global pools too - hash := tx.Hash() - pool.all.Remove(hash) - - // Update the account nonce to the dropped transaction - pool.pendingNonces.setIfLower(addr, tx.Nonce()) - log.Trace("Removed fairness-exceeding pending transaction", "hash", hash) - } - pool.priced.Removed(len(caps)) - pendingGauge.Dec(int64(len(caps))) - if pool.locals.contains(addr) { - localGauge.Dec(int64(len(caps))) - } - pending-- - } - } - } - pendingRateLimitMeter.Mark(int64(pendingBeforeCap - pending)) -} - -// truncateQueue drops the oldest transactions in the queue if the pool is above the global queue limit. -func (pool *TxPool) truncateQueue() { - queued := uint64(0) - for _, list := range pool.queue { - queued += uint64(list.Len()) - } - if queued <= pool.config.GlobalQueue { - return - } - - // Sort all accounts with queued transactions by heartbeat - addresses := make(addressesByHeartbeat, 0, len(pool.queue)) - for addr := range pool.queue { - if !pool.locals.contains(addr) { // don't drop locals - addresses = append(addresses, addressByHeartbeat{addr, pool.beats[addr]}) - } - } - sort.Sort(sort.Reverse(addresses)) - - // Drop transactions until the total is below the limit or only locals remain - for drop := queued - pool.config.GlobalQueue; drop > 0 && len(addresses) > 0; { - addr := addresses[len(addresses)-1] - list := pool.queue[addr.address] - - addresses = addresses[:len(addresses)-1] - - // Drop all transactions if they are less than the overflow - if size := uint64(list.Len()); size <= drop { - for _, tx := range list.Flatten() { - pool.removeTx(tx.Hash(), true) - } - drop -= size - queuedRateLimitMeter.Mark(int64(size)) - continue - } - // Otherwise drop only last few transactions - txs := list.Flatten() - for i := len(txs) - 1; i >= 0 && drop > 0; i-- { - pool.removeTx(txs[i].Hash(), true) - drop-- - queuedRateLimitMeter.Mark(1) - } - } -} - -// demoteUnexecutables removes invalid and processed transactions from the pools -// executable/pending queue and any subsequent transactions that become unexecutable -// are moved back into the future queue. -// -// Note: transactions are not marked as removed in the priced list because re-heaping -// is always explicitly triggered by SetBaseFee and it would be unnecessary and wasteful -// to trigger a re-heap is this function -func (pool *TxPool) demoteUnexecutables() { - // Iterate over all accounts and demote any non-executable transactions - gasLimit := pool.currentHead.Load().GasLimit - for addr, list := range pool.pending { - nonce := pool.currentState.GetNonce(addr) - - // Drop all transactions that are deemed too old (low nonce) - olds := list.Forward(nonce) - for _, tx := range olds { - hash := tx.Hash() - pool.all.Remove(hash) - log.Trace("Removed old pending transaction", "hash", hash) - } - // Drop all transactions that are too costly (low balance or out of gas), and queue any invalids back for later - drops, invalids := list.Filter(pool.currentState.GetBalance(addr), gasLimit) - for _, tx := range drops { - hash := tx.Hash() - log.Trace("Removed unpayable pending transaction", "hash", hash) - pool.all.Remove(hash) - } - pendingNofundsMeter.Mark(int64(len(drops))) - - for _, tx := range invalids { - hash := tx.Hash() - log.Trace("Demoting pending transaction", "hash", hash) - - // Internal shuffle shouldn't touch the lookup set. - pool.enqueueTx(hash, tx, false, false) - } - pendingGauge.Dec(int64(len(olds) + len(drops) + len(invalids))) - if pool.locals.contains(addr) { - localGauge.Dec(int64(len(olds) + len(drops) + len(invalids))) - } - // If there's a gap in front, alert (should never happen) and postpone all transactions - if list.Len() > 0 && list.txs.Get(nonce) == nil { - gapped := list.Cap(0) - for _, tx := range gapped { - hash := tx.Hash() - log.Error("Demoting invalidated transaction", "hash", hash) - - // Internal shuffle shouldn't touch the lookup set. - pool.enqueueTx(hash, tx, false, false) - } - pendingGauge.Dec(int64(len(gapped))) - } - // Delete the entire pending entry if it became empty. - if list.Empty() { - delete(pool.pending, addr) - } - } -} - -// addressByHeartbeat is an account address tagged with its last activity timestamp. -type addressByHeartbeat struct { - address common.Address - heartbeat time.Time -} - -type addressesByHeartbeat []addressByHeartbeat - -func (a addressesByHeartbeat) Len() int { return len(a) } -func (a addressesByHeartbeat) Less(i, j int) bool { return a[i].heartbeat.Before(a[j].heartbeat) } -func (a addressesByHeartbeat) Swap(i, j int) { a[i], a[j] = a[j], a[i] } - -// accountSet is simply a set of addresses to check for existence, and a signer -// capable of deriving addresses from transactions. -type accountSet struct { - accounts map[common.Address]struct{} - signer types.Signer - cache *[]common.Address -} - -// newAccountSet creates a new address set with an associated signer for sender -// derivations. -func newAccountSet(signer types.Signer, addrs ...common.Address) *accountSet { - as := &accountSet{ - accounts: make(map[common.Address]struct{}, len(addrs)), - signer: signer, - } - for _, addr := range addrs { - as.add(addr) - } - return as -} - -// contains checks if a given address is contained within the set. -func (as *accountSet) contains(addr common.Address) bool { - _, exist := as.accounts[addr] - return exist -} - -// containsTx checks if the sender of a given tx is within the set. If the sender -// cannot be derived, this method returns false. -func (as *accountSet) containsTx(tx *types.Transaction) bool { - if addr, err := types.Sender(as.signer, tx); err == nil { - return as.contains(addr) - } - return false -} - -// add inserts a new address into the set to track. -func (as *accountSet) add(addr common.Address) { - as.accounts[addr] = struct{}{} - as.cache = nil -} - -// addTx adds the sender of tx into the set. -func (as *accountSet) addTx(tx *types.Transaction) { - if addr, err := types.Sender(as.signer, tx); err == nil { - as.add(addr) - } -} - -// flatten returns the list of addresses within this set, also caching it for later -// reuse. The returned slice should not be changed! -func (as *accountSet) flatten() []common.Address { - if as.cache == nil { - accounts := make([]common.Address, 0, len(as.accounts)) - for account := range as.accounts { - accounts = append(accounts, account) - } - as.cache = &accounts - } - return *as.cache -} - -// merge adds all addresses from the 'other' set into 'as'. -func (as *accountSet) merge(other *accountSet) { - for addr := range other.accounts { - as.accounts[addr] = struct{}{} - } - as.cache = nil -} - -// lookup is used internally by TxPool to track transactions while allowing -// lookup without mutex contention. -// -// Note, although this type is properly protected against concurrent access, it -// is **not** a type that should ever be mutated or even exposed outside of the -// transaction pool, since its internal state is tightly coupled with the pools -// internal mechanisms. The sole purpose of the type is to permit out-of-bound -// peeking into the pool in TxPool.Get without having to acquire the widely scoped -// TxPool.mu mutex. -// -// This lookup set combines the notion of "local transactions", which is useful -// to build upper-level structure. -type lookup struct { - slots int - lock sync.RWMutex - locals map[common.Hash]*types.Transaction - remotes map[common.Hash]*types.Transaction -} - -// newLookup returns a new lookup structure. -func newLookup() *lookup { - return &lookup{ - locals: make(map[common.Hash]*types.Transaction), - remotes: make(map[common.Hash]*types.Transaction), - } -} - -// Range calls f on each key and value present in the map. The callback passed -// should return the indicator whether the iteration needs to be continued. -// Callers need to specify which set (or both) to be iterated. -func (t *lookup) Range(f func(hash common.Hash, tx *types.Transaction, local bool) bool, local bool, remote bool) { - t.lock.RLock() - defer t.lock.RUnlock() - - if local { - for key, value := range t.locals { - if !f(key, value, true) { - return - } - } - } - if remote { - for key, value := range t.remotes { - if !f(key, value, false) { - return - } - } - } -} - -// Get returns a transaction if it exists in the lookup, or nil if not found. -func (t *lookup) Get(hash common.Hash) *types.Transaction { - t.lock.RLock() - defer t.lock.RUnlock() - - if tx := t.locals[hash]; tx != nil { - return tx - } - return t.remotes[hash] -} - -// GetLocal returns a transaction if it exists in the lookup, or nil if not found. -func (t *lookup) GetLocal(hash common.Hash) *types.Transaction { - t.lock.RLock() - defer t.lock.RUnlock() - - return t.locals[hash] -} - -// GetRemote returns a transaction if it exists in the lookup, or nil if not found. -func (t *lookup) GetRemote(hash common.Hash) *types.Transaction { - t.lock.RLock() - defer t.lock.RUnlock() - - return t.remotes[hash] -} - -// Count returns the current number of transactions in the lookup. -func (t *lookup) Count() int { - t.lock.RLock() - defer t.lock.RUnlock() - - return len(t.locals) + len(t.remotes) -} - -// LocalCount returns the current number of local transactions in the lookup. -func (t *lookup) LocalCount() int { - t.lock.RLock() - defer t.lock.RUnlock() - - return len(t.locals) -} - -// RemoteCount returns the current number of remote transactions in the lookup. -func (t *lookup) RemoteCount() int { - t.lock.RLock() - defer t.lock.RUnlock() - - return len(t.remotes) -} - -// Slots returns the current number of slots used in the lookup. -func (t *lookup) Slots() int { - t.lock.RLock() - defer t.lock.RUnlock() - - return t.slots -} - -// Add adds a transaction to the lookup. -func (t *lookup) Add(tx *types.Transaction, local bool) { - t.lock.Lock() - defer t.lock.Unlock() - - t.slots += numSlots(tx) - slotsGauge.Update(int64(t.slots)) - - if local { - t.locals[tx.Hash()] = tx - } else { - t.remotes[tx.Hash()] = tx - } -} - -// Remove removes a transaction from the lookup. -func (t *lookup) Remove(hash common.Hash) { - t.lock.Lock() - defer t.lock.Unlock() - - tx, ok := t.locals[hash] - if !ok { - tx, ok = t.remotes[hash] - } - if !ok { - log.Error("No transaction found to be deleted", "hash", hash) - return - } - t.slots -= numSlots(tx) - slotsGauge.Update(int64(t.slots)) - - delete(t.locals, hash) - delete(t.remotes, hash) -} - -// RemoteToLocals migrates the transactions belongs to the given locals to locals -// set. The assumption is held the locals set is thread-safe to be used. -func (t *lookup) RemoteToLocals(locals *accountSet) int { - t.lock.Lock() - defer t.lock.Unlock() - - var migrated int - for hash, tx := range t.remotes { - if locals.containsTx(tx) { - t.locals[hash] = tx - delete(t.remotes, hash) - migrated += 1 - } - } - return migrated -} - -// RemotesBelowTip finds all remote transactions below the given tip threshold. -func (t *lookup) RemotesBelowTip(threshold *big.Int) types.Transactions { - found := make(types.Transactions, 0, 128) - t.Range(func(hash common.Hash, tx *types.Transaction, local bool) bool { - if tx.GasTipCapIntCmp(threshold) < 0 { - found = append(found, tx) - } - return true - }, false, true) // Only iterate remotes - return found -} - -// numSlots calculates the number of slots needed for a single transaction. -func numSlots(tx *types.Transaction) int { - return int((tx.Size() + txSlotSize - 1) / txSlotSize) + return TxStatusUnknown } diff --git a/core/types/transaction.go b/core/types/transaction.go index b7cb36b602..579fc00d10 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -555,10 +555,10 @@ func (s *TxByPriceAndTime) Pop() interface{} { // transactions in a profit-maximizing sorted order, while supporting removing // entire batches of transactions for non-executable accounts. type TransactionsByPriceAndNonce struct { - txs map[common.Address]Transactions // Per account nonce-sorted list of transactions - heads TxByPriceAndTime // Next transaction for each unique account (price heap) - signer Signer // Signer for the set of transactions - baseFee *big.Int // Current base fee + txs map[common.Address][]*Transaction // Per account nonce-sorted list of transactions + heads TxByPriceAndTime // Next transaction for each unique account (price heap) + signer Signer // Signer for the set of transactions + baseFee *big.Int // Current base fee } // NewTransactionsByPriceAndNonce creates a transaction set that can retrieve @@ -566,7 +566,7 @@ type TransactionsByPriceAndNonce struct { // // Note, the input map is reowned so the caller should not interact any more with // if after providing it to the constructor. -func NewTransactionsByPriceAndNonce(signer Signer, txs map[common.Address]Transactions, baseFee *big.Int) *TransactionsByPriceAndNonce { +func NewTransactionsByPriceAndNonce(signer Signer, txs map[common.Address][]*Transaction, baseFee *big.Int) *TransactionsByPriceAndNonce { // Initialize a price and received time based heap with the head transactions heads := make(TxByPriceAndTime, 0, len(txs)) for from, accTxs := range txs { diff --git a/core/types/transaction_signing.go b/core/types/transaction_signing.go index 59dd2e76eb..8633440824 100644 --- a/core/types/transaction_signing.go +++ b/core/types/transaction_signing.go @@ -57,9 +57,9 @@ func MakeSigner(config *params.ChainConfig, blockNumber *big.Int, blockTime uint } // LatestSigner returns the 'most permissive' Signer available for the given chain -// configuration. Specifically, this enables support of EIP-155 replay protection and -// EIP-2930 access list transactions when their respective forks are scheduled to occur at -// any block number in the chain config. +// configuration. Specifically, this enables support of all types of transacrions +// when their respective forks are scheduled to occur at any block number (or time) +// in the chain config. // // Use this in transaction-handling code where the current block number is unknown. If you // have the current block number available, use MakeSigner instead. diff --git a/core/types/transaction_test.go b/core/types/transaction_test.go index f6442c5a35..dbe77fa603 100644 --- a/core/types/transaction_test.go +++ b/core/types/transaction_test.go @@ -281,7 +281,7 @@ func testTransactionPriceNonceSort(t *testing.T, baseFee *big.Int) { signer := LatestSignerForChainID(common.Big1) // Generate a batch of transactions with overlapping values, but shifted nonces - groups := map[common.Address]Transactions{} + groups := map[common.Address][]*Transaction{} expectedCount := 0 for start, key := range keys { addr := crypto.PubkeyToAddress(key.PublicKey) @@ -368,7 +368,7 @@ func TestTransactionTimeSort(t *testing.T) { signer := HomesteadSigner{} // Generate a batch of transactions with overlapping prices, but different creation times - groups := map[common.Address]Transactions{} + groups := map[common.Address][]*Transaction{} for start, key := range keys { addr := crypto.PubkeyToAddress(key.PublicKey) diff --git a/eth/api_backend.go b/eth/api_backend.go index 927dcdef10..02d1946e8e 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -294,7 +294,7 @@ func (b *EthAPIBackend) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscri } func (b *EthAPIBackend) SendTx(ctx context.Context, signedTx *types.Transaction) error { - return b.eth.txPool.AddLocal(signedTx) + return b.eth.txPool.Add([]*txpool.Transaction{&txpool.Transaction{Tx: signedTx}}, true, false)[0] } func (b *EthAPIBackend) GetPoolTransactions() (types.Transactions, error) { @@ -307,7 +307,10 @@ func (b *EthAPIBackend) GetPoolTransactions() (types.Transactions, error) { } func (b *EthAPIBackend) GetPoolTransaction(hash common.Hash) *types.Transaction { - return b.eth.txPool.Get(hash) + if tx := b.eth.txPool.Get(hash); tx != nil { + return tx.Tx + } + return nil } func (b *EthAPIBackend) GetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error) { @@ -319,24 +322,24 @@ func (b *EthAPIBackend) GetPoolNonce(ctx context.Context, addr common.Address) ( return b.eth.txPool.Nonce(addr), nil } -func (b *EthAPIBackend) Stats() (pending int, queued int) { +func (b *EthAPIBackend) Stats() (runnable int, blocked int) { return b.eth.txPool.Stats() } -func (b *EthAPIBackend) TxPoolContent() (map[common.Address]types.Transactions, map[common.Address]types.Transactions) { - return b.eth.TxPool().Content() +func (b *EthAPIBackend) TxPoolContent() (map[common.Address][]*types.Transaction, map[common.Address][]*types.Transaction) { + return b.eth.txPool.Content() } -func (b *EthAPIBackend) TxPoolContentFrom(addr common.Address) (types.Transactions, types.Transactions) { - return b.eth.TxPool().ContentFrom(addr) +func (b *EthAPIBackend) TxPoolContentFrom(addr common.Address) ([]*types.Transaction, []*types.Transaction) { + return b.eth.txPool.ContentFrom(addr) } func (b *EthAPIBackend) TxPool() *txpool.TxPool { - return b.eth.TxPool() + return b.eth.txPool } func (b *EthAPIBackend) SubscribeNewTxsEvent(ch chan<- core.NewTxsEvent) event.Subscription { - return b.eth.TxPool().SubscribeNewTxsEvent(ch) + return b.eth.txPool.SubscribeNewTxsEvent(ch) } func (b *EthAPIBackend) SyncProgress() ethereum.SyncProgress { diff --git a/eth/backend.go b/eth/backend.go index b3338918be..8d6977205e 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -35,6 +35,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state/pruner" "github.com/ethereum/go-ethereum/core/txpool" + "github.com/ethereum/go-ethereum/core/txpool/legacypool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/downloader" @@ -66,7 +67,8 @@ type Ethereum struct { config *ethconfig.Config // Handlers - txPool *txpool.TxPool + txPool *txpool.TxPool + blockchain *core.BlockChain handler *handler ethDialCandidates enode.Iterator @@ -206,8 +208,12 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { if config.TxPool.Journal != "" { config.TxPool.Journal = stack.ResolvePath(config.TxPool.Journal) } - eth.txPool = txpool.New(config.TxPool, eth.blockchain.Config(), eth.blockchain) + legacyPool := legacypool.New(config.TxPool, eth.blockchain) + eth.txPool, err = txpool.New(new(big.Int).SetUint64(config.TxPool.PriceLimit), eth.blockchain, []txpool.SubPool{legacyPool}) + if err != nil { + return nil, err + } // Permit the downloader to use the trie cache allowance during fast sync cacheLimit := cacheConfig.TrieCleanLimit + cacheConfig.TrieDirtyLimit + cacheConfig.SnapshotLimit if eth.handler, err = newHandler(&handlerConfig{ @@ -512,7 +518,7 @@ func (s *Ethereum) Stop() error { // Then stop everything else. s.bloomIndexer.Close() close(s.closeBloomHandler) - s.txPool.Stop() + s.txPool.Close() s.miner.Close() s.blockchain.Stop() s.engine.Close() diff --git a/eth/catalyst/api_test.go b/eth/catalyst/api_test.go index 5bab7ba186..254a510ab4 100644 --- a/eth/catalyst/api_test.go +++ b/eth/catalyst/api_test.go @@ -35,6 +35,7 @@ import ( beaconConsensus "github.com/ethereum/go-ethereum/consensus/beacon" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth" @@ -106,7 +107,7 @@ func TestEth2AssembleBlock(t *testing.T) { if err != nil { t.Fatalf("error signing transaction, err=%v", err) } - ethservice.TxPool().AddLocal(tx) + ethservice.TxPool().Add([]*txpool.Transaction{{Tx: tx}}, true, false) blockParams := engine.PayloadAttributes{ Timestamp: blocks[9].Time() + 5, } @@ -142,7 +143,12 @@ func TestEth2AssembleBlockWithAnotherBlocksTxs(t *testing.T) { api := NewConsensusAPI(ethservice) // Put the 10th block's tx in the pool and produce a new block - api.eth.TxPool().AddRemotesSync(blocks[9].Transactions()) + txs := blocks[9].Transactions() + wrapped := make([]*txpool.Transaction, len(txs)) + for i, tx := range txs { + wrapped[i] = &txpool.Transaction{Tx: tx} + } + api.eth.TxPool().Add(wrapped, false, true) blockParams := engine.PayloadAttributes{ Timestamp: blocks[8].Time() + 5, } @@ -181,7 +187,12 @@ func TestEth2PrepareAndGetPayload(t *testing.T) { api := NewConsensusAPI(ethservice) // Put the 10th block's tx in the pool and produce a new block - ethservice.TxPool().AddLocals(blocks[9].Transactions()) + txs := blocks[9].Transactions() + wrapped := make([]*txpool.Transaction, len(txs)) + for i, tx := range txs { + wrapped[i] = &txpool.Transaction{Tx: tx} + } + ethservice.TxPool().Add(wrapped, true, false) blockParams := engine.PayloadAttributes{ Timestamp: blocks[8].Time() + 5, } @@ -303,7 +314,7 @@ func TestEth2NewBlock(t *testing.T) { statedb, _ := ethservice.BlockChain().StateAt(parent.Root()) nonce := statedb.GetNonce(testAddr) tx, _ := types.SignTx(types.NewContractCreation(nonce, new(big.Int), 1000000, big.NewInt(2*params.InitialBaseFee), logCode), types.LatestSigner(ethservice.BlockChain().Config()), testKey) - ethservice.TxPool().AddLocal(tx) + ethservice.TxPool().Add([]*txpool.Transaction{{Tx: tx}}, true, false) execData, err := assembleWithTransactions(api, parent.Hash(), &engine.PayloadAttributes{ Timestamp: parent.Time() + 5, @@ -472,7 +483,7 @@ func TestFullAPI(t *testing.T) { statedb, _ := ethservice.BlockChain().StateAt(parent.Root) nonce := statedb.GetNonce(testAddr) tx, _ := types.SignTx(types.NewContractCreation(nonce, new(big.Int), 1000000, big.NewInt(2*params.InitialBaseFee), logCode), types.LatestSigner(ethservice.BlockChain().Config()), testKey) - ethservice.TxPool().AddLocal(tx) + ethservice.TxPool().Add([]*txpool.Transaction{{Tx: tx}}, true, false) } setupBlocks(t, ethservice, 10, parent, callback, nil) @@ -598,7 +609,7 @@ func TestNewPayloadOnInvalidChain(t *testing.T) { GasPrice: big.NewInt(2 * params.InitialBaseFee), Data: logCode, }) - ethservice.TxPool().AddRemotesSync([]*types.Transaction{tx}) + ethservice.TxPool().Add([]*txpool.Transaction{{Tx: tx}}, false, true) var ( params = engine.PayloadAttributes{ Timestamp: parent.Time + 1, @@ -1272,7 +1283,7 @@ func setupBodies(t *testing.T) (*node.Node, *eth.Ethereum, []*types.Block) { statedb, _ := ethservice.BlockChain().StateAt(parent.Root) nonce := statedb.GetNonce(testAddr) tx, _ := types.SignTx(types.NewContractCreation(nonce, new(big.Int), 1000000, big.NewInt(2*params.InitialBaseFee), logCode), types.LatestSigner(ethservice.BlockChain().Config()), testKey) - ethservice.TxPool().AddLocal(tx) + ethservice.TxPool().Add([]*txpool.Transaction{{Tx: tx}}, true, false) } withdrawals := make([][]*types.Withdrawal, 10) diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index a98d9ee4aa..432ec8ffbb 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -27,7 +27,7 @@ import ( "github.com/ethereum/go-ethereum/consensus/clique" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/txpool" + "github.com/ethereum/go-ethereum/core/txpool/legacypool" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/ethdb" @@ -71,7 +71,7 @@ var Defaults = Config{ SnapshotCache: 102, FilterLogCacheSize: 32, Miner: miner.DefaultConfig, - TxPool: txpool.DefaultConfig, + TxPool: legacypool.DefaultConfig, RPCGasCap: 50000000, RPCEVMTimeout: 5 * time.Second, GPO: FullNodeGPO, @@ -139,7 +139,7 @@ type Config struct { Miner miner.Config // Transaction pool options - TxPool txpool.Config + TxPool legacypool.Config // Gas Price Oracle options GPO gasprice.Config diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index 0e2f037740..5e1d75e3b5 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -7,7 +7,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/txpool" + "github.com/ethereum/go-ethereum/core/txpool/legacypool" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/miner" @@ -47,7 +47,7 @@ func (c Config) MarshalTOML() (interface{}, error) { Preimages bool FilterLogCacheSize int Miner miner.Config - TxPool txpool.Config + TxPool legacypool.Config GPO gasprice.Config EnablePreimageRecording bool DocRoot string `toml:"-"` @@ -133,7 +133,7 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { Preimages *bool FilterLogCacheSize *int Miner *miner.Config - TxPool *txpool.Config + TxPool *legacypool.Config GPO *gasprice.Config EnablePreimageRecording *bool DocRoot *string `toml:"-"` diff --git a/eth/fetcher/tx_fetcher.go b/eth/fetcher/tx_fetcher.go index 39727e0079..6387273403 100644 --- a/eth/fetcher/tx_fetcher.go +++ b/eth/fetcher/tx_fetcher.go @@ -169,9 +169,9 @@ type TxFetcher struct { alternates map[common.Hash]map[string]struct{} // In-flight transaction alternate origins if retrieval fails // Callbacks - hasTx func(common.Hash) bool // Retrieves a tx from the local txpool - addTxs func([]*types.Transaction) []error // Insert a batch of transactions into local txpool - fetchTxs func(string, []common.Hash) error // Retrieves a set of txs from a remote peer + hasTx func(common.Hash) bool // Retrieves a tx from the local txpool + addTxs func([]*txpool.Transaction) []error // Insert a batch of transactions into local txpool + fetchTxs func(string, []common.Hash) error // Retrieves a set of txs from a remote peer step chan struct{} // Notification channel when the fetcher loop iterates clock mclock.Clock // Time wrapper to simulate in tests @@ -180,14 +180,14 @@ type TxFetcher struct { // NewTxFetcher creates a transaction fetcher to retrieve transaction // based on hash announcements. -func NewTxFetcher(hasTx func(common.Hash) bool, addTxs func([]*types.Transaction) []error, fetchTxs func(string, []common.Hash) error) *TxFetcher { +func NewTxFetcher(hasTx func(common.Hash) bool, addTxs func([]*txpool.Transaction) []error, fetchTxs func(string, []common.Hash) error) *TxFetcher { return NewTxFetcherForTests(hasTx, addTxs, fetchTxs, mclock.System{}, nil) } // NewTxFetcherForTests is a testing method to mock out the realtime clock with // a simulated version and the internal randomness with a deterministic one. func NewTxFetcherForTests( - hasTx func(common.Hash) bool, addTxs func([]*types.Transaction) []error, fetchTxs func(string, []common.Hash) error, + hasTx func(common.Hash) bool, addTxs func([]*txpool.Transaction) []error, fetchTxs func(string, []common.Hash) error, clock mclock.Clock, rand *mrand.Rand) *TxFetcher { return &TxFetcher{ notify: make(chan *txAnnounce), @@ -294,7 +294,12 @@ func (f *TxFetcher) Enqueue(peer string, txs []*types.Transaction, direct bool) otherreject int64 ) batch := txs[i:end] - for j, err := range f.addTxs(batch) { + + wrapped := make([]*txpool.Transaction, len(batch)) + for j, tx := range batch { + wrapped[j] = &txpool.Transaction{Tx: tx} + } + for j, err := range f.addTxs(wrapped) { // Track the transaction hash if the price is too low for us. // Avoid re-request this transaction when we receive another // announcement. diff --git a/eth/fetcher/tx_fetcher_test.go b/eth/fetcher/tx_fetcher_test.go index 1715def99c..c5805d6ef3 100644 --- a/eth/fetcher/tx_fetcher_test.go +++ b/eth/fetcher/tx_fetcher_test.go @@ -378,7 +378,7 @@ func TestTransactionFetcherCleanup(t *testing.T) { init: func() *TxFetcher { return NewTxFetcher( func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { + func(txs []*txpool.Transaction) []error { return make([]error, len(txs)) }, func(string, []common.Hash) error { return nil }, @@ -417,7 +417,7 @@ func TestTransactionFetcherCleanupEmpty(t *testing.T) { init: func() *TxFetcher { return NewTxFetcher( func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { + func(txs []*txpool.Transaction) []error { return make([]error, len(txs)) }, func(string, []common.Hash) error { return nil }, @@ -455,7 +455,7 @@ func TestTransactionFetcherMissingRescheduling(t *testing.T) { init: func() *TxFetcher { return NewTxFetcher( func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { + func(txs []*txpool.Transaction) []error { return make([]error, len(txs)) }, func(string, []common.Hash) error { return nil }, @@ -501,7 +501,7 @@ func TestTransactionFetcherMissingCleanup(t *testing.T) { init: func() *TxFetcher { return NewTxFetcher( func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { + func(txs []*txpool.Transaction) []error { return make([]error, len(txs)) }, func(string, []common.Hash) error { return nil }, @@ -539,7 +539,7 @@ func TestTransactionFetcherBroadcasts(t *testing.T) { init: func() *TxFetcher { return NewTxFetcher( func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { + func(txs []*txpool.Transaction) []error { return make([]error, len(txs)) }, func(string, []common.Hash) error { return nil }, @@ -644,7 +644,7 @@ func TestTransactionFetcherTimeoutRescheduling(t *testing.T) { init: func() *TxFetcher { return NewTxFetcher( func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { + func(txs []*txpool.Transaction) []error { return make([]error, len(txs)) }, func(string, []common.Hash) error { return nil }, @@ -865,7 +865,7 @@ func TestTransactionFetcherUnderpricedDedup(t *testing.T) { init: func() *TxFetcher { return NewTxFetcher( func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { + func(txs []*txpool.Transaction) []error { errs := make([]error, len(txs)) for i := 0; i < len(errs); i++ { if i%2 == 0 { @@ -938,7 +938,7 @@ func TestTransactionFetcherUnderpricedDoSProtection(t *testing.T) { init: func() *TxFetcher { return NewTxFetcher( func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { + func(txs []*txpool.Transaction) []error { errs := make([]error, len(txs)) for i := 0; i < len(errs); i++ { errs[i] = txpool.ErrUnderpriced @@ -964,7 +964,7 @@ func TestTransactionFetcherOutOfBoundDeliveries(t *testing.T) { init: func() *TxFetcher { return NewTxFetcher( func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { + func(txs []*txpool.Transaction) []error { return make([]error, len(txs)) }, func(string, []common.Hash) error { return nil }, @@ -1017,7 +1017,7 @@ func TestTransactionFetcherDrop(t *testing.T) { init: func() *TxFetcher { return NewTxFetcher( func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { + func(txs []*txpool.Transaction) []error { return make([]error, len(txs)) }, func(string, []common.Hash) error { return nil }, @@ -1083,7 +1083,7 @@ func TestTransactionFetcherDropRescheduling(t *testing.T) { init: func() *TxFetcher { return NewTxFetcher( func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { + func(txs []*txpool.Transaction) []error { return make([]error, len(txs)) }, func(string, []common.Hash) error { return nil }, @@ -1128,7 +1128,7 @@ func TestTransactionFetcherFuzzCrash01(t *testing.T) { init: func() *TxFetcher { return NewTxFetcher( func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { + func(txs []*txpool.Transaction) []error { return make([]error, len(txs)) }, func(string, []common.Hash) error { return nil }, @@ -1155,7 +1155,7 @@ func TestTransactionFetcherFuzzCrash02(t *testing.T) { init: func() *TxFetcher { return NewTxFetcher( func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { + func(txs []*txpool.Transaction) []error { return make([]error, len(txs)) }, func(string, []common.Hash) error { return nil }, @@ -1184,7 +1184,7 @@ func TestTransactionFetcherFuzzCrash03(t *testing.T) { init: func() *TxFetcher { return NewTxFetcher( func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { + func(txs []*txpool.Transaction) []error { return make([]error, len(txs)) }, func(string, []common.Hash) error { return nil }, @@ -1217,7 +1217,7 @@ func TestTransactionFetcherFuzzCrash04(t *testing.T) { init: func() *TxFetcher { return NewTxFetcher( func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { + func(txs []*txpool.Transaction) []error { return make([]error, len(txs)) }, func(string, []common.Hash) error { diff --git a/eth/handler.go b/eth/handler.go index f0b043166e..46113fbc2f 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/consensus/beacon" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/forkid" + "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/fetcher" @@ -59,14 +60,14 @@ type txPool interface { // Get retrieves the transaction from local txpool with given // tx hash. - Get(hash common.Hash) *types.Transaction + Get(hash common.Hash) *txpool.Transaction - // AddRemotes should add the given transactions to the pool. - AddRemotes([]*types.Transaction) []error + // Add should add the given transactions to the pool. + Add(txs []*txpool.Transaction, local bool, sync bool) []error // Pending should return pending transactions. // The slice should be modifiable by the caller. - Pending(enforceTips bool) map[common.Address]types.Transactions + Pending(enforceTips bool) map[common.Address][]*types.Transaction // SubscribeNewTxsEvent should return an event subscription of // NewTxsEvent and send events to the given channel. @@ -274,7 +275,10 @@ func newHandler(config *handlerConfig) (*handler, error) { } return p.RequestTxs(hashes) } - h.txFetcher = fetcher.NewTxFetcher(h.txpool.Has, h.txpool.AddRemotes, fetchTx) + addTxs := func(txs []*txpool.Transaction) []error { + return h.txpool.Add(txs, false, false) + } + h.txFetcher = fetcher.NewTxFetcher(h.txpool.Has, addTxs, fetchTx) h.chainSync = newChainSyncer(h) return h, nil } diff --git a/eth/handler_eth_test.go b/eth/handler_eth_test.go index fd7df23403..a9ce83e2ee 100644 --- a/eth/handler_eth_test.go +++ b/eth/handler_eth_test.go @@ -28,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/forkid" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/downloader" @@ -307,15 +308,15 @@ func testSendTransactions(t *testing.T, protocol uint) { handler := newTestHandler() defer handler.close() - insert := make([]*types.Transaction, 100) + insert := make([]*txpool.Transaction, 100) for nonce := range insert { tx := types.NewTransaction(uint64(nonce), common.Address{}, big.NewInt(0), 100000, big.NewInt(0), make([]byte, 10240)) tx, _ = types.SignTx(tx, types.HomesteadSigner{}, testKey) - insert[nonce] = tx + insert[nonce] = &txpool.Transaction{Tx: tx} } - go handler.txpool.AddRemotes(insert) // Need goroutine to not block on feed - time.Sleep(250 * time.Millisecond) // Wait until tx events get out of the system (can't use events, tx broadcaster races with peer join) + go handler.txpool.Add(insert, false, false) // Need goroutine to not block on feed + time.Sleep(250 * time.Millisecond) // Wait until tx events get out of the system (can't use events, tx broadcaster races with peer join) // Create a source handler to send messages through and a sink peer to receive them p2pSrc, p2pSink := p2p.MsgPipe() @@ -375,8 +376,8 @@ func testSendTransactions(t *testing.T, protocol uint) { } } for _, tx := range insert { - if _, ok := seen[tx.Hash()]; !ok { - t.Errorf("missing transaction: %x", tx.Hash()) + if _, ok := seen[tx.Tx.Hash()]; !ok { + t.Errorf("missing transaction: %x", tx.Tx.Hash()) } } } @@ -433,14 +434,14 @@ func testTransactionPropagation(t *testing.T, protocol uint) { defer sub.Unsubscribe() } // Fill the source pool with transactions and wait for them at the sinks - txs := make([]*types.Transaction, 1024) + txs := make([]*txpool.Transaction, 1024) for nonce := range txs { tx := types.NewTransaction(uint64(nonce), common.Address{}, big.NewInt(0), 100000, big.NewInt(0), nil) tx, _ = types.SignTx(tx, types.HomesteadSigner{}, testKey) - txs[nonce] = tx + txs[nonce] = &txpool.Transaction{Tx: tx} } - source.txpool.AddRemotes(txs) + source.txpool.Add(txs, false, false) // Iterate through all the sinks and ensure they all got the transactions for i := range sinks { diff --git a/eth/handler_test.go b/eth/handler_test.go index 8939e53a95..2f776e874b 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -26,6 +26,7 @@ import ( "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" @@ -71,32 +72,40 @@ func (p *testTxPool) Has(hash common.Hash) bool { // Get retrieves the transaction from local txpool with given // tx hash. -func (p *testTxPool) Get(hash common.Hash) *types.Transaction { +func (p *testTxPool) Get(hash common.Hash) *txpool.Transaction { p.lock.Lock() defer p.lock.Unlock() - return p.pool[hash] + if tx := p.pool[hash]; tx != nil { + return &txpool.Transaction{Tx: tx} + } + return nil } -// AddRemotes appends a batch of transactions to the pool, and notifies any +// Add appends a batch of transactions to the pool, and notifies any // listeners if the addition channel is non nil -func (p *testTxPool) AddRemotes(txs []*types.Transaction) []error { +func (p *testTxPool) Add(txs []*txpool.Transaction, local bool, sync bool) []error { + unwrapped := make([]*types.Transaction, len(txs)) + for i, tx := range txs { + unwrapped[i] = tx.Tx + } p.lock.Lock() defer p.lock.Unlock() - for _, tx := range txs { + for _, tx := range unwrapped { p.pool[tx.Hash()] = tx } - p.txFeed.Send(core.NewTxsEvent{Txs: txs}) - return make([]error, len(txs)) + + p.txFeed.Send(core.NewTxsEvent{Txs: unwrapped}) + return make([]error, len(unwrapped)) } // Pending returns all the transactions known to the pool -func (p *testTxPool) Pending(enforceTips bool) map[common.Address]types.Transactions { +func (p *testTxPool) Pending(enforceTips bool) map[common.Address][]*types.Transaction { p.lock.RLock() defer p.lock.RUnlock() - batches := make(map[common.Address]types.Transactions) + batches := make(map[common.Address][]*types.Transaction) for _, tx := range p.pool { from, _ := types.Sender(types.HomesteadSigner{}, tx) batches[from] = append(batches[from], tx) diff --git a/eth/protocols/eth/broadcast.go b/eth/protocols/eth/broadcast.go index 3045303f22..c431aa4005 100644 --- a/eth/protocols/eth/broadcast.go +++ b/eth/protocols/eth/broadcast.go @@ -81,8 +81,8 @@ func (p *Peer) broadcastTransactions() { ) for i := 0; i < len(queue) && size < maxTxPacketSize; i++ { if tx := p.txpool.Get(queue[i]); tx != nil { - txs = append(txs, tx) - size += common.StorageSize(tx.Size()) + txs = append(txs, tx.Tx) + size += common.StorageSize(tx.Tx.Size()) } hashesCount++ } @@ -151,8 +151,8 @@ func (p *Peer) announceTransactions() { for count = 0; count < len(queue) && size < maxTxPacketSize; count++ { if tx := p.txpool.Get(queue[count]); tx != nil { pending = append(pending, queue[count]) - pendingTypes = append(pendingTypes, tx.Type()) - pendingSizes = append(pendingSizes, uint32(tx.Size())) + pendingTypes = append(pendingTypes, tx.Tx.Type()) + pendingSizes = append(pendingSizes, uint32(tx.Tx.Size())) size += common.HashLength } } diff --git a/eth/protocols/eth/handler.go b/eth/protocols/eth/handler.go index 3136864270..7f51d4f5cd 100644 --- a/eth/protocols/eth/handler.go +++ b/eth/protocols/eth/handler.go @@ -23,7 +23,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" @@ -90,7 +90,7 @@ type Backend interface { // TxPool defines the methods needed by the protocol handler to serve transactions. type TxPool interface { // Get retrieves the transaction from the local txpool with the given hash. - Get(hash common.Hash) *types.Transaction + Get(hash common.Hash) *txpool.Transaction } // MakeProtocols constructs the P2P protocol definitions for `eth`. diff --git a/eth/protocols/eth/handler_test.go b/eth/protocols/eth/handler_test.go index 21782103ca..3f1dc9fe76 100644 --- a/eth/protocols/eth/handler_test.go +++ b/eth/protocols/eth/handler_test.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/txpool" + "github.com/ethereum/go-ethereum/core/txpool/legacypool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" @@ -113,19 +114,22 @@ func newTestBackendWithGenerator(blocks int, shanghai bool, generator func(int, for _, block := range bs { chain.StateCache().TrieDB().Commit(block.Root(), false) } - txconfig := txpool.DefaultConfig + txconfig := legacypool.DefaultConfig txconfig.Journal = "" // Don't litter the disk with test journals + pool := legacypool.New(txconfig, chain) + txpool, _ := txpool.New(new(big.Int).SetUint64(txconfig.PriceLimit), chain, []txpool.SubPool{pool}) + return &testBackend{ db: db, chain: chain, - txpool: txpool.New(txconfig, params.TestChainConfig, chain), + txpool: txpool, } } // close tears down the transaction pool and chain behind the mock backend. func (b *testBackend) close() { - b.txpool.Stop() + b.txpool.Close() b.chain.Stop() } diff --git a/eth/protocols/eth/handlers.go b/eth/protocols/eth/handlers.go index 74e514b863..f9fbf72b7b 100644 --- a/eth/protocols/eth/handlers.go +++ b/eth/protocols/eth/handlers.go @@ -498,7 +498,7 @@ func answerGetPooledTransactions(backend Backend, query GetPooledTransactionsPac continue } // If known, encode and queue for response packet - if encoded, err := rlp.EncodeToBytes(tx); err != nil { + if encoded, err := rlp.EncodeToBytes(tx.Tx); err != nil { log.Error("Failed to encode transaction", "err", err) } else { hashes = append(hashes, hash) diff --git a/event/multisub.go b/event/multisub.go new file mode 100644 index 0000000000..5c8d2df48c --- /dev/null +++ b/event/multisub.go @@ -0,0 +1,50 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package event + +// JoinSubscriptions joins multiple subscriptions to be able to track them as +// one entity and collectively cancel them of consume any errors from them. +func JoinSubscriptions(subs ...Subscription) Subscription { + return NewSubscription(func(unsubbed <-chan struct{}) error { + // Unsubscribe all subscriptions before returning + defer func() { + for _, sub := range subs { + sub.Unsubscribe() + } + }() + // Wait for an error on any of the subscriptions and propagate up + errc := make(chan error, len(subs)) + for i := range subs { + go func(sub Subscription) { + select { + case err := <-sub.Err(): + if err != nil { + errc <- err + } + case <-unsubbed: + } + }(subs[i]) + } + + select { + case err := <-errc: + return err + case <-unsubbed: + return nil + } + }) +} diff --git a/event/multisub_test.go b/event/multisub_test.go new file mode 100644 index 0000000000..c92bcfae9b --- /dev/null +++ b/event/multisub_test.go @@ -0,0 +1,175 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package event + +import ( + "testing" + "time" +) + +func TestMultisub(t *testing.T) { + // Create a double subscription and ensure events propagate through + var ( + feed1 Feed + feed2 Feed + ) + sink1 := make(chan int, 1) + sink2 := make(chan int, 1) + + sub1 := feed1.Subscribe(sink1) + sub2 := feed2.Subscribe(sink2) + + sub := JoinSubscriptions(sub1, sub2) + + feed1.Send(1) + select { + case n := <-sink1: + if n != 1 { + t.Errorf("sink 1 delivery mismatch: have %d, want %d", n, 1) + } + default: + t.Error("sink 1 missing delivery") + } + + feed2.Send(2) + select { + case n := <-sink2: + if n != 2 { + t.Errorf("sink 2 delivery mismatch: have %d, want %d", n, 2) + } + default: + t.Error("sink 2 missing delivery") + } + // Unsubscribe and ensure no more events are delivered + sub.Unsubscribe() + select { + case <-sub.Err(): + case <-time.After(50 * time.Millisecond): + t.Error("multisub didn't propagate closure") + } + + feed1.Send(11) + select { + case n := <-sink1: + t.Errorf("sink 1 unexpected delivery: %d", n) + default: + } + + feed2.Send(22) + select { + case n := <-sink2: + t.Errorf("sink 2 unexpected delivery: %d", n) + default: + } +} + +func TestMutisubPartialUnsubscribe(t *testing.T) { + // Create a double subscription but terminate one half, ensuring no error + // is propagated yet up to the outer subscription + var ( + feed1 Feed + feed2 Feed + ) + sink1 := make(chan int, 1) + sink2 := make(chan int, 1) + + sub1 := feed1.Subscribe(sink1) + sub2 := feed2.Subscribe(sink2) + + sub := JoinSubscriptions(sub1, sub2) + + sub1.Unsubscribe() + select { + case <-sub.Err(): + t.Error("multisub propagated closure") + case <-time.After(50 * time.Millisecond): + } + // Ensure that events cross only the second feed + feed1.Send(1) + select { + case n := <-sink1: + t.Errorf("sink 1 unexpected delivery: %d", n) + default: + } + + feed2.Send(2) + select { + case n := <-sink2: + if n != 2 { + t.Errorf("sink 2 delivery mismatch: have %d, want %d", n, 2) + } + default: + t.Error("sink 2 missing delivery") + } + // Unsubscribe and ensure no more events are delivered + sub.Unsubscribe() + select { + case <-sub.Err(): + case <-time.After(50 * time.Millisecond): + t.Error("multisub didn't propagate closure") + } + + feed1.Send(11) + select { + case n := <-sink1: + t.Errorf("sink 1 unexpected delivery: %d", n) + default: + } + + feed2.Send(22) + select { + case n := <-sink2: + t.Errorf("sink 2 unexpected delivery: %d", n) + default: + } +} + +func TestMultisubFullUnsubscribe(t *testing.T) { + // Create a double subscription and terminate the multi sub, ensuring an + // error is propagated up. + var ( + feed1 Feed + feed2 Feed + ) + sink1 := make(chan int, 1) + sink2 := make(chan int, 1) + + sub1 := feed1.Subscribe(sink1) + sub2 := feed2.Subscribe(sink2) + + sub := JoinSubscriptions(sub1, sub2) + sub.Unsubscribe() + select { + case <-sub.Err(): + case <-time.After(50 * time.Millisecond): + t.Error("multisub didn't propagate closure") + } + // Ensure no more events are delivered + feed1.Send(1) + select { + case n := <-sink1: + t.Errorf("sink 1 unexpected delivery: %d", n) + default: + } + + feed2.Send(2) + select { + case n := <-sink2: + t.Errorf("sink 2 unexpected delivery: %d", n) + default: + } +} diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go index f80f111472..7c503dc391 100644 --- a/internal/ethapi/api_test.go +++ b/internal/ethapi/api_test.go @@ -366,10 +366,10 @@ func (b testBackend) GetPoolNonce(ctx context.Context, addr common.Address) (uin panic("implement me") } func (b testBackend) Stats() (pending int, queued int) { panic("implement me") } -func (b testBackend) TxPoolContent() (map[common.Address]types.Transactions, map[common.Address]types.Transactions) { +func (b testBackend) TxPoolContent() (map[common.Address][]*types.Transaction, map[common.Address][]*types.Transaction) { panic("implement me") } -func (b testBackend) TxPoolContentFrom(addr common.Address) (types.Transactions, types.Transactions) { +func (b testBackend) TxPoolContentFrom(addr common.Address) ([]*types.Transaction, []*types.Transaction) { panic("implement me") } func (b testBackend) SubscribeNewTxsEvent(events chan<- core.NewTxsEvent) event.Subscription { diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index 918b3b6309..458fb811ed 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -80,8 +80,8 @@ type Backend interface { GetPoolTransaction(txHash common.Hash) *types.Transaction GetPoolNonce(ctx context.Context, addr common.Address) (uint64, error) Stats() (pending int, queued int) - TxPoolContent() (map[common.Address]types.Transactions, map[common.Address]types.Transactions) - TxPoolContentFrom(addr common.Address) (types.Transactions, types.Transactions) + TxPoolContent() (map[common.Address][]*types.Transaction, map[common.Address][]*types.Transaction) + TxPoolContentFrom(addr common.Address) ([]*types.Transaction, []*types.Transaction) SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription ChainConfig() *params.ChainConfig diff --git a/internal/ethapi/transaction_args_test.go b/internal/ethapi/transaction_args_test.go index 0868f8762c..9161d5e681 100644 --- a/internal/ethapi/transaction_args_test.go +++ b/internal/ethapi/transaction_args_test.go @@ -325,10 +325,10 @@ func (b *backendMock) GetPoolNonce(ctx context.Context, addr common.Address) (ui return 0, nil } func (b *backendMock) Stats() (pending int, queued int) { return 0, 0 } -func (b *backendMock) TxPoolContent() (map[common.Address]types.Transactions, map[common.Address]types.Transactions) { +func (b *backendMock) TxPoolContent() (map[common.Address][]*types.Transaction, map[common.Address][]*types.Transaction) { return nil, nil } -func (b *backendMock) TxPoolContentFrom(addr common.Address) (types.Transactions, types.Transactions) { +func (b *backendMock) TxPoolContentFrom(addr common.Address) ([]*types.Transaction, []*types.Transaction) { return nil, nil } func (b *backendMock) SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription { return nil } diff --git a/internal/flags/categories.go b/internal/flags/categories.go index c2db6c6c1d..ac71931879 100644 --- a/internal/flags/categories.go +++ b/internal/flags/categories.go @@ -23,7 +23,8 @@ const ( LightCategory = "LIGHT CLIENT" DevCategory = "DEVELOPER CHAIN" EthashCategory = "ETHASH" - TxPoolCategory = "TRANSACTION POOL" + TxPoolCategory = "TRANSACTION POOL (EVM)" + BlobPoolCategory = "TRANSACTION POOL (BLOB)" PerfCategory = "PERFORMANCE TUNING" AccountCategory = "ACCOUNT" APICategory = "API AND CONSOLE" diff --git a/les/api_backend.go b/les/api_backend.go index 97665757b1..dee3e3bfd9 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -224,11 +224,11 @@ func (b *LesApiBackend) Stats() (pending int, queued int) { return b.eth.txPool.Stats(), 0 } -func (b *LesApiBackend) TxPoolContent() (map[common.Address]types.Transactions, map[common.Address]types.Transactions) { +func (b *LesApiBackend) TxPoolContent() (map[common.Address][]*types.Transaction, map[common.Address][]*types.Transaction) { return b.eth.txPool.Content() } -func (b *LesApiBackend) TxPoolContentFrom(addr common.Address) (types.Transactions, types.Transactions) { +func (b *LesApiBackend) TxPoolContentFrom(addr common.Address) ([]*types.Transaction, []*types.Transaction) { return b.eth.txPool.ContentFrom(addr) } diff --git a/les/server_requests.go b/les/server_requests.go index 033b11d793..29f8f63ce8 100644 --- a/les/server_requests.go +++ b/les/server_requests.go @@ -518,12 +518,7 @@ func handleSendTx(msg Decoder) (serveRequestFn, uint64, uint64, error) { hash := tx.Hash() stats[i] = txStatus(backend, hash) if stats[i].Status == txpool.TxStatusUnknown { - addFn := backend.TxPool().AddRemotes - // Add txs synchronously for testing purpose - if backend.AddTxsSync() { - addFn = backend.TxPool().AddRemotesSync - } - if errs := addFn([]*types.Transaction{tx}); errs[0] != nil { + if errs := backend.TxPool().Add([]*txpool.Transaction{{Tx: tx}}, false, backend.AddTxsSync()); errs[0] != nil { stats[i].Error = errs[0].Error() continue } @@ -556,7 +551,7 @@ func handleGetTxStatus(msg Decoder) (serveRequestFn, uint64, uint64, error) { func txStatus(b serverBackend, hash common.Hash) light.TxStatus { var stat light.TxStatus // Looking the transaction in txpool first. - stat.Status = b.TxPool().Status([]common.Hash{hash})[0] + stat.Status = b.TxPool().Status(hash) // If the transaction is unknown to the pool, try looking it up locally. if stat.Status == txpool.TxStatusUnknown { diff --git a/les/test_helper.go b/les/test_helper.go index 44a454eaec..057903557a 100644 --- a/les/test_helper.go +++ b/les/test_helper.go @@ -37,6 +37,7 @@ import ( "github.com/ethereum/go-ethereum/core/forkid" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/txpool" + "github.com/ethereum/go-ethereum/core/txpool/legacypool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/ethconfig" @@ -232,9 +233,11 @@ func newTestServerHandler(blocks int, indexers []*core.ChainIndexer, db ethdb.Da simulation := backends.NewSimulatedBackendWithDatabase(db, gspec.Alloc, 100000000) prepare(blocks, simulation) - txpoolConfig := txpool.DefaultConfig + txpoolConfig := legacypool.DefaultConfig txpoolConfig.Journal = "" - txpool := txpool.New(txpoolConfig, gspec.Config, simulation.Blockchain()) + + pool := legacypool.New(txpoolConfig, simulation.Blockchain()) + txpool, _ := txpool.New(new(big.Int).SetUint64(txpoolConfig.PriceLimit), simulation.Blockchain(), []txpool.SubPool{pool}) server := &LesServer{ lesCommons: lesCommons{ diff --git a/light/txpool.go b/light/txpool.go index 8d2a189c01..b792d70b14 100644 --- a/light/txpool.go +++ b/light/txpool.go @@ -494,29 +494,29 @@ func (pool *TxPool) GetTransactions() (txs types.Transactions, err error) { // Content retrieves the data content of the transaction pool, returning all the // pending as well as queued transactions, grouped by account and nonce. -func (pool *TxPool) Content() (map[common.Address]types.Transactions, map[common.Address]types.Transactions) { +func (pool *TxPool) Content() (map[common.Address][]*types.Transaction, map[common.Address][]*types.Transaction) { pool.mu.RLock() defer pool.mu.RUnlock() // Retrieve all the pending transactions and sort by account and by nonce - pending := make(map[common.Address]types.Transactions) + pending := make(map[common.Address][]*types.Transaction) for _, tx := range pool.pending { account, _ := types.Sender(pool.signer, tx) pending[account] = append(pending[account], tx) } // There are no queued transactions in a light pool, just return an empty map - queued := make(map[common.Address]types.Transactions) + queued := make(map[common.Address][]*types.Transaction) return pending, queued } // ContentFrom retrieves the data content of the transaction pool, returning the // pending as well as queued transactions of this address, grouped by nonce. -func (pool *TxPool) ContentFrom(addr common.Address) (types.Transactions, types.Transactions) { +func (pool *TxPool) ContentFrom(addr common.Address) ([]*types.Transaction, []*types.Transaction) { pool.mu.RLock() defer pool.mu.RUnlock() // Retrieve the pending transactions and sort by nonce - var pending types.Transactions + var pending []*types.Transaction for _, tx := range pool.pending { account, _ := types.Sender(pool.signer, tx) if account != addr { @@ -525,7 +525,7 @@ func (pool *TxPool) ContentFrom(addr common.Address) (types.Transactions, types. pending = append(pending, tx) } // There are no queued transactions in a light pool, just return an empty map - return pending, types.Transactions{} + return pending, []*types.Transaction{} } // RemoveTransactions removes all given transactions from the pool. diff --git a/miner/miner_test.go b/miner/miner_test.go index 67d038d684..3f84446ff6 100644 --- a/miner/miner_test.go +++ b/miner/miner_test.go @@ -29,10 +29,12 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/txpool" + "github.com/ethereum/go-ethereum/core/txpool/legacypool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" ) @@ -61,11 +63,16 @@ func (m *mockBackend) StateAtBlock(block *types.Block, reexec uint64, base *stat } type testBlockChain struct { + config *params.ChainConfig statedb *state.StateDB gasLimit uint64 chainHeadFeed *event.Feed } +func (bc *testBlockChain) Config() *params.ChainConfig { + return bc.config +} + func (bc *testBlockChain) CurrentBlock() *types.Header { return &types.Header{ Number: new(big.Int), @@ -265,10 +272,12 @@ func createMiner(t *testing.T) (*Miner, *event.TypeMux, func(skipMiner bool)) { t.Fatalf("can't create new chain %v", err) } statedb, _ := state.New(types.EmptyRootHash, state.NewDatabase(chainDB), nil) - blockchain := &testBlockChain{statedb, 10000000, new(event.Feed)} + blockchain := &testBlockChain{chainConfig, statedb, 10000000, new(event.Feed)} - pool := txpool.New(testTxPoolConfig, chainConfig, blockchain) - backend := NewMockBackend(bc, pool) + pool := legacypool.New(testTxPoolConfig, blockchain) + txpool, _ := txpool.New(new(big.Int).SetUint64(testTxPoolConfig.PriceLimit), blockchain, []txpool.SubPool{pool}) + + backend := NewMockBackend(bc, txpool) // Create event Mux mux := new(event.TypeMux) // Create Miner @@ -276,7 +285,7 @@ func createMiner(t *testing.T) (*Miner, *event.TypeMux, func(skipMiner bool)) { cleanup := func(skipMiner bool) { bc.Stop() engine.Close() - pool.Stop() + txpool.Close() if !skipMiner { miner.Close() } diff --git a/miner/stress/clique/main.go b/miner/stress/clique/main.go index 4742dc4385..53ff2450c5 100644 --- a/miner/stress/clique/main.go +++ b/miner/stress/clique/main.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/common/fdlimit" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/txpool" + "github.com/ethereum/go-ethereum/core/txpool/legacypool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth" @@ -132,7 +133,7 @@ func main() { if err != nil { panic(err) } - if err := backend.TxPool().AddLocal(tx); err != nil { + if err := backend.TxPool().Add([]*txpool.Transaction{{Tx: tx}}, true, false); err != nil { panic(err) } nonces[index]++ @@ -206,7 +207,7 @@ func makeSealer(genesis *core.Genesis) (*node.Node, *eth.Ethereum, error) { SyncMode: downloader.FullSync, DatabaseCache: 256, DatabaseHandles: 256, - TxPool: txpool.DefaultConfig, + TxPool: legacypool.DefaultConfig, GPO: ethconfig.Defaults.GPO, Miner: miner.Config{ GasCeil: genesis.GasLimit * 11 / 10, diff --git a/miner/worker.go b/miner/worker.go index d61f8dcada..fb30142aa6 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -533,7 +533,7 @@ func (w *worker) mainLoop() { if gp := w.current.gasPool; gp != nil && gp.Gas() < params.TxGas { continue } - txs := make(map[common.Address]types.Transactions, len(ev.Txs)) + txs := make(map[common.Address][]*types.Transaction, len(ev.Txs)) for _, tx := range ev.Txs { acc, _ := types.Sender(w.current.signer, tx) txs[acc] = append(txs[acc], tx) @@ -904,7 +904,13 @@ func (w *worker) fillTransactions(interrupt *atomic.Int32, env *environment) err // Split the pending transactions into locals and remotes // Fill the block with all available pending transactions. pending := w.eth.TxPool().Pending(true) - localTxs, remoteTxs := make(map[common.Address]types.Transactions), pending + /*blobtxs := w.eth.BlobPool().Pending( + uint256.MustFromBig(env.header.BaseFee), + uint256.MustFromBig(misc.CalcBlobFee(*env.header.ExcessDataGas)), + ) + log.Trace("Side-effect log, much wow", "blobs", len(blobtxs))*/ + + localTxs, remoteTxs := make(map[common.Address][]*types.Transaction), pending for _, account := range w.eth.TxPool().Locals() { if txs := remoteTxs[account]; len(txs) > 0 { delete(remoteTxs, account) diff --git a/miner/worker_test.go b/miner/worker_test.go index d58382e1ec..80557d99bf 100644 --- a/miner/worker_test.go +++ b/miner/worker_test.go @@ -30,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/txpool" + "github.com/ethereum/go-ethereum/core/txpool/legacypool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" @@ -49,7 +50,7 @@ const ( var ( // Test chain configurations - testTxPoolConfig txpool.Config + testTxPoolConfig legacypool.Config ethashChainConfig *params.ChainConfig cliqueChainConfig *params.ChainConfig @@ -62,7 +63,7 @@ var ( testUserAddress = crypto.PubkeyToAddress(testUserKey.PublicKey) // Test transactions - pendingTxs []*types.Transaction + pendingTxs []*txpool.Transaction newTxs []*types.Transaction testConfig = &Config{ @@ -72,7 +73,7 @@ var ( ) func init() { - testTxPoolConfig = txpool.DefaultConfig + testTxPoolConfig = legacypool.DefaultConfig testTxPoolConfig.Journal = "" ethashChainConfig = new(params.ChainConfig) *ethashChainConfig = *params.TestChainConfig @@ -92,7 +93,7 @@ func init() { Gas: params.TxGas, GasPrice: big.NewInt(params.InitialBaseFee), }) - pendingTxs = append(pendingTxs, tx1) + pendingTxs = append(pendingTxs, &txpool.Transaction{Tx: tx1}) tx2 := types.MustSignNewTx(testBankKey, signer, &types.LegacyTx{ Nonce: 1, @@ -132,10 +133,13 @@ func newTestWorkerBackend(t *testing.T, chainConfig *params.ChainConfig, engine if err != nil { t.Fatalf("core.NewBlockChain failed: %v", err) } + pool := legacypool.New(testTxPoolConfig, chain) + txpool, _ := txpool.New(new(big.Int).SetUint64(testTxPoolConfig.PriceLimit), chain, []txpool.SubPool{pool}) + return &testWorkerBackend{ db: db, chain: chain, - txPool: txpool.New(testTxPoolConfig, chainConfig, chain), + txPool: txpool, genesis: gspec, } } @@ -156,7 +160,7 @@ func (b *testWorkerBackend) newRandomTx(creation bool) *types.Transaction { func newTestWorker(t *testing.T, chainConfig *params.ChainConfig, engine consensus.Engine, db ethdb.Database, blocks int) (*worker, *testWorkerBackend) { backend := newTestWorkerBackend(t, chainConfig, engine, db, blocks) - backend.txPool.AddLocals(pendingTxs) + backend.txPool.Add(pendingTxs, true, false) w := newWorker(testConfig, chainConfig, engine, backend, new(event.TypeMux), nil, false) w.setEtherbase(testBankAddress) return w, backend @@ -190,8 +194,8 @@ func TestGenerateAndImportBlock(t *testing.T) { w.start() for i := 0; i < 5; i++ { - b.txPool.AddLocal(b.newRandomTx(true)) - b.txPool.AddLocal(b.newRandomTx(false)) + b.txPool.Add([]*txpool.Transaction{{Tx: b.newRandomTx(true)}}, true, false) + b.txPool.Add([]*txpool.Transaction{{Tx: b.newRandomTx(false)}}, true, false) select { case ev := <-sub.Chan(): diff --git a/params/protocol_params.go b/params/protocol_params.go index c662e4a3f3..9a0b8115b1 100644 --- a/params/protocol_params.go +++ b/params/protocol_params.go @@ -160,6 +160,8 @@ const ( RefundQuotient uint64 = 2 RefundQuotientEIP3529 uint64 = 5 + BlobTxBytesPerFieldElement = 32 // Size in bytes of a field element + BlobTxFieldElementsPerBlob = 4096 // Number of field elements stored in a single data blob BlobTxHashVersion = 0x01 // Version byte of the commitment hash BlobTxMaxDataGasPerBlock = 1 << 19 // Maximum consumable data gas for data blobs per block BlobTxTargetDataGasPerBlock = 1 << 18 // Target consumable data gas for data blobs per block (for 1559-like pricing) diff --git a/tests/fuzzers/les/les-fuzzer.go b/tests/fuzzers/les/les-fuzzer.go index c203c87f81..b4599025e8 100644 --- a/tests/fuzzers/les/les-fuzzer.go +++ b/tests/fuzzers/les/les-fuzzer.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/txpool" + "github.com/ethereum/go-ethereum/core/txpool/legacypool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" @@ -128,6 +129,9 @@ type fuzzer struct { } func newFuzzer(input []byte) *fuzzer { + pool := legacypool.New(legacypool.DefaultConfig, chain) + txpool, _ := txpool.New(new(big.Int).SetUint64(legacypool.DefaultConfig.PriceLimit), chain, []txpool.SubPool{pool}) + return &fuzzer{ chain: chain, chainLen: testChainLen, @@ -138,7 +142,7 @@ func newFuzzer(input []byte) *fuzzer { chtKeys: chtKeys, bloomKeys: bloomKeys, nonce: uint64(len(txHashes)), - pool: txpool.New(txpool.DefaultConfig, params.TestChainConfig, chain), + pool: txpool, input: bytes.NewReader(input), } } diff --git a/tests/fuzzers/txfetcher/txfetcher_fuzzer.go b/tests/fuzzers/txfetcher/txfetcher_fuzzer.go index d1d6fdc665..56b6b1e64e 100644 --- a/tests/fuzzers/txfetcher/txfetcher_fuzzer.go +++ b/tests/fuzzers/txfetcher/txfetcher_fuzzer.go @@ -25,6 +25,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth/fetcher" ) @@ -79,7 +80,7 @@ func Fuzz(input []byte) int { f := fetcher.NewTxFetcherForTests( func(common.Hash) bool { return false }, - func(txs []*types.Transaction) []error { + func(txs []*txpool.Transaction) []error { return make([]error, len(txs)) }, func(string, []common.Hash) error { return nil }, From 36ca59f1ec4c9777ef7483b5f4389aa92e060cd2 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Sun, 18 Jun 2023 09:09:00 -0400 Subject: [PATCH 03/90] build: use slices package for sorting (#27486) --- build/update-license.go | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/build/update-license.go b/build/update-license.go index f61536470a..52a54bf66a 100644 --- a/build/update-license.go +++ b/build/update-license.go @@ -46,12 +46,13 @@ import ( "path/filepath" "regexp" "runtime" - "sort" "strconv" "strings" "sync" "text/template" "time" + + "golang.org/x/exp/slices" ) var ( @@ -152,13 +153,6 @@ func (i info) gpl() bool { return false } -// authors implements the sort.Interface for strings in case-insensitive mode. -type authors []string - -func (as authors) Len() int { return len(as) } -func (as authors) Less(i, j int) bool { return strings.ToLower(as[i]) < strings.ToLower(as[j]) } -func (as authors) Swap(i, j int) { as[i], as[j] = as[j], as[i] } - func main() { var ( files = getFiles() @@ -299,7 +293,9 @@ func writeAuthors(files []string) { } } // Write sorted list of authors back to the file. - sort.Sort(authors(list)) + slices.SortFunc(list, func(a, b string) bool { + return strings.ToLower(a) < strings.ToLower(b) + }) content := new(bytes.Buffer) content.WriteString(authorsFileHeader) for _, a := range list { From 46ec972c9c56a4e0d97d812f2eaf9e3657c66276 Mon Sep 17 00:00:00 2001 From: jin <35813306+lochjin@users.noreply.github.com> Date: Mon, 19 Jun 2023 04:43:53 +0800 Subject: [PATCH 04/90] core/txpool/legacypool: reheap the tx list if london fork not enabled (#27481) This change ensures Reheap will be called even before the London fork activates. Since Reheap would otherwise only be called through `SetBaseFee` after London, the list would just keep growing if the fork was not enabled or not reached yet. --- core/txpool/legacypool/legacypool.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/core/txpool/legacypool/legacypool.go b/core/txpool/legacypool/legacypool.go index 3525747c72..fc6fc43bdd 100644 --- a/core/txpool/legacypool/legacypool.go +++ b/core/txpool/legacypool/legacypool.go @@ -1206,9 +1206,13 @@ func (pool *LegacyPool) runReorg(done chan struct{}, reset *txpoolResetRequest, // because of another transaction (e.g. higher gas price). if reset != nil { pool.demoteUnexecutables() - if reset.newHead != nil && pool.chainconfig.IsLondon(new(big.Int).Add(reset.newHead.Number, big.NewInt(1))) { - pendingBaseFee := misc.CalcBaseFee(pool.chainconfig, reset.newHead) - pool.priced.SetBaseFee(pendingBaseFee) + if reset.newHead != nil { + if pool.chainconfig.IsLondon(new(big.Int).Add(reset.newHead.Number, big.NewInt(1))) { + pendingBaseFee := misc.CalcBaseFee(pool.chainconfig, reset.newHead) + pool.priced.SetBaseFee(pendingBaseFee) + } else { + pool.priced.Reheap() + } } // Update all accounts to the latest known pending nonce nonces := make(map[common.Address]uint64, len(pool.pending)) From 289c6c3b157a5bf7111a667bc0d760bd1a6911a6 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Mon, 19 Jun 2023 01:48:12 -0400 Subject: [PATCH 05/90] p2p: use slices package for sorting (#27494) Co-authored-by: Felix Lange --- p2p/discover/ntp.go | 12 ++---------- p2p/discover/table_util_test.go | 10 +++++----- p2p/discover/v4_lookup_test.go | 6 +++--- p2p/discover/v5_udp_test.go | 6 +++--- p2p/dnsdisc/tree.go | 6 +++--- p2p/peer.go | 4 ++-- p2p/protocol.go | 12 ++++++------ p2p/server.go | 4 ++-- 8 files changed, 26 insertions(+), 34 deletions(-) diff --git a/p2p/discover/ntp.go b/p2p/discover/ntp.go index 48ceffe95b..3f9157808f 100644 --- a/p2p/discover/ntp.go +++ b/p2p/discover/ntp.go @@ -22,10 +22,10 @@ package discover import ( "fmt" "net" - "sort" "time" "github.com/ethereum/go-ethereum/log" + "golang.org/x/exp/slices" ) const ( @@ -33,14 +33,6 @@ const ( ntpChecks = 3 // Number of measurements to do against the NTP server ) -// durationSlice attaches the methods of sort.Interface to []time.Duration, -// sorting in increasing order. -type durationSlice []time.Duration - -func (s durationSlice) Len() int { return len(s) } -func (s durationSlice) Less(i, j int) bool { return s[i] < s[j] } -func (s durationSlice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } - // checkClockDrift queries an NTP server for clock drifts and warns the user if // one large enough is detected. func checkClockDrift() { @@ -109,7 +101,7 @@ func sntpDrift(measurements int) (time.Duration, error) { drifts = append(drifts, sent.Sub(t)+elapsed/2) } // Calculate average drift (drop two extremities to avoid outliers) - sort.Sort(durationSlice(drifts)) + slices.Sort(drifts) drift := time.Duration(0) for i := 1; i < len(drifts)-1; i++ { diff --git a/p2p/discover/table_util_test.go b/p2p/discover/table_util_test.go index 5b5e9a0431..52544a2c93 100644 --- a/p2p/discover/table_util_test.go +++ b/p2p/discover/table_util_test.go @@ -24,12 +24,12 @@ import ( "fmt" "math/rand" "net" - "sort" "sync" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/enr" + "golang.org/x/exp/slices" ) var nullNode *enode.Node @@ -217,14 +217,14 @@ func nodeEqual(n1 *enode.Node, n2 *enode.Node) bool { } func sortByID(nodes []*enode.Node) { - sort.Slice(nodes, func(i, j int) bool { - return string(nodes[i].ID().Bytes()) < string(nodes[j].ID().Bytes()) + slices.SortFunc(nodes, func(a, b *enode.Node) bool { + return string(a.ID().Bytes()) < string(b.ID().Bytes()) }) } func sortedByDistanceTo(distbase enode.ID, slice []*node) bool { - return sort.SliceIsSorted(slice, func(i, j int) bool { - return enode.DistCmp(distbase, slice[i].ID(), slice[j].ID()) < 0 + return slices.IsSortedFunc(slice, func(a, b *node) bool { + return enode.DistCmp(distbase, a.ID(), b.ID()) < 0 }) } diff --git a/p2p/discover/v4_lookup_test.go b/p2p/discover/v4_lookup_test.go index a00de9ca18..83c4626288 100644 --- a/p2p/discover/v4_lookup_test.go +++ b/p2p/discover/v4_lookup_test.go @@ -20,13 +20,13 @@ import ( "crypto/ecdsa" "fmt" "net" - "sort" "testing" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/p2p/discover/v4wire" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/enr" + "golang.org/x/exp/slices" ) func TestUDPv4_Lookup(t *testing.T) { @@ -302,8 +302,8 @@ func (tn *preminedTestnet) closest(n int) (nodes []*enode.Node) { nodes = append(nodes, tn.node(d, i)) } } - sort.Slice(nodes, func(i, j int) bool { - return enode.DistCmp(tn.target.id(), nodes[i].ID(), nodes[j].ID()) < 0 + slices.SortFunc(nodes, func(a, b *enode.Node) bool { + return enode.DistCmp(tn.target.id(), a.ID(), b.ID()) < 0 }) return nodes[:n] } diff --git a/p2p/discover/v5_udp_test.go b/p2p/discover/v5_udp_test.go index 392a95b61e..887d46c497 100644 --- a/p2p/discover/v5_udp_test.go +++ b/p2p/discover/v5_udp_test.go @@ -24,7 +24,6 @@ import ( "math/rand" "net" "reflect" - "sort" "testing" "time" @@ -35,6 +34,7 @@ import ( "github.com/ethereum/go-ethereum/p2p/enr" "github.com/ethereum/go-ethereum/rlp" "github.com/stretchr/testify/require" + "golang.org/x/exp/slices" ) // Real sockets, real crypto: this test checks end-to-end connectivity for UDPv5. @@ -61,8 +61,8 @@ func TestUDPv5_lookupE2E(t *testing.T) { for i := range nodes { expectedResult[i] = nodes[i].Self() } - sort.Slice(expectedResult, func(i, j int) bool { - return enode.DistCmp(target.ID(), expectedResult[i].ID(), expectedResult[j].ID()) < 0 + slices.SortFunc(expectedResult, func(a, b *enode.Node) bool { + return enode.DistCmp(target.ID(), a.ID(), b.ID()) < 0 }) // Do the lookup. diff --git a/p2p/dnsdisc/tree.go b/p2p/dnsdisc/tree.go index a3f426e428..f91aff458f 100644 --- a/p2p/dnsdisc/tree.go +++ b/p2p/dnsdisc/tree.go @@ -23,7 +23,6 @@ import ( "encoding/base64" "fmt" "io" - "sort" "strings" "github.com/ethereum/go-ethereum/crypto" @@ -31,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/p2p/enr" "github.com/ethereum/go-ethereum/rlp" "golang.org/x/crypto/sha3" + "golang.org/x/exp/slices" ) // Tree is a merkle tree of node records. @@ -214,8 +214,8 @@ func (t *Tree) build(entries []entry) entry { } func sortByID(nodes []*enode.Node) []*enode.Node { - sort.Slice(nodes, func(i, j int) bool { - return bytes.Compare(nodes[i].ID().Bytes(), nodes[j].ID().Bytes()) < 0 + slices.SortFunc(nodes, func(a, b *enode.Node) bool { + return bytes.Compare(a.ID().Bytes(), b.ID().Bytes()) < 0 }) return nodes } diff --git a/p2p/peer.go b/p2p/peer.go index a4b22fb6d4..23ac117749 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -21,7 +21,6 @@ import ( "fmt" "io" "net" - "sort" "sync" "time" @@ -32,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/enr" "github.com/ethereum/go-ethereum/rlp" + "golang.org/x/exp/slices" ) var ( @@ -375,7 +375,7 @@ func countMatchingProtocols(protocols []Protocol, caps []Cap) int { // matchProtocols creates structures for matching named subprotocols. func matchProtocols(protocols []Protocol, caps []Cap, rw MsgReadWriter) map[string]*protoRW { - sort.Sort(capsByNameAndVersion(caps)) + slices.SortFunc(caps, Cap.Less) offset := baseProtocolLength result := make(map[string]*protoRW) diff --git a/p2p/protocol.go b/p2p/protocol.go index fa23a087c2..4b3cca6d4a 100644 --- a/p2p/protocol.go +++ b/p2p/protocol.go @@ -77,10 +77,10 @@ func (cap Cap) String() string { return fmt.Sprintf("%s/%d", cap.Name, cap.Version) } -type capsByNameAndVersion []Cap - -func (cs capsByNameAndVersion) Len() int { return len(cs) } -func (cs capsByNameAndVersion) Swap(i, j int) { cs[i], cs[j] = cs[j], cs[i] } -func (cs capsByNameAndVersion) Less(i, j int) bool { - return cs[i].Name < cs[j].Name || (cs[i].Name == cs[j].Name && cs[i].Version < cs[j].Version) +// Less defines the canonical sorting order of capabilities. +func (cap Cap) Less(other Cap) bool { + if cap.Name == other.Name { + return cap.Version < other.Version + } + return cap.Name < other.Name } diff --git a/p2p/server.go b/p2p/server.go index f7bf948b69..bdfeb74c67 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -24,7 +24,6 @@ import ( "errors" "fmt" "net" - "sort" "sync" "sync/atomic" "time" @@ -39,6 +38,7 @@ import ( "github.com/ethereum/go-ethereum/p2p/enr" "github.com/ethereum/go-ethereum/p2p/nat" "github.com/ethereum/go-ethereum/p2p/netutil" + "golang.org/x/exp/slices" ) const ( @@ -498,7 +498,7 @@ func (srv *Server) setupLocalNode() error { for _, p := range srv.Protocols { srv.ourHandshake.Caps = append(srv.ourHandshake.Caps, p.cap()) } - sort.Sort(capsByNameAndVersion(srv.ourHandshake.Caps)) + slices.SortFunc(srv.ourHandshake.Caps, Cap.Less) // Create the local node. db, err := enode.OpenDB(srv.NodeDatabase) From 5d75123cb7804feb385f025eafba83594076e6a2 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Mon, 19 Jun 2023 01:48:47 -0400 Subject: [PATCH 06/90] ethdb/dbtest: use slices package for sorting (#27491) --- ethdb/dbtest/testsuite.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ethdb/dbtest/testsuite.go b/ethdb/dbtest/testsuite.go index d82dbd6992..ac10ddde93 100644 --- a/ethdb/dbtest/testsuite.go +++ b/ethdb/dbtest/testsuite.go @@ -24,6 +24,7 @@ import ( "testing" "github.com/ethereum/go-ethereum/ethdb" + "golang.org/x/exp/slices" ) // TestDatabaseSuite runs a suite of tests against a KeyValueStore database @@ -526,7 +527,7 @@ func makeDataset(size, ksize, vsize int, order bool) ([][]byte, [][]byte) { vals = append(vals, randBytes(vsize)) } if order { - sort.Slice(keys, func(i, j int) bool { return bytes.Compare(keys[i], keys[j]) < 0 }) + slices.SortFunc(keys, func(a, b []byte) bool { return bytes.Compare(a, b) < 0 }) } return keys, vals } From 168d0cc3b3e9073a542f0b3c45f0d8e19ccf380f Mon Sep 17 00:00:00 2001 From: John Chase <68833933+joohhnnn@users.noreply.github.com> Date: Mon, 19 Jun 2023 13:50:56 +0800 Subject: [PATCH 07/90] internal/jsre/deps: fix typos in web3.js (#27500) --- internal/jsre/deps/web3.js | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/internal/jsre/deps/web3.js b/internal/jsre/deps/web3.js index a291218ec5..7a09fddab0 100644 --- a/internal/jsre/deps/web3.js +++ b/internal/jsre/deps/web3.js @@ -518,7 +518,7 @@ var f = require('./formatters'); var SolidityType = require('./type'); /** - * SolidityTypeAddress is a prootype that represents address type + * SolidityTypeAddress is a prototype that represents address type * It matches: * address * address[] @@ -546,7 +546,7 @@ var f = require('./formatters'); var SolidityType = require('./type'); /** - * SolidityTypeBool is a prootype that represents bool type + * SolidityTypeBool is a prototype that represents bool type * It matches: * bool * bool[] @@ -1146,7 +1146,7 @@ var f = require('./formatters'); var SolidityType = require('./type'); /** - * SolidityTypeInt is a prootype that represents int type + * SolidityTypeInt is a prototype that represents int type * It matches: * int * int[] @@ -1334,7 +1334,7 @@ var f = require('./formatters'); var SolidityType = require('./type'); /** - * SolidityTypeReal is a prootype that represents real type + * SolidityTypeReal is a prototype that represents real type * It matches: * real * real[] @@ -1647,7 +1647,7 @@ var f = require('./formatters'); var SolidityType = require('./type'); /** - * SolidityTypeUInt is a prootype that represents uint type + * SolidityTypeUInt is a prototype that represents uint type * It matches: * uint * uint[] @@ -1681,7 +1681,7 @@ var f = require('./formatters'); var SolidityType = require('./type'); /** - * SolidityTypeUReal is a prootype that represents ureal type + * SolidityTypeUReal is a prototype that represents ureal type * It matches: * ureal * ureal[] @@ -2307,7 +2307,7 @@ var toChecksumAddress = function (address) { }; /** - * Transforms given string to valid 20 bytes-length addres with 0x prefix + * Transforms given string to valid 20 bytes-length address with 0x prefix * * @method toAddress * @param {String} address From a71b9b9ffad052210137b1c4d6014a40cfd1046a Mon Sep 17 00:00:00 2001 From: hero5512 Date: Mon, 19 Jun 2023 01:52:40 -0400 Subject: [PATCH 08/90] core/txpool/legacypool: remove redundant check for floatingRatio == 0 (#27477) floatingRatio is a constant and always non-zero. So there is no need to check for == 0. --- core/txpool/legacypool/list.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/txpool/legacypool/list.go b/core/txpool/legacypool/list.go index d3b6768f81..d5d24c85a5 100644 --- a/core/txpool/legacypool/list.go +++ b/core/txpool/legacypool/list.go @@ -590,7 +590,7 @@ func (l *pricedList) underpricedFor(h *priceHeap, tx *types.Transaction) bool { func (l *pricedList) Discard(slots int, force bool) (types.Transactions, bool) { drop := make(types.Transactions, 0, slots) // Remote underpriced transactions to drop for slots > 0 { - if len(l.urgent.list)*floatingRatio > len(l.floating.list)*urgentRatio || floatingRatio == 0 { + if len(l.urgent.list)*floatingRatio > len(l.floating.list)*urgentRatio { // Discard stale transactions if found during cleanup tx := heap.Pop(&l.urgent).(*types.Transaction) if l.all.GetRemote(tx.Hash()) == nil { // Removed or migrated From e4660a1181d19c6c50ebf77eb3ff3865948c2d63 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Mon, 19 Jun 2023 01:58:04 -0400 Subject: [PATCH 09/90] core/txpool/legacypool: handle missing head in reset (#27479) Fixes #27301, a crash that could occur during txpool reorg handling. --- core/txpool/legacypool/legacypool.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/core/txpool/legacypool/legacypool.go b/core/txpool/legacypool/legacypool.go index fc6fc43bdd..3464fe212f 100644 --- a/core/txpool/legacypool/legacypool.go +++ b/core/txpool/legacypool/legacypool.go @@ -1283,6 +1283,14 @@ func (pool *LegacyPool) reset(oldHead, newHead *types.Header) { "old", oldHead.Hash(), "oldnum", oldNum, "new", newHead.Hash(), "newnum", newNum) // We still need to update the current state s.th. the lost transactions can be readded by the user } else { + if add == nil { + // if the new head is nil, it means that something happened between + // the firing of newhead-event and _now_: most likely a + // reorg caused by sync-reversion or explicit sethead back to an + // earlier block. + log.Warn("New head missing in txpool reset", "number", newHead.Number, "hash", newHead.Hash()) + return + } for rem.NumberU64() > add.NumberU64() { discarded = append(discarded, rem.Transactions()...) if rem = pool.chain.GetBlock(rem.ParentHash(), rem.NumberU64()-1); rem == nil { From f0b5af74a34439d95bfd174f6d034a8df070a588 Mon Sep 17 00:00:00 2001 From: Koichi Shiraishi Date: Mon, 19 Jun 2023 15:06:58 +0900 Subject: [PATCH 10/90] rpc: avoid use of cgo by hard-coding maxPathSize (#27447) Package rpc uses cgo to find the maximum UNIX domain socket path length. If exceeded, a warning is printed. This is the only use of cgo in this package. It seems excessive to depend on cgo just for this warning, so we now hard-code the usual limit for Linux instead. --------- Co-authored-by: Felix Lange --- rpc/constants_unix.go | 34 ---------------------------------- rpc/constants_unix_nocgo.go | 26 -------------------------- rpc/ipc_unix.go | 10 ++++++++-- 3 files changed, 8 insertions(+), 62 deletions(-) delete mode 100644 rpc/constants_unix.go delete mode 100644 rpc/constants_unix_nocgo.go diff --git a/rpc/constants_unix.go b/rpc/constants_unix.go deleted file mode 100644 index 1f04d15d7f..0000000000 --- a/rpc/constants_unix.go +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright 2019 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -//go:build darwin || dragonfly || freebsd || linux || nacl || netbsd || openbsd || solaris -// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris - -package rpc - -/* -#include - -int max_socket_path_size() { -struct sockaddr_un s; -return sizeof(s.sun_path); -} -*/ -import "C" - -var ( - max_path_size = C.max_socket_path_size() -) diff --git a/rpc/constants_unix_nocgo.go b/rpc/constants_unix_nocgo.go deleted file mode 100644 index a62e4ee529..0000000000 --- a/rpc/constants_unix_nocgo.go +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright 2019 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -//go:build !cgo && !windows -// +build !cgo,!windows - -package rpc - -var ( - // On Linux, sun_path is 108 bytes in size - // see http://man7.org/linux/man-pages/man7/unix.7.html - max_path_size = 108 -) diff --git a/rpc/ipc_unix.go b/rpc/ipc_unix.go index 9876347708..33c1cad549 100644 --- a/rpc/ipc_unix.go +++ b/rpc/ipc_unix.go @@ -29,11 +29,17 @@ import ( "github.com/ethereum/go-ethereum/log" ) +const ( + // On Linux, sun_path is 108 bytes in size + // see http://man7.org/linux/man-pages/man7/unix.7.html + maxPathSize = int(108) +) + // ipcListen will create a Unix socket on the given endpoint. func ipcListen(endpoint string) (net.Listener, error) { // account for null-terminator too - if len(endpoint)+1 > int(max_path_size) { - log.Warn(fmt.Sprintf("The ipc endpoint is longer than %d characters. ", max_path_size-1), + if len(endpoint)+1 > maxPathSize { + log.Warn(fmt.Sprintf("The ipc endpoint is longer than %d characters. ", maxPathSize-1), "endpoint", endpoint) } From 311b742c840471d084c298fe268a9a934d1e5e1d Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Mon, 19 Jun 2023 02:34:47 -0400 Subject: [PATCH 11/90] accounts/keystore: use slices package for sorting (#27485) Co-authored-by: Felix Lange --- accounts/keystore/account_cache.go | 14 +++++++------- accounts/keystore/account_cache_test.go | 4 ++-- accounts/keystore/keystore_test.go | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/accounts/keystore/account_cache.go b/accounts/keystore/account_cache.go index 12f92d2619..a88b0fa5f3 100644 --- a/accounts/keystore/account_cache.go +++ b/accounts/keystore/account_cache.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" + "golang.org/x/exp/slices" ) // Minimum amount of time between cache reloads. This limit applies if the platform does @@ -38,11 +39,10 @@ import ( // exist yet, the code will attempt to create a watcher at most this often. const minReloadInterval = 2 * time.Second -type accountsByURL []accounts.Account - -func (s accountsByURL) Len() int { return len(s) } -func (s accountsByURL) Less(i, j int) bool { return s[i].URL.Cmp(s[j].URL) < 0 } -func (s accountsByURL) Swap(i, j int) { s[i], s[j] = s[j], s[i] } +// byURL defines the sorting order for accounts. +func byURL(a, b accounts.Account) bool { + return a.URL.Cmp(b.URL) < 0 +} // AmbiguousAddrError is returned when attempting to unlock // an address for which more than one file exists. @@ -67,7 +67,7 @@ type accountCache struct { keydir string watcher *watcher mu sync.Mutex - all accountsByURL + all []accounts.Account byAddr map[common.Address][]accounts.Account throttle *time.Timer notify chan struct{} @@ -194,7 +194,7 @@ func (ac *accountCache) find(a accounts.Account) (accounts.Account, error) { default: err := &AmbiguousAddrError{Addr: a.Address, Matches: make([]accounts.Account, len(matches))} copy(err.Matches, matches) - sort.Sort(accountsByURL(err.Matches)) + slices.SortFunc(err.Matches, byURL) return accounts.Account{}, err } } diff --git a/accounts/keystore/account_cache_test.go b/accounts/keystore/account_cache_test.go index 7605d0fbf0..3847e9daf6 100644 --- a/accounts/keystore/account_cache_test.go +++ b/accounts/keystore/account_cache_test.go @@ -23,7 +23,6 @@ import ( "os" "path/filepath" "reflect" - "sort" "testing" "time" @@ -31,6 +30,7 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/common" + "golang.org/x/exp/slices" ) var ( @@ -203,7 +203,7 @@ func TestCacheAddDeleteOrder(t *testing.T) { // Check that the account list is sorted by filename. wantAccounts := make([]accounts.Account, len(accs)) copy(wantAccounts, accs) - sort.Sort(accountsByURL(wantAccounts)) + slices.SortFunc(wantAccounts, byURL) list := cache.accounts() if !reflect.DeepEqual(list, wantAccounts) { t.Fatalf("got accounts: %s\nwant %s", spew.Sdump(accs), spew.Sdump(wantAccounts)) diff --git a/accounts/keystore/keystore_test.go b/accounts/keystore/keystore_test.go index f90d809b55..6f0527ef87 100644 --- a/accounts/keystore/keystore_test.go +++ b/accounts/keystore/keystore_test.go @@ -20,7 +20,6 @@ import ( "math/rand" "os" "runtime" - "sort" "strings" "sync" "sync/atomic" @@ -31,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/event" + "golang.org/x/exp/slices" ) var testSigData = make([]byte, 32) @@ -424,7 +424,7 @@ func checkAccounts(t *testing.T, live map[common.Address]accounts.Account, walle for _, account := range live { liveList = append(liveList, account) } - sort.Sort(accountsByURL(liveList)) + slices.SortFunc(liveList, byURL) for j, wallet := range wallets { if accs := wallet.Accounts(); len(accs) != 1 { t.Errorf("wallet %d: contains invalid number of accounts: have %d, want 1", j, len(accs)) From 4544dc5f32fef0b822d855c64176e872ae387634 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Mon, 19 Jun 2023 02:42:49 -0400 Subject: [PATCH 12/90] cmd/devp2p: use slices package for sorting (#27487) --- cmd/devp2p/dns_route53.go | 10 +++++----- cmd/devp2p/nodeset.go | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cmd/devp2p/dns_route53.go b/cmd/devp2p/dns_route53.go index 6c97fa937e..eae6f70f58 100644 --- a/cmd/devp2p/dns_route53.go +++ b/cmd/devp2p/dns_route53.go @@ -20,7 +20,6 @@ import ( "context" "errors" "fmt" - "sort" "strconv" "strings" "time" @@ -33,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/dnsdisc" "github.com/urfave/cli/v2" + "golang.org/x/exp/slices" ) const ( @@ -288,11 +288,11 @@ func makeDeletionChanges(records map[string]recordSet, keep map[string]string) [ // sortChanges ensures DNS changes are in leaf-added -> root-changed -> leaf-deleted order. func sortChanges(changes []types.Change) { score := map[string]int{"CREATE": 1, "UPSERT": 2, "DELETE": 3} - sort.Slice(changes, func(i, j int) bool { - if changes[i].Action == changes[j].Action { - return *changes[i].ResourceRecordSet.Name < *changes[j].ResourceRecordSet.Name + slices.SortFunc(changes, func(a, b types.Change) bool { + if a.Action == b.Action { + return *a.ResourceRecordSet.Name < *b.ResourceRecordSet.Name } - return score[string(changes[i].Action)] < score[string(changes[j].Action)] + return score[string(a.Action)] < score[string(b.Action)] }) } diff --git a/cmd/devp2p/nodeset.go b/cmd/devp2p/nodeset.go index 33c39f4b9e..c040c1a908 100644 --- a/cmd/devp2p/nodeset.go +++ b/cmd/devp2p/nodeset.go @@ -21,11 +21,11 @@ import ( "encoding/json" "fmt" "os" - "sort" "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/p2p/enode" + "golang.org/x/exp/slices" ) const jsonIndent = " " @@ -77,8 +77,8 @@ func (ns nodeSet) nodes() []*enode.Node { result = append(result, n.N) } // Sort by ID. - sort.Slice(result, func(i, j int) bool { - return bytes.Compare(result[i].ID().Bytes(), result[j].ID().Bytes()) < 0 + slices.SortFunc(result, func(a, b *enode.Node) bool { + return bytes.Compare(a.ID().Bytes(), b.ID().Bytes()) < 0 }) return result } @@ -103,8 +103,8 @@ func (ns nodeSet) topN(n int) nodeSet { for _, v := range ns { byscore = append(byscore, v) } - sort.Slice(byscore, func(i, j int) bool { - return byscore[i].Score >= byscore[j].Score + slices.SortFunc(byscore, func(a, b nodeJSON) bool { + return a.Score >= b.Score }) result := make(nodeSet, n) for _, v := range byscore[:n] { From cd3b8c3d780e466701095192312f18c6642464a8 Mon Sep 17 00:00:00 2001 From: John Chase <68833933+joohhnnn@users.noreply.github.com> Date: Mon, 19 Jun 2023 14:44:54 +0800 Subject: [PATCH 13/90] cmd/evm: fix typos in documentation (#27501) --- cmd/evm/testdata/13/readme.md | 2 +- cmd/evm/testdata/2/readme.md | 2 +- cmd/evm/testdata/23/readme.md | 2 +- cmd/evm/testdata/3/readme.md | 2 +- cmd/evm/testdata/4/readme.md | 2 +- cmd/evm/testdata/5/readme.md | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/evm/testdata/13/readme.md b/cmd/evm/testdata/13/readme.md index 64f52fc9a9..889975d47e 100644 --- a/cmd/evm/testdata/13/readme.md +++ b/cmd/evm/testdata/13/readme.md @@ -1,4 +1,4 @@ ## Input transactions in RLP form -This testdata folder is used to examplify how transaction input can be provided in rlp form. +This testdata folder is used to exemplify how transaction input can be provided in rlp form. Please see the README in `evm` folder for how this is performed. \ No newline at end of file diff --git a/cmd/evm/testdata/2/readme.md b/cmd/evm/testdata/2/readme.md index c116f0e792..4bcf0f0fa0 100644 --- a/cmd/evm/testdata/2/readme.md +++ b/cmd/evm/testdata/2/readme.md @@ -1 +1 @@ -These files examplify a selfdestruct to the `0`-address. \ No newline at end of file +These files exemplify a selfdestruct to the `0`-address. \ No newline at end of file diff --git a/cmd/evm/testdata/23/readme.md b/cmd/evm/testdata/23/readme.md index 85fe8db66c..f31b64de2f 100644 --- a/cmd/evm/testdata/23/readme.md +++ b/cmd/evm/testdata/23/readme.md @@ -1 +1 @@ -These files examplify how to sign a transaction using the pre-EIP155 scheme. +These files exemplify how to sign a transaction using the pre-EIP155 scheme. diff --git a/cmd/evm/testdata/3/readme.md b/cmd/evm/testdata/3/readme.md index 499f03d7aa..aa1f041df2 100644 --- a/cmd/evm/testdata/3/readme.md +++ b/cmd/evm/testdata/3/readme.md @@ -1,2 +1,2 @@ -These files examplify a transition where a transaction (excuted on block 5) requests +These files exemplify a transition where a transaction (excuted on block 5) requests the blockhash for block `1`. diff --git a/cmd/evm/testdata/4/readme.md b/cmd/evm/testdata/4/readme.md index 08840d37bd..9918edfb20 100644 --- a/cmd/evm/testdata/4/readme.md +++ b/cmd/evm/testdata/4/readme.md @@ -1,3 +1,3 @@ -These files examplify a transition where a transaction (excuted on block 5) requests +These files exemplify a transition where a transaction (excuted on block 5) requests the blockhash for block `4`, but where the hash for that block is missing. It's expected that executing these should cause `exit` with errorcode `4`. diff --git a/cmd/evm/testdata/5/readme.md b/cmd/evm/testdata/5/readme.md index e2b608face..1a84afaab6 100644 --- a/cmd/evm/testdata/5/readme.md +++ b/cmd/evm/testdata/5/readme.md @@ -1 +1 @@ -These files examplify a transition where there are no transcations, two ommers, at block `N-1` (delta 1) and `N-2` (delta 2). \ No newline at end of file +These files exemplify a transition where there are no transactions, two ommers, at block `N-1` (delta 1) and `N-2` (delta 2). \ No newline at end of file From 760fd0c79b83bd5e7da1c3cc6e5bef1764037459 Mon Sep 17 00:00:00 2001 From: Sanghee Choi <32831939+pengin7384@users.noreply.github.com> Date: Mon, 19 Jun 2023 15:47:00 +0900 Subject: [PATCH 14/90] consensus/clique: fix typo in comment (#27503) --- consensus/clique/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/consensus/clique/api.go b/consensus/clique/api.go index cb270d321d..374b50692d 100644 --- a/consensus/clique/api.go +++ b/consensus/clique/api.go @@ -205,7 +205,7 @@ func (sb *blockNumberOrHashOrRLP) UnmarshalJSON(data []byte) error { } // GetSigner returns the signer for a specific clique block. -// Can be called with either a blocknumber, blockhash or an rlp encoded blob. +// Can be called with a block number, a block hash or a rlp encoded blob. // The RLP encoded blob can either be a block or a header. func (api *API) GetSigner(rlpOrBlockNr *blockNumberOrHashOrRLP) (common.Address, error) { if len(rlpOrBlockNr.RLP) == 0 { From 4367ab499fd69def921cdbf2da547f1245edda52 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Mon, 19 Jun 2023 02:53:15 -0400 Subject: [PATCH 15/90] metrics: use slices package for sorting (#27493) Co-authored-by: Felix Lange --- metrics/resetting_timer.go | 12 +++--------- metrics/sample.go | 15 +++++---------- metrics/writer.go | 18 ++++++------------ metrics/writer_test.go | 7 ++++--- 4 files changed, 18 insertions(+), 34 deletions(-) diff --git a/metrics/resetting_timer.go b/metrics/resetting_timer.go index e5327d3bd3..924ba11052 100644 --- a/metrics/resetting_timer.go +++ b/metrics/resetting_timer.go @@ -2,9 +2,10 @@ package metrics import ( "math" - "sort" "sync" "time" + + "golang.org/x/exp/slices" ) // Initial slice capacity for the values stored in a ResettingTimer @@ -186,7 +187,7 @@ func (t *ResettingTimerSnapshot) Mean() float64 { } func (t *ResettingTimerSnapshot) calc(percentiles []float64) { - sort.Sort(Int64Slice(t.values)) + slices.Sort(t.values) count := len(t.values) if count > 0 { @@ -232,10 +233,3 @@ func (t *ResettingTimerSnapshot) calc(percentiles []float64) { t.calculated = true } - -// Int64Slice attaches the methods of sort.Interface to []int64, sorting in increasing order. -type Int64Slice []int64 - -func (s Int64Slice) Len() int { return len(s) } -func (s Int64Slice) Less(i, j int) bool { return s[i] < s[j] } -func (s Int64Slice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } diff --git a/metrics/sample.go b/metrics/sample.go index afcaa21184..252a878f58 100644 --- a/metrics/sample.go +++ b/metrics/sample.go @@ -3,9 +3,10 @@ package metrics import ( "math" "math/rand" - "sort" "sync" "time" + + "golang.org/x/exp/slices" ) const rescaleThreshold = time.Hour @@ -282,17 +283,17 @@ func SampleMin(values []int64) int64 { } // SamplePercentiles returns an arbitrary percentile of the slice of int64. -func SamplePercentile(values int64Slice, p float64) float64 { +func SamplePercentile(values []int64, p float64) float64 { return SamplePercentiles(values, []float64{p})[0] } // SamplePercentiles returns a slice of arbitrary percentiles of the slice of // int64. -func SamplePercentiles(values int64Slice, ps []float64) []float64 { +func SamplePercentiles(values []int64, ps []float64) []float64 { scores := make([]float64, len(ps)) size := len(values) if size > 0 { - sort.Sort(values) + slices.Sort(values) for i, p := range ps { pos := p * float64(size+1) if pos < 1.0 { @@ -633,9 +634,3 @@ func (h *expDecaySampleHeap) down(i, n int) { i = j } } - -type int64Slice []int64 - -func (p int64Slice) Len() int { return len(p) } -func (p int64Slice) Less(i, j int) bool { return p[i] < p[j] } -func (p int64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } diff --git a/metrics/writer.go b/metrics/writer.go index 256fbd14c9..3d60485b6b 100644 --- a/metrics/writer.go +++ b/metrics/writer.go @@ -3,8 +3,9 @@ package metrics import ( "fmt" "io" - "sort" "time" + + "golang.org/x/exp/slices" ) // Write sorts writes each metric in the given registry periodically to the @@ -18,12 +19,12 @@ func Write(r Registry, d time.Duration, w io.Writer) { // WriteOnce sorts and writes metrics in the given registry to the given // io.Writer. func WriteOnce(r Registry, w io.Writer) { - var namedMetrics namedMetricSlice + var namedMetrics []namedMetric r.Each(func(name string, i interface{}) { namedMetrics = append(namedMetrics, namedMetric{name, i}) }) - sort.Sort(namedMetrics) + slices.SortFunc(namedMetrics, namedMetric.less) for _, namedMetric := range namedMetrics { switch metric := namedMetric.m.(type) { case Counter: @@ -91,13 +92,6 @@ type namedMetric struct { m interface{} } -// namedMetricSlice is a slice of namedMetrics that implements sort.Interface. -type namedMetricSlice []namedMetric - -func (nms namedMetricSlice) Len() int { return len(nms) } - -func (nms namedMetricSlice) Swap(i, j int) { nms[i], nms[j] = nms[j], nms[i] } - -func (nms namedMetricSlice) Less(i, j int) bool { - return nms[i].name < nms[j].name +func (m namedMetric) less(other namedMetric) bool { + return m.name < other.name } diff --git a/metrics/writer_test.go b/metrics/writer_test.go index 1aacc28712..a4c92addc9 100644 --- a/metrics/writer_test.go +++ b/metrics/writer_test.go @@ -1,19 +1,20 @@ package metrics import ( - "sort" "testing" + + "golang.org/x/exp/slices" ) func TestMetricsSorting(t *testing.T) { - var namedMetrics = namedMetricSlice{ + var namedMetrics = []namedMetric{ {name: "zzz"}, {name: "bbb"}, {name: "fff"}, {name: "ggg"}, } - sort.Sort(namedMetrics) + slices.SortFunc(namedMetrics, namedMetric.less) for i, name := range []string{"bbb", "fff", "ggg", "zzz"} { if namedMetrics[i].name != name { t.Fail() From a8482127091edbcf3de4b515fd97ce904ac2acb8 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Mon, 19 Jun 2023 03:04:16 -0400 Subject: [PATCH 16/90] consensus/clique: use slices package for sorting (#27488) Also adds Address.Less for sorting use in other packages. --------- Co-authored-by: Felix Lange --- common/types.go | 5 +++++ consensus/clique/snapshot.go | 11 ++--------- consensus/clique/snapshot_test.go | 4 ++-- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/common/types.go b/common/types.go index bdfd964237..31515f52fa 100644 --- a/common/types.go +++ b/common/types.go @@ -226,6 +226,11 @@ func IsHexAddress(s string) bool { return len(s) == 2*AddressLength && isHex(s) } +// Less compares two addresses. +func (a Address) Less(other Address) bool { + return bytes.Compare(a[:], other[:]) < 0 +} + // Bytes gets the string representation of the underlying address. func (a Address) Bytes() []byte { return a[:] } diff --git a/consensus/clique/snapshot.go b/consensus/clique/snapshot.go index e5efa5108f..0f1b9f4b3a 100644 --- a/consensus/clique/snapshot.go +++ b/consensus/clique/snapshot.go @@ -19,7 +19,6 @@ package clique import ( "bytes" "encoding/json" - "sort" "time" "github.com/ethereum/go-ethereum/common" @@ -29,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" + "golang.org/x/exp/slices" ) // Vote represents a single vote that an authorized signer made to modify the @@ -62,13 +62,6 @@ type Snapshot struct { Tally map[common.Address]Tally `json:"tally"` // Current vote tally to avoid recalculating } -// signersAscending implements the sort interface to allow sorting a list of addresses -type signersAscending []common.Address - -func (s signersAscending) Len() int { return len(s) } -func (s signersAscending) Less(i, j int) bool { return bytes.Compare(s[i][:], s[j][:]) < 0 } -func (s signersAscending) Swap(i, j int) { s[i], s[j] = s[j], s[i] } - // newSnapshot creates a new snapshot with the specified startup parameters. This // method does not initialize the set of recent signers, so only ever use if for // the genesis block. @@ -315,7 +308,7 @@ func (s *Snapshot) signers() []common.Address { for sig := range s.Signers { sigs = append(sigs, sig) } - sort.Sort(signersAscending(sigs)) + slices.SortFunc(sigs, common.Address.Less) return sigs } diff --git a/consensus/clique/snapshot_test.go b/consensus/clique/snapshot_test.go index 66e667276f..f30900f274 100644 --- a/consensus/clique/snapshot_test.go +++ b/consensus/clique/snapshot_test.go @@ -21,7 +21,6 @@ import ( "crypto/ecdsa" "fmt" "math/big" - "sort" "testing" "github.com/ethereum/go-ethereum/common" @@ -31,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" + "golang.org/x/exp/slices" ) // testerAccountPool is a pool to maintain currently active tester accounts, @@ -53,7 +53,7 @@ func (ap *testerAccountPool) checkpoint(header *types.Header, signers []string) for i, signer := range signers { auths[i] = ap.address(signer) } - sort.Sort(signersAscending(auths)) + slices.SortFunc(auths, common.Address.Less) for i, auth := range auths { copy(header.Extra[extraVanity+i*common.AddressLength:], auth.Bytes()) } From 87e510d96324386c5275f92030bee59c52f98ca3 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Mon, 19 Jun 2023 05:33:48 -0400 Subject: [PATCH 17/90] internal/ethapi, les: use slices package for sorting (#27492) Co-authored-by: Felix Lange --- internal/ethapi/api_test.go | 13 +++---------- les/servingqueue.go | 36 +++++++++++------------------------- les/utils/limiter.go | 29 ++++++++--------------------- 3 files changed, 22 insertions(+), 56 deletions(-) diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go index 7c503dc391..3e58e54f97 100644 --- a/internal/ethapi/api_test.go +++ b/internal/ethapi/api_test.go @@ -17,7 +17,6 @@ package ethapi import ( - "bytes" "context" "crypto/ecdsa" "encoding/json" @@ -25,7 +24,6 @@ import ( "hash" "math/big" "reflect" - "sort" "testing" "time" @@ -48,6 +46,7 @@ import ( "github.com/ethereum/go-ethereum/rpc" "github.com/stretchr/testify/require" "golang.org/x/crypto/sha3" + "golang.org/x/exp/slices" ) func TestTransaction_RoundTripRpcJSON(t *testing.T) { @@ -649,19 +648,13 @@ type Account struct { addr common.Address } -type Accounts []Account - -func (a Accounts) Len() int { return len(a) } -func (a Accounts) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a Accounts) Less(i, j int) bool { return bytes.Compare(a[i].addr.Bytes(), a[j].addr.Bytes()) < 0 } - -func newAccounts(n int) (accounts Accounts) { +func newAccounts(n int) (accounts []Account) { for i := 0; i < n; i++ { key, _ := crypto.GenerateKey() addr := crypto.PubkeyToAddress(key.PublicKey) accounts = append(accounts, Account{key: key, addr: addr}) } - sort.Sort(accounts) + slices.SortFunc(accounts, func(a, b Account) bool { return a.addr.Less(b.addr) }) return accounts } diff --git a/les/servingqueue.go b/les/servingqueue.go index b4b53d8df5..c25925403a 100644 --- a/les/servingqueue.go +++ b/les/servingqueue.go @@ -17,12 +17,12 @@ package les import ( - "sort" "sync" "sync/atomic" "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/common/prque" + "golang.org/x/exp/slices" ) // servingQueue allows running tasks in a limited number of threads and puts the @@ -180,35 +180,19 @@ func (sq *servingQueue) threadController() { } } -type ( - // peerTasks lists the tasks received from a given peer when selecting peers to freeze - peerTasks struct { - peer *clientPeer - list []*servingTask - sumTime uint64 - priority float64 - } - // peerList is a sortable list of peerTasks - peerList []*peerTasks -) - -func (l peerList) Len() int { - return len(l) -} - -func (l peerList) Less(i, j int) bool { - return l[i].priority < l[j].priority -} - -func (l peerList) Swap(i, j int) { - l[i], l[j] = l[j], l[i] +// peerTasks lists the tasks received from a given peer when selecting peers to freeze +type peerTasks struct { + peer *clientPeer + list []*servingTask + sumTime uint64 + priority float64 } // freezePeers selects the peers with the worst priority queued tasks and freezes // them until burstTime goes under burstDropLimit or all peers are frozen func (sq *servingQueue) freezePeers() { peerMap := make(map[*clientPeer]*peerTasks) - var peerList peerList + var peerList []*peerTasks if sq.best != nil { sq.queue.Push(sq.best, sq.best.priority) } @@ -231,7 +215,9 @@ func (sq *servingQueue) freezePeers() { tasks.list = append(tasks.list, task) tasks.sumTime += task.expTime } - sort.Sort(peerList) + slices.SortFunc(peerList, func(a, b *peerTasks) bool { + return a.priority < b.priority + }) drop := true for _, tasks := range peerList { if drop { diff --git a/les/utils/limiter.go b/les/utils/limiter.go index 84d186efd6..155bd80150 100644 --- a/les/utils/limiter.go +++ b/les/utils/limiter.go @@ -17,10 +17,10 @@ package utils import ( - "sort" "sync" "github.com/ethereum/go-ethereum/p2p/enode" + "golang.org/x/exp/slices" ) const maxSelectionWeight = 1000000000 // maximum selection weight of each individual node/address group @@ -340,24 +340,9 @@ func (l *Limiter) Stop() { l.cond.Signal() } -type ( - dropList []dropListItem - dropListItem struct { - nq *nodeQueue - priority float64 - } -) - -func (l dropList) Len() int { - return len(l) -} - -func (l dropList) Less(i, j int) bool { - return l[i].priority < l[j].priority -} - -func (l dropList) Swap(i, j int) { - l[i], l[j] = l[j], l[i] +type dropListItem struct { + nq *nodeQueue + priority float64 } // dropRequests selects the nodes with the highest queued request cost to selection @@ -366,7 +351,7 @@ func (l dropList) Swap(i, j int) { func (l *Limiter) dropRequests() { var ( sumValue float64 - list dropList + list []dropListItem ) for _, nq := range l.nodes { sumValue += nq.value @@ -384,7 +369,9 @@ func (l *Limiter) dropRequests() { priority: w / float64(nq.sumCost), }) } - sort.Sort(list) + slices.SortFunc(list, func(a, b dropListItem) bool { + return a.priority < b.priority + }) for _, item := range list { for _, request := range item.nq.queue { close(request.process) From 50ecb16de04456542e4e5b1d8ac517f3ef0a7db9 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Mon, 19 Jun 2023 05:41:31 -0400 Subject: [PATCH 18/90] tests, trie: use slices package for sorting (#27496) Co-authored-by: Felix Lange --- tests/fuzzers/rangeproof/rangeproof-fuzzer.go | 14 ++-- tests/fuzzers/stacktrie/trie_fuzzer.go | 21 ++---- trie/iterator_test.go | 4 ++ trie/proof_test.go | 72 +++++++++---------- trie/trienode/node.go | 8 ++- 5 files changed, 52 insertions(+), 67 deletions(-) diff --git a/tests/fuzzers/rangeproof/rangeproof-fuzzer.go b/tests/fuzzers/rangeproof/rangeproof-fuzzer.go index 2881c7a7c2..19bd39df98 100644 --- a/tests/fuzzers/rangeproof/rangeproof-fuzzer.go +++ b/tests/fuzzers/rangeproof/rangeproof-fuzzer.go @@ -21,12 +21,12 @@ import ( "encoding/binary" "fmt" "io" - "sort" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb/memorydb" "github.com/ethereum/go-ethereum/trie" + "golang.org/x/exp/slices" ) type kv struct { @@ -34,12 +34,6 @@ type kv struct { t bool } -type entrySlice []*kv - -func (p entrySlice) Len() int { return len(p) } -func (p entrySlice) Less(i, j int) bool { return bytes.Compare(p[i].k, p[j].k) < 0 } -func (p entrySlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } - type fuzzer struct { input io.Reader exhausted bool @@ -97,14 +91,16 @@ func (f *fuzzer) fuzz() int { if f.exhausted { return 0 // input too short } - var entries entrySlice + var entries []*kv for _, kv := range vals { entries = append(entries, kv) } if len(entries) <= 1 { return 0 } - sort.Sort(entries) + slices.SortFunc(entries, func(a, b *kv) bool { + return bytes.Compare(a.k, b.k) < 0 + }) var ok = 0 for { diff --git a/tests/fuzzers/stacktrie/trie_fuzzer.go b/tests/fuzzers/stacktrie/trie_fuzzer.go index 0099e9e169..77e602f807 100644 --- a/tests/fuzzers/stacktrie/trie_fuzzer.go +++ b/tests/fuzzers/stacktrie/trie_fuzzer.go @@ -23,7 +23,6 @@ import ( "fmt" "hash" "io" - "sort" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" @@ -33,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/trienode" "golang.org/x/crypto/sha3" + "golang.org/x/exp/slices" ) type fuzzer struct { @@ -104,19 +104,6 @@ func (b *spongeBatch) Replay(w ethdb.KeyValueWriter) error { return nil } type kv struct { k, v []byte } -type kvs []kv - -func (k kvs) Len() int { - return len(k) -} - -func (k kvs) Less(i, j int) bool { - return bytes.Compare(k[i].k, k[j].k) < 0 -} - -func (k kvs) Swap(i, j int) { - k[j], k[i] = k[i], k[j] -} // Fuzz is the fuzzing entry-point. // The function must return @@ -156,7 +143,7 @@ func (f *fuzzer) fuzz() int { trieB = trie.NewStackTrie(func(owner common.Hash, path []byte, hash common.Hash, blob []byte) { rawdb.WriteTrieNode(spongeB, owner, path, hash, blob, dbB.Scheme()) }) - vals kvs + vals []kv useful bool maxElements = 10000 // operate on unique keys only @@ -192,7 +179,9 @@ func (f *fuzzer) fuzz() int { dbA.Commit(rootA, false) // Stacktrie requires sorted insertion - sort.Sort(vals) + slices.SortFunc(vals, func(a, b kv) bool { + return bytes.Compare(a.k, b.k) < 0 + }) for _, kv := range vals { if f.debugging { fmt.Printf("{\"%#x\" , \"%#x\"} // stacktrie.Update\n", kv.k, kv.v) diff --git a/trie/iterator_test.go b/trie/iterator_test.go index 8003858e5b..ccf5ea0258 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -84,6 +84,10 @@ type kv struct { t bool } +func (k *kv) less(other *kv) bool { + return bytes.Compare(k.k, other.k) < 0 +} + func TestIteratorLargeData(t *testing.T) { trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) vals := make(map[string]*kv) diff --git a/trie/proof_test.go b/trie/proof_test.go index 69e3f8e9ce..015df39751 100644 --- a/trie/proof_test.go +++ b/trie/proof_test.go @@ -22,13 +22,13 @@ import ( "encoding/binary" "fmt" mrand "math/rand" - "sort" "testing" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb/memorydb" + "golang.org/x/exp/slices" ) // Prng is a pseudo random number generator seeded by strong randomness. @@ -165,21 +165,15 @@ func TestMissingKeyProof(t *testing.T) { } } -type entrySlice []*kv - -func (p entrySlice) Len() int { return len(p) } -func (p entrySlice) Less(i, j int) bool { return bytes.Compare(p[i].k, p[j].k) < 0 } -func (p entrySlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } - // TestRangeProof tests normal range proof with both edge proofs // as the existent proof. The test cases are generated randomly. func TestRangeProof(t *testing.T) { trie, vals := randomTrie(4096) - var entries entrySlice + var entries []*kv for _, kv := range vals { entries = append(entries, kv) } - sort.Sort(entries) + slices.SortFunc(entries, (*kv).less) for i := 0; i < 500; i++ { start := mrand.Intn(len(entries)) end := mrand.Intn(len(entries)-start) + start + 1 @@ -208,11 +202,11 @@ func TestRangeProof(t *testing.T) { // The test cases are generated randomly. func TestRangeProofWithNonExistentProof(t *testing.T) { trie, vals := randomTrie(4096) - var entries entrySlice + var entries []*kv for _, kv := range vals { entries = append(entries, kv) } - sort.Sort(entries) + slices.SortFunc(entries, (*kv).less) for i := 0; i < 500; i++ { start := mrand.Intn(len(entries)) end := mrand.Intn(len(entries)-start) + start + 1 @@ -280,11 +274,11 @@ func TestRangeProofWithNonExistentProof(t *testing.T) { // - There exists a gap between the last element and the right edge proof func TestRangeProofWithInvalidNonExistentProof(t *testing.T) { trie, vals := randomTrie(4096) - var entries entrySlice + var entries []*kv for _, kv := range vals { entries = append(entries, kv) } - sort.Sort(entries) + slices.SortFunc(entries, (*kv).less) // Case 1 start, end := 100, 200 @@ -337,11 +331,11 @@ func TestRangeProofWithInvalidNonExistentProof(t *testing.T) { // non-existent one. func TestOneElementRangeProof(t *testing.T) { trie, vals := randomTrie(4096) - var entries entrySlice + var entries []*kv for _, kv := range vals { entries = append(entries, kv) } - sort.Sort(entries) + slices.SortFunc(entries, (*kv).less) // One element with existent edge proof, both edge proofs // point to the SAME key. @@ -424,11 +418,11 @@ func TestOneElementRangeProof(t *testing.T) { // The edge proofs can be nil. func TestAllElementsProof(t *testing.T) { trie, vals := randomTrie(4096) - var entries entrySlice + var entries []*kv for _, kv := range vals { entries = append(entries, kv) } - sort.Sort(entries) + slices.SortFunc(entries, (*kv).less) var k [][]byte var v [][]byte @@ -474,13 +468,13 @@ func TestAllElementsProof(t *testing.T) { func TestSingleSideRangeProof(t *testing.T) { for i := 0; i < 64; i++ { trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) - var entries entrySlice + var entries []*kv for i := 0; i < 4096; i++ { value := &kv{randBytes(32), randBytes(20), false} trie.MustUpdate(value.k, value.v) entries = append(entries, value) } - sort.Sort(entries) + slices.SortFunc(entries, (*kv).less) var cases = []int{0, 1, 50, 100, 1000, 2000, len(entries) - 1} for _, pos := range cases { @@ -509,13 +503,13 @@ func TestSingleSideRangeProof(t *testing.T) { func TestReverseSingleSideRangeProof(t *testing.T) { for i := 0; i < 64; i++ { trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) - var entries entrySlice + var entries []*kv for i := 0; i < 4096; i++ { value := &kv{randBytes(32), randBytes(20), false} trie.MustUpdate(value.k, value.v) entries = append(entries, value) } - sort.Sort(entries) + slices.SortFunc(entries, (*kv).less) var cases = []int{0, 1, 50, 100, 1000, 2000, len(entries) - 1} for _, pos := range cases { @@ -545,11 +539,11 @@ func TestReverseSingleSideRangeProof(t *testing.T) { // The prover is expected to detect the error. func TestBadRangeProof(t *testing.T) { trie, vals := randomTrie(4096) - var entries entrySlice + var entries []*kv for _, kv := range vals { entries = append(entries, kv) } - sort.Sort(entries) + slices.SortFunc(entries, (*kv).less) for i := 0; i < 500; i++ { start := mrand.Intn(len(entries)) @@ -648,11 +642,11 @@ func TestGappedRangeProof(t *testing.T) { // TestSameSideProofs tests the element is not in the range covered by proofs func TestSameSideProofs(t *testing.T) { trie, vals := randomTrie(4096) - var entries entrySlice + var entries []*kv for _, kv := range vals { entries = append(entries, kv) } - sort.Sort(entries) + slices.SortFunc(entries, (*kv).less) pos := 1000 first := decreaseKey(common.CopyBytes(entries[pos].k)) @@ -690,13 +684,13 @@ func TestSameSideProofs(t *testing.T) { func TestHasRightElement(t *testing.T) { trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) - var entries entrySlice + var entries []*kv for i := 0; i < 4096; i++ { value := &kv{randBytes(32), randBytes(20), false} trie.MustUpdate(value.k, value.v) entries = append(entries, value) } - sort.Sort(entries) + slices.SortFunc(entries, (*kv).less) var cases = []struct { start int @@ -764,11 +758,11 @@ func TestHasRightElement(t *testing.T) { // The first edge proof must be a non-existent proof. func TestEmptyRangeProof(t *testing.T) { trie, vals := randomTrie(4096) - var entries entrySlice + var entries []*kv for _, kv := range vals { entries = append(entries, kv) } - sort.Sort(entries) + slices.SortFunc(entries, (*kv).less) var cases = []struct { pos int @@ -799,11 +793,11 @@ func TestEmptyRangeProof(t *testing.T) { func TestBloatedProof(t *testing.T) { // Use a small trie trie, kvs := nonRandomTrie(100) - var entries entrySlice + var entries []*kv for _, kv := range kvs { entries = append(entries, kv) } - sort.Sort(entries) + slices.SortFunc(entries, (*kv).less) var keys [][]byte var vals [][]byte @@ -833,11 +827,11 @@ func TestBloatedProof(t *testing.T) { // noop technically, but practically should be rejected. func TestEmptyValueRangeProof(t *testing.T) { trie, values := randomTrie(512) - var entries entrySlice + var entries []*kv for _, kv := range values { entries = append(entries, kv) } - sort.Sort(entries) + slices.SortFunc(entries, (*kv).less) // Create a new entry with a slightly modified key mid := len(entries) / 2 @@ -877,11 +871,11 @@ func TestEmptyValueRangeProof(t *testing.T) { // practically should be rejected. func TestAllElementsEmptyValueRangeProof(t *testing.T) { trie, values := randomTrie(512) - var entries entrySlice + var entries []*kv for _, kv := range values { entries = append(entries, kv) } - sort.Sort(entries) + slices.SortFunc(entries, (*kv).less) // Create a new entry with a slightly modified key mid := len(entries) / 2 @@ -983,11 +977,11 @@ func BenchmarkVerifyRangeProof5000(b *testing.B) { benchmarkVerifyRangeProof(b, func benchmarkVerifyRangeProof(b *testing.B, size int) { trie, vals := randomTrie(8192) - var entries entrySlice + var entries []*kv for _, kv := range vals { entries = append(entries, kv) } - sort.Sort(entries) + slices.SortFunc(entries, (*kv).less) start := 2 end := start + size @@ -1020,11 +1014,11 @@ func BenchmarkVerifyRangeNoProof1000(b *testing.B) { benchmarkVerifyRangeNoProof func benchmarkVerifyRangeNoProof(b *testing.B, size int) { trie, vals := randomTrie(size) - var entries entrySlice + var entries []*kv for _, kv := range vals { entries = append(entries, kv) } - sort.Sort(entries) + slices.SortFunc(entries, (*kv).less) var keys [][]byte var values [][]byte diff --git a/trie/trienode/node.go b/trie/trienode/node.go index 8152eab6c0..b233f517f0 100644 --- a/trie/trienode/node.go +++ b/trie/trienode/node.go @@ -18,10 +18,10 @@ package trienode import ( "fmt" - "sort" "strings" "github.com/ethereum/go-ethereum/common" + "golang.org/x/exp/slices" ) // Node is a wrapper which contains the encoded blob of the trie node and its @@ -100,12 +100,14 @@ func NewNodeSet(owner common.Hash) *NodeSet { // ForEachWithOrder iterates the nodes with the order from bottom to top, // right to left, nodes with the longest path will be iterated first. func (set *NodeSet) ForEachWithOrder(callback func(path string, n *Node)) { - var paths sort.StringSlice + var paths []string for path := range set.Nodes { paths = append(paths, path) } // Bottom-up, longest path first - sort.Sort(sort.Reverse(paths)) + slices.SortFunc(paths, func(a, b string) bool { + return a > b // Sort in reverse order + }) for _, path := range paths { callback(path, set.Nodes[path].Unwrap()) } From 091c25d983291eda64a7ea9e42cfe8c3274b655c Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Mon, 19 Jun 2023 14:08:13 +0200 Subject: [PATCH 19/90] go.mod: update kzg libraries to use big-endian (#27510) * go.mod: update kzg libraries to use big-endian * go.sum: ran go mod tidy * core/testdata/precompiles: fix blob verification test * core/testdata/precompiles: fix blob verification test --- .../vm/testdata/precompiles/pointEvaluation.json | 16 ++++++++-------- go.mod | 4 ++-- go.sum | 10 ++++------ 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/core/vm/testdata/precompiles/pointEvaluation.json b/core/vm/testdata/precompiles/pointEvaluation.json index 2fee5baf24..93fc66d836 100644 --- a/core/vm/testdata/precompiles/pointEvaluation.json +++ b/core/vm/testdata/precompiles/pointEvaluation.json @@ -1,9 +1,9 @@ [ - { - "Input": "013c03613f6fc558fb7e61e75602241ed9a2f04e36d8670aadd286e71b5ca9cc420000000000000000000000000000000000000000000000000000000000000031e5a2356cbc2ef6a733eae8d54bf48719ae3d990017ca787c419c7d369f8e3c83fac17c3f237fc51f90e2c660eb202a438bc2025baded5cd193c1a018c5885bc9281ba704d5566082e851235c7be763b2a99adff965e0a121ee972ebc472d02944a74f5c6243e14052e105124b70bf65faf85ad3a494325e269fad097842cba", - "Expected": "000000000000000000000000000000000000000000000000000000000000100073eda753299d7d483339d80809a1d80553bda402fffe5bfeffffffff00000001", - "Name": "pointEvaluation1", - "Gas": 50000, - "NoBenchmark": false - } - ] \ No newline at end of file + { + "Input": "01d18459b334ffe8e2226eef1db874fda6db2bdd9357268b39220af2d59464fb564c0a11a0f704f4fc3e8acfe0f8245f0ad1347b378fbf96e206da11a5d3630624d25032e67a7e6a4910df5834b8fe70e6bcfeeac0352434196bdf4b2485d5a1978a0d595c823c05947b1156175e72634a377808384256e9921ebf72181890be2d6b58d4a73a880541d1656875654806942307f266e636553e94006d11423f2688945ff3bdf515859eba1005c1a7708d620a94d91a1c0c285f9584e75ec2f82a", + "Expected": "000000000000000000000000000000000000000000000000000000000000100073eda753299d7d483339d80809a1d80553bda402fffe5bfeffffffff00000001", + "Name": "pointEvaluation1", + "Gas": 50000, + "NoBenchmark": false + } +] diff --git a/go.mod b/go.mod index 1cd835798f..253e47d1e4 100644 --- a/go.mod +++ b/go.mod @@ -14,12 +14,12 @@ require ( github.com/cloudflare/cloudflare-go v0.14.0 github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811 github.com/consensys/gnark-crypto v0.10.0 - github.com/crate-crypto/go-kzg-4844 v0.2.0 + github.com/crate-crypto/go-kzg-4844 v0.3.0 github.com/davecgh/go-spew v1.1.1 github.com/deckarep/golang-set/v2 v2.1.0 github.com/docker/docker v1.6.2 github.com/dop251/goja v0.0.0-20230605162241-28ee0ee714f3 - github.com/ethereum/c-kzg-4844 v0.2.0 + github.com/ethereum/c-kzg-4844 v0.3.0 github.com/fatih/color v1.7.0 github.com/fjl/gencodec v0.0.0-20230517082657-f9840df7b83e github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 diff --git a/go.sum b/go.sum index eb819cd68e..c257b615aa 100644 --- a/go.sum +++ b/go.sum @@ -86,8 +86,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHH github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crate-crypto/go-ipa v0.0.0-20220523130400-f11357ae11c7 h1:6IrxszG5G+O7zhtkWxq6+unVvnrm1fqV2Pe+T95DUzw= github.com/crate-crypto/go-ipa v0.0.0-20220523130400-f11357ae11c7/go.mod h1:gFnFS95y8HstDP6P9pPwzrxOOC5TRDkwbM+ao15ChAI= -github.com/crate-crypto/go-kzg-4844 v0.2.0 h1:UVuHOE+5tIWrim4zf/Xaa43+MIsDCPyW76QhUpiMGj4= -github.com/crate-crypto/go-kzg-4844 v0.2.0/go.mod h1:SBP7ikXEgDnUPONgm33HtuDZEDtWa3L4QtN1ocJSEQ4= +github.com/crate-crypto/go-kzg-4844 v0.3.0 h1:UBlWE0CgyFqqzTI+IFyCzA7A3Zw4iip6uzRv5NIXG0A= +github.com/crate-crypto/go-kzg-4844 v0.3.0/go.mod h1:SBP7ikXEgDnUPONgm33HtuDZEDtWa3L4QtN1ocJSEQ4= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -114,8 +114,6 @@ github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5O github.com/docker/docker v1.6.2 h1:HlFGsy+9/xrgMmhmN+NGhCc5SHGJ7I+kHosRR1xc/aI= github.com/docker/docker v1.6.2/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/dop251/goja v0.0.0-20211022113120-dc8c55024d06/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= -github.com/dop251/goja v0.0.0-20230122112309-96b1610dd4f7 h1:kgvzE5wLsLa7XKfV85VZl40QXaMCaeFtHpPwJ8fhotY= -github.com/dop251/goja v0.0.0-20230122112309-96b1610dd4f7/go.mod h1:yRkwfj0CBpOGre+TwBsqPV0IH0Pk73e4PXJOeNDboGs= github.com/dop251/goja v0.0.0-20230605162241-28ee0ee714f3 h1:+3HCtB74++ClLy8GgjUQYeC8R4ILzVcIe8+5edAJJnE= github.com/dop251/goja v0.0.0-20230605162241-28ee0ee714f3/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4= github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= @@ -127,8 +125,8 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= -github.com/ethereum/c-kzg-4844 v0.2.0 h1:+cUvymlnoDDQgMInp25Bo3OmLajmmY8mLJ/tLjqd77Q= -github.com/ethereum/c-kzg-4844 v0.2.0/go.mod h1:WI2Nd82DMZAAZI1wV2neKGost9EKjvbpQR9OqE5Qqa8= +github.com/ethereum/c-kzg-4844 v0.3.0 h1:3Y3hD6l5i0dEYsBL50C+Om644kve3pNqoAcvE26o9zI= +github.com/ethereum/c-kzg-4844 v0.3.0/go.mod h1:WI2Nd82DMZAAZI1wV2neKGost9EKjvbpQR9OqE5Qqa8= github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= From ceca4578ca008b288bd4b3b02de5a252b3900f12 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Mon, 19 Jun 2023 22:28:40 +0800 Subject: [PATCH 20/90] trie: remove parameter 'fromLevel' in Prove (#27512) This removes the feature where top nodes of the proof can be elided. It was intended to be used by the LES server, to save bandwidth when the client had already fetched parts of the state and only needed some extra nodes to complete the proof. Alas, it never got implemented in the client. --- core/state/database.go | 2 +- core/state/snapshot/generate.go | 4 +- core/state/statedb.go | 4 +- eth/protocols/snap/handler.go | 8 +- eth/protocols/snap/sync_test.go | 16 +-- internal/ethapi/api.go | 2 +- les/handler_test.go | 8 +- les/server_requests.go | 4 +- light/odr_test.go | 2 +- light/trie.go | 2 +- tests/fuzzers/rangeproof/rangeproof-fuzzer.go | 4 +- tests/fuzzers/trie/trie-fuzzer.go | 2 +- trie/proof.go | 10 +- trie/proof_test.go | 106 +++++++++--------- trie/tracer_test.go | 2 +- trie/trie_test.go | 2 +- 16 files changed, 87 insertions(+), 91 deletions(-) diff --git a/core/state/database.go b/core/state/database.go index ace462165c..304c04f963 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -123,7 +123,7 @@ type Trie interface { // If the trie does not contain a value for key, the returned proof contains all // nodes of the longest existing prefix of the key (at least the root), ending // with the node that proves the absence of the key. - Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) error + Prove(key []byte, proofDb ethdb.KeyValueWriter) error } // NewDatabase creates a backing store for state. The returned database is safe for diff --git a/core/state/snapshot/generate.go b/core/state/snapshot/generate.go index f42f1e7808..78aa23a5de 100644 --- a/core/state/snapshot/generate.go +++ b/core/state/snapshot/generate.go @@ -256,7 +256,7 @@ func (dl *diskLayer) proveRange(ctx *generatorContext, trieId *trie.ID, prefix [ if origin == nil { origin = common.Hash{}.Bytes() } - if err := tr.Prove(origin, 0, proof); err != nil { + if err := tr.Prove(origin, proof); err != nil { log.Debug("Failed to prove range", "kind", kind, "origin", origin, "err", err) return &proofResult{ keys: keys, @@ -267,7 +267,7 @@ func (dl *diskLayer) proveRange(ctx *generatorContext, trieId *trie.ID, prefix [ }, nil } if last != nil { - if err := tr.Prove(last, 0, proof); err != nil { + if err := tr.Prove(last, proof); err != nil { log.Debug("Failed to prove range", "kind", kind, "last", last, "err", err) return &proofResult{ keys: keys, diff --git a/core/state/statedb.go b/core/state/statedb.go index b1b52cd7f1..90b32e3fc0 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -332,7 +332,7 @@ func (s *StateDB) GetProof(addr common.Address) ([][]byte, error) { // GetProofByHash returns the Merkle proof for a given account. func (s *StateDB) GetProofByHash(addrHash common.Hash) ([][]byte, error) { var proof proofList - err := s.trie.Prove(addrHash[:], 0, &proof) + err := s.trie.Prove(addrHash[:], &proof) return proof, err } @@ -346,7 +346,7 @@ func (s *StateDB) GetStorageProof(a common.Address, key common.Hash) ([][]byte, return nil, errors.New("storage trie for requested address does not exist") } var proof proofList - err = trie.Prove(crypto.Keccak256(key.Bytes()), 0, &proof) + err = trie.Prove(crypto.Keccak256(key.Bytes()), &proof) if err != nil { return nil, err } diff --git a/eth/protocols/snap/handler.go b/eth/protocols/snap/handler.go index 55781ac54b..1c6d80d354 100644 --- a/eth/protocols/snap/handler.go +++ b/eth/protocols/snap/handler.go @@ -322,12 +322,12 @@ func ServiceGetAccountRangeQuery(chain *core.BlockChain, req *GetAccountRangePac // Generate the Merkle proofs for the first and last account proof := light.NewNodeSet() - if err := tr.Prove(req.Origin[:], 0, proof); err != nil { + if err := tr.Prove(req.Origin[:], proof); err != nil { log.Warn("Failed to prove account range", "origin", req.Origin, "err", err) return nil, nil } if last != (common.Hash{}) { - if err := tr.Prove(last[:], 0, proof); err != nil { + if err := tr.Prove(last[:], proof); err != nil { log.Warn("Failed to prove account range", "last", last, "err", err) return nil, nil } @@ -428,12 +428,12 @@ func ServiceGetStorageRangesQuery(chain *core.BlockChain, req *GetStorageRangesP return nil, nil } proof := light.NewNodeSet() - if err := stTrie.Prove(origin[:], 0, proof); err != nil { + if err := stTrie.Prove(origin[:], proof); err != nil { log.Warn("Failed to prove storage range", "origin", req.Origin, "err", err) return nil, nil } if last != (common.Hash{}) { - if err := stTrie.Prove(last[:], 0, proof); err != nil { + if err := stTrie.Prove(last[:], proof); err != nil { log.Warn("Failed to prove storage range", "last", last, "err", err) return nil, nil } diff --git a/eth/protocols/snap/sync_test.go b/eth/protocols/snap/sync_test.go index 58cde8d1eb..23af204162 100644 --- a/eth/protocols/snap/sync_test.go +++ b/eth/protocols/snap/sync_test.go @@ -273,12 +273,12 @@ func createAccountRequestResponse(t *testPeer, root common.Hash, origin common.H // Actually, we need to supply proofs either way! This seems to be an implementation // quirk in go-ethereum proof := light.NewNodeSet() - if err := t.accountTrie.Prove(origin[:], 0, proof); err != nil { + if err := t.accountTrie.Prove(origin[:], proof); err != nil { t.logger.Error("Could not prove inexistence of origin", "origin", origin, "error", err) } if len(keys) > 0 { lastK := (keys[len(keys)-1])[:] - if err := t.accountTrie.Prove(lastK, 0, proof); err != nil { + if err := t.accountTrie.Prove(lastK, proof); err != nil { t.logger.Error("Could not prove last item", "error", err) } } @@ -358,12 +358,12 @@ func createStorageRequestResponse(t *testPeer, root common.Hash, accounts []comm // Here's a potential gotcha: when constructing the proof, we cannot // use the 'origin' slice directly, but must use the full 32-byte // hash form. - if err := stTrie.Prove(originHash[:], 0, proof); err != nil { + if err := stTrie.Prove(originHash[:], proof); err != nil { t.logger.Error("Could not prove inexistence of origin", "origin", originHash, "error", err) } if len(keys) > 0 { lastK := (keys[len(keys)-1])[:] - if err := stTrie.Prove(lastK, 0, proof); err != nil { + if err := stTrie.Prove(lastK, proof); err != nil { t.logger.Error("Could not prove last item", "error", err) } } @@ -416,13 +416,13 @@ func createStorageRequestResponseAlwaysProve(t *testPeer, root common.Hash, acco // Here's a potential gotcha: when constructing the proof, we cannot // use the 'origin' slice directly, but must use the full 32-byte // hash form. - if err := stTrie.Prove(origin[:], 0, proof); err != nil { + if err := stTrie.Prove(origin[:], proof); err != nil { t.logger.Error("Could not prove inexistence of origin", "origin", origin, "error", err) } if len(keys) > 0 { lastK := (keys[len(keys)-1])[:] - if err := stTrie.Prove(lastK, 0, proof); err != nil { + if err := stTrie.Prove(lastK, proof); err != nil { t.logger.Error("Could not prove last item", "error", err) } } @@ -594,12 +594,12 @@ func TestSyncBloatedProof(t *testing.T) { } // The proofs proof := light.NewNodeSet() - if err := t.accountTrie.Prove(origin[:], 0, proof); err != nil { + if err := t.accountTrie.Prove(origin[:], proof); err != nil { t.logger.Error("Could not prove origin", "origin", origin, "error", err) } // The bloat: add proof of every single element for _, entry := range t.accountValues { - if err := t.accountTrie.Prove(entry.k, 0, proof); err != nil { + if err := t.accountTrie.Prove(entry.k, proof); err != nil { t.logger.Error("Could not prove item", "error", err) } } diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 80756b64b8..823ac6c975 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -705,7 +705,7 @@ func (s *BlockChainAPI) GetProof(ctx context.Context, address common.Address, st continue } var proof proofList - if err := storageTrie.Prove(crypto.Keccak256(key.Bytes()), 0, &proof); err != nil { + if err := storageTrie.Prove(crypto.Keccak256(key.Bytes()), &proof); err != nil { return nil, err } storageProof[i] = StorageResult{storageKeys[i], diff --git a/les/handler_test.go b/les/handler_test.go index a3854dff96..d0878dcd99 100644 --- a/les/handler_test.go +++ b/les/handler_test.go @@ -414,7 +414,7 @@ func testGetProofs(t *testing.T, protocol int) { Key: crypto.Keccak256(acc[:]), } proofreqs = append(proofreqs, req) - trie.Prove(crypto.Keccak256(acc[:]), 0, proofsV2) + trie.Prove(crypto.Keccak256(acc[:]), proofsV2) } } // Send the proof request and verify the response @@ -458,7 +458,7 @@ func testGetStaleProof(t *testing.T, protocol int) { if wantOK { proofsV2 := light.NewNodeSet() t, _ := trie.New(trie.StateTrieID(header.Root), trie.NewDatabase(server.db)) - t.Prove(account, 0, proofsV2) + t.Prove(account, proofsV2) expected = proofsV2.NodeList() } if err := expectResponse(rawPeer.app, ProofsV2Msg, 42, testBufLimit, expected); err != nil { @@ -514,7 +514,7 @@ func testGetCHTProofs(t *testing.T, protocol int) { } root := light.GetChtRoot(server.db, 0, bc.GetHeaderByNumber(config.ChtSize-1).Hash()) trie, _ := trie.New(trie.TrieID(root), trie.NewDatabase(rawdb.NewTable(server.db, string(rawdb.ChtTablePrefix)))) - trie.Prove(key, 0, &proofsV2.Proofs) + trie.Prove(key, &proofsV2.Proofs) // Assemble the requests for the different protocols requestsV2 := []HelperTrieReq{{ Type: htCanonical, @@ -579,7 +579,7 @@ func testGetBloombitsProofs(t *testing.T, protocol int) { root := light.GetBloomTrieRoot(server.db, 0, bc.GetHeaderByNumber(config.BloomTrieSize-1).Hash()) trie, _ := trie.New(trie.TrieID(root), trie.NewDatabase(rawdb.NewTable(server.db, string(rawdb.BloomTrieTablePrefix)))) - trie.Prove(key, 0, &proofs.Proofs) + trie.Prove(key, &proofs.Proofs) // Send the proof request and verify the response sendRequest(rawPeer.app, GetHelperTrieProofsMsg, 42, requests) diff --git a/les/server_requests.go b/les/server_requests.go index 29f8f63ce8..f6208db443 100644 --- a/les/server_requests.go +++ b/les/server_requests.go @@ -436,7 +436,7 @@ func handleGetProofs(msg Decoder) (serveRequestFn, uint64, uint64, error) { } } // Prove the user's request from the account or storage trie - if err := trie.Prove(request.Key, request.FromLevel, nodes); err != nil { + if err := trie.Prove(request.Key, nodes); err != nil { p.Log().Warn("Failed to prove state request", "block", header.Number, "hash", header.Hash(), "err", err) continue } @@ -480,7 +480,7 @@ func handleGetHelperTrieProofs(msg Decoder) (serveRequestFn, uint64, uint64, err // the headers with no valid proof. Keep the compatibility for // legacy les protocol and drop this hack when the les2/3 are // not supported. - err := auxTrie.Prove(request.Key, request.FromLevel, nodes) + err := auxTrie.Prove(request.Key, nodes) if p.version >= lpv4 && err != nil { return nil } diff --git a/light/odr_test.go b/light/odr_test.go index 0df1fbf2f5..90fdea81dc 100644 --- a/light/odr_test.go +++ b/light/odr_test.go @@ -95,7 +95,7 @@ func (odr *testOdr) Retrieve(ctx context.Context, req OdrRequest) error { panic(err) } nodes := NewNodeSet() - t.Prove(req.Key, 0, nodes) + t.Prove(req.Key, nodes) req.Proof = nodes case *CodeRequest: req.Data = rawdb.ReadCode(odr.sdb, req.Hash) diff --git a/light/trie.go b/light/trie.go index 105a11fb31..888598ca92 100644 --- a/light/trie.go +++ b/light/trie.go @@ -192,7 +192,7 @@ func (t *odrTrie) GetKey(sha []byte) []byte { return nil } -func (t *odrTrie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) error { +func (t *odrTrie) Prove(key []byte, proofDb ethdb.KeyValueWriter) error { return errors.New("not implemented, needs client/server interface split") } diff --git a/tests/fuzzers/rangeproof/rangeproof-fuzzer.go b/tests/fuzzers/rangeproof/rangeproof-fuzzer.go index 19bd39df98..d5aee206da 100644 --- a/tests/fuzzers/rangeproof/rangeproof-fuzzer.go +++ b/tests/fuzzers/rangeproof/rangeproof-fuzzer.go @@ -113,10 +113,10 @@ func (f *fuzzer) fuzz() int { break } proof := memorydb.New() - if err := tr.Prove(entries[start].k, 0, proof); err != nil { + if err := tr.Prove(entries[start].k, proof); err != nil { panic(fmt.Sprintf("Failed to prove the first node %v", err)) } - if err := tr.Prove(entries[end-1].k, 0, proof); err != nil { + if err := tr.Prove(entries[end-1].k, proof); err != nil { panic(fmt.Sprintf("Failed to prove the last node %v", err)) } var keys [][]byte diff --git a/tests/fuzzers/trie/trie-fuzzer.go b/tests/fuzzers/trie/trie-fuzzer.go index 7be41b57cb..0926d04764 100644 --- a/tests/fuzzers/trie/trie-fuzzer.go +++ b/tests/fuzzers/trie/trie-fuzzer.go @@ -187,7 +187,7 @@ func runRandTest(rt randTest) error { return errors.New("hash mismatch in opItercheckhash") } case opProve: - rt[i].err = tr.Prove(step.key, 0, proofDb{}) + rt[i].err = tr.Prove(step.key, proofDb{}) } // Abort the test on error. if rt[i].err != nil { diff --git a/trie/proof.go b/trie/proof.go index a25263d82e..818e338892 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -33,7 +33,7 @@ import ( // If the trie does not contain a value for key, the returned proof contains all // nodes of the longest existing prefix of the key (at least the root node), ending // with the node that proves the absence of the key. -func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) error { +func (t *Trie) Prove(key []byte, proofDb ethdb.KeyValueWriter) error { // Collect all nodes on the path to key. var ( prefix []byte @@ -81,10 +81,6 @@ func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) e defer returnHasherToPool(hasher) for i, n := range nodes { - if fromLevel > 0 { - fromLevel-- - continue - } var hn node n, hn = hasher.proofHash(n) if hash, ok := hn.(hashNode); ok || i == 0 { @@ -107,8 +103,8 @@ func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) e // If the trie does not contain a value for key, the returned proof contains all // nodes of the longest existing prefix of the key (at least the root node), ending // with the node that proves the absence of the key. -func (t *StateTrie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) error { - return t.trie.Prove(key, fromLevel, proofDb) +func (t *StateTrie) Prove(key []byte, proofDb ethdb.KeyValueWriter) error { + return t.trie.Prove(key, proofDb) } // VerifyProof checks merkle proofs. The given proof must contain the value for diff --git a/trie/proof_test.go b/trie/proof_test.go index 015df39751..6af2c3e472 100644 --- a/trie/proof_test.go +++ b/trie/proof_test.go @@ -57,7 +57,7 @@ func makeProvers(trie *Trie) []func(key []byte) *memorydb.Database { // Create a direct trie based Merkle prover provers = append(provers, func(key []byte) *memorydb.Database { proof := memorydb.New() - trie.Prove(key, 0, proof) + trie.Prove(key, proof) return proof }) // Create a leaf iterator based Merkle prover @@ -150,7 +150,7 @@ func TestMissingKeyProof(t *testing.T) { for i, key := range []string{"a", "j", "l", "z"} { proof := memorydb.New() - trie.Prove([]byte(key), 0, proof) + trie.Prove([]byte(key), proof) if proof.Len() != 1 { t.Errorf("test %d: proof should have one element", i) @@ -179,10 +179,10 @@ func TestRangeProof(t *testing.T) { end := mrand.Intn(len(entries)-start) + start + 1 proof := memorydb.New() - if err := trie.Prove(entries[start].k, 0, proof); err != nil { + if err := trie.Prove(entries[start].k, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } - if err := trie.Prove(entries[end-1].k, 0, proof); err != nil { + if err := trie.Prove(entries[end-1].k, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } var keys [][]byte @@ -230,10 +230,10 @@ func TestRangeProofWithNonExistentProof(t *testing.T) { if bytes.Compare(last, entries[end-1].k) < 0 { continue } - if err := trie.Prove(first, 0, proof); err != nil { + if err := trie.Prove(first, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } - if err := trie.Prove(last, 0, proof); err != nil { + if err := trie.Prove(last, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } var keys [][]byte @@ -251,10 +251,10 @@ func TestRangeProofWithNonExistentProof(t *testing.T) { proof := memorydb.New() first := common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000").Bytes() last := common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").Bytes() - if err := trie.Prove(first, 0, proof); err != nil { + if err := trie.Prove(first, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } - if err := trie.Prove(last, 0, proof); err != nil { + if err := trie.Prove(last, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } var k [][]byte @@ -285,10 +285,10 @@ func TestRangeProofWithInvalidNonExistentProof(t *testing.T) { first := decreaseKey(common.CopyBytes(entries[start].k)) proof := memorydb.New() - if err := trie.Prove(first, 0, proof); err != nil { + if err := trie.Prove(first, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } - if err := trie.Prove(entries[end-1].k, 0, proof); err != nil { + if err := trie.Prove(entries[end-1].k, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } start = 105 // Gap created @@ -307,10 +307,10 @@ func TestRangeProofWithInvalidNonExistentProof(t *testing.T) { start, end = 100, 200 last := increaseKey(common.CopyBytes(entries[end-1].k)) proof = memorydb.New() - if err := trie.Prove(entries[start].k, 0, proof); err != nil { + if err := trie.Prove(entries[start].k, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } - if err := trie.Prove(last, 0, proof); err != nil { + if err := trie.Prove(last, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } end = 195 // Capped slice @@ -341,7 +341,7 @@ func TestOneElementRangeProof(t *testing.T) { // point to the SAME key. start := 1000 proof := memorydb.New() - if err := trie.Prove(entries[start].k, 0, proof); err != nil { + if err := trie.Prove(entries[start].k, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } _, err := VerifyRangeProof(trie.Hash(), entries[start].k, entries[start].k, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof) @@ -353,10 +353,10 @@ func TestOneElementRangeProof(t *testing.T) { start = 1000 first := decreaseKey(common.CopyBytes(entries[start].k)) proof = memorydb.New() - if err := trie.Prove(first, 0, proof); err != nil { + if err := trie.Prove(first, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } - if err := trie.Prove(entries[start].k, 0, proof); err != nil { + if err := trie.Prove(entries[start].k, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } _, err = VerifyRangeProof(trie.Hash(), first, entries[start].k, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof) @@ -368,10 +368,10 @@ func TestOneElementRangeProof(t *testing.T) { start = 1000 last := increaseKey(common.CopyBytes(entries[start].k)) proof = memorydb.New() - if err := trie.Prove(entries[start].k, 0, proof); err != nil { + if err := trie.Prove(entries[start].k, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } - if err := trie.Prove(last, 0, proof); err != nil { + if err := trie.Prove(last, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } _, err = VerifyRangeProof(trie.Hash(), entries[start].k, last, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof) @@ -383,10 +383,10 @@ func TestOneElementRangeProof(t *testing.T) { start = 1000 first, last = decreaseKey(common.CopyBytes(entries[start].k)), increaseKey(common.CopyBytes(entries[start].k)) proof = memorydb.New() - if err := trie.Prove(first, 0, proof); err != nil { + if err := trie.Prove(first, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } - if err := trie.Prove(last, 0, proof); err != nil { + if err := trie.Prove(last, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } _, err = VerifyRangeProof(trie.Hash(), first, last, [][]byte{entries[start].k}, [][]byte{entries[start].v}, proof) @@ -402,10 +402,10 @@ func TestOneElementRangeProof(t *testing.T) { first = common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000").Bytes() last = entry.k proof = memorydb.New() - if err := tinyTrie.Prove(first, 0, proof); err != nil { + if err := tinyTrie.Prove(first, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } - if err := tinyTrie.Prove(last, 0, proof); err != nil { + if err := tinyTrie.Prove(last, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } _, err = VerifyRangeProof(tinyTrie.Hash(), first, last, [][]byte{entry.k}, [][]byte{entry.v}, proof) @@ -437,10 +437,10 @@ func TestAllElementsProof(t *testing.T) { // With edge proofs, it should still work. proof := memorydb.New() - if err := trie.Prove(entries[0].k, 0, proof); err != nil { + if err := trie.Prove(entries[0].k, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } - if err := trie.Prove(entries[len(entries)-1].k, 0, proof); err != nil { + if err := trie.Prove(entries[len(entries)-1].k, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } _, err = VerifyRangeProof(trie.Hash(), k[0], k[len(k)-1], k, v, proof) @@ -452,10 +452,10 @@ func TestAllElementsProof(t *testing.T) { proof = memorydb.New() first := common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000000").Bytes() last := common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").Bytes() - if err := trie.Prove(first, 0, proof); err != nil { + if err := trie.Prove(first, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } - if err := trie.Prove(last, 0, proof); err != nil { + if err := trie.Prove(last, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } _, err = VerifyRangeProof(trie.Hash(), first, last, k, v, proof) @@ -479,10 +479,10 @@ func TestSingleSideRangeProof(t *testing.T) { var cases = []int{0, 1, 50, 100, 1000, 2000, len(entries) - 1} for _, pos := range cases { proof := memorydb.New() - if err := trie.Prove(common.Hash{}.Bytes(), 0, proof); err != nil { + if err := trie.Prove(common.Hash{}.Bytes(), proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } - if err := trie.Prove(entries[pos].k, 0, proof); err != nil { + if err := trie.Prove(entries[pos].k, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } k := make([][]byte, 0) @@ -514,11 +514,11 @@ func TestReverseSingleSideRangeProof(t *testing.T) { var cases = []int{0, 1, 50, 100, 1000, 2000, len(entries) - 1} for _, pos := range cases { proof := memorydb.New() - if err := trie.Prove(entries[pos].k, 0, proof); err != nil { + if err := trie.Prove(entries[pos].k, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } last := common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") - if err := trie.Prove(last.Bytes(), 0, proof); err != nil { + if err := trie.Prove(last.Bytes(), proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } k := make([][]byte, 0) @@ -549,10 +549,10 @@ func TestBadRangeProof(t *testing.T) { start := mrand.Intn(len(entries)) end := mrand.Intn(len(entries)-start) + start + 1 proof := memorydb.New() - if err := trie.Prove(entries[start].k, 0, proof); err != nil { + if err := trie.Prove(entries[start].k, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } - if err := trie.Prove(entries[end-1].k, 0, proof); err != nil { + if err := trie.Prove(entries[end-1].k, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } var keys [][]byte @@ -618,10 +618,10 @@ func TestGappedRangeProof(t *testing.T) { } first, last := 2, 8 proof := memorydb.New() - if err := trie.Prove(entries[first].k, 0, proof); err != nil { + if err := trie.Prove(entries[first].k, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } - if err := trie.Prove(entries[last-1].k, 0, proof); err != nil { + if err := trie.Prove(entries[last-1].k, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } var keys [][]byte @@ -654,10 +654,10 @@ func TestSameSideProofs(t *testing.T) { last := decreaseKey(common.CopyBytes(entries[pos].k)) proof := memorydb.New() - if err := trie.Prove(first, 0, proof); err != nil { + if err := trie.Prove(first, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } - if err := trie.Prove(last, 0, proof); err != nil { + if err := trie.Prove(last, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } _, err := VerifyRangeProof(trie.Hash(), first, last, [][]byte{entries[pos].k}, [][]byte{entries[pos].v}, proof) @@ -670,10 +670,10 @@ func TestSameSideProofs(t *testing.T) { last = increaseKey(last) proof = memorydb.New() - if err := trie.Prove(first, 0, proof); err != nil { + if err := trie.Prove(first, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } - if err := trie.Prove(last, 0, proof); err != nil { + if err := trie.Prove(last, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } _, err = VerifyRangeProof(trie.Hash(), first, last, [][]byte{entries[pos].k}, [][]byte{entries[pos].v}, proof) @@ -718,23 +718,23 @@ func TestHasRightElement(t *testing.T) { ) if c.start == -1 { firstKey, start = common.Hash{}.Bytes(), 0 - if err := trie.Prove(firstKey, 0, proof); err != nil { + if err := trie.Prove(firstKey, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } } else { firstKey = entries[c.start].k - if err := trie.Prove(entries[c.start].k, 0, proof); err != nil { + if err := trie.Prove(entries[c.start].k, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } } if c.end == -1 { lastKey, end = common.HexToHash("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff").Bytes(), len(entries) - if err := trie.Prove(lastKey, 0, proof); err != nil { + if err := trie.Prove(lastKey, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } } else { lastKey = entries[c.end-1].k - if err := trie.Prove(entries[c.end-1].k, 0, proof); err != nil { + if err := trie.Prove(entries[c.end-1].k, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } } @@ -774,7 +774,7 @@ func TestEmptyRangeProof(t *testing.T) { for _, c := range cases { proof := memorydb.New() first := increaseKey(common.CopyBytes(entries[c.pos].k)) - if err := trie.Prove(first, 0, proof); err != nil { + if err := trie.Prove(first, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } _, err := VerifyRangeProof(trie.Hash(), first, nil, nil, nil, proof) @@ -805,7 +805,7 @@ func TestBloatedProof(t *testing.T) { // In the 'malicious' case, we add proofs for every single item // (but only one key/value pair used as leaf) for i, entry := range entries { - trie.Prove(entry.k, 0, proof) + trie.Prove(entry.k, proof) if i == 50 { keys = append(keys, entry.k) vals = append(vals, entry.v) @@ -814,8 +814,8 @@ func TestBloatedProof(t *testing.T) { // For reference, we use the same function, but _only_ prove the first // and last element want := memorydb.New() - trie.Prove(keys[0], 0, want) - trie.Prove(keys[len(keys)-1], 0, want) + trie.Prove(keys[0], want) + trie.Prove(keys[len(keys)-1], want) if _, err := VerifyRangeProof(trie.Hash(), keys[0], keys[len(keys)-1], keys, vals, proof); err != nil { t.Fatalf("expected bloated proof to succeed, got %v", err) @@ -848,10 +848,10 @@ func TestEmptyValueRangeProof(t *testing.T) { start, end := 1, len(entries)-1 proof := memorydb.New() - if err := trie.Prove(entries[start].k, 0, proof); err != nil { + if err := trie.Prove(entries[start].k, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } - if err := trie.Prove(entries[end-1].k, 0, proof); err != nil { + if err := trie.Prove(entries[end-1].k, proof); err != nil { t.Fatalf("Failed to prove the last node %v", err) } var keys [][]byte @@ -943,7 +943,7 @@ func BenchmarkProve(b *testing.B) { for i := 0; i < b.N; i++ { kv := vals[keys[i%len(keys)]] proofs := memorydb.New() - if trie.Prove(kv.k, 0, proofs); proofs.Len() == 0 { + if trie.Prove(kv.k, proofs); proofs.Len() == 0 { b.Fatalf("zero length proof for %x", kv.k) } } @@ -957,7 +957,7 @@ func BenchmarkVerifyProof(b *testing.B) { for k := range vals { keys = append(keys, k) proof := memorydb.New() - trie.Prove([]byte(k), 0, proof) + trie.Prove([]byte(k), proof) proofs = append(proofs, proof) } @@ -986,10 +986,10 @@ func benchmarkVerifyRangeProof(b *testing.B, size int) { start := 2 end := start + size proof := memorydb.New() - if err := trie.Prove(entries[start].k, 0, proof); err != nil { + if err := trie.Prove(entries[start].k, proof); err != nil { b.Fatalf("Failed to prove the first node %v", err) } - if err := trie.Prove(entries[end-1].k, 0, proof); err != nil { + if err := trie.Prove(entries[end-1].k, proof); err != nil { b.Fatalf("Failed to prove the last node %v", err) } var keys [][]byte @@ -1088,10 +1088,10 @@ func TestRangeProofKeysWithSharedPrefix(t *testing.T) { proof := memorydb.New() start := common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000000000") end := common.Hex2Bytes("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff") - if err := trie.Prove(start, 0, proof); err != nil { + if err := trie.Prove(start, proof); err != nil { t.Fatalf("failed to prove start: %v", err) } - if err := trie.Prove(end, 0, proof); err != nil { + if err := trie.Prove(end, proof); err != nil { t.Fatalf("failed to prove end: %v", err) } diff --git a/trie/tracer_test.go b/trie/tracer_test.go index 5f61a1d6b3..32949a5a99 100644 --- a/trie/tracer_test.go +++ b/trie/tracer_test.go @@ -241,7 +241,7 @@ func TestAccessListLeak(t *testing.T) { { func(tr *Trie) { for _, val := range standard { - tr.Prove([]byte(val.k), 0, rawdb.NewMemoryDatabase()) + tr.Prove([]byte(val.k), rawdb.NewMemoryDatabase()) } }, }, diff --git a/trie/trie_test.go b/trie/trie_test.go index 70de460227..bd21a572d2 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -490,7 +490,7 @@ func runRandTest(rt randTest) bool { continue } proofDb := rawdb.NewMemoryDatabase() - err := tr.Prove(step.key, 0, proofDb) + err := tr.Prove(step.key, proofDb) if err != nil { rt[i].err = fmt.Errorf("failed for proving key %#x, %v", step.key, err) } From 9a167c45d1650653e9618977cb08c6f453bb6f70 Mon Sep 17 00:00:00 2001 From: hero5512 Date: Mon, 19 Jun 2023 17:30:39 -0400 Subject: [PATCH 21/90] core/txpool/legacypool: narrow down the scope of the variable (#27471) Variables discarded, included can be defined closer to their usage. Co-authored-by: Martin Holst Swende --- core/txpool/legacypool/legacypool.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/txpool/legacypool/legacypool.go b/core/txpool/legacypool/legacypool.go index 3464fe212f..02cd53ab96 100644 --- a/core/txpool/legacypool/legacypool.go +++ b/core/txpool/legacypool/legacypool.go @@ -1262,7 +1262,6 @@ func (pool *LegacyPool) reset(oldHead, newHead *types.Header) { log.Debug("Skipping deep transaction reorg", "depth", depth) } else { // Reorg seems shallow enough to pull in all transactions into memory - var discarded, included types.Transactions var ( rem = pool.chain.GetBlock(oldHead.Hash(), oldHead.Number.Uint64()) add = pool.chain.GetBlock(newHead.Hash(), newHead.Number.Uint64()) @@ -1291,6 +1290,7 @@ func (pool *LegacyPool) reset(oldHead, newHead *types.Header) { log.Warn("New head missing in txpool reset", "number", newHead.Number, "hash", newHead.Hash()) return } + var discarded, included types.Transactions for rem.NumberU64() > add.NumberU64() { discarded = append(discarded, rem.Transactions()...) if rem = pool.chain.GetBlock(rem.ParentHash(), rem.NumberU64()-1); rem == nil { From b1ef0bfe031ea1a02cacaa1be814422ef598ca8a Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Mon, 19 Jun 2023 17:38:57 -0400 Subject: [PATCH 22/90] eth: use slices package for sorting (#27490) Also adds Hash.Less method for sorting purposes. --------- Co-authored-by: Felix Lange --- common/types.go | 5 +++ eth/api_debug_test.go | 12 ++----- eth/gasprice/feehistory.go | 25 +++++---------- eth/gasprice/gasprice.go | 53 +++++++++--------------------- eth/protocols/snap/sync_test.go | 57 ++++++++++++++++----------------- eth/tracers/api_test.go | 13 ++------ 6 files changed, 61 insertions(+), 104 deletions(-) diff --git a/common/types.go b/common/types.go index 31515f52fa..93fb090454 100644 --- a/common/types.go +++ b/common/types.go @@ -65,6 +65,11 @@ func BigToHash(b *big.Int) Hash { return BytesToHash(b.Bytes()) } // If b is larger than len(h), b will be cropped from the left. func HexToHash(s string) Hash { return BytesToHash(FromHex(s)) } +// Less compares two hashes. +func (h Hash) Less(other Hash) bool { + return bytes.Compare(h[:], other[:]) < 0 +} + // Bytes gets the byte representation of the underlying hash. func (h Hash) Bytes() []byte { return h[:] } diff --git a/eth/api_debug_test.go b/eth/api_debug_test.go index dec1b34ddc..c7f0d4ac90 100644 --- a/eth/api_debug_test.go +++ b/eth/api_debug_test.go @@ -21,7 +21,6 @@ import ( "fmt" "math/big" "reflect" - "sort" "testing" "github.com/davecgh/go-spew/spew" @@ -31,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/trie" + "golang.org/x/exp/slices" ) var dumper = spew.ConfigState{Indent: " "} @@ -58,12 +58,6 @@ func accountRangeTest(t *testing.T, trie *state.Trie, statedb *state.StateDB, st return result } -type resultHash []common.Hash - -func (h resultHash) Len() int { return len(h) } -func (h resultHash) Swap(i, j int) { h[i], h[j] = h[j], h[i] } -func (h resultHash) Less(i, j int) bool { return bytes.Compare(h[i].Bytes(), h[j].Bytes()) < 0 } - func TestAccountRange(t *testing.T) { t.Parallel() @@ -97,7 +91,7 @@ func TestAccountRange(t *testing.T) { firstResult := accountRangeTest(t, &trie, state, common.Hash{}, AccountRangeMaxResults, AccountRangeMaxResults) secondResult := accountRangeTest(t, &trie, state, common.BytesToHash(firstResult.Next), AccountRangeMaxResults, AccountRangeMaxResults) - hList := make(resultHash, 0) + hList := make([]common.Hash, 0) for addr1 := range firstResult.Accounts { // If address is empty, then it makes no sense to compare // them as they might be two different accounts. @@ -111,7 +105,7 @@ func TestAccountRange(t *testing.T) { } // Test to see if it's possible to recover from the middle of the previous // set and get an even split between the first and second sets. - sort.Sort(hList) + slices.SortFunc(hList, common.Hash.Less) middleH := hList[AccountRangeMaxResults/2] middleResult := accountRangeTest(t, &trie, state, middleH, AccountRangeMaxResults, AccountRangeMaxResults) missing, infirst, insecond := 0, 0, 0 diff --git a/eth/gasprice/feehistory.go b/eth/gasprice/feehistory.go index ee0b0ebcca..c048377062 100644 --- a/eth/gasprice/feehistory.go +++ b/eth/gasprice/feehistory.go @@ -23,7 +23,6 @@ import ( "fmt" "math" "math/big" - "sort" "sync/atomic" "github.com/ethereum/go-ethereum/common" @@ -31,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" + "golang.org/x/exp/slices" ) var ( @@ -69,20 +69,9 @@ type processedFees struct { } // txGasAndReward is sorted in ascending order based on reward -type ( - txGasAndReward struct { - gasUsed uint64 - reward *big.Int - } - sortGasAndReward []txGasAndReward -) - -func (s sortGasAndReward) Len() int { return len(s) } -func (s sortGasAndReward) Swap(i, j int) { - s[i], s[j] = s[j], s[i] -} -func (s sortGasAndReward) Less(i, j int) bool { - return s[i].reward.Cmp(s[j].reward) < 0 +type txGasAndReward struct { + gasUsed uint64 + reward *big.Int } // processBlock takes a blockFees structure with the blockNumber, the header and optionally @@ -117,12 +106,14 @@ func (oracle *Oracle) processBlock(bf *blockFees, percentiles []float64) { return } - sorter := make(sortGasAndReward, len(bf.block.Transactions())) + sorter := make([]txGasAndReward, len(bf.block.Transactions())) for i, tx := range bf.block.Transactions() { reward, _ := tx.EffectiveGasTip(bf.block.BaseFee()) sorter[i] = txGasAndReward{gasUsed: bf.receipts[i].GasUsed, reward: reward} } - sort.Stable(sorter) + slices.SortStableFunc(sorter, func(a, b txGasAndReward) bool { + return a.reward.Cmp(b.reward) < 0 + }) var txIndex int sumGasUsed := sorter[0].gasUsed diff --git a/eth/gasprice/gasprice.go b/eth/gasprice/gasprice.go index a3dd83d79f..9eccd72428 100644 --- a/eth/gasprice/gasprice.go +++ b/eth/gasprice/gasprice.go @@ -19,7 +19,6 @@ package gasprice import ( "context" "math/big" - "sort" "sync" "github.com/ethereum/go-ethereum/common" @@ -30,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" + "golang.org/x/exp/slices" ) const sampleNumber = 3 // Number of transactions sampled in a block @@ -208,7 +208,7 @@ func (oracle *Oracle) SuggestTipCap(ctx context.Context) (*big.Int, error) { } price := lastPrice if len(results) > 0 { - sort.Sort(bigIntArray(results)) + slices.SortFunc(results, func(a, b *big.Int) bool { return a.Cmp(b) < 0 }) price = results[(len(results)-1)*oracle.percentile/100] } if price.Cmp(oracle.maxPrice) > 0 { @@ -227,30 +227,6 @@ type results struct { err error } -type txSorter struct { - txs []*types.Transaction - baseFee *big.Int -} - -func newSorter(txs []*types.Transaction, baseFee *big.Int) *txSorter { - return &txSorter{ - txs: txs, - baseFee: baseFee, - } -} - -func (s *txSorter) Len() int { return len(s.txs) } -func (s *txSorter) Swap(i, j int) { - s.txs[i], s.txs[j] = s.txs[j], s.txs[i] -} -func (s *txSorter) Less(i, j int) bool { - // It's okay to discard the error because a tx would never be - // accepted into a block with an invalid effective tip. - tip1, _ := s.txs[i].EffectiveGasTip(s.baseFee) - tip2, _ := s.txs[j].EffectiveGasTip(s.baseFee) - return tip1.Cmp(tip2) < 0 -} - // getBlockValues calculates the lowest transaction gas price in a given block // and sends it to the result channel. If the block is empty or all transactions // are sent by the miner itself(it doesn't make any sense to include this kind of @@ -267,14 +243,21 @@ func (oracle *Oracle) getBlockValues(ctx context.Context, blockNum uint64, limit signer := types.MakeSigner(oracle.backend.ChainConfig(), block.Number(), block.Time()) // Sort the transaction by effective tip in ascending sort. - txs := make([]*types.Transaction, len(block.Transactions())) - copy(txs, block.Transactions()) - sorter := newSorter(txs, block.BaseFee()) - sort.Sort(sorter) + txs := block.Transactions() + sortedTxs := make([]*types.Transaction, len(txs)) + copy(sortedTxs, txs) + baseFee := block.BaseFee() + slices.SortFunc(sortedTxs, func(a, b *types.Transaction) bool { + // It's okay to discard the error because a tx would never be + // accepted into a block with an invalid effective tip. + tip1, _ := a.EffectiveGasTip(baseFee) + tip2, _ := b.EffectiveGasTip(baseFee) + return tip1.Cmp(tip2) < 0 + }) var prices []*big.Int - for _, tx := range sorter.txs { - tip, _ := tx.EffectiveGasTip(block.BaseFee()) + for _, tx := range sortedTxs { + tip, _ := tx.EffectiveGasTip(baseFee) if ignoreUnder != nil && tip.Cmp(ignoreUnder) == -1 { continue } @@ -291,9 +274,3 @@ func (oracle *Oracle) getBlockValues(ctx context.Context, blockNum uint64, limit case <-quit: } } - -type bigIntArray []*big.Int - -func (s bigIntArray) Len() int { return len(s) } -func (s bigIntArray) Less(i, j int) bool { return s[i].Cmp(s[j]) < 0 } -func (s bigIntArray) Swap(i, j int) { s[i], s[j] = s[j], s[i] } diff --git a/eth/protocols/snap/sync_test.go b/eth/protocols/snap/sync_test.go index 23af204162..6efe0c8325 100644 --- a/eth/protocols/snap/sync_test.go +++ b/eth/protocols/snap/sync_test.go @@ -22,7 +22,6 @@ import ( "encoding/binary" "fmt" "math/big" - "sort" "sync" "testing" "time" @@ -38,6 +37,7 @@ import ( "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/trienode" "golang.org/x/crypto/sha3" + "golang.org/x/exp/slices" ) func TestHashing(t *testing.T) { @@ -127,9 +127,9 @@ type testPeer struct { remote *Syncer logger log.Logger accountTrie *trie.Trie - accountValues entrySlice + accountValues []*kv storageTries map[common.Hash]*trie.Trie - storageValues map[common.Hash]entrySlice + storageValues map[common.Hash][]*kv accountRequestHandler accountHandlerFunc storageRequestHandler storageHandlerFunc @@ -1321,12 +1321,9 @@ type kv struct { k, v []byte } -// Some helpers for sorting -type entrySlice []*kv - -func (p entrySlice) Len() int { return len(p) } -func (p entrySlice) Less(i, j int) bool { return bytes.Compare(p[i].k, p[j].k) < 0 } -func (p entrySlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } +func (k *kv) less(other *kv) bool { + return bytes.Compare(k.k, other.k) < 0 +} func key32(i uint64) []byte { key := make([]byte, 32) @@ -1367,11 +1364,11 @@ func getCodeByHash(hash common.Hash) []byte { } // makeAccountTrieNoStorage spits out a trie, along with the leafs -func makeAccountTrieNoStorage(n int) (string, *trie.Trie, entrySlice) { +func makeAccountTrieNoStorage(n int) (string, *trie.Trie, []*kv) { var ( db = trie.NewDatabase(rawdb.NewMemoryDatabase()) accTrie = trie.NewEmpty(db) - entries entrySlice + entries []*kv ) for i := uint64(1); i <= uint64(n); i++ { value, _ := rlp.EncodeToBytes(&types.StateAccount{ @@ -1385,7 +1382,7 @@ func makeAccountTrieNoStorage(n int) (string, *trie.Trie, entrySlice) { accTrie.MustUpdate(elem.k, elem.v) entries = append(entries, elem) } - sort.Sort(entries) + slices.SortFunc(entries, (*kv).less) // Commit the state changes into db and re-create the trie // for accessing later. @@ -1399,9 +1396,9 @@ func makeAccountTrieNoStorage(n int) (string, *trie.Trie, entrySlice) { // makeBoundaryAccountTrie constructs an account trie. Instead of filling // accounts normally, this function will fill a few accounts which have // boundary hash. -func makeBoundaryAccountTrie(n int) (string, *trie.Trie, entrySlice) { +func makeBoundaryAccountTrie(n int) (string, *trie.Trie, []*kv) { var ( - entries entrySlice + entries []*kv boundaries []common.Hash db = trie.NewDatabase(rawdb.NewMemoryDatabase()) @@ -1447,7 +1444,7 @@ func makeBoundaryAccountTrie(n int) (string, *trie.Trie, entrySlice) { accTrie.MustUpdate(elem.k, elem.v) entries = append(entries, elem) } - sort.Sort(entries) + slices.SortFunc(entries, (*kv).less) // Commit the state changes into db and re-create the trie // for accessing later. @@ -1460,14 +1457,14 @@ func makeBoundaryAccountTrie(n int) (string, *trie.Trie, entrySlice) { // makeAccountTrieWithStorageWithUniqueStorage creates an account trie where each accounts // has a unique storage set. -func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) (string, *trie.Trie, entrySlice, map[common.Hash]*trie.Trie, map[common.Hash]entrySlice) { +func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) (string, *trie.Trie, []*kv, map[common.Hash]*trie.Trie, map[common.Hash][]*kv) { var ( db = trie.NewDatabase(rawdb.NewMemoryDatabase()) accTrie = trie.NewEmpty(db) - entries entrySlice + entries []*kv storageRoots = make(map[common.Hash]common.Hash) storageTries = make(map[common.Hash]*trie.Trie) - storageEntries = make(map[common.Hash]entrySlice) + storageEntries = make(map[common.Hash][]*kv) nodes = trienode.NewMergedNodeSet() ) // Create n accounts in the trie @@ -1494,7 +1491,7 @@ func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) storageRoots[common.BytesToHash(key)] = stRoot storageEntries[common.BytesToHash(key)] = stEntries } - sort.Sort(entries) + slices.SortFunc(entries, (*kv).less) // Commit account trie root, set := accTrie.Commit(true) @@ -1515,14 +1512,14 @@ func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) } // makeAccountTrieWithStorage spits out a trie, along with the leafs -func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (string, *trie.Trie, entrySlice, map[common.Hash]*trie.Trie, map[common.Hash]entrySlice) { +func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (string, *trie.Trie, []*kv, map[common.Hash]*trie.Trie, map[common.Hash][]*kv) { var ( db = trie.NewDatabase(rawdb.NewMemoryDatabase()) accTrie = trie.NewEmpty(db) - entries entrySlice + entries []*kv storageRoots = make(map[common.Hash]common.Hash) storageTries = make(map[common.Hash]*trie.Trie) - storageEntries = make(map[common.Hash]entrySlice) + storageEntries = make(map[common.Hash][]*kv) nodes = trienode.NewMergedNodeSet() ) // Create n accounts in the trie @@ -1536,7 +1533,7 @@ func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (strin var ( stRoot common.Hash stNodes *trienode.NodeSet - stEntries entrySlice + stEntries []*kv ) if boundary { stRoot, stNodes, stEntries = makeBoundaryStorageTrie(common.BytesToHash(key), slots, db) @@ -1559,7 +1556,7 @@ func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (strin storageRoots[common.BytesToHash(key)] = stRoot storageEntries[common.BytesToHash(key)] = stEntries } - sort.Sort(entries) + slices.SortFunc(entries, (*kv).less) // Commit account trie root, set := accTrie.Commit(true) @@ -1588,9 +1585,9 @@ func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (strin // makeStorageTrieWithSeed fills a storage trie with n items, returning the // not-yet-committed trie and the sorted entries. The seeds can be used to ensure // that tries are unique. -func makeStorageTrieWithSeed(owner common.Hash, n, seed uint64, db *trie.Database) (common.Hash, *trienode.NodeSet, entrySlice) { +func makeStorageTrieWithSeed(owner common.Hash, n, seed uint64, db *trie.Database) (common.Hash, *trienode.NodeSet, []*kv) { trie, _ := trie.New(trie.StorageTrieID(types.EmptyRootHash, owner, types.EmptyRootHash), db) - var entries entrySlice + var entries []*kv for i := uint64(1); i <= n; i++ { // store 'x' at slot 'x' slotValue := key32(i + seed) @@ -1603,7 +1600,7 @@ func makeStorageTrieWithSeed(owner common.Hash, n, seed uint64, db *trie.Databas trie.MustUpdate(elem.k, elem.v) entries = append(entries, elem) } - sort.Sort(entries) + slices.SortFunc(entries, (*kv).less) root, nodes := trie.Commit(false) return root, nodes, entries } @@ -1611,9 +1608,9 @@ func makeStorageTrieWithSeed(owner common.Hash, n, seed uint64, db *trie.Databas // makeBoundaryStorageTrie constructs a storage trie. Instead of filling // storage slots normally, this function will fill a few slots which have // boundary hash. -func makeBoundaryStorageTrie(owner common.Hash, n int, db *trie.Database) (common.Hash, *trienode.NodeSet, entrySlice) { +func makeBoundaryStorageTrie(owner common.Hash, n int, db *trie.Database) (common.Hash, *trienode.NodeSet, []*kv) { var ( - entries entrySlice + entries []*kv boundaries []common.Hash trie, _ = trie.New(trie.StorageTrieID(types.EmptyRootHash, owner, types.EmptyRootHash), db) ) @@ -1654,7 +1651,7 @@ func makeBoundaryStorageTrie(owner common.Hash, n int, db *trie.Database) (commo trie.MustUpdate(elem.k, elem.v) entries = append(entries, elem) } - sort.Sort(entries) + slices.SortFunc(entries, (*kv).less) root, nodes := trie.Commit(false) return root, nodes, entries } diff --git a/eth/tracers/api_test.go b/eth/tracers/api_test.go index fdb02b1895..11d7ff4b69 100644 --- a/eth/tracers/api_test.go +++ b/eth/tracers/api_test.go @@ -17,7 +17,6 @@ package tracers import ( - "bytes" "context" "crypto/ecdsa" "encoding/json" @@ -25,7 +24,6 @@ import ( "fmt" "math/big" "reflect" - "sort" "sync/atomic" "testing" "time" @@ -45,6 +43,7 @@ import ( "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" + "golang.org/x/exp/slices" ) var ( @@ -785,19 +784,13 @@ type Account struct { addr common.Address } -type Accounts []Account - -func (a Accounts) Len() int { return len(a) } -func (a Accounts) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a Accounts) Less(i, j int) bool { return bytes.Compare(a[i].addr.Bytes(), a[j].addr.Bytes()) < 0 } - -func newAccounts(n int) (accounts Accounts) { +func newAccounts(n int) (accounts []Account) { for i := 0; i < n; i++ { key, _ := crypto.GenerateKey() addr := crypto.PubkeyToAddress(key.PublicKey) accounts = append(accounts, Account{key: key, addr: addr}) } - sort.Sort(accounts) + slices.SortFunc(accounts, func(a, b Account) bool { return a.addr.Less(b.addr) }) return accounts } From 84b05d4f343dea9d89862d0bbaa7e9d0c3f0acd5 Mon Sep 17 00:00:00 2001 From: Delweng Date: Tue, 20 Jun 2023 16:42:13 +0800 Subject: [PATCH 23/90] internal/build: ignore some files in FindMainPackages (#27521) --- internal/build/util.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/build/util.go b/internal/build/util.go index 3a0b384e5e..5c77b236dc 100644 --- a/internal/build/util.go +++ b/internal/build/util.go @@ -197,6 +197,9 @@ func FindMainPackages(dir string) []string { } for _, cmd := range cmds { pkgdir := filepath.Join(dir, cmd.Name()) + if !cmd.IsDir() { + continue + } pkgs, err := parser.ParseDir(token.NewFileSet(), pkgdir, nil, parser.PackageClauseOnly) if err != nil { log.Fatal(err) From 154b016b6ce1c1b508463be05e091085ca3f557a Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Tue, 20 Jun 2023 05:58:47 -0400 Subject: [PATCH 24/90] core: use slices package for sorting (#27489) Co-authored-by: Felix Lange --- core/blockchain.go | 6 ++--- core/forkid/forkid.go | 6 ++--- core/mkalloc.go | 20 +++++++--------- core/rawdb/accessors_chain.go | 23 +++++++----------- core/rawdb/chain_iterator_test.go | 8 ++++--- core/state/snapshot/difflayer.go | 6 ++--- core/state/snapshot/iterator_fast.go | 28 +++++++--------------- core/state/snapshot/sort.go | 36 ---------------------------- 8 files changed, 39 insertions(+), 94 deletions(-) delete mode 100644 core/state/snapshot/sort.go diff --git a/core/blockchain.go b/core/blockchain.go index ec8b789c58..becd39a1fa 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -23,7 +23,6 @@ import ( "io" "math/big" "runtime" - "sort" "strings" "sync" "sync/atomic" @@ -48,6 +47,7 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + "golang.org/x/exp/slices" ) var ( @@ -1015,8 +1015,8 @@ func (bc *BlockChain) procFutureBlocks() { } } if len(blocks) > 0 { - sort.Slice(blocks, func(i, j int) bool { - return blocks[i].NumberU64() < blocks[j].NumberU64() + slices.SortFunc(blocks, func(a, b *types.Block) bool { + return a.NumberU64() < b.NumberU64() }) // Insert one by one as chain insertion needs contiguous ancestry between blocks for i := range blocks { diff --git a/core/forkid/forkid.go b/core/forkid/forkid.go index f536019dac..8964558845 100644 --- a/core/forkid/forkid.go +++ b/core/forkid/forkid.go @@ -24,13 +24,13 @@ import ( "math" "math/big" "reflect" - "sort" "strings" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" + "golang.org/x/exp/slices" ) var ( @@ -270,8 +270,8 @@ func gatherForks(config *params.ChainConfig) ([]uint64, []uint64) { } } } - sort.Slice(forksByBlock, func(i, j int) bool { return forksByBlock[i] < forksByBlock[j] }) - sort.Slice(forksByTime, func(i, j int) bool { return forksByTime[i] < forksByTime[j] }) + slices.Sort(forksByBlock) + slices.Sort(forksByTime) // Deduplicate fork identifiers applying multiple forks for i := 1; i < len(forksByBlock); i++ { diff --git a/core/mkalloc.go b/core/mkalloc.go index e4c2ec0d83..0e7355f631 100644 --- a/core/mkalloc.go +++ b/core/mkalloc.go @@ -30,32 +30,28 @@ import ( "fmt" "math/big" "os" - "sort" "strconv" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/rlp" + "golang.org/x/exp/slices" ) type allocItem struct{ Addr, Balance *big.Int } -type allocList []allocItem - -func (a allocList) Len() int { return len(a) } -func (a allocList) Less(i, j int) bool { return a[i].Addr.Cmp(a[j].Addr) < 0 } -func (a allocList) Swap(i, j int) { a[i], a[j] = a[j], a[i] } - -func makelist(g *core.Genesis) allocList { - a := make(allocList, 0, len(g.Alloc)) +func makelist(g *core.Genesis) []allocItem { + items := make([]allocItem, 0, len(g.Alloc)) for addr, account := range g.Alloc { if len(account.Storage) > 0 || len(account.Code) > 0 || account.Nonce != 0 { panic(fmt.Sprintf("can't encode account %x", addr)) } bigAddr := new(big.Int).SetBytes(addr.Bytes()) - a = append(a, allocItem{bigAddr, account.Balance}) + items = append(items, allocItem{bigAddr, account.Balance}) } - sort.Sort(a) - return a + slices.SortFunc(items, func(a, b allocItem) bool { + return a.Addr.Cmp(b.Addr) < 0 + }) + return items } func makealloc(g *core.Genesis) string { diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index b479655b0b..dd5425eec7 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -22,7 +22,6 @@ import ( "errors" "fmt" "math/big" - "sort" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" @@ -31,6 +30,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" + "golang.org/x/exp/slices" ) // ReadCanonicalHash retrieves the hash assigned to a canonical block number. @@ -836,23 +836,13 @@ type badBlock struct { Body *types.Body } -// badBlockList implements the sort interface to allow sorting a list of -// bad blocks by their number in the reverse order. -type badBlockList []*badBlock - -func (s badBlockList) Len() int { return len(s) } -func (s badBlockList) Less(i, j int) bool { - return s[i].Header.Number.Uint64() < s[j].Header.Number.Uint64() -} -func (s badBlockList) Swap(i, j int) { s[i], s[j] = s[j], s[i] } - // ReadBadBlock retrieves the bad block with the corresponding block hash. func ReadBadBlock(db ethdb.Reader, hash common.Hash) *types.Block { blob, err := db.Get(badBlockKey) if err != nil { return nil } - var badBlocks badBlockList + var badBlocks []*badBlock if err := rlp.DecodeBytes(blob, &badBlocks); err != nil { return nil } @@ -871,7 +861,7 @@ func ReadAllBadBlocks(db ethdb.Reader) []*types.Block { if err != nil { return nil } - var badBlocks badBlockList + var badBlocks []*badBlock if err := rlp.DecodeBytes(blob, &badBlocks); err != nil { return nil } @@ -889,7 +879,7 @@ func WriteBadBlock(db ethdb.KeyValueStore, block *types.Block) { if err != nil { log.Warn("Failed to load old bad blocks", "error", err) } - var badBlocks badBlockList + var badBlocks []*badBlock if len(blob) > 0 { if err := rlp.DecodeBytes(blob, &badBlocks); err != nil { log.Crit("Failed to decode old bad blocks", "error", err) @@ -905,7 +895,10 @@ func WriteBadBlock(db ethdb.KeyValueStore, block *types.Block) { Header: block.Header(), Body: block.Body(), }) - sort.Sort(sort.Reverse(badBlocks)) + slices.SortFunc(badBlocks, func(a, b *badBlock) bool { + // Note: sorting in descending number order. + return a.Header.Number.Uint64() >= b.Header.Number.Uint64() + }) if len(badBlocks) > badBlockToKeep { badBlocks = badBlocks[:badBlockToKeep] } diff --git a/core/rawdb/chain_iterator_test.go b/core/rawdb/chain_iterator_test.go index e1f5159753..fd405e9d69 100644 --- a/core/rawdb/chain_iterator_test.go +++ b/core/rawdb/chain_iterator_test.go @@ -19,12 +19,12 @@ package rawdb import ( "math/big" "reflect" - "sort" "sync" "testing" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "golang.org/x/exp/slices" ) func TestChainIterator(t *testing.T) { @@ -92,9 +92,11 @@ func TestChainIterator(t *testing.T) { } } if !c.reverse { - sort.Ints(numbers) + slices.Sort(numbers) } else { - sort.Sort(sort.Reverse(sort.IntSlice(numbers))) + slices.SortFunc(numbers, func(a, b int) bool { + return a > b // Sort descending + }) } if !reflect.DeepEqual(numbers, c.expect) { t.Fatalf("Case %d failed, visit element mismatch, want %v, got %v", i, c.expect, numbers) diff --git a/core/state/snapshot/difflayer.go b/core/state/snapshot/difflayer.go index 8869a84716..b10b43b1ae 100644 --- a/core/state/snapshot/difflayer.go +++ b/core/state/snapshot/difflayer.go @@ -21,7 +21,6 @@ import ( "fmt" "math" "math/rand" - "sort" "sync" "sync/atomic" "time" @@ -30,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rlp" bloomfilter "github.com/holiman/bloomfilter/v2" + "golang.org/x/exp/slices" ) var ( @@ -525,7 +525,7 @@ func (dl *diffLayer) AccountList() []common.Hash { dl.accountList = append(dl.accountList, hash) } } - sort.Sort(hashes(dl.accountList)) + slices.SortFunc(dl.accountList, common.Hash.Less) dl.memory += uint64(len(dl.accountList) * common.HashLength) return dl.accountList } @@ -563,7 +563,7 @@ func (dl *diffLayer) StorageList(accountHash common.Hash) ([]common.Hash, bool) for k := range storageMap { storageList = append(storageList, k) } - sort.Sort(hashes(storageList)) + slices.SortFunc(storageList, common.Hash.Less) dl.storageList[accountHash] = storageList dl.memory += uint64(len(dl.storageList)*common.HashLength + common.HashLength) return storageList, destructed diff --git a/core/state/snapshot/iterator_fast.go b/core/state/snapshot/iterator_fast.go index 1a042c7cd3..339f930ffd 100644 --- a/core/state/snapshot/iterator_fast.go +++ b/core/state/snapshot/iterator_fast.go @@ -22,6 +22,7 @@ import ( "sort" "github.com/ethereum/go-ethereum/common" + "golang.org/x/exp/slices" ) // weightedIterator is a iterator with an assigned weight. It is used to prioritise @@ -32,18 +33,10 @@ type weightedIterator struct { priority int } -// weightedIterators is a set of iterators implementing the sort.Interface. -type weightedIterators []*weightedIterator - -// Len implements sort.Interface, returning the number of active iterators. -func (its weightedIterators) Len() int { return len(its) } - -// Less implements sort.Interface, returning which of two iterators in the stack -// is before the other. -func (its weightedIterators) Less(i, j int) bool { +func (it *weightedIterator) Less(other *weightedIterator) bool { // Order the iterators primarily by the account hashes - hashI := its[i].it.Hash() - hashJ := its[j].it.Hash() + hashI := it.it.Hash() + hashJ := other.it.Hash() switch bytes.Compare(hashI[:], hashJ[:]) { case -1: @@ -52,12 +45,7 @@ func (its weightedIterators) Less(i, j int) bool { return false } // Same account/storage-slot in multiple layers, split by priority - return its[i].priority < its[j].priority -} - -// Swap implements sort.Interface, swapping two entries in the iterator stack. -func (its weightedIterators) Swap(i, j int) { - its[i], its[j] = its[j], its[i] + return it.priority < other.priority } // fastIterator is a more optimized multi-layer iterator which maintains a @@ -69,7 +57,7 @@ type fastIterator struct { curAccount []byte curSlot []byte - iterators weightedIterators + iterators []*weightedIterator initiated bool account bool fail error @@ -167,7 +155,9 @@ func (fi *fastIterator) init() { } } // Re-sort the entire list - sort.Sort(fi.iterators) + slices.SortFunc(fi.iterators, func(a, b *weightedIterator) bool { + return a.Less(b) + }) fi.initiated = false } diff --git a/core/state/snapshot/sort.go b/core/state/snapshot/sort.go deleted file mode 100644 index 88841231d9..0000000000 --- a/core/state/snapshot/sort.go +++ /dev/null @@ -1,36 +0,0 @@ -// Copyright 2019 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package snapshot - -import ( - "bytes" - - "github.com/ethereum/go-ethereum/common" -) - -// hashes is a helper to implement sort.Interface. -type hashes []common.Hash - -// Len is the number of elements in the collection. -func (hs hashes) Len() int { return len(hs) } - -// Less reports whether the element with index i should sort before the element -// with index j. -func (hs hashes) Less(i, j int) bool { return bytes.Compare(hs[i][:], hs[j][:]) < 0 } - -// Swap swaps the elements with indexes i and j. -func (hs hashes) Swap(i, j int) { hs[i], hs[j] = hs[j], hs[i] } From 1affc1c08d921c2ac70cee9d0a88d3cc378fa947 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 20 Jun 2023 13:51:59 +0200 Subject: [PATCH 25/90] core/txpool: remove use of errors.Join function (#27523) his function was added in Go 1.20, but our compatibility target is Go 1.19. --- core/txpool/txpool.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/core/txpool/txpool.go b/core/txpool/txpool.go index f2c94563bc..24105babc9 100644 --- a/core/txpool/txpool.go +++ b/core/txpool/txpool.go @@ -17,7 +17,7 @@ package txpool import ( - "errors" + "fmt" "math/big" "github.com/ethereum/go-ethereum/common" @@ -88,13 +88,20 @@ func (p *TxPool) Close() error { // Terminate the reset loop and wait for it to finish errc := make(chan error) p.quit <- errc - errs = append(errs, <-errc) + if err := <-errc; err != nil { + errs = append(errs, err) + } // Terminate each subpool for _, subpool := range p.subpools { - errs = append(errs, subpool.Close()) + if err := subpool.Close(); err != nil { + errs = append(errs, err) + } } - return errors.Join(errs...) + if len(errs) > 0 { + return fmt.Errorf("subpool close errors: %v", errs) + } + return nil } // loop is the transaction pool's main event loop, waiting for and reacting to From 8c288b528d3c9653c84321387e78687ed02134d7 Mon Sep 17 00:00:00 2001 From: jwasinger Date: Tue, 20 Jun 2023 14:40:18 +0200 Subject: [PATCH 26/90] internal/ethapi: use same state for each invocation within EstimateGas (#27505) EstimateGas repeatedly executes a transaction, performing a binary search with multiple gas prices to determine proper pricing. Each call retrieves a new copy of the state (https://github.com/ethereum/go-ethereum/blob/master/internal/ethapi/api.go#L1017) . Because the pending/latest state can change during the execution of EstimateGas, this can potentially cause strange behavior (as noted here: https://github.com/ethereum/go-ethereum/pull/27502#issue-1761957009). This PR modifies EstimateGas to retrieve the state once and use a copy of it for every call invocation it does. --- internal/ethapi/api.go | 32 +++++++++++++++++++++----------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 823ac6c975..48c6e38e88 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1011,13 +1011,7 @@ func (context *ChainContext) GetHeader(hash common.Hash, number uint64) *types.H return header } -func DoCall(ctx context.Context, b Backend, args TransactionArgs, blockNrOrHash rpc.BlockNumberOrHash, overrides *StateOverride, blockOverrides *BlockOverrides, timeout time.Duration, globalGasCap uint64) (*core.ExecutionResult, error) { - defer func(start time.Time) { log.Debug("Executing EVM call finished", "runtime", time.Since(start)) }(time.Now()) - - state, header, err := b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) - if state == nil || err != nil { - return nil, err - } +func doCall(ctx context.Context, b Backend, args TransactionArgs, state *state.StateDB, header *types.Header, overrides *StateOverride, blockOverrides *BlockOverrides, timeout time.Duration, globalGasCap uint64) (*core.ExecutionResult, error) { if err := overrides.Apply(state); err != nil { return nil, err } @@ -1068,6 +1062,17 @@ func DoCall(ctx context.Context, b Backend, args TransactionArgs, blockNrOrHash return result, nil } +func DoCall(ctx context.Context, b Backend, args TransactionArgs, blockNrOrHash rpc.BlockNumberOrHash, overrides *StateOverride, blockOverrides *BlockOverrides, timeout time.Duration, globalGasCap uint64) (*core.ExecutionResult, error) { + defer func(start time.Time) { log.Debug("Executing EVM call finished", "runtime", time.Since(start)) }(time.Now()) + + state, header, err := b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) + if state == nil || err != nil { + return nil, err + } + + return doCall(ctx, b, args, state, header, overrides, blockOverrides, timeout, globalGasCap) +} + func newRevertError(result *core.ExecutionResult) *revertError { reason, errUnpack := abi.UnpackRevert(result.Revert()) err := errors.New("execution reverted") @@ -1187,10 +1192,10 @@ func DoEstimateGas(ctx context.Context, b Backend, args TransactionArgs, blockNr cap = hi // Create a helper to check if a gas allowance results in an executable transaction - executable := func(gas uint64) (bool, *core.ExecutionResult, error) { + executable := func(gas uint64, state *state.StateDB, header *types.Header) (bool, *core.ExecutionResult, error) { args.Gas = (*hexutil.Uint64)(&gas) - result, err := DoCall(ctx, b, args, blockNrOrHash, nil, nil, 0, gasCap) + result, err := doCall(ctx, b, args, state, header, nil, nil, 0, gasCap) if err != nil { if errors.Is(err, core.ErrIntrinsicGas) { return true, nil, nil // Special case, raise gas limit @@ -1199,10 +1204,15 @@ func DoEstimateGas(ctx context.Context, b Backend, args TransactionArgs, blockNr } return result.Failed(), result, nil } + state, header, err := b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) + if state == nil || err != nil { + return 0, err + } // Execute the binary search and hone in on an executable gas limit for lo+1 < hi { + s := state.Copy() mid := (hi + lo) / 2 - failed, _, err := executable(mid) + failed, _, err := executable(mid, s, header) // If the error is not nil(consensus error), it means the provided message // call or transaction will never be accepted no matter how much gas it is @@ -1218,7 +1228,7 @@ func DoEstimateGas(ctx context.Context, b Backend, args TransactionArgs, blockNr } // Reject the transaction as invalid if it still fails at the highest allowance if hi == cap { - failed, result, err := executable(hi) + failed, result, err := executable(hi, state, header) if err != nil { return 0, err } From 6d2aeb43d516fbe2cafd9e65df7ccbd885f861d3 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Wed, 21 Jun 2023 03:31:45 +0800 Subject: [PATCH 27/90] cmd, core/state, eth, tests, trie: improve state reader (#27428) The state availability is checked during the creation of a state reader. - In hash-based database, if the specified root node does not exist on disk disk, then the state reader won't be created and an error will be returned. - In path-based database, if the specified state layer is not available, then the state reader won't be created and an error will be returned. This change also contains a stricter semantics regarding the `Commit` operation: once it has been performed, the trie is no longer usable, and certain operations will return an error. --- cmd/evm/internal/t8ntool/execution.go | 8 ++- cmd/geth/dbcmd.go | 6 +- cmd/geth/snapshot.go | 26 +++++-- core/state/database.go | 5 +- core/state/dump.go | 13 +++- core/state/iterator.go | 11 ++- core/state/pruner/pruner.go | 10 ++- core/state/snapshot/generate.go | 7 +- core/state/state_test.go | 12 ++-- core/state/statedb.go | 24 +++++-- core/state/statedb_test.go | 97 ++++++++++++++++++-------- core/state/sync_test.go | 8 ++- eth/api_debug.go | 16 ++++- eth/api_debug_test.go | 26 +++---- eth/protocols/snap/sync_test.go | 4 +- light/trie.go | 10 ++- light/trie_test.go | 12 +++- tests/fuzzers/stacktrie/trie_fuzzer.go | 2 +- tests/fuzzers/trie/trie-fuzzer.go | 2 +- tests/state_test_util.go | 9 ++- trie/database_wrap.go | 4 +- trie/errors.go | 6 ++ trie/iterator_test.go | 41 ++++++----- trie/proof.go | 4 ++ trie/proof_test.go | 2 +- trie/secure_trie.go | 12 +++- trie/sync_test.go | 52 +++++++++++--- trie/tracer_test.go | 8 +-- trie/trie.go | 56 ++++++++++++--- trie/trie_reader.go | 25 ++++--- trie/trie_test.go | 8 +-- trie/triedb/hashdb/database.go | 9 ++- 32 files changed, 384 insertions(+), 151 deletions(-) diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index 5633198d39..fb85794b76 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -19,7 +19,6 @@ package t8ntool import ( "fmt" "math/big" - "os" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" @@ -269,7 +268,6 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, // Commit block root, err := statedb.Commit(chainConfig.IsEIP158(vmContext.BlockNumber)) if err != nil { - fmt.Fprintf(os.Stderr, "Could not commit state: %v", err) return nil, nil, NewError(ErrorEVM, fmt.Errorf("could not commit state: %v", err)) } execRs := &ExecutionResult{ @@ -288,6 +286,12 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, h := types.DeriveSha(types.Withdrawals(pre.Env.Withdrawals), trie.NewStackTrie(nil)) execRs.WithdrawalsRoot = &h } + // Re-create statedb instance with new root upon the updated database + // for accessing latest states. + statedb, err = state.New(root, statedb.Database(), nil) + if err != nil { + return nil, nil, NewError(ErrorEVM, fmt.Errorf("could not reopen state: %v", err)) + } return statedb, execRs, nil } diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index b409b19260..aa7939851d 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -519,8 +519,12 @@ func dbDumpTrie(ctx *cli.Context) error { if err != nil { return err } + trieIt, err := theTrie.NodeIterator(start) + if err != nil { + return err + } var count int64 - it := trie.NewIterator(theTrie.NodeIterator(start)) + it := trie.NewIterator(trieIt) for it.Next() { if max > 0 && count == max { fmt.Printf("Exiting after %d values\n", count) diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 7b6b939be3..67837261c6 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -292,7 +292,12 @@ func traverseState(ctx *cli.Context) error { lastReport time.Time start = time.Now() ) - accIter := trie.NewIterator(t.NodeIterator(nil)) + acctIt, err := t.NodeIterator(nil) + if err != nil { + log.Error("Failed to open iterator", "root", root, "err", err) + return err + } + accIter := trie.NewIterator(acctIt) for accIter.Next() { accounts += 1 var acc types.StateAccount @@ -307,7 +312,12 @@ func traverseState(ctx *cli.Context) error { log.Error("Failed to open storage trie", "root", acc.Root, "err", err) return err } - storageIter := trie.NewIterator(storageTrie.NodeIterator(nil)) + storageIt, err := storageTrie.NodeIterator(nil) + if err != nil { + log.Error("Failed to open storage iterator", "root", acc.Root, "err", err) + return err + } + storageIter := trie.NewIterator(storageIt) for storageIter.Next() { slots += 1 } @@ -385,7 +395,11 @@ func traverseRawState(ctx *cli.Context) error { hasher = crypto.NewKeccakState() got = make([]byte, 32) ) - accIter := t.NodeIterator(nil) + accIter, err := t.NodeIterator(nil) + if err != nil { + log.Error("Failed to open iterator", "root", root, "err", err) + return err + } for accIter.Next(true) { nodes += 1 node := accIter.Hash() @@ -422,7 +436,11 @@ func traverseRawState(ctx *cli.Context) error { log.Error("Failed to open storage trie", "root", acc.Root, "err", err) return errors.New("missing storage trie") } - storageIter := storageTrie.NodeIterator(nil) + storageIter, err := storageTrie.NodeIterator(nil) + if err != nil { + log.Error("Failed to open storage iterator", "root", acc.Root, "err", err) + return err + } for storageIter.Next(true) { nodes += 1 node := storageIter.Hash() diff --git a/core/state/database.go b/core/state/database.go index 304c04f963..2c8a7afee6 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -113,8 +113,9 @@ type Trie interface { Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet) // NodeIterator returns an iterator that returns nodes of the trie. Iteration - // starts at the key after the given start key. - NodeIterator(startKey []byte) trie.NodeIterator + // starts at the key after the given start key. And error will be returned + // if fails to create node iterator. + NodeIterator(startKey []byte) (trie.NodeIterator, error) // Prove constructs a Merkle proof for key. The result contains all encoded nodes // on the path to the value at key. The value itself is also included in the last diff --git a/core/state/dump.go b/core/state/dump.go index 96a2ac3d3e..2d9c65e473 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -140,7 +140,11 @@ func (s *StateDB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey [] log.Info("Trie dumping started", "root", s.trie.Hash()) c.OnRoot(s.trie.Hash()) - it := trie.NewIterator(s.trie.NodeIterator(conf.Start)) + trieIt, err := s.trie.NodeIterator(conf.Start) + if err != nil { + return nil + } + it := trie.NewIterator(trieIt) for it.Next() { var data types.StateAccount if err := rlp.DecodeBytes(it.Value, &data); err != nil { @@ -178,7 +182,12 @@ func (s *StateDB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey [] log.Error("Failed to load storage trie", "err", err) continue } - storageIt := trie.NewIterator(tr.NodeIterator(nil)) + trieIt, err := tr.NodeIterator(nil) + if err != nil { + log.Error("Failed to create trie iterator", "err", err) + continue + } + storageIt := trie.NewIterator(trieIt) for storageIt.Next() { _, content, _, err := rlp.Split(storageIt.Value) if err != nil { diff --git a/core/state/iterator.go b/core/state/iterator.go index 79bb650abc..7802263b75 100644 --- a/core/state/iterator.go +++ b/core/state/iterator.go @@ -74,8 +74,12 @@ func (it *nodeIterator) step() error { return nil } // Initialize the iterator if we've just started + var err error if it.stateIt == nil { - it.stateIt = it.state.trie.NodeIterator(nil) + it.stateIt, err = it.state.trie.NodeIterator(nil) + if err != nil { + return err + } } // If we had data nodes previously, we surely have at least state nodes if it.dataIt != nil { @@ -113,7 +117,10 @@ func (it *nodeIterator) step() error { if err != nil { return err } - it.dataIt = dataTrie.NodeIterator(nil) + it.dataIt, err = dataTrie.NodeIterator(nil) + if err != nil { + return err + } if !it.dataIt.Next(true) { it.dataIt = nil } diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index aed002f905..1ee5a0e625 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -420,7 +420,10 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error { if err != nil { return err } - accIter := t.NodeIterator(nil) + accIter, err := t.NodeIterator(nil) + if err != nil { + return err + } for accIter.Next(true) { hash := accIter.Hash() @@ -441,7 +444,10 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error { if err != nil { return err } - storageIter := storageTrie.NodeIterator(nil) + storageIter, err := storageTrie.NodeIterator(nil) + if err != nil { + return err + } for storageIter.Next(true) { hash := storageIter.Hash() if hash != (common.Hash{}) { diff --git a/core/state/snapshot/generate.go b/core/state/snapshot/generate.go index 78aa23a5de..1ec82b9442 100644 --- a/core/state/snapshot/generate.go +++ b/core/state/snapshot/generate.go @@ -382,8 +382,6 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, trieId *trie.ID, prefi } var ( trieMore bool - nodeIt = tr.NodeIterator(origin) - iter = trie.NewIterator(nodeIt) kvkeys, kvvals = result.keys, result.vals // counters @@ -397,7 +395,12 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, trieId *trie.ID, prefi start = time.Now() internal time.Duration ) + nodeIt, err := tr.NodeIterator(origin) + if err != nil { + return false, nil, err + } nodeIt.AddResolver(resolver) + iter := trie.NewIterator(nodeIt) for iter.Next() { if last != nil && bytes.Compare(iter.Key, last) > 0 { diff --git a/core/state/state_test.go b/core/state/state_test.go index 15e6037265..555e1f7454 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -43,7 +43,8 @@ func newStateTest() *stateTest { func TestDump(t *testing.T) { db := rawdb.NewMemoryDatabase() - sdb, _ := New(types.EmptyRootHash, NewDatabaseWithConfig(db, &trie.Config{Preimages: true}), nil) + tdb := NewDatabaseWithConfig(db, &trie.Config{Preimages: true}) + sdb, _ := New(types.EmptyRootHash, tdb, nil) s := &stateTest{db: db, state: sdb} // generate a few entries @@ -57,9 +58,10 @@ func TestDump(t *testing.T) { // write some of them to the trie s.state.updateStateObject(obj1) s.state.updateStateObject(obj2) - s.state.Commit(false) + root, _ := s.state.Commit(false) // check that DumpToCollector contains the state objects that are in trie + s.state, _ = New(root, tdb, nil) got := string(s.state.Dump(nil)) want := `{ "root": "71edff0130dd2385947095001c73d9e28d862fc286fca2b922ca6f6f3cddfdd2", @@ -95,7 +97,8 @@ func TestDump(t *testing.T) { func TestIterativeDump(t *testing.T) { db := rawdb.NewMemoryDatabase() - sdb, _ := New(types.EmptyRootHash, NewDatabaseWithConfig(db, &trie.Config{Preimages: true}), nil) + tdb := NewDatabaseWithConfig(db, &trie.Config{Preimages: true}) + sdb, _ := New(types.EmptyRootHash, tdb, nil) s := &stateTest{db: db, state: sdb} // generate a few entries @@ -111,7 +114,8 @@ func TestIterativeDump(t *testing.T) { // write some of them to the trie s.state.updateStateObject(obj1) s.state.updateStateObject(obj2) - s.state.Commit(false) + root, _ := s.state.Commit(false) + s.state, _ = New(root, tdb, nil) b := &bytes.Buffer{} s.state.IterativeDump(nil, json.NewEncoder(b)) diff --git a/core/state/statedb.go b/core/state/statedb.go index 90b32e3fc0..f8457c6de3 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -56,8 +56,14 @@ func (n *proofList) Delete(key []byte) error { // StateDB structs within the ethereum protocol are used to store anything // within the merkle trie. StateDBs take care of caching and storing // nested states. It's the general query interface to retrieve: +// // * Contracts // * Accounts +// +// Once the state is committed, tries cached in stateDB (including account +// trie, storage tries) will no longer be functional. A new state instance +// must be created with new root and updated database for accessing post- +// commit states. type StateDB struct { db Database prefetcher *triePrefetcher @@ -680,19 +686,23 @@ func (s *StateDB) CreateAccount(addr common.Address) { } } -func (db *StateDB) ForEachStorage(addr common.Address, cb func(key, value common.Hash) bool) error { - so := db.getStateObject(addr) +func (s *StateDB) ForEachStorage(addr common.Address, cb func(key, value common.Hash) bool) error { + so := s.getStateObject(addr) if so == nil { return nil } - tr, err := so.getTrie(db.db) + tr, err := so.getTrie(s.db) if err != nil { return err } - it := trie.NewIterator(tr.NodeIterator(nil)) + trieIt, err := tr.NodeIterator(nil) + if err != nil { + return err + } + it := trie.NewIterator(trieIt) for it.Next() { - key := common.BytesToHash(db.trie.GetKey(it.Key)) + key := common.BytesToHash(s.trie.GetKey(it.Key)) if value, dirty := so.dirtyStorage[key]; dirty { if !cb(key, value) { return nil @@ -977,6 +987,10 @@ func (s *StateDB) clearJournalAndRefund() { } // Commit writes the state to the underlying in-memory trie database. +// Once the state is committed, tries cached in stateDB (including account +// trie, storage tries) will no longer be functional. A new state instance +// must be created with new root and updated database for accessing post- +// commit states. func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { // Short circuit in case any database failure occurred earlier. if s.dbErr != nil { diff --git a/core/state/statedb_test.go b/core/state/statedb_test.go index 82118fc5f4..44eace01d8 100644 --- a/core/state/statedb_test.go +++ b/core/state/statedb_test.go @@ -19,6 +19,7 @@ package state import ( "bytes" "encoding/binary" + "errors" "fmt" "math" "math/big" @@ -521,7 +522,8 @@ func TestCopyOfCopy(t *testing.T) { // // See https://github.com/ethereum/go-ethereum/issues/20106. func TestCopyCommitCopy(t *testing.T) { - state, _ := New(types.EmptyRootHash, NewDatabase(rawdb.NewMemoryDatabase()), nil) + tdb := NewDatabase(rawdb.NewMemoryDatabase()) + state, _ := New(types.EmptyRootHash, tdb, nil) // Create an account and check if the retrieved balance is correct addr := common.HexToAddress("0xaffeaffeaffeaffeaffeaffeaffeaffeaffeaffe") @@ -558,20 +560,6 @@ func TestCopyCommitCopy(t *testing.T) { if val := copyOne.GetCommittedState(addr, skey); val != (common.Hash{}) { t.Fatalf("first copy pre-commit committed storage slot mismatch: have %x, want %x", val, common.Hash{}) } - - copyOne.Commit(false) - if balance := copyOne.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { - t.Fatalf("first copy post-commit balance mismatch: have %v, want %v", balance, 42) - } - if code := copyOne.GetCode(addr); !bytes.Equal(code, []byte("hello")) { - t.Fatalf("first copy post-commit code mismatch: have %x, want %x", code, []byte("hello")) - } - if val := copyOne.GetState(addr, skey); val != sval { - t.Fatalf("first copy post-commit non-committed storage slot mismatch: have %x, want %x", val, sval) - } - if val := copyOne.GetCommittedState(addr, skey); val != sval { - t.Fatalf("first copy post-commit committed storage slot mismatch: have %x, want %x", val, sval) - } // Copy the copy and check the balance once more copyTwo := copyOne.Copy() if balance := copyTwo.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { @@ -583,8 +571,23 @@ func TestCopyCommitCopy(t *testing.T) { if val := copyTwo.GetState(addr, skey); val != sval { t.Fatalf("second copy non-committed storage slot mismatch: have %x, want %x", val, sval) } - if val := copyTwo.GetCommittedState(addr, skey); val != sval { - t.Fatalf("second copy post-commit committed storage slot mismatch: have %x, want %x", val, sval) + if val := copyTwo.GetCommittedState(addr, skey); val != (common.Hash{}) { + t.Fatalf("second copy committed storage slot mismatch: have %x, want %x", val, sval) + } + // Commit state, ensure states can be loaded from disk + root, _ := state.Commit(false) + state, _ = New(root, tdb, nil) + if balance := state.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { + t.Fatalf("state post-commit balance mismatch: have %v, want %v", balance, 42) + } + if code := state.GetCode(addr); !bytes.Equal(code, []byte("hello")) { + t.Fatalf("state post-commit code mismatch: have %x, want %x", code, []byte("hello")) + } + if val := state.GetState(addr, skey); val != sval { + t.Fatalf("state post-commit non-committed storage slot mismatch: have %x, want %x", val, sval) + } + if val := state.GetCommittedState(addr, skey); val != sval { + t.Fatalf("state post-commit committed storage slot mismatch: have %x, want %x", val, sval) } } @@ -644,19 +647,6 @@ func TestCopyCopyCommitCopy(t *testing.T) { if val := copyTwo.GetCommittedState(addr, skey); val != (common.Hash{}) { t.Fatalf("second copy pre-commit committed storage slot mismatch: have %x, want %x", val, common.Hash{}) } - copyTwo.Commit(false) - if balance := copyTwo.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { - t.Fatalf("second copy post-commit balance mismatch: have %v, want %v", balance, 42) - } - if code := copyTwo.GetCode(addr); !bytes.Equal(code, []byte("hello")) { - t.Fatalf("second copy post-commit code mismatch: have %x, want %x", code, []byte("hello")) - } - if val := copyTwo.GetState(addr, skey); val != sval { - t.Fatalf("second copy post-commit non-committed storage slot mismatch: have %x, want %x", val, sval) - } - if val := copyTwo.GetCommittedState(addr, skey); val != sval { - t.Fatalf("second copy post-commit committed storage slot mismatch: have %x, want %x", val, sval) - } // Copy the copy-copy and check the balance once more copyThree := copyTwo.Copy() if balance := copyThree.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { @@ -668,11 +658,56 @@ func TestCopyCopyCommitCopy(t *testing.T) { if val := copyThree.GetState(addr, skey); val != sval { t.Fatalf("third copy non-committed storage slot mismatch: have %x, want %x", val, sval) } - if val := copyThree.GetCommittedState(addr, skey); val != sval { + if val := copyThree.GetCommittedState(addr, skey); val != (common.Hash{}) { t.Fatalf("third copy committed storage slot mismatch: have %x, want %x", val, sval) } } +// TestCommitCopy tests the copy from a committed state is not functional. +func TestCommitCopy(t *testing.T) { + state, _ := New(types.EmptyRootHash, NewDatabase(rawdb.NewMemoryDatabase()), nil) + + // Create an account and check if the retrieved balance is correct + addr := common.HexToAddress("0xaffeaffeaffeaffeaffeaffeaffeaffeaffeaffe") + skey := common.HexToHash("aaa") + sval := common.HexToHash("bbb") + + state.SetBalance(addr, big.NewInt(42)) // Change the account trie + state.SetCode(addr, []byte("hello")) // Change an external metadata + state.SetState(addr, skey, sval) // Change the storage trie + + if balance := state.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { + t.Fatalf("initial balance mismatch: have %v, want %v", balance, 42) + } + if code := state.GetCode(addr); !bytes.Equal(code, []byte("hello")) { + t.Fatalf("initial code mismatch: have %x, want %x", code, []byte("hello")) + } + if val := state.GetState(addr, skey); val != sval { + t.Fatalf("initial non-committed storage slot mismatch: have %x, want %x", val, sval) + } + if val := state.GetCommittedState(addr, skey); val != (common.Hash{}) { + t.Fatalf("initial committed storage slot mismatch: have %x, want %x", val, common.Hash{}) + } + // Copy the committed state database, the copied one is not functional. + state.Commit(true) + copied := state.Copy() + if balance := copied.GetBalance(addr); balance.Cmp(big.NewInt(0)) != 0 { + t.Fatalf("unexpected balance: have %v", balance) + } + if code := copied.GetCode(addr); code != nil { + t.Fatalf("unexpected code: have %x", code) + } + if val := copied.GetState(addr, skey); val != (common.Hash{}) { + t.Fatalf("unexpected storage slot: have %x", val) + } + if val := copied.GetCommittedState(addr, skey); val != (common.Hash{}) { + t.Fatalf("unexpected storage slot: have %x", val) + } + if !errors.Is(copied.Error(), trie.ErrCommitted) { + t.Fatalf("unexpected state error, %v", copied.Error()) + } +} + // TestDeleteCreateRevert tests a weird state transition corner case that we hit // while changing the internals of StateDB. The workflow is that a contract is // self-destructed, then in a follow-up transaction (but same block) it's created diff --git a/core/state/sync_test.go b/core/state/sync_test.go index 6e9d9342ee..cac0ee6e48 100644 --- a/core/state/sync_test.go +++ b/core/state/sync_test.go @@ -109,7 +109,7 @@ func checkTrieConsistency(db ethdb.Database, root common.Hash) error { if err != nil { return err } - it := trie.NodeIterator(nil) + it := trie.MustNodeIterator(nil) for it.Next(true) { } return it.Error() @@ -566,6 +566,10 @@ func TestIncompleteStateSync(t *testing.T) { addedPaths []string addedHashes []common.Hash ) + reader, err := srcDb.TrieDB().Reader(srcRoot) + if err != nil { + t.Fatalf("state is not available %x", srcRoot) + } nodeQueue := make(map[string]stateElement) codeQueue := make(map[common.Hash]struct{}) paths, nodes, codes := sched.Missing(1) @@ -603,7 +607,7 @@ func TestIncompleteStateSync(t *testing.T) { results := make([]trie.NodeSyncResult, 0, len(nodeQueue)) for path, element := range nodeQueue { owner, inner := trie.ResolvePath([]byte(element.path)) - data, err := srcDb.TrieDB().Reader(srcRoot).Node(owner, inner, element.hash) + data, err := reader.Node(owner, inner, element.hash) if err != nil { t.Fatalf("failed to retrieve node data for %x", element.hash) } diff --git a/eth/api_debug.go b/eth/api_debug.go index 6f9daadd6b..c25e024cb2 100644 --- a/eth/api_debug.go +++ b/eth/api_debug.go @@ -235,7 +235,11 @@ func (api *DebugAPI) StorageRangeAt(ctx context.Context, blockNrOrHash rpc.Block } func storageRangeAt(st state.Trie, start []byte, maxResult int) (StorageRangeResult, error) { - it := trie.NewIterator(st.NodeIterator(start)) + trieIt, err := st.NodeIterator(start) + if err != nil { + return StorageRangeResult{}, err + } + it := trie.NewIterator(trieIt) result := StorageRangeResult{Storage: storageMap{}} for i := 0; i < maxResult && it.Next(); i++ { _, content, _, err := rlp.Split(it.Value) @@ -326,7 +330,15 @@ func (api *DebugAPI) getModifiedAccounts(startBlock, endBlock *types.Block) ([]c if err != nil { return nil, err } - diff, _ := trie.NewDifferenceIterator(oldTrie.NodeIterator([]byte{}), newTrie.NodeIterator([]byte{})) + oldIt, err := oldTrie.NodeIterator([]byte{}) + if err != nil { + return nil, err + } + newIt, err := newTrie.NodeIterator([]byte{}) + if err != nil { + return nil, err + } + diff, _ := trie.NewDifferenceIterator(oldIt, newIt) iter := trie.NewIterator(diff) var dirty []common.Address diff --git a/eth/api_debug_test.go b/eth/api_debug_test.go index c7f0d4ac90..ee54e4c3c8 100644 --- a/eth/api_debug_test.go +++ b/eth/api_debug_test.go @@ -62,34 +62,34 @@ func TestAccountRange(t *testing.T) { t.Parallel() var ( - statedb = state.NewDatabaseWithConfig(rawdb.NewMemoryDatabase(), &trie.Config{Preimages: true}) - state, _ = state.New(types.EmptyRootHash, statedb, nil) - addrs = [AccountRangeMaxResults * 2]common.Address{} - m = map[common.Address]bool{} + statedb = state.NewDatabaseWithConfig(rawdb.NewMemoryDatabase(), &trie.Config{Preimages: true}) + sdb, _ = state.New(types.EmptyRootHash, statedb, nil) + addrs = [AccountRangeMaxResults * 2]common.Address{} + m = map[common.Address]bool{} ) for i := range addrs { hash := common.HexToHash(fmt.Sprintf("%x", i)) addr := common.BytesToAddress(crypto.Keccak256Hash(hash.Bytes()).Bytes()) addrs[i] = addr - state.SetBalance(addrs[i], big.NewInt(1)) + sdb.SetBalance(addrs[i], big.NewInt(1)) if _, ok := m[addr]; ok { t.Fatalf("bad") } else { m[addr] = true } } - state.Commit(true) - root := state.IntermediateRoot(true) + root, _ := sdb.Commit(true) + sdb, _ = state.New(root, statedb, nil) trie, err := statedb.OpenTrie(root) if err != nil { t.Fatal(err) } - accountRangeTest(t, &trie, state, common.Hash{}, AccountRangeMaxResults/2, AccountRangeMaxResults/2) + accountRangeTest(t, &trie, sdb, common.Hash{}, AccountRangeMaxResults/2, AccountRangeMaxResults/2) // test pagination - firstResult := accountRangeTest(t, &trie, state, common.Hash{}, AccountRangeMaxResults, AccountRangeMaxResults) - secondResult := accountRangeTest(t, &trie, state, common.BytesToHash(firstResult.Next), AccountRangeMaxResults, AccountRangeMaxResults) + firstResult := accountRangeTest(t, &trie, sdb, common.Hash{}, AccountRangeMaxResults, AccountRangeMaxResults) + secondResult := accountRangeTest(t, &trie, sdb, common.BytesToHash(firstResult.Next), AccountRangeMaxResults, AccountRangeMaxResults) hList := make([]common.Hash, 0) for addr1 := range firstResult.Accounts { @@ -107,7 +107,7 @@ func TestAccountRange(t *testing.T) { // set and get an even split between the first and second sets. slices.SortFunc(hList, common.Hash.Less) middleH := hList[AccountRangeMaxResults/2] - middleResult := accountRangeTest(t, &trie, state, middleH, AccountRangeMaxResults, AccountRangeMaxResults) + middleResult := accountRangeTest(t, &trie, sdb, middleH, AccountRangeMaxResults, AccountRangeMaxResults) missing, infirst, insecond := 0, 0, 0 for h := range middleResult.Accounts { if _, ok := firstResult.Accounts[h]; ok { @@ -136,8 +136,10 @@ func TestEmptyAccountRange(t *testing.T) { statedb = state.NewDatabase(rawdb.NewMemoryDatabase()) st, _ = state.New(types.EmptyRootHash, statedb, nil) ) + // Commit(although nothing to flush) and re-init the statedb st.Commit(true) - st.IntermediateRoot(true) + st, _ = state.New(types.EmptyRootHash, statedb, nil) + results := st.IteratorDump(&state.DumpConfig{ SkipCode: true, SkipStorage: true, diff --git a/eth/protocols/snap/sync_test.go b/eth/protocols/snap/sync_test.go index 6efe0c8325..173f1da604 100644 --- a/eth/protocols/snap/sync_test.go +++ b/eth/protocols/snap/sync_test.go @@ -1664,7 +1664,7 @@ func verifyTrie(db ethdb.KeyValueStore, root common.Hash, t *testing.T) { t.Fatal(err) } accounts, slots := 0, 0 - accIt := trie.NewIterator(accTrie.NodeIterator(nil)) + accIt := trie.NewIterator(accTrie.MustNodeIterator(nil)) for accIt.Next() { var acc struct { Nonce uint64 @@ -1682,7 +1682,7 @@ func verifyTrie(db ethdb.KeyValueStore, root common.Hash, t *testing.T) { if err != nil { t.Fatal(err) } - storeIt := trie.NewIterator(storeTrie.NodeIterator(nil)) + storeIt := trie.NewIterator(storeTrie.MustNodeIterator(nil)) for storeIt.Next() { slots++ } diff --git a/light/trie.go b/light/trie.go index 888598ca92..e0675f98c9 100644 --- a/light/trie.go +++ b/light/trie.go @@ -184,8 +184,8 @@ func (t *odrTrie) Hash() common.Hash { return t.trie.Hash() } -func (t *odrTrie) NodeIterator(startkey []byte) trie.NodeIterator { - return newNodeIterator(t, startkey) +func (t *odrTrie) NodeIterator(startkey []byte) (trie.NodeIterator, error) { + return newNodeIterator(t, startkey), nil } func (t *odrTrie) GetKey(sha []byte) []byte { @@ -248,7 +248,11 @@ func newNodeIterator(t *odrTrie, startkey []byte) trie.NodeIterator { }) } it.do(func() error { - it.NodeIterator = it.t.trie.NodeIterator(startkey) + var err error + it.NodeIterator, err = it.t.trie.NodeIterator(startkey) + if err != nil { + return err + } return it.NodeIterator.Error() }) return it diff --git a/light/trie_test.go b/light/trie_test.go index 0a24609cbd..ad7d769c84 100644 --- a/light/trie_test.go +++ b/light/trie_test.go @@ -62,8 +62,16 @@ func TestNodeIterator(t *testing.T) { } func diffTries(t1, t2 state.Trie) error { - i1 := trie.NewIterator(t1.NodeIterator(nil)) - i2 := trie.NewIterator(t2.NodeIterator(nil)) + trieIt1, err := t1.NodeIterator(nil) + if err != nil { + return err + } + trieIt2, err := t2.NodeIterator(nil) + if err != nil { + return err + } + i1 := trie.NewIterator(trieIt1) + i2 := trie.NewIterator(trieIt2) for i1.Next() && i2.Next() { if !bytes.Equal(i1.Key, i2.Key) { spew.Dump(i2) diff --git a/tests/fuzzers/stacktrie/trie_fuzzer.go b/tests/fuzzers/stacktrie/trie_fuzzer.go index 77e602f807..d7be54eda0 100644 --- a/tests/fuzzers/stacktrie/trie_fuzzer.go +++ b/tests/fuzzers/stacktrie/trie_fuzzer.go @@ -221,7 +221,7 @@ func (f *fuzzer) fuzz() int { panic(fmt.Sprintf("roots differ: (trie) %x != %x (stacktrie)", rootA, rootC)) } trieA, _ = trie.New(trie.TrieID(rootA), dbA) - iterA := trieA.NodeIterator(nil) + iterA := trieA.MustNodeIterator(nil) for iterA.Next(true) { if iterA.Hash() == (common.Hash{}) { if _, present := nodeset[string(iterA.Path())]; present { diff --git a/tests/fuzzers/trie/trie-fuzzer.go b/tests/fuzzers/trie/trie-fuzzer.go index 0926d04764..33edc2a18a 100644 --- a/tests/fuzzers/trie/trie-fuzzer.go +++ b/tests/fuzzers/trie/trie-fuzzer.go @@ -179,7 +179,7 @@ func runRandTest(rt randTest) error { origin = hash case opItercheckhash: checktr := trie.NewEmpty(triedb) - it := trie.NewIterator(tr.NodeIterator(nil)) + it := trie.NewIterator(tr.MustNodeIterator(nil)) for it.Next() { checktr.MustUpdate(it.Key, it.Value) } diff --git a/tests/state_test_util.go b/tests/state_test_util.go index eec91b67a7..667e951b66 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -204,6 +204,11 @@ func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config, snapshotter bo if logs := rlpHash(statedb.Logs()); logs != common.Hash(post.Logs) { return snaps, statedb, fmt.Errorf("post state logs hash mismatch: got %x, want %x", logs, post.Logs) } + // Re-init the post-state instance for further operation + statedb, err = state.New(root, statedb.Database(), snaps) + if err != nil { + return nil, nil, err + } return snaps, statedb, nil } @@ -275,9 +280,7 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh // the coinbase gets no txfee, so isn't created, and thus needs to be touched statedb.AddBalance(block.Coinbase(), new(big.Int)) // Commit block - statedb.Commit(config.IsEIP158(block.Number())) - // And _now_ get the state root - root := statedb.IntermediateRoot(config.IsEIP158(block.Number())) + root, _ := statedb.Commit(config.IsEIP158(block.Number())) return snaps, statedb, root, err } diff --git a/trie/database_wrap.go b/trie/database_wrap.go index b05f56847a..30835b7d25 100644 --- a/trie/database_wrap.go +++ b/trie/database_wrap.go @@ -113,8 +113,8 @@ func NewDatabaseWithConfig(diskdb ethdb.Database, config *Config) *Database { } // Reader returns a reader for accessing all trie nodes with provided state root. -// Nil is returned in case the state is not available. -func (db *Database) Reader(blockRoot common.Hash) Reader { +// An error will be returned if the requested state is not available. +func (db *Database) Reader(blockRoot common.Hash) (Reader, error) { return db.backend.(*hashdb.Database).Reader(blockRoot) } diff --git a/trie/errors.go b/trie/errors.go index bd82b950aa..7be7041c7f 100644 --- a/trie/errors.go +++ b/trie/errors.go @@ -17,11 +17,17 @@ package trie import ( + "errors" "fmt" "github.com/ethereum/go-ethereum/common" ) +// ErrCommitted is returned when a already committed trie is requested for usage. +// The potential usages can be `Get`, `Update`, `Delete`, `NodeIterator`, `Prove` +// and so on. +var ErrCommitted = errors.New("trie is already committed") + // MissingNodeError is returned by the trie functions (Get, Update, Delete) // in the case where a trie node is not present in the local database. It contains // information necessary for retrieving the missing node. diff --git a/trie/iterator_test.go b/trie/iterator_test.go index ccf5ea0258..5c7f9c98fb 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -34,7 +34,7 @@ import ( func TestEmptyIterator(t *testing.T) { trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) - iter := trie.NodeIterator(nil) + iter := trie.MustNodeIterator(nil) seen := make(map[string]struct{}) for iter.Next(true) { @@ -67,7 +67,7 @@ func TestIterator(t *testing.T) { trie, _ = New(TrieID(root), db) found := make(map[string]string) - it := NewIterator(trie.NodeIterator(nil)) + it := NewIterator(trie.MustNodeIterator(nil)) for it.Next() { found[string(it.Key)] = string(it.Value) } @@ -101,7 +101,7 @@ func TestIteratorLargeData(t *testing.T) { vals[string(value2.k)] = value2 } - it := NewIterator(trie.NodeIterator(nil)) + it := NewIterator(trie.MustNodeIterator(nil)) for it.Next() { vals[string(it.Key)].t = true } @@ -139,7 +139,7 @@ func testNodeIteratorCoverage(t *testing.T, scheme string) { // Gather all the node hashes found by the iterator var elements = make(map[common.Hash]iterationElement) - for it := trie.NodeIterator(nil); it.Next(true); { + for it := trie.MustNodeIterator(nil); it.Next(true); { if it.Hash() != (common.Hash{}) { elements[it.Hash()] = iterationElement{ hash: it.Hash(), @@ -149,8 +149,12 @@ func testNodeIteratorCoverage(t *testing.T, scheme string) { } } // Cross check the hashes and the database itself + reader, err := nodeDb.Reader(trie.Hash()) + if err != nil { + t.Fatalf("state is not available %x", trie.Hash()) + } for _, element := range elements { - if blob, err := nodeDb.Reader(trie.Hash()).Node(common.Hash{}, element.path, element.hash); err != nil { + if blob, err := reader.Node(common.Hash{}, element.path, element.hash); err != nil { t.Errorf("failed to retrieve reported node %x: %v", element.hash, err) } else if !bytes.Equal(blob, element.blob) { t.Errorf("node blob is different, want %v got %v", element.blob, blob) @@ -210,19 +214,19 @@ func TestIteratorSeek(t *testing.T) { } // Seek to the middle. - it := NewIterator(trie.NodeIterator([]byte("fab"))) + it := NewIterator(trie.MustNodeIterator([]byte("fab"))) if err := checkIteratorOrder(testdata1[4:], it); err != nil { t.Fatal(err) } // Seek to a non-existent key. - it = NewIterator(trie.NodeIterator([]byte("barc"))) + it = NewIterator(trie.MustNodeIterator([]byte("barc"))) if err := checkIteratorOrder(testdata1[1:], it); err != nil { t.Fatal(err) } // Seek beyond the end. - it = NewIterator(trie.NodeIterator([]byte("z"))) + it = NewIterator(trie.MustNodeIterator([]byte("z"))) if err := checkIteratorOrder(nil, it); err != nil { t.Fatal(err) } @@ -264,7 +268,7 @@ func TestDifferenceIterator(t *testing.T) { trieb, _ = New(TrieID(rootB), dbb) found := make(map[string]string) - di, _ := NewDifferenceIterator(triea.NodeIterator(nil), trieb.NodeIterator(nil)) + di, _ := NewDifferenceIterator(triea.MustNodeIterator(nil), trieb.MustNodeIterator(nil)) it := NewIterator(di) for it.Next() { found[string(it.Key)] = string(it.Value) @@ -305,7 +309,7 @@ func TestUnionIterator(t *testing.T) { dbb.Update(rootB, types.EmptyRootHash, trienode.NewWithNodeSet(nodesB)) trieb, _ = New(TrieID(rootB), dbb) - di, _ := NewUnionIterator([]NodeIterator{triea.NodeIterator(nil), trieb.NodeIterator(nil)}) + di, _ := NewUnionIterator([]NodeIterator{triea.MustNodeIterator(nil), trieb.MustNodeIterator(nil)}) it := NewIterator(di) all := []struct{ k, v string }{ @@ -344,7 +348,7 @@ func TestIteratorNoDups(t *testing.T) { for _, val := range testdata1 { tr.MustUpdate([]byte(val.k), []byte(val.v)) } - checkIteratorNoDups(t, tr.NodeIterator(nil), nil) + checkIteratorNoDups(t, tr.MustNodeIterator(nil), nil) } // This test checks that nodeIterator.Next can be retried after inserting missing trie nodes. @@ -369,7 +373,7 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool, scheme string) { tdb.Commit(root, false) } tr, _ = New(TrieID(root), tdb) - wantNodeCount := checkIteratorNoDups(t, tr.NodeIterator(nil), nil) + wantNodeCount := checkIteratorNoDups(t, tr.MustNodeIterator(nil), nil) var ( paths [][]byte @@ -428,7 +432,7 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool, scheme string) { } // Iterate until the error is hit. seen := make(map[string]bool) - it := tr.NodeIterator(nil) + it := tr.MustNodeIterator(nil) checkIteratorNoDups(t, it, seen) missing, ok := it.Error().(*MissingNodeError) if !ok || missing.NodeHash != rhash { @@ -496,7 +500,7 @@ func testIteratorContinueAfterSeekError(t *testing.T, memonly bool, scheme strin } // Create a new iterator that seeks to "bars". Seeking can't proceed because // the node is missing. - it := tr.NodeIterator([]byte("bars")) + it := tr.MustNodeIterator([]byte("bars")) missing, ok := it.Error().(*MissingNodeError) if !ok { t.Fatal("want MissingNodeError, got", it.Error()) @@ -602,7 +606,10 @@ func makeLargeTestTrie() (*Database, *StateTrie, *loggingDb) { } root, nodes := trie.Commit(false) triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + triedb.Commit(root, false) + // Return the generated trie + trie, _ = NewStateTrie(TrieID(root), triedb) return triedb, trie, logDb } @@ -614,8 +621,8 @@ func TestNodeIteratorLargeTrie(t *testing.T) { // Do a seek operation trie.NodeIterator(common.FromHex("0x77667766776677766778855885885885")) // master: 24 get operations - // this pr: 5 get operations - if have, want := logDb.getCount, uint64(5); have != want { + // this pr: 6 get operations + if have, want := logDb.getCount, uint64(6); have != want { t.Fatalf("Too many lookups during seek, have %d want %d", have, want) } } @@ -646,7 +653,7 @@ func testIteratorNodeBlob(t *testing.T, scheme string) { var found = make(map[common.Hash][]byte) trie, _ = New(TrieID(root), triedb) - it := trie.NodeIterator(nil) + it := trie.MustNodeIterator(nil) for it.Next(true) { if it.Hash() == (common.Hash{}) { continue diff --git a/trie/proof.go b/trie/proof.go index 818e338892..a463c80b48 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -34,6 +34,10 @@ import ( // nodes of the longest existing prefix of the key (at least the root node), ending // with the node that proves the absence of the key. func (t *Trie) Prove(key []byte, proofDb ethdb.KeyValueWriter) error { + // Short circuit if the trie is already committed and not usable. + if t.committed { + return ErrCommitted + } // Collect all nodes on the path to key. var ( prefix []byte diff --git a/trie/proof_test.go b/trie/proof_test.go index 6af2c3e472..d8effd0d04 100644 --- a/trie/proof_test.go +++ b/trie/proof_test.go @@ -63,7 +63,7 @@ func makeProvers(trie *Trie) []func(key []byte) *memorydb.Database { // Create a leaf iterator based Merkle prover provers = append(provers, func(key []byte) *memorydb.Database { proof := memorydb.New() - if it := NewIterator(trie.NodeIterator(key)); it.Next() && bytes.Equal(key, it.Key) { + if it := NewIterator(trie.MustNodeIterator(key)); it.Next() && bytes.Equal(key, it.Key) { for _, p := range it.Prove() { proof.Put(crypto.Keccak256(p), p) } diff --git a/trie/secure_trie.go b/trie/secure_trie.go index 62666c7f0c..a6d5676197 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -250,12 +250,18 @@ func (t *StateTrie) Copy() *StateTrie { } } -// NodeIterator returns an iterator that returns nodes of the underlying trie. Iteration -// starts at the key after the given start key. -func (t *StateTrie) NodeIterator(start []byte) NodeIterator { +// NodeIterator returns an iterator that returns nodes of the underlying trie. +// Iteration starts at the key after the given start key. +func (t *StateTrie) NodeIterator(start []byte) (NodeIterator, error) { return t.trie.NodeIterator(start) } +// MustNodeIterator is a wrapper of NodeIterator and will omit any encountered +// error but just print out an error message. +func (t *StateTrie) MustNodeIterator(start []byte) NodeIterator { + return t.trie.MustNodeIterator(start) +} + // hashKey returns the hash of key as an ephemeral buffer. // The caller must not hold onto the return value because it will become // invalid on the next call to hashKey or secKey. diff --git a/trie/sync_test.go b/trie/sync_test.go index 7cd4058176..ec49beef04 100644 --- a/trie/sync_test.go +++ b/trie/sync_test.go @@ -94,7 +94,7 @@ func checkTrieConsistency(db ethdb.Database, scheme string, root common.Hash) er if err != nil { return nil // Consider a non existent state consistent } - it := trie.NodeIterator(nil) + it := trie.MustNodeIterator(nil) for it.Next(true) { } return it.Error() @@ -159,12 +159,16 @@ func testIterativeSync(t *testing.T, count int, bypath bool, scheme string) { syncPath: NewSyncPath([]byte(paths[i])), }) } + reader, err := srcDb.Reader(srcTrie.Hash()) + if err != nil { + t.Fatalf("State is not available %x", srcTrie.Hash()) + } for len(elements) > 0 { results := make([]NodeSyncResult, len(elements)) if !bypath { for i, element := range elements { owner, inner := ResolvePath([]byte(element.path)) - data, err := srcDb.Reader(srcTrie.Hash()).Node(owner, inner, element.hash) + data, err := reader.Node(owner, inner, element.hash) if err != nil { t.Fatalf("failed to retrieve node data for hash %x: %v", element.hash, err) } @@ -230,12 +234,16 @@ func testIterativeDelayedSync(t *testing.T, scheme string) { syncPath: NewSyncPath([]byte(paths[i])), }) } + reader, err := srcDb.Reader(srcTrie.Hash()) + if err != nil { + t.Fatalf("State is not available %x", srcTrie.Hash()) + } for len(elements) > 0 { // Sync only half of the scheduled nodes results := make([]NodeSyncResult, len(elements)/2+1) for i, element := range elements[:len(results)] { owner, inner := ResolvePath([]byte(element.path)) - data, err := srcDb.Reader(srcTrie.Hash()).Node(owner, inner, element.hash) + data, err := reader.Node(owner, inner, element.hash) if err != nil { t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err) } @@ -295,12 +303,16 @@ func testIterativeRandomSync(t *testing.T, count int, scheme string) { syncPath: NewSyncPath([]byte(paths[i])), } } + reader, err := srcDb.Reader(srcTrie.Hash()) + if err != nil { + t.Fatalf("State is not available %x", srcTrie.Hash()) + } for len(queue) > 0 { // Fetch all the queued nodes in a random order results := make([]NodeSyncResult, 0, len(queue)) for path, element := range queue { owner, inner := ResolvePath([]byte(element.path)) - data, err := srcDb.Reader(srcTrie.Hash()).Node(owner, inner, element.hash) + data, err := reader.Node(owner, inner, element.hash) if err != nil { t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err) } @@ -358,12 +370,16 @@ func testIterativeRandomDelayedSync(t *testing.T, scheme string) { syncPath: NewSyncPath([]byte(path)), } } + reader, err := srcDb.Reader(srcTrie.Hash()) + if err != nil { + t.Fatalf("State is not available %x", srcTrie.Hash()) + } for len(queue) > 0 { // Sync only half of the scheduled nodes, even those in random order results := make([]NodeSyncResult, 0, len(queue)/2+1) for path, element := range queue { owner, inner := ResolvePath([]byte(element.path)) - data, err := srcDb.Reader(srcTrie.Hash()).Node(owner, inner, element.hash) + data, err := reader.Node(owner, inner, element.hash) if err != nil { t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err) } @@ -426,13 +442,16 @@ func testDuplicateAvoidanceSync(t *testing.T, scheme string) { syncPath: NewSyncPath([]byte(paths[i])), }) } + reader, err := srcDb.Reader(srcTrie.Hash()) + if err != nil { + t.Fatalf("State is not available %x", srcTrie.Hash()) + } requested := make(map[common.Hash]struct{}) - for len(elements) > 0 { results := make([]NodeSyncResult, len(elements)) for i, element := range elements { owner, inner := ResolvePath([]byte(element.path)) - data, err := srcDb.Reader(srcTrie.Hash()).Node(owner, inner, element.hash) + data, err := reader.Node(owner, inner, element.hash) if err != nil { t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err) } @@ -501,12 +520,16 @@ func testIncompleteSync(t *testing.T, scheme string) { syncPath: NewSyncPath([]byte(paths[i])), }) } + reader, err := srcDb.Reader(srcTrie.Hash()) + if err != nil { + t.Fatalf("State is not available %x", srcTrie.Hash()) + } for len(elements) > 0 { // Fetch a batch of trie nodes results := make([]NodeSyncResult, len(elements)) for i, element := range elements { owner, inner := ResolvePath([]byte(element.path)) - data, err := srcDb.Reader(srcTrie.Hash()).Node(owner, inner, element.hash) + data, err := reader.Node(owner, inner, element.hash) if err != nil { t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err) } @@ -585,12 +608,15 @@ func testSyncOrdering(t *testing.T, scheme string) { }) reqs = append(reqs, NewSyncPath([]byte(paths[i]))) } - + reader, err := srcDb.Reader(srcTrie.Hash()) + if err != nil { + t.Fatalf("State is not available %x", srcTrie.Hash()) + } for len(elements) > 0 { results := make([]NodeSyncResult, len(elements)) for i, element := range elements { owner, inner := ResolvePath([]byte(element.path)) - data, err := srcDb.Reader(srcTrie.Hash()).Node(owner, inner, element.hash) + data, err := reader.Node(owner, inner, element.hash) if err != nil { t.Fatalf("failed to retrieve node data for %x: %v", element.hash, err) } @@ -649,11 +675,15 @@ func syncWith(t *testing.T, root common.Hash, db ethdb.Database, srcDb *Database syncPath: NewSyncPath([]byte(paths[i])), }) } + reader, err := srcDb.Reader(root) + if err != nil { + t.Fatalf("State is not available %x", root) + } for len(elements) > 0 { results := make([]NodeSyncResult, len(elements)) for i, element := range elements { owner, inner := ResolvePath([]byte(element.path)) - data, err := srcDb.Reader(root).Node(owner, inner, element.hash) + data, err := reader.Node(owner, inner, element.hash) if err != nil { t.Fatalf("failed to retrieve node data for hash %x: %v", element.hash, err) } diff --git a/trie/tracer_test.go b/trie/tracer_test.go index 32949a5a99..4cec53688e 100644 --- a/trie/tracer_test.go +++ b/trie/tracer_test.go @@ -226,14 +226,14 @@ func TestAccessListLeak(t *testing.T) { }{ { func(tr *Trie) { - it := tr.NodeIterator(nil) + it := tr.MustNodeIterator(nil) for it.Next(true) { } }, }, { func(tr *Trie) { - it := NewIterator(tr.NodeIterator(nil)) + it := NewIterator(tr.MustNodeIterator(nil)) for it.Next() { } }, @@ -300,7 +300,7 @@ func compareSet(setA, setB map[string]struct{}) bool { func forNodes(tr *Trie) map[string][]byte { var ( - it = tr.NodeIterator(nil) + it = tr.MustNodeIterator(nil) nodes = make(map[string][]byte) ) for it.Next(true) { @@ -319,7 +319,7 @@ func iterNodes(db *Database, root common.Hash) map[string][]byte { func forHashedNodes(tr *Trie) map[string][]byte { var ( - it = tr.NodeIterator(nil) + it = tr.MustNodeIterator(nil) nodes = make(map[string][]byte) ) for it.Next(true) { diff --git a/trie/trie.go b/trie/trie.go index e63b2d2830..cf5d181ff7 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -39,6 +39,10 @@ type Trie struct { root node owner common.Hash + // Flag whether the commit operation is already performed. If so the + // trie is not usable(latest states is invisible). + committed bool + // Keep track of the number leaves which have been inserted since the last // hashing operation. This number will not directly map to the number of // actually unhashed nodes. @@ -60,11 +64,12 @@ func (t *Trie) newFlag() nodeFlag { // Copy returns a copy of Trie. func (t *Trie) Copy() *Trie { return &Trie{ - root: t.root, - owner: t.owner, - unhashed: t.unhashed, - reader: t.reader, - tracer: t.tracer.copy(), + root: t.root, + owner: t.owner, + committed: t.committed, + unhashed: t.unhashed, + reader: t.reader, + tracer: t.tracer.copy(), } } @@ -74,7 +79,7 @@ func (t *Trie) Copy() *Trie { // zero hash or the sha3 hash of an empty string, then trie is initially // empty, otherwise, the root node must be present in database or returns // a MissingNodeError if not. -func New(id *ID, db NodeReader) (*Trie, error) { +func New(id *ID, db *Database) (*Trie, error) { reader, err := newTrieReader(id.StateRoot, id.Owner, db) if err != nil { return nil, err @@ -100,10 +105,24 @@ func NewEmpty(db *Database) *Trie { return tr } +// MustNodeIterator is a wrapper of NodeIterator and will omit any encountered +// error but just print out an error message. +func (t *Trie) MustNodeIterator(start []byte) NodeIterator { + it, err := t.NodeIterator(start) + if err != nil { + log.Error("Unhandled trie error in Trie.NodeIterator", "err", err) + } + return it +} + // NodeIterator returns an iterator that returns nodes of the trie. Iteration starts at // the key after the given start key. -func (t *Trie) NodeIterator(start []byte) NodeIterator { - return newNodeIterator(t, start) +func (t *Trie) NodeIterator(start []byte) (NodeIterator, error) { + // Short circuit if the trie is already committed and not usable. + if t.committed { + return nil, ErrCommitted + } + return newNodeIterator(t, start), nil } // MustGet is a wrapper of Get and will omit any encountered error but just @@ -122,6 +141,10 @@ func (t *Trie) MustGet(key []byte) []byte { // If the requested node is not present in trie, no error will be returned. // If the trie is corrupted, a MissingNodeError is returned. func (t *Trie) Get(key []byte) ([]byte, error) { + // Short circuit if the trie is already committed and not usable. + if t.committed { + return nil, ErrCommitted + } value, newroot, didResolve, err := t.get(t.root, keybytesToHex(key), 0) if err == nil && didResolve { t.root = newroot @@ -181,6 +204,10 @@ func (t *Trie) MustGetNode(path []byte) ([]byte, int) { // If the requested node is not present in trie, no error will be returned. // If the trie is corrupted, a MissingNodeError is returned. func (t *Trie) GetNode(path []byte) ([]byte, int, error) { + // Short circuit if the trie is already committed and not usable. + if t.committed { + return nil, 0, ErrCommitted + } item, newroot, resolved, err := t.getNode(t.root, compactToHex(path), 0) if err != nil { return nil, resolved, err @@ -273,6 +300,10 @@ func (t *Trie) MustUpdate(key, value []byte) { // If the requested node is not present in trie, no error will be returned. // If the trie is corrupted, a MissingNodeError is returned. func (t *Trie) Update(key, value []byte) error { + // Short circuit if the trie is already committed and not usable. + if t.committed { + return ErrCommitted + } return t.update(key, value) } @@ -387,6 +418,10 @@ func (t *Trie) MustDelete(key []byte) { // If the requested node is not present in trie, no error will be returned. // If the trie is corrupted, a MissingNodeError is returned. func (t *Trie) Delete(key []byte) error { + // Short circuit if the trie is already committed and not usable. + if t.committed { + return ErrCommitted + } t.unhashed++ k := keybytesToHex(key) _, n, err := t.delete(t.root, nil, k) @@ -574,7 +609,9 @@ func (t *Trie) Hash() common.Hash { // be created with new root and updated trie database for following usage func (t *Trie) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet) { defer t.tracer.reset() - + defer func() { + t.committed = true + }() nodes := trienode.NewNodeSet(t.owner) t.tracer.markDeletions(nodes) @@ -621,4 +658,5 @@ func (t *Trie) Reset() { t.owner = common.Hash{} t.unhashed = 0 t.tracer.reset() + t.committed = false } diff --git a/trie/trie_reader.go b/trie/trie_reader.go index 51855320a6..d42adad2c2 100644 --- a/trie/trie_reader.go +++ b/trie/trie_reader.go @@ -17,9 +17,9 @@ package trie import ( - "fmt" - "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/log" ) // Reader wraps the Node method of a backing trie store. @@ -30,13 +30,6 @@ type Reader interface { Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) } -// NodeReader wraps all the necessary functions for accessing trie node. -type NodeReader interface { - // Reader returns a reader for accessing all trie nodes with provided - // state root. Nil is returned in case the state is not available. - Reader(root common.Hash) Reader -} - // trieReader is a wrapper of the underlying node reader. It's not safe // for concurrent usage. type trieReader struct { @@ -46,10 +39,16 @@ type trieReader struct { } // newTrieReader initializes the trie reader with the given node reader. -func newTrieReader(stateRoot, owner common.Hash, db NodeReader) (*trieReader, error) { - reader := db.Reader(stateRoot) - if reader == nil { - return nil, fmt.Errorf("state not found #%x", stateRoot) +func newTrieReader(stateRoot, owner common.Hash, db *Database) (*trieReader, error) { + if stateRoot == (common.Hash{}) || stateRoot == types.EmptyRootHash { + if stateRoot == (common.Hash{}) { + log.Error("Zero state root hash!") + } + return &trieReader{owner: owner}, nil + } + reader, err := db.Reader(stateRoot) + if err != nil { + return nil, &MissingNodeError{Owner: owner, NodeHash: stateRoot, err: err} } return &trieReader{owner: owner, reader: reader}, nil } diff --git a/trie/trie_test.go b/trie/trie_test.go index bd21a572d2..e9278e4670 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -521,7 +521,7 @@ func runRandTest(rt randTest) bool { origin = root case opItercheckhash: checktr := NewEmpty(triedb) - it := NewIterator(tr.NodeIterator(nil)) + it := NewIterator(tr.MustNodeIterator(nil)) for it.Next() { checktr.MustUpdate(it.Key, it.Value) } @@ -530,8 +530,8 @@ func runRandTest(rt randTest) bool { } case opNodeDiff: var ( - origIter = origTrie.NodeIterator(nil) - curIter = tr.NodeIterator(nil) + origIter = origTrie.MustNodeIterator(nil) + curIter = tr.MustNodeIterator(nil) origSeen = make(map[string]struct{}) curSeen = make(map[string]struct{}) ) @@ -710,7 +710,7 @@ func TestTinyTrie(t *testing.T) { t.Errorf("3: got %x, exp %x", root, exp) } checktr := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) - it := NewIterator(trie.NodeIterator(nil)) + it := NewIterator(trie.MustNodeIterator(nil)) for it.Next() { checktr.MustUpdate(it.Key, it.Value) } diff --git a/trie/triedb/hashdb/database.go b/trie/triedb/hashdb/database.go index eaccfe944a..ed20ff9cde 100644 --- a/trie/triedb/hashdb/database.go +++ b/trie/triedb/hashdb/database.go @@ -18,6 +18,7 @@ package hashdb import ( "errors" + "fmt" "reflect" "sync" "time" @@ -621,8 +622,12 @@ func (db *Database) Scheme() string { } // Reader retrieves a node reader belonging to the given state root. -func (db *Database) Reader(root common.Hash) *reader { - return &reader{db: db} +// An error will be returned if the requested state is not available. +func (db *Database) Reader(root common.Hash) (*reader, error) { + if _, err := db.Node(root); err != nil { + return nil, fmt.Errorf("state %#x is not available, %v", root, err) + } + return &reader{db: db}, nil } // reader is a state reader of Database which implements the Reader interface. From 713fc8bbe679905daeee4881cfa3f070bee8b2af Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 21 Jun 2023 12:08:12 -0400 Subject: [PATCH 28/90] ethdb/pebble: fsync for batch writes (#27522) This is likely the culprit behind several data corruption issues, e.g. where data has been written to the freezer, but the deletion from pebble does not go through due to process crash. --- ethdb/pebble/pebble.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethdb/pebble/pebble.go b/ethdb/pebble/pebble.go index b43529eddb..f0efee8aa6 100644 --- a/ethdb/pebble/pebble.go +++ b/ethdb/pebble/pebble.go @@ -535,7 +535,7 @@ func (b *batch) Write() error { if b.db.closed { return pebble.ErrClosed } - return b.b.Commit(pebble.NoSync) + return b.b.Commit(pebble.Sync) } // Reset resets the batch for reuse. From fd5d2ef0a6d9eac7542ead4bfbc9b5f0f399eb10 Mon Sep 17 00:00:00 2001 From: James Prestwich Date: Wed, 21 Jun 2023 09:11:11 -0700 Subject: [PATCH 29/90] internal/ethapi: quantity-encode storage keys in eth_getProof response (#27309) This changes the eth_getProof method implementation to re-encode the requested storage keys, canonicalizing them in the response. For backwards-compatibility reasons, go-ethereum accepts non-canonical hex keys. Accepting them is fine, but we should not mirror invalid inputs into the output. Closes #27306 --------- Co-authored-by: Martin Holst Swende Co-authored-by: Felix Lange --- ethclient/gethclient/gethclient_test.go | 36 +++++++++++++++++++- internal/ethapi/api.go | 45 ++++++++++++++++--------- 2 files changed, 64 insertions(+), 17 deletions(-) diff --git a/ethclient/gethclient/gethclient_test.go b/ethclient/gethclient/gethclient_test.go index 7ff5836f81..c2c11d0770 100644 --- a/ethclient/gethclient/gethclient_test.go +++ b/ethclient/gethclient/gethclient_test.go @@ -105,6 +105,9 @@ func TestGethClient(t *testing.T) { { "TestGetProof", func(t *testing.T) { testGetProof(t, client) }, + }, { + "TestGetProofCanonicalizeKeys", + func(t *testing.T) { testGetProofCanonicalizeKeys(t, client) }, }, { "TestGCStats", func(t *testing.T) { testGCStats(t, client) }, @@ -218,6 +221,7 @@ func testGetProof(t *testing.T, client *rpc.Client) { if result.Balance.Cmp(balance) != 0 { t.Fatalf("invalid balance, want: %v got: %v", balance, result.Balance) } + // test storage if len(result.StorageProof) != 1 { t.Fatalf("invalid storage proof, want 1 proof, got %v proof(s)", len(result.StorageProof)) @@ -228,7 +232,37 @@ func testGetProof(t *testing.T, client *rpc.Client) { t.Fatalf("invalid storage proof value, want: %v, got: %v", slotValue, proof.Value.Bytes()) } if proof.Key != testSlot.String() { - t.Fatalf("invalid storage proof key, want: %v, got: %v", testSlot.String(), proof.Key) + t.Fatalf("invalid storage proof key, want: %q, got: %q", testSlot.String(), proof.Key) + } +} + +func testGetProofCanonicalizeKeys(t *testing.T, client *rpc.Client) { + ec := New(client) + + // Tests with non-canon input for storage keys. + // Here we check that the storage key is canonicalized. + result, err := ec.GetProof(context.Background(), testAddr, []string{"0x0dEadbeef"}, nil) + if err != nil { + t.Fatal(err) + } + if result.StorageProof[0].Key != "0xdeadbeef" { + t.Fatalf("wrong storage key encoding in proof: %q", result.StorageProof[0].Key) + } + if result, err = ec.GetProof(context.Background(), testAddr, []string{"0x000deadbeef"}, nil); err != nil { + t.Fatal(err) + } + if result.StorageProof[0].Key != "0xdeadbeef" { + t.Fatalf("wrong storage key encoding in proof: %q", result.StorageProof[0].Key) + } + + // If the requested storage key is 32 bytes long, it will be returned as is. + hashSizedKey := "0x00000000000000000000000000000000000000000000000000000000deadbeef" + result, err = ec.GetProof(context.Background(), testAddr, []string{hashSizedKey}, nil) + if err != nil { + t.Fatal(err) + } + if result.StorageProof[0].Key != hashSizedKey { + t.Fatalf("wrong storage key encoding in proof: %q", result.StorageProof[0].Key) } } diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 48c6e38e88..ec29cf28f0 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -672,19 +672,21 @@ func (n *proofList) Delete(key []byte) error { func (s *BlockChainAPI) GetProof(ctx context.Context, address common.Address, storageKeys []string, blockNrOrHash rpc.BlockNumberOrHash) (*AccountResult, error) { var ( keys = make([]common.Hash, len(storageKeys)) + keyLengths = make([]int, len(storageKeys)) storageProof = make([]StorageResult, len(storageKeys)) storageTrie state.Trie storageHash = types.EmptyRootHash codeHash = types.EmptyCodeHash ) - // Greedily deserialize all keys. This prevents state access on invalid input + // Deserialize all keys. This prevents state access on invalid input. for i, hexKey := range storageKeys { - if key, err := decodeHash(hexKey); err != nil { + var err error + keys[i], keyLengths[i], err = decodeHash(hexKey) + if err != nil { return nil, err - } else { - keys[i] = key } } + state, _, err := s.b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) if state == nil || err != nil { return nil, err @@ -692,28 +694,39 @@ func (s *BlockChainAPI) GetProof(ctx context.Context, address common.Address, st if storageTrie, err = state.StorageTrie(address); err != nil { return nil, err } - // if we have a storageTrie, the account exists and we must update + + // If we have a storageTrie, the account exists and we must update // the storage root hash and the code hash. if storageTrie != nil { storageHash = storageTrie.Hash() codeHash = state.GetCodeHash(address) } - // create the proof for the storageKeys + // Create the proofs for the storageKeys. for i, key := range keys { + // Output key encoding is a bit special: if the input was a 32-byte hash, it is + // returned as such. Otherwise, we apply the QUANTITY encoding mandated by the + // JSON-RPC spec for getProof. This behavior exists to preserve backwards + // compatibility with older client versions. + var outputKey string + if keyLengths[i] != 32 { + outputKey = hexutil.EncodeBig(key.Big()) + } else { + outputKey = hexutil.Encode(key[:]) + } + if storageTrie == nil { - storageProof[i] = StorageResult{storageKeys[i], &hexutil.Big{}, []string{}} + storageProof[i] = StorageResult{outputKey, &hexutil.Big{}, []string{}} continue } var proof proofList if err := storageTrie.Prove(crypto.Keccak256(key.Bytes()), &proof); err != nil { return nil, err } - storageProof[i] = StorageResult{storageKeys[i], - (*hexutil.Big)(state.GetState(address, key).Big()), - proof} + value := (*hexutil.Big)(state.GetState(address, key).Big()) + storageProof[i] = StorageResult{outputKey, value, proof} } - // create the accountProof + // Create the accountProof. accountProof, proofErr := state.GetProof(address) if proofErr != nil { return nil, proofErr @@ -732,7 +745,7 @@ func (s *BlockChainAPI) GetProof(ctx context.Context, address common.Address, st // decodeHash parses a hex-encoded 32-byte hash. The input may optionally // be prefixed by 0x and can have a byte length up to 32. -func decodeHash(s string) (common.Hash, error) { +func decodeHash(s string) (h common.Hash, inputLength int, err error) { if strings.HasPrefix(s, "0x") || strings.HasPrefix(s, "0X") { s = s[2:] } @@ -741,12 +754,12 @@ func decodeHash(s string) (common.Hash, error) { } b, err := hex.DecodeString(s) if err != nil { - return common.Hash{}, errors.New("hex string invalid") + return common.Hash{}, 0, errors.New("hex string invalid") } if len(b) > 32 { - return common.Hash{}, errors.New("hex string too long, want at most 32 bytes") + return common.Hash{}, len(b), errors.New("hex string too long, want at most 32 bytes") } - return common.BytesToHash(b), nil + return common.BytesToHash(b), len(b), nil } // GetHeaderByNumber returns the requested canonical block header. @@ -876,7 +889,7 @@ func (s *BlockChainAPI) GetStorageAt(ctx context.Context, address common.Address if state == nil || err != nil { return nil, err } - key, err := decodeHash(hexKey) + key, _, err := decodeHash(hexKey) if err != nil { return nil, fmt.Errorf("unable to decode storage key: %s", err) } From 5520cd97a157c5df1d6c7fe70000f31350177a8d Mon Sep 17 00:00:00 2001 From: Exca-DK <85954505+Exca-DK@users.noreply.github.com> Date: Thu, 22 Jun 2023 10:06:14 +0200 Subject: [PATCH 30/90] p2p/discover: swap verification order in discv4 ping handler (#27532) In all other UDPv4 methods, the deadline is checked first. It seems weird to me that ping is an exception. Deadline comparison is also less resource intensive. Co-authored-by: Exca-DK --- p2p/discover/v4_udp.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/p2p/discover/v4_udp.go b/p2p/discover/v4_udp.go index d61a52c395..ab335e2ced 100644 --- a/p2p/discover/v4_udp.go +++ b/p2p/discover/v4_udp.go @@ -643,13 +643,13 @@ type packetHandlerV4 struct { func (t *UDPv4) verifyPing(h *packetHandlerV4, from *net.UDPAddr, fromID enode.ID, fromKey v4wire.Pubkey) error { req := h.Packet.(*v4wire.Ping) + if v4wire.Expired(req.Expiration) { + return errExpired + } senderKey, err := v4wire.DecodePubkey(crypto.S256(), fromKey) if err != nil { return err } - if v4wire.Expired(req.Expiration) { - return errExpired - } h.senderKey = senderKey return nil } From 699243f8ae870fa2ddbc5cb919fda85313e4684a Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Thu, 22 Jun 2023 14:52:52 +0200 Subject: [PATCH 31/90] core/state, light, trie: add UpdateContractCode to the Trie interface (#27476) Verkle trees store the code inside the trie. This PR changes the interface to pass the code, as well as the dirty flag to tell the trie package if the code is dirty and needs to be updated. This is a no-op for the MPT and the odr trie. --- core/state/database.go | 4 ++++ core/state/statedb.go | 1 + light/trie.go | 4 ++++ trie/secure_trie.go | 4 ++++ 4 files changed, 13 insertions(+) diff --git a/core/state/database.go b/core/state/database.go index 2c8a7afee6..b16536fff5 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -93,6 +93,10 @@ type Trie interface { // in the trie with provided address. UpdateAccount(address common.Address, account *types.StateAccount) error + // UpdateContractCode abstracts code write to the trie. It is expected + // to be moved to the stateWriter interface when the latter is ready. + UpdateContractCode(address common.Address, codeHash common.Hash, code []byte) error + // DeleteStorage removes any existing value for key from the trie. If a node // was not found in the database, a trie.MissingNodeError is returned. DeleteStorage(addr common.Address, key []byte) error diff --git a/core/state/statedb.go b/core/state/statedb.go index f8457c6de3..9ea6afc401 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -1012,6 +1012,7 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { if obj := s.stateObjects[addr]; !obj.deleted { // Write any contract code associated with the state object if obj.code != nil && obj.dirtyCode { + s.trie.UpdateContractCode(obj.Address(), common.BytesToHash(obj.CodeHash()), obj.code) rawdb.WriteCode(codeWriter, common.BytesToHash(obj.CodeHash()), obj.code) obj.dirtyCode = false } diff --git a/light/trie.go b/light/trie.go index e0675f98c9..b90b43c1dd 100644 --- a/light/trie.go +++ b/light/trie.go @@ -147,6 +147,10 @@ func (t *odrTrie) UpdateAccount(address common.Address, acc *types.StateAccount) }) } +func (t *odrTrie) UpdateContractCode(_ common.Address, _ common.Hash, _ []byte) error { + return nil +} + func (t *odrTrie) UpdateStorage(_ common.Address, key, value []byte) error { key = crypto.Keccak256(key) v, _ := rlp.EncodeToBytes(value) diff --git a/trie/secure_trie.go b/trie/secure_trie.go index a6d5676197..1df0b6a8f9 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -176,6 +176,10 @@ func (t *StateTrie) UpdateAccount(address common.Address, acc *types.StateAccoun return nil } +func (t *StateTrie) UpdateContractCode(_ common.Address, _ common.Hash, _ []byte) error { + return nil +} + // MustDelete removes any existing value for key from the trie. This function // will omit any encountered error but just print out an error message. func (t *StateTrie) MustDelete(key []byte) { From 942ba4ddaa0e2591b77a42f40059babbaaa44154 Mon Sep 17 00:00:00 2001 From: Francisco de Borja Aranda Castillejo Date: Fri, 23 Jun 2023 19:02:05 +0200 Subject: [PATCH 32/90] crypto/secp256k1: define NDEBUG only if not defined (#27550) --- crypto/secp256k1/secp256.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crypto/secp256k1/secp256.go b/crypto/secp256k1/secp256.go index c9c01b3209..61abc1eaf0 100644 --- a/crypto/secp256k1/secp256.go +++ b/crypto/secp256k1/secp256.go @@ -21,11 +21,14 @@ package secp256k1 # define USE_SCALAR_8X32 #endif +#ifndef NDEBUG +# define NDEBUG +#endif + #define USE_ENDOMORPHISM #define USE_NUM_NONE #define USE_FIELD_INV_BUILTIN #define USE_SCALAR_INV_BUILTIN -#define NDEBUG #include "./libsecp256k1/src/secp256k1.c" #include "./libsecp256k1/src/modules/recovery/main_impl.h" #include "ext.h" From 2754b197c935ee63101cbbca2752338246384fec Mon Sep 17 00:00:00 2001 From: Sanghee Choi <32831939+pengin7384@users.noreply.github.com> Date: Mon, 26 Jun 2023 00:33:50 +0900 Subject: [PATCH 33/90] common/math: fix typo in comment (#27561) --- common/math/big.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/math/big.go b/common/math/big.go index 1c2afa749a..013c0ba4b6 100644 --- a/common/math/big.go +++ b/common/math/big.go @@ -82,7 +82,7 @@ func (i *HexOrDecimal256) MarshalText() ([]byte, error) { // it however accepts either "0x"-prefixed (hex encoded) or non-prefixed (decimal) type Decimal256 big.Int -// NewHexOrDecimal256 creates a new Decimal256 +// NewDecimal256 creates a new Decimal256 func NewDecimal256(x int64) *Decimal256 { b := big.NewInt(x) d := Decimal256(*b) From 80441779d44052feac67d3dae9f93352ee0491b3 Mon Sep 17 00:00:00 2001 From: Delweng Date: Tue, 27 Jun 2023 14:11:06 +0800 Subject: [PATCH 34/90] eth/protocols: put protocols in order (#27543) Signed-off-by: jsvisa --- eth/protocols/eth/protocol.go | 29 ++++++++++++++--------------- 1 file changed, 14 insertions(+), 15 deletions(-) diff --git a/eth/protocols/eth/protocol.go b/eth/protocols/eth/protocol.go index 0d4b368988..4b9f5ad6ba 100644 --- a/eth/protocols/eth/protocol.go +++ b/eth/protocols/eth/protocol.go @@ -59,13 +59,13 @@ const ( GetBlockBodiesMsg = 0x05 BlockBodiesMsg = 0x06 NewBlockMsg = 0x07 + NewPooledTransactionHashesMsg = 0x08 + GetPooledTransactionsMsg = 0x09 + PooledTransactionsMsg = 0x0a GetNodeDataMsg = 0x0d NodeDataMsg = 0x0e GetReceiptsMsg = 0x0f ReceiptsMsg = 0x10 - NewPooledTransactionHashesMsg = 0x08 - GetPooledTransactionsMsg = 0x09 - PooledTransactionsMsg = 0x0a ) var ( @@ -363,6 +363,17 @@ func (*BlockBodiesPacket) Kind() byte { return BlockBodiesMsg } func (*NewBlockPacket) Name() string { return "NewBlock" } func (*NewBlockPacket) Kind() byte { return NewBlockMsg } +func (*NewPooledTransactionHashesPacket66) Name() string { return "NewPooledTransactionHashes" } +func (*NewPooledTransactionHashesPacket66) Kind() byte { return NewPooledTransactionHashesMsg } +func (*NewPooledTransactionHashesPacket68) Name() string { return "NewPooledTransactionHashes" } +func (*NewPooledTransactionHashesPacket68) Kind() byte { return NewPooledTransactionHashesMsg } + +func (*GetPooledTransactionsPacket) Name() string { return "GetPooledTransactions" } +func (*GetPooledTransactionsPacket) Kind() byte { return GetPooledTransactionsMsg } + +func (*PooledTransactionsPacket) Name() string { return "PooledTransactions" } +func (*PooledTransactionsPacket) Kind() byte { return PooledTransactionsMsg } + func (*GetNodeDataPacket) Name() string { return "GetNodeData" } func (*GetNodeDataPacket) Kind() byte { return GetNodeDataMsg } @@ -374,15 +385,3 @@ func (*GetReceiptsPacket) Kind() byte { return GetReceiptsMsg } func (*ReceiptsPacket) Name() string { return "Receipts" } func (*ReceiptsPacket) Kind() byte { return ReceiptsMsg } - -func (*NewPooledTransactionHashesPacket66) Name() string { return "NewPooledTransactionHashes" } -func (*NewPooledTransactionHashesPacket66) Kind() byte { return NewPooledTransactionHashesMsg } - -func (*NewPooledTransactionHashesPacket68) Name() string { return "NewPooledTransactionHashes" } -func (*NewPooledTransactionHashesPacket68) Kind() byte { return NewPooledTransactionHashesMsg } - -func (*GetPooledTransactionsPacket) Name() string { return "GetPooledTransactions" } -func (*GetPooledTransactionsPacket) Kind() byte { return GetPooledTransactionsMsg } - -func (*PooledTransactionsPacket) Name() string { return "PooledTransactions" } -func (*PooledTransactionsPacket) Kind() byte { return PooledTransactionsMsg } From b4bc9b0db693d99d62e91c660194d37ffd94f94a Mon Sep 17 00:00:00 2001 From: puhtaytow <18026645+puhtaytow@users.noreply.github.com> Date: Tue, 27 Jun 2023 08:12:03 +0200 Subject: [PATCH 35/90] cmd/geth: error message should not be capitalised (#27549) error message should not be capitalized / consistency --- cmd/geth/verkle.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/geth/verkle.go b/cmd/geth/verkle.go index e856eaeb9c..7766c458e6 100644 --- a/cmd/geth/verkle.go +++ b/cmd/geth/verkle.go @@ -100,7 +100,7 @@ func checkChildren(root verkle.VerkleNode, resolver verkle.NodeResolverFn) error return nil } } - return errors.New("Both balance and nonce are 0") + return errors.New("both balance and nonce are 0") case verkle.Empty: // nothing to do default: From 4b90c4488d6d1847357eb0414a3a7ee850046b27 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Tue, 27 Jun 2023 15:19:02 +0800 Subject: [PATCH 36/90] ethdb/pebble: use sync mode for pebble writes (#27615) --- ethdb/pebble/pebble.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ethdb/pebble/pebble.go b/ethdb/pebble/pebble.go index f0efee8aa6..a06f59bcfa 100644 --- a/ethdb/pebble/pebble.go +++ b/ethdb/pebble/pebble.go @@ -279,7 +279,7 @@ func (d *Database) Put(key []byte, value []byte) error { if d.closed { return pebble.ErrClosed } - return d.db.Set(key, value, pebble.NoSync) + return d.db.Set(key, value, pebble.Sync) } // Delete removes the key from the key-value store. From d7ea278fe38840a14474de310cdaefdc3abe6557 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Tue, 27 Jun 2023 09:25:08 +0200 Subject: [PATCH 37/90] core/vm: move TSTORE,TLOAD to correct opcode nums (#27613) * core/vm: move TSTORE,TLOAD to correct opcode nums * core/vm: cleanup --- core/vm/opcodes.go | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/core/vm/opcodes.go b/core/vm/opcodes.go index ed074674b2..3cdeab8ff9 100644 --- a/core/vm/opcodes.go +++ b/core/vm/opcodes.go @@ -117,6 +117,8 @@ const ( MSIZE OpCode = 0x59 GAS OpCode = 0x5a JUMPDEST OpCode = 0x5b + TLOAD OpCode = 0x5c + TSTORE OpCode = 0x5d PUSH0 OpCode = 0x5f ) @@ -205,12 +207,6 @@ const ( LOG4 ) -// 0xb0 range. -const ( - TLOAD OpCode = 0xb3 - TSTORE OpCode = 0xb4 -) - // 0xf0 range - closures. const ( CREATE OpCode = 0xf0 @@ -304,6 +300,8 @@ var opCodeToString = map[OpCode]string{ MSIZE: "MSIZE", GAS: "GAS", JUMPDEST: "JUMPDEST", + TLOAD: "TLOAD", + TSTORE: "TSTORE", PUSH0: "PUSH0", // 0x60 range - pushes. @@ -383,10 +381,6 @@ var opCodeToString = map[OpCode]string{ LOG3: "LOG3", LOG4: "LOG4", - // 0xb0 range. - TLOAD: "TLOAD", - TSTORE: "TSTORE", - // 0xf0 range - closures. CREATE: "CREATE", CALL: "CALL", @@ -477,6 +471,8 @@ var stringToOp = map[string]OpCode{ "MSIZE": MSIZE, "GAS": GAS, "JUMPDEST": JUMPDEST, + "TLOAD": TLOAD, + "TSTORE": TSTORE, "PUSH0": PUSH0, "PUSH1": PUSH1, "PUSH2": PUSH2, @@ -547,8 +543,6 @@ var stringToOp = map[string]OpCode{ "LOG2": LOG2, "LOG3": LOG3, "LOG4": LOG4, - "TLOAD": TLOAD, - "TSTORE": TSTORE, "CREATE": CREATE, "CREATE2": CREATE2, "CALL": CALL, From 900591299f5a8ef61a09722205dfab03cc6ecae5 Mon Sep 17 00:00:00 2001 From: Sanghee Choi <32831939+pengin7384@users.noreply.github.com> Date: Tue, 27 Jun 2023 17:27:40 +0900 Subject: [PATCH 38/90] common/lru: add test case for BasicLRU.Peek (#27559) --- common/lru/basiclru_test.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/common/lru/basiclru_test.go b/common/lru/basiclru_test.go index e2d3559f55..29812bda15 100644 --- a/common/lru/basiclru_test.go +++ b/common/lru/basiclru_test.go @@ -170,6 +170,20 @@ func TestBasicLRUContains(t *testing.T) { } } +// Test that Peek doesn't update recent-ness +func TestBasicLRUPeek(t *testing.T) { + cache := NewBasicLRU[int, int](2) + cache.Add(1, 1) + cache.Add(2, 2) + if v, ok := cache.Peek(1); !ok || v != 1 { + t.Errorf("1 should be set to 1") + } + cache.Add(3, 3) + if cache.Contains(1) { + t.Errorf("should not have updated recent-ness of 1") + } +} + func BenchmarkLRU(b *testing.B) { var ( capacity = 1000 From d73eb87979021e63cb5867319c8ef07a9ba8a7d2 Mon Sep 17 00:00:00 2001 From: aaronbuchwald Date: Tue, 27 Jun 2023 04:29:19 -0400 Subject: [PATCH 39/90] core: log message if DeriveFields returns error (#27530) This error will never happen, but if it ever does happen, we will at least see it. --- core/blockchain.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/core/blockchain.go b/core/blockchain.go index becd39a1fa..1110e3e5f1 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -2048,7 +2048,9 @@ func (bc *BlockChain) recoverAncestors(block *types.Block) (common.Hash, error) // the processing of a block. These logs are later announced as deleted or reborn. func (bc *BlockChain) collectLogs(b *types.Block, removed bool) []*types.Log { receipts := rawdb.ReadRawReceipts(bc.db, b.Hash(), b.NumberU64()) - receipts.DeriveFields(bc.chainConfig, b.Hash(), b.NumberU64(), b.Time(), b.BaseFee(), b.Transactions()) + if err := receipts.DeriveFields(bc.chainConfig, b.Hash(), b.NumberU64(), b.Time(), b.BaseFee(), b.Transactions()); err != nil { + log.Error("Failed to derive block receipts fields", "hash", b.Hash(), "number", b.NumberU64(), "err", err) + } var logs []*types.Log for _, receipt := range receipts { From c7b099b2ea58b3c2b594ae77b607142451ae9129 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Tue, 27 Jun 2023 14:36:38 +0200 Subject: [PATCH 40/90] trie, core/state: revert error removal in (*state.Trie).Commit (#27544) * trie, core/state: revert error removal in (*state.Trie).Commit * Gary's nitpick :) Co-Authored-By: rjl493456442 --------- Co-authored-by: rjl493456442 --- core/state/database.go | 2 +- core/state/snapshot/generate.go | 5 ++++- core/state/snapshot/generate_test.go | 4 ++-- core/state/state_object.go | 5 ++++- core/state/statedb.go | 5 ++++- eth/protocols/snap/sync_test.go | 12 ++++++------ light/postprocess.go | 12 ++++++++---- light/trie.go | 4 ++-- tests/fuzzers/stacktrie/trie_fuzzer.go | 5 ++++- tests/fuzzers/trie/trie-fuzzer.go | 5 ++++- trie/iterator_test.go | 18 +++++++++--------- trie/secure_trie.go | 2 +- trie/secure_trie_test.go | 2 +- trie/sync_test.go | 6 +++--- trie/tracer_test.go | 18 +++++++++--------- trie/trie.go | 8 ++++---- trie/trie_test.go | 24 ++++++++++++------------ 17 files changed, 78 insertions(+), 59 deletions(-) diff --git a/core/state/database.go b/core/state/database.go index b16536fff5..5ca2366360 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -114,7 +114,7 @@ type Trie interface { // The returned nodeset can be nil if the trie is clean(nothing to commit). // Once the trie is committed, it's not usable anymore. A new trie must // be created with new root and updated trie database for following usage - Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet) + Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet, error) // NodeIterator returns an iterator that returns nodes of the trie. Iteration // starts at the key after the given start key. And error will be returned diff --git a/core/state/snapshot/generate.go b/core/state/snapshot/generate.go index 1ec82b9442..f5db0ff921 100644 --- a/core/state/snapshot/generate.go +++ b/core/state/snapshot/generate.go @@ -361,7 +361,10 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, trieId *trie.ID, prefi for i, key := range result.keys { snapTrie.Update(key, result.vals[i]) } - root, nodes := snapTrie.Commit(false) + root, nodes, err := snapTrie.Commit(false) + if err != nil { + return false, nil, err + } if nodes != nil { tdb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) tdb.Commit(root, false) diff --git a/core/state/snapshot/generate_test.go b/core/state/snapshot/generate_test.go index 837280f291..20a389e477 100644 --- a/core/state/snapshot/generate_test.go +++ b/core/state/snapshot/generate_test.go @@ -191,7 +191,7 @@ func (t *testHelper) makeStorageTrie(owner common.Hash, keys []string, vals []st if !commit { return stTrie.Hash() } - root, nodes := stTrie.Commit(false) + root, nodes, _ := stTrie.Commit(false) if nodes != nil { t.nodes.Merge(nodes) } @@ -199,7 +199,7 @@ func (t *testHelper) makeStorageTrie(owner common.Hash, keys []string, vals []st } func (t *testHelper) Commit() common.Hash { - root, nodes := t.accTrie.Commit(true) + root, nodes, _ := t.accTrie.Commit(true) if nodes != nil { t.nodes.Merge(nodes) } diff --git a/core/state/state_object.go b/core/state/state_object.go index 606441d2a6..148136fc2c 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -366,7 +366,10 @@ func (s *stateObject) commitTrie(db Database) (*trienode.NodeSet, error) { if metrics.EnabledExpensive { defer func(start time.Time) { s.db.StorageCommits += time.Since(start) }(time.Now()) } - root, nodes := tr.Commit(false) + root, nodes, err := tr.Commit(false) + if err != nil { + return nil, err + } s.data.Root = root return nodes, nil } diff --git a/core/state/statedb.go b/core/state/statedb.go index 9ea6afc401..998fb176df 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -1051,7 +1051,10 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { if metrics.EnabledExpensive { start = time.Now() } - root, set := s.trie.Commit(true) + root, set, err := s.trie.Commit(true) + if err != nil { + return common.Hash{}, err + } // Merge the dirty nodes of account trie into global set if set != nil { if err := nodes.Merge(set); err != nil { diff --git a/eth/protocols/snap/sync_test.go b/eth/protocols/snap/sync_test.go index 173f1da604..c1875cc721 100644 --- a/eth/protocols/snap/sync_test.go +++ b/eth/protocols/snap/sync_test.go @@ -1386,7 +1386,7 @@ func makeAccountTrieNoStorage(n int) (string, *trie.Trie, []*kv) { // Commit the state changes into db and re-create the trie // for accessing later. - root, nodes := accTrie.Commit(false) + root, nodes, _ := accTrie.Commit(false) db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) accTrie, _ = trie.New(trie.StateTrieID(root), db) @@ -1448,7 +1448,7 @@ func makeBoundaryAccountTrie(n int) (string, *trie.Trie, []*kv) { // Commit the state changes into db and re-create the trie // for accessing later. - root, nodes := accTrie.Commit(false) + root, nodes, _ := accTrie.Commit(false) db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) accTrie, _ = trie.New(trie.StateTrieID(root), db) @@ -1494,7 +1494,7 @@ func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) slices.SortFunc(entries, (*kv).less) // Commit account trie - root, set := accTrie.Commit(true) + root, set, _ := accTrie.Commit(true) nodes.Merge(set) // Commit gathered dirty nodes into database @@ -1559,7 +1559,7 @@ func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (strin slices.SortFunc(entries, (*kv).less) // Commit account trie - root, set := accTrie.Commit(true) + root, set, _ := accTrie.Commit(true) nodes.Merge(set) // Commit gathered dirty nodes into database @@ -1601,7 +1601,7 @@ func makeStorageTrieWithSeed(owner common.Hash, n, seed uint64, db *trie.Databas entries = append(entries, elem) } slices.SortFunc(entries, (*kv).less) - root, nodes := trie.Commit(false) + root, nodes, _ := trie.Commit(false) return root, nodes, entries } @@ -1652,7 +1652,7 @@ func makeBoundaryStorageTrie(owner common.Hash, n int, db *trie.Database) (commo entries = append(entries, elem) } slices.SortFunc(entries, (*kv).less) - root, nodes := trie.Commit(false) + root, nodes, _ := trie.Commit(false) return root, nodes, entries } diff --git a/light/postprocess.go b/light/postprocess.go index 001df209b7..7ef797ff47 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -214,7 +214,10 @@ func (c *ChtIndexerBackend) Process(ctx context.Context, header *types.Header) e // Commit implements core.ChainIndexerBackend func (c *ChtIndexerBackend) Commit() error { - root, nodes := c.trie.Commit(false) + root, nodes, err := c.trie.Commit(false) + if err != nil { + return err + } // Commit trie changes into trie database in case it's not nil. if nodes != nil { if err := c.triedb.Update(root, c.originRoot, trienode.NewWithNodeSet(nodes)); err != nil { @@ -225,7 +228,6 @@ func (c *ChtIndexerBackend) Commit() error { } } // Re-create trie with newly generated root and updated database. - var err error c.trie, err = trie.New(trie.TrieID(root), c.triedb) if err != nil { return err @@ -465,7 +467,10 @@ func (b *BloomTrieIndexerBackend) Commit() error { return terr } } - root, nodes := b.trie.Commit(false) + root, nodes, err := b.trie.Commit(false) + if err != nil { + return err + } // Commit trie changes into trie database in case it's not nil. if nodes != nil { if err := b.triedb.Update(root, b.originRoot, trienode.NewWithNodeSet(nodes)); err != nil { @@ -476,7 +481,6 @@ func (b *BloomTrieIndexerBackend) Commit() error { } } // Re-create trie with newly generated root and updated database. - var err error b.trie, err = trie.New(trie.TrieID(root), b.triedb) if err != nil { return err diff --git a/light/trie.go b/light/trie.go index b90b43c1dd..b411c299f2 100644 --- a/light/trie.go +++ b/light/trie.go @@ -174,9 +174,9 @@ func (t *odrTrie) DeleteAccount(address common.Address) error { }) } -func (t *odrTrie) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet) { +func (t *odrTrie) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet, error) { if t.trie == nil { - return t.id.Root, nil + return t.id.Root, nil, nil } return t.trie.Commit(collectLeaf) } diff --git a/tests/fuzzers/stacktrie/trie_fuzzer.go b/tests/fuzzers/stacktrie/trie_fuzzer.go index d7be54eda0..a6b1b3bc79 100644 --- a/tests/fuzzers/stacktrie/trie_fuzzer.go +++ b/tests/fuzzers/stacktrie/trie_fuzzer.go @@ -171,7 +171,10 @@ func (f *fuzzer) fuzz() int { return 0 } // Flush trie -> database - rootA, nodes := trieA.Commit(false) + rootA, nodes, err := trieA.Commit(false) + if err != nil { + panic(err) + } if nodes != nil { dbA.Update(rootA, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) } diff --git a/tests/fuzzers/trie/trie-fuzzer.go b/tests/fuzzers/trie/trie-fuzzer.go index 33edc2a18a..004fb64372 100644 --- a/tests/fuzzers/trie/trie-fuzzer.go +++ b/tests/fuzzers/trie/trie-fuzzer.go @@ -165,7 +165,10 @@ func runRandTest(rt randTest) error { case opHash: tr.Hash() case opCommit: - hash, nodes := tr.Commit(false) + hash, nodes, err := tr.Commit(false) + if err != nil { + return err + } if nodes != nil { if err := triedb.Update(hash, origin, trienode.NewWithNodeSet(nodes)); err != nil { return err diff --git a/trie/iterator_test.go b/trie/iterator_test.go index 5c7f9c98fb..8f9849abbd 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -62,7 +62,7 @@ func TestIterator(t *testing.T) { all[val.k] = val.v trie.MustUpdate([]byte(val.k), []byte(val.v)) } - root, nodes := trie.Commit(false) + root, nodes, _ := trie.Commit(false) db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) trie, _ = New(TrieID(root), db) @@ -254,7 +254,7 @@ func TestDifferenceIterator(t *testing.T) { for _, val := range testdata1 { triea.MustUpdate([]byte(val.k), []byte(val.v)) } - rootA, nodesA := triea.Commit(false) + rootA, nodesA, _ := triea.Commit(false) dba.Update(rootA, types.EmptyRootHash, trienode.NewWithNodeSet(nodesA)) triea, _ = New(TrieID(rootA), dba) @@ -263,7 +263,7 @@ func TestDifferenceIterator(t *testing.T) { for _, val := range testdata2 { trieb.MustUpdate([]byte(val.k), []byte(val.v)) } - rootB, nodesB := trieb.Commit(false) + rootB, nodesB, _ := trieb.Commit(false) dbb.Update(rootB, types.EmptyRootHash, trienode.NewWithNodeSet(nodesB)) trieb, _ = New(TrieID(rootB), dbb) @@ -296,7 +296,7 @@ func TestUnionIterator(t *testing.T) { for _, val := range testdata1 { triea.MustUpdate([]byte(val.k), []byte(val.v)) } - rootA, nodesA := triea.Commit(false) + rootA, nodesA, _ := triea.Commit(false) dba.Update(rootA, types.EmptyRootHash, trienode.NewWithNodeSet(nodesA)) triea, _ = New(TrieID(rootA), dba) @@ -305,7 +305,7 @@ func TestUnionIterator(t *testing.T) { for _, val := range testdata2 { trieb.MustUpdate([]byte(val.k), []byte(val.v)) } - rootB, nodesB := trieb.Commit(false) + rootB, nodesB, _ := trieb.Commit(false) dbb.Update(rootB, types.EmptyRootHash, trienode.NewWithNodeSet(nodesB)) trieb, _ = New(TrieID(rootB), dbb) @@ -367,7 +367,7 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool, scheme string) { for _, val := range testdata1 { tr.MustUpdate([]byte(val.k), []byte(val.v)) } - root, nodes := tr.Commit(false) + root, nodes, _ := tr.Commit(false) tdb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) if !memonly { tdb.Commit(root, false) @@ -477,7 +477,7 @@ func testIteratorContinueAfterSeekError(t *testing.T, memonly bool, scheme strin for _, val := range testdata1 { ctr.MustUpdate([]byte(val.k), []byte(val.v)) } - root, nodes := ctr.Commit(false) + root, nodes, _ := ctr.Commit(false) for path, n := range nodes.Nodes { if n.Hash == barNodeHash { barNodePath = []byte(path) @@ -604,7 +604,7 @@ func makeLargeTestTrie() (*Database, *StateTrie, *loggingDb) { val = crypto.Keccak256(val) trie.MustUpdate(key, val) } - root, nodes := trie.Commit(false) + root, nodes, _ := trie.Commit(false) triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) triedb.Commit(root, false) @@ -647,7 +647,7 @@ func testIteratorNodeBlob(t *testing.T, scheme string) { all[val.k] = val.v trie.MustUpdate([]byte(val.k), []byte(val.v)) } - root, nodes := trie.Commit(false) + root, nodes, _ := trie.Commit(false) triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) triedb.Commit(root, false) diff --git a/trie/secure_trie.go b/trie/secure_trie.go index 1df0b6a8f9..7f0685e306 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -223,7 +223,7 @@ func (t *StateTrie) GetKey(shaKey []byte) []byte { // All cached preimages will be also flushed if preimages recording is enabled. // Once the trie is committed, it's not usable anymore. A new trie must // be created with new root and updated trie database for following usage -func (t *StateTrie) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet) { +func (t *StateTrie) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet, error) { // Write all the pre-images to the actual disk database if len(t.getSecKeyCache()) > 0 { if t.preimages != nil { diff --git a/trie/secure_trie_test.go b/trie/secure_trie_test.go index e4f6bca97d..4c70cbaa44 100644 --- a/trie/secure_trie_test.go +++ b/trie/secure_trie_test.go @@ -60,7 +60,7 @@ func makeTestStateTrie() (*Database, *StateTrie, map[string][]byte) { trie.MustUpdate(key, val) } } - root, nodes := trie.Commit(false) + root, nodes, _ := trie.Commit(false) if err := triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)); err != nil { panic(fmt.Errorf("failed to commit db %v", err)) } diff --git a/trie/sync_test.go b/trie/sync_test.go index ec49beef04..cfb7261a04 100644 --- a/trie/sync_test.go +++ b/trie/sync_test.go @@ -56,7 +56,7 @@ func makeTestTrie(scheme string) (ethdb.Database, *Database, *StateTrie, map[str trie.MustUpdate(key, val) } } - root, nodes := trie.Commit(false) + root, nodes, _ := trie.Commit(false) if err := triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)); err != nil { panic(fmt.Errorf("failed to commit db %v", err)) } @@ -739,7 +739,7 @@ func testSyncMovingTarget(t *testing.T, scheme string) { srcTrie.MustUpdate(key, val) diff[string(key)] = val } - root, nodes := srcTrie.Commit(false) + root, nodes, _ := srcTrie.Commit(false) if err := srcDb.Update(root, preRoot, trienode.NewWithNodeSet(nodes)); err != nil { panic(err) } @@ -764,7 +764,7 @@ func testSyncMovingTarget(t *testing.T, scheme string) { srcTrie.MustUpdate([]byte(k), val) reverted[k] = val } - root, nodes = srcTrie.Commit(false) + root, nodes, _ = srcTrie.Commit(false) if err := srcDb.Update(root, preRoot, trienode.NewWithNodeSet(nodes)); err != nil { panic(err) } diff --git a/trie/tracer_test.go b/trie/tracer_test.go index 4cec53688e..ca9d64766b 100644 --- a/trie/tracer_test.go +++ b/trie/tracer_test.go @@ -70,7 +70,7 @@ func testTrieTracer(t *testing.T, vals []struct{ k, v string }) { } insertSet := copySet(trie.tracer.inserts) // copy before commit deleteSet := copySet(trie.tracer.deletes) // copy before commit - root, nodes := trie.Commit(false) + root, nodes, _ := trie.Commit(false) db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) seen := setKeys(iterNodes(db, root)) @@ -136,7 +136,7 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) { for _, val := range vals { trie.MustUpdate([]byte(val.k), []byte(val.v)) } - root, nodes := trie.Commit(false) + root, nodes, _ := trie.Commit(false) db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) trie, _ = New(TrieID(root), db) @@ -151,7 +151,7 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) { for _, val := range vals { trie.MustUpdate([]byte(val.k), randBytes(32)) } - root, nodes = trie.Commit(false) + root, nodes, _ = trie.Commit(false) db.Update(root, parent, trienode.NewWithNodeSet(nodes)) trie, _ = New(TrieID(root), db) @@ -169,7 +169,7 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) { keys = append(keys, string(key)) trie.MustUpdate(key, randBytes(32)) } - root, nodes = trie.Commit(false) + root, nodes, _ = trie.Commit(false) db.Update(root, parent, trienode.NewWithNodeSet(nodes)) trie, _ = New(TrieID(root), db) @@ -184,7 +184,7 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) { for _, key := range keys { trie.MustUpdate([]byte(key), nil) } - root, nodes = trie.Commit(false) + root, nodes, _ = trie.Commit(false) db.Update(root, parent, trienode.NewWithNodeSet(nodes)) trie, _ = New(TrieID(root), db) @@ -199,7 +199,7 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) { for _, val := range vals { trie.MustUpdate([]byte(val.k), nil) } - root, nodes = trie.Commit(false) + root, nodes, _ = trie.Commit(false) db.Update(root, parent, trienode.NewWithNodeSet(nodes)) trie, _ = New(TrieID(root), db) @@ -218,7 +218,7 @@ func TestAccessListLeak(t *testing.T) { for _, val := range standard { trie.MustUpdate([]byte(val.k), []byte(val.v)) } - root, nodes := trie.Commit(false) + root, nodes, _ := trie.Commit(false) db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) var cases = []struct { @@ -268,7 +268,7 @@ func TestTinyTree(t *testing.T) { for _, val := range tiny { trie.MustUpdate([]byte(val.k), randBytes(32)) } - root, set := trie.Commit(false) + root, set, _ := trie.Commit(false) db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(set)) parent := root @@ -277,7 +277,7 @@ func TestTinyTree(t *testing.T) { for _, val := range tiny { trie.MustUpdate([]byte(val.k), []byte(val.v)) } - root, set = trie.Commit(false) + root, set, _ = trie.Commit(false) db.Update(root, parent, trienode.NewWithNodeSet(set)) trie, _ = New(TrieID(root), db) diff --git a/trie/trie.go b/trie/trie.go index cf5d181ff7..8fbd4e64a3 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -607,7 +607,7 @@ func (t *Trie) Hash() common.Hash { // The returned nodeset can be nil if the trie is clean (nothing to commit). // Once the trie is committed, it's not usable anymore. A new trie must // be created with new root and updated trie database for following usage -func (t *Trie) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet) { +func (t *Trie) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet, error) { defer t.tracer.reset() defer func() { t.committed = true @@ -619,7 +619,7 @@ func (t *Trie) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet) { // - The trie was empty and no update happens // - The trie was non-empty and all nodes are dropped if t.root == nil { - return types.EmptyRootHash, nodes + return types.EmptyRootHash, nodes, nil } // Derive the hash for all dirty nodes first. We hold the assumption // in the following procedure that all nodes are hashed. @@ -631,10 +631,10 @@ func (t *Trie) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet) { // Replace the root node with the origin hash in order to // ensure all resolved nodes are dropped after the commit. t.root = hashedNode - return rootHash, nil + return rootHash, nil, nil } t.root = newCommitter(nodes, t.tracer, collectLeaf).Commit(t.root) - return rootHash, nodes + return rootHash, nodes, nil } // hashRoot calculates the root hash of the given trie diff --git a/trie/trie_test.go b/trie/trie_test.go index e9278e4670..3997c4565d 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -88,7 +88,7 @@ func testMissingNode(t *testing.T, memonly bool, scheme string) { trie := NewEmpty(triedb) updateString(trie, "120000", "qwerqwerqwerqwerqwerqwerqwerqwer") updateString(trie, "123456", "asdfasdfasdfasdfasdfasdfasdfasdf") - root, nodes := trie.Commit(false) + root, nodes, _ := trie.Commit(false) triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) if !memonly { @@ -177,7 +177,7 @@ func TestInsert(t *testing.T) { updateString(trie, "A", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") exp = common.HexToHash("d23786fb4a010da3ce639d66d5e904a11dbc02746d1ce25029e53290cabf28ab") - root, _ = trie.Commit(false) + root, _, _ = trie.Commit(false) if root != exp { t.Errorf("case 2: exp %x got %x", exp, root) } @@ -202,7 +202,7 @@ func TestGet(t *testing.T) { if i == 1 { return } - root, nodes := trie.Commit(false) + root, nodes, _ := trie.Commit(false) db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) trie, _ = New(TrieID(root), db) } @@ -274,7 +274,7 @@ func TestReplication(t *testing.T) { for _, val := range vals { updateString(trie, val.k, val.v) } - root, nodes := trie.Commit(false) + root, nodes, _ := trie.Commit(false) db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) // create a new trie on top of the database and check that lookups work. @@ -287,7 +287,7 @@ func TestReplication(t *testing.T) { t.Errorf("trie2 doesn't have %q => %q", kv.k, kv.v) } } - hash, nodes := trie2.Commit(false) + hash, nodes, _ := trie2.Commit(false) if hash != root { t.Errorf("root failure. expected %x got %x", root, hash) } @@ -501,7 +501,7 @@ func runRandTest(rt randTest) bool { case opHash: tr.Hash() case opCommit: - root, nodes := tr.Commit(true) + root, nodes, _ := tr.Commit(true) if nodes != nil { triedb.Update(root, origin, trienode.NewWithNodeSet(nodes)) } @@ -734,7 +734,7 @@ func TestCommitAfterHash(t *testing.T) { if exp != root { t.Errorf("got %x, exp %x", root, exp) } - root, _ = trie.Commit(false) + root, _, _ = trie.Commit(false) if exp != root { t.Errorf("got %x, exp %x", root, exp) } @@ -837,7 +837,7 @@ func TestCommitSequence(t *testing.T) { trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i]) } // Flush trie -> database - root, nodes := trie.Commit(false) + root, nodes, _ := trie.Commit(false) db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) // Flush memdb -> disk (sponge) db.Commit(root, false) @@ -878,7 +878,7 @@ func TestCommitSequenceRandomBlobs(t *testing.T) { trie.MustUpdate(key, val) } // Flush trie -> database - root, nodes := trie.Commit(false) + root, nodes, _ := trie.Commit(false) db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) // Flush memdb -> disk (sponge) db.Commit(root, false) @@ -917,7 +917,7 @@ func TestCommitSequenceStackTrie(t *testing.T) { stTrie.Update(key, val) } // Flush trie -> database - root, nodes := trie.Commit(false) + root, nodes, _ := trie.Commit(false) // Flush memdb -> disk (sponge) db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) db.Commit(root, false) @@ -965,7 +965,7 @@ func TestCommitSequenceSmallRoot(t *testing.T) { trie.Update(key, []byte{0x1}) stTrie.Update(key, []byte{0x1}) // Flush trie -> database - root, nodes := trie.Commit(false) + root, nodes, _ := trie.Commit(false) // Flush memdb -> disk (sponge) db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) db.Commit(root, false) @@ -1138,7 +1138,7 @@ func benchmarkDerefRootFixedSize(b *testing.B, addresses [][20]byte, accounts [] trie.MustUpdate(crypto.Keccak256(addresses[i][:]), accounts[i]) } h := trie.Hash() - root, nodes := trie.Commit(false) + root, nodes, _ := trie.Commit(false) triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) b.StartTimer() triedb.Dereference(h) From eed7983c7c0b0e76f1121368ace3e7e0efeb202b Mon Sep 17 00:00:00 2001 From: hero5512 Date: Tue, 27 Jun 2023 13:28:18 -0400 Subject: [PATCH 41/90] core/txpool/legacypool: fix typo (#27620) --- core/txpool/legacypool/legacypool.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/txpool/legacypool/legacypool.go b/core/txpool/legacypool/legacypool.go index 02cd53ab96..5b2f4eea09 100644 --- a/core/txpool/legacypool/legacypool.go +++ b/core/txpool/legacypool/legacypool.go @@ -432,7 +432,7 @@ func (pool *LegacyPool) SetGasTip(tip *big.Int) { } pool.priced.Removed(len(drop)) } - log.Info("LEgacy pool tip threshold updated", "tip", tip) + log.Info("Legacy pool tip threshold updated", "tip", tip) } // Nonce returns the next nonce of an account, with all transactions executable From f5d3d486e459dce29130576ae88f2324ad586b50 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Wed, 28 Jun 2023 11:06:20 +0200 Subject: [PATCH 42/90] eth: send big transactions by announce/retrieve only (#27618) * eth: send big transactions by announce/retrieve only * Update eth/handler.go Co-authored-by: Martin Holst Swende * eth: remove superfluous bracket * eth: add whitespace --------- Co-authored-by: Martin Holst Swende --- eth/handler.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/eth/handler.go b/eth/handler.go index 46113fbc2f..6a8487649e 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -45,6 +45,11 @@ const ( // txChanSize is the size of channel listening to NewTxsEvent. // The number is referenced from the size of tx pool. txChanSize = 4096 + + // txMaxBroadcastSize is the max size of a transaction that will be broadcasted. + // All transactions with a higher size will be announced and need to be fetched + // by the peer. + txMaxBroadcastSize = 4096 ) var ( @@ -564,8 +569,12 @@ func (h *handler) BroadcastTransactions(txs types.Transactions) { // Broadcast transactions to a batch of peers not knowing about it for _, tx := range txs { peers := h.peers.peersWithoutTransaction(tx.Hash()) + + var numDirect int + if tx.Size() <= txMaxBroadcastSize { + numDirect = int(math.Sqrt(float64(len(peers)))) + } // Send the tx unconditionally to a subset of our peers - numDirect := int(math.Sqrt(float64(len(peers)))) for _, peer := range peers[:numDirect] { txset[peer] = append(txset[peer], tx.Hash()) } From 85b8d1c06c49342966cad2bbdc17d0dc28b66ffd Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Wed, 28 Jun 2023 11:08:48 +0200 Subject: [PATCH 43/90] params, trie: add verkle fork management + upgrade go-verkle (#27464) * params, trie: add verkle fork management + upgrade go-verkle * remove the two verkle files * core, eth, params: add missing function * Gary's feedback * remove trie/utils/verkle.go * add verkle block override --------- Co-authored-by: Gary Rong --- cmd/geth/config.go | 4 ++++ cmd/geth/main.go | 1 + cmd/geth/verkle.go | 4 ++-- cmd/utils/flags.go | 5 +++++ core/genesis.go | 4 ++++ core/vm/jump_table_export.go | 2 ++ eth/backend.go | 3 +++ eth/ethconfig/config.go | 3 +++ eth/ethconfig/gen_config.go | 6 ++++++ eth/tracers/api.go | 4 ++++ go.mod | 8 ++++---- go.sum | 9 +++++++++ les/client.go | 3 +++ params/config.go | 19 +++++++++++++++++++ 14 files changed, 69 insertions(+), 6 deletions(-) diff --git a/cmd/geth/config.go b/cmd/geth/config.go index 42ded49322..6edcd2f012 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -170,6 +170,10 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { v := ctx.Uint64(utils.OverrideCancun.Name) cfg.Eth.OverrideCancun = &v } + if ctx.IsSet(utils.OverrideVerkle.Name) { + v := ctx.Uint64(utils.OverrideVerkle.Name) + cfg.Eth.OverrideVerkle = &v + } backend, eth := utils.RegisterEthService(stack, &cfg.Eth) // Configure log filter RPC API. diff --git a/cmd/geth/main.go b/cmd/geth/main.go index dcdf0045ce..ede10ed909 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -65,6 +65,7 @@ var ( utils.USBFlag, utils.SmartCardDaemonPathFlag, utils.OverrideCancun, + utils.OverrideVerkle, utils.EnablePersonal, utils.TxPoolLocalsFlag, utils.TxPoolNoLocalsFlag, diff --git a/cmd/geth/verkle.go b/cmd/geth/verkle.go index 7766c458e6..9ba2b41671 100644 --- a/cmd/geth/verkle.go +++ b/cmd/geth/verkle.go @@ -74,7 +74,7 @@ func checkChildren(root verkle.VerkleNode, resolver verkle.NodeResolverFn) error switch node := root.(type) { case *verkle.InternalNode: for i, child := range node.Children() { - childC := child.ComputeCommitment().Bytes() + childC := child.Commit().Bytes() childS, err := resolver(childC[:]) if bytes.Equal(childC[:], zero[:]) { @@ -86,7 +86,7 @@ func checkChildren(root verkle.VerkleNode, resolver verkle.NodeResolverFn) error // depth is set to 0, the tree isn't rebuilt so it's not a problem childN, err := verkle.ParseNode(childS, 0, childC[:]) if err != nil { - return fmt.Errorf("decode error child %x in db: %w", child.ComputeCommitment().Bytes(), err) + return fmt.Errorf("decode error child %x in db: %w", child.Commitment().Bytes(), err) } if err := checkChildren(childN, resolver); err != nil { return fmt.Errorf("%x%w", i, err) // write the path to the erroring node diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index de4e58a3b1..7bb47f1282 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -268,6 +268,11 @@ var ( Usage: "Manually specify the Cancun fork timestamp, overriding the bundled setting", Category: flags.EthCategory, } + OverrideVerkle = &cli.Uint64Flag{ + Name: "override.verkle", + Usage: "Manually specify the Verkle fork timestamp, overriding the bundled setting", + Category: flags.EthCategory, + } // Light server and client settings LightServeFlag = &cli.IntFlag{ Name: "light.serve", diff --git a/core/genesis.go b/core/genesis.go index 1e56845d8a..ab5623ad00 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -266,6 +266,7 @@ func (e *GenesisMismatchError) Error() string { // ChainOverrides contains the changes to chain config. type ChainOverrides struct { OverrideCancun *uint64 + OverrideVerkle *uint64 } // SetupGenesisBlock writes or updates the genesis block in db. @@ -294,6 +295,9 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, triedb *trie.Database, gen if overrides != nil && overrides.OverrideCancun != nil { config.CancunTime = overrides.OverrideCancun } + if overrides != nil && overrides.OverrideVerkle != nil { + config.VerkleTime = overrides.OverrideVerkle + } } } // Just commit the new block if there is no stored genesis block. diff --git a/core/vm/jump_table_export.go b/core/vm/jump_table_export.go index 0d61b00ede..9b160ad18b 100644 --- a/core/vm/jump_table_export.go +++ b/core/vm/jump_table_export.go @@ -26,6 +26,8 @@ import ( // the rules. func LookupInstructionSet(rules params.Rules) (JumpTable, error) { switch { + case rules.IsVerkle: + return newShanghaiInstructionSet(), errors.New("verkle-fork not defined yet") case rules.IsPrague: return newShanghaiInstructionSet(), errors.New("prague-fork not defined yet") case rules.IsCancun: diff --git a/eth/backend.go b/eth/backend.go index 8d6977205e..cc555f0c23 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -199,6 +199,9 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { if config.OverrideCancun != nil { overrides.OverrideCancun = config.OverrideCancun } + if config.OverrideVerkle != nil { + overrides.OverrideVerkle = config.OverrideVerkle + } eth.blockchain, err = core.NewBlockChain(chainDb, cacheConfig, config.Genesis, &overrides, eth.engine, vmConfig, eth.shouldPreserve, &config.TxLookupLimit) if err != nil { return nil, err diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 432ec8ffbb..b3f7932d7f 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -162,6 +162,9 @@ type Config struct { // OverrideCancun (TODO: remove after the fork) OverrideCancun *uint64 `toml:",omitempty"` + + // OverrideVerkle (TODO: remove after the fork) + OverrideVerkle *uint64 `toml:",omitempty"` } // CreateConsensusEngine creates a consensus engine for the given chain config. diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index 5e1d75e3b5..b41c445fe6 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -55,6 +55,7 @@ func (c Config) MarshalTOML() (interface{}, error) { RPCEVMTimeout time.Duration RPCTxFeeCap float64 OverrideCancun *uint64 `toml:",omitempty"` + OverrideVerkle *uint64 `toml:",omitempty"` } var enc Config enc.Genesis = c.Genesis @@ -96,6 +97,7 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.RPCEVMTimeout = c.RPCEVMTimeout enc.RPCTxFeeCap = c.RPCTxFeeCap enc.OverrideCancun = c.OverrideCancun + enc.OverrideVerkle = c.OverrideVerkle return &enc, nil } @@ -141,6 +143,7 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { RPCEVMTimeout *time.Duration RPCTxFeeCap *float64 OverrideCancun *uint64 `toml:",omitempty"` + OverrideVerkle *uint64 `toml:",omitempty"` } var dec Config if err := unmarshal(&dec); err != nil { @@ -263,5 +266,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.OverrideCancun != nil { c.OverrideCancun = dec.OverrideCancun } + if dec.OverrideVerkle != nil { + c.OverrideVerkle = dec.OverrideVerkle + } return nil } diff --git a/eth/tracers/api.go b/eth/tracers/api.go index 5e90180df8..740a38ab9f 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -1020,6 +1020,10 @@ func overrideConfig(original *params.ChainConfig, override *params.ChainConfig) copy.PragueTime = timestamp canon = false } + if timestamp := override.VerkleTime; timestamp != nil { + copy.VerkleTime = timestamp + canon = false + } return copy, canon } diff --git a/go.mod b/go.mod index 253e47d1e4..5090cf1a98 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 github.com/fsnotify/fsnotify v1.6.0 github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff - github.com/gballet/go-verkle v0.0.0-20220902153445-097bd83b7732 + github.com/gballet/go-verkle v0.0.0-20230607174250-df487255f46b github.com/go-stack/stack v1.8.1 github.com/gofrs/flock v0.8.1 github.com/golang-jwt/jwt/v4 v4.3.0 @@ -62,8 +62,8 @@ require ( github.com/urfave/cli/v2 v2.17.2-0.20221006022127-8f469abc00aa golang.org/x/crypto v0.9.0 golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc - golang.org/x/sync v0.2.0 - golang.org/x/sys v0.8.0 + golang.org/x/sync v0.3.0 + golang.org/x/sys v0.9.0 golang.org/x/text v0.9.0 golang.org/x/time v0.3.0 golang.org/x/tools v0.9.1 @@ -90,7 +90,7 @@ require ( github.com/cockroachdb/redact v1.1.3 // indirect github.com/consensys/bavard v0.1.13 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect - github.com/crate-crypto/go-ipa v0.0.0-20220523130400-f11357ae11c7 // indirect + github.com/crate-crypto/go-ipa v0.0.0-20230601170251-1830d0757c80 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect github.com/deepmap/oapi-codegen v1.8.2 // indirect github.com/dlclark/regexp2 v1.7.0 // indirect diff --git a/go.sum b/go.sum index c257b615aa..58b40cdb01 100644 --- a/go.sum +++ b/go.sum @@ -86,6 +86,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHH github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crate-crypto/go-ipa v0.0.0-20220523130400-f11357ae11c7 h1:6IrxszG5G+O7zhtkWxq6+unVvnrm1fqV2Pe+T95DUzw= github.com/crate-crypto/go-ipa v0.0.0-20220523130400-f11357ae11c7/go.mod h1:gFnFS95y8HstDP6P9pPwzrxOOC5TRDkwbM+ao15ChAI= +github.com/crate-crypto/go-ipa v0.0.0-20230601170251-1830d0757c80 h1:DuBDHVjgGMPki7bAyh91+3cF1Vh34sAEdH8JQgbc2R0= +github.com/crate-crypto/go-ipa v0.0.0-20230601170251-1830d0757c80/go.mod h1:gzbVz57IDJgQ9rLQwfSk696JGWof8ftznEL9GoAv3NI= github.com/crate-crypto/go-kzg-4844 v0.3.0 h1:UBlWE0CgyFqqzTI+IFyCzA7A3Zw4iip6uzRv5NIXG0A= github.com/crate-crypto/go-kzg-4844 v0.3.0/go.mod h1:SBP7ikXEgDnUPONgm33HtuDZEDtWa3L4QtN1ocJSEQ4= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -146,6 +148,8 @@ github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqG github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= github.com/gballet/go-verkle v0.0.0-20220902153445-097bd83b7732 h1:AB7YjNrzlVHsYz06zCULVV2zYCEft82P86dSmtwxKL0= github.com/gballet/go-verkle v0.0.0-20220902153445-097bd83b7732/go.mod h1:o/XfIXWi4/GqbQirfRm5uTbXMG5NpqxkxblnbZ+QM9I= +github.com/gballet/go-verkle v0.0.0-20230607174250-df487255f46b h1:vMT47RYsrftsHSTQhqXwC3BYflo38OLC3Y4LtXtLyU0= +github.com/gballet/go-verkle v0.0.0-20230607174250-df487255f46b/go.mod h1:CDncRYVRSDqwakm282WEkjfaAj1hxU/v5RXxk5nXOiI= github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= github.com/getsentry/sentry-go v0.12.0/go.mod h1:NSap0JBYWzHND8oMbyi0+XZhUalc1TBdRL1M71JZW2c= @@ -525,8 +529,11 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -568,6 +575,8 @@ golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/les/client.go b/les/client.go index 0072eb8a1a..78bd05c77b 100644 --- a/les/client.go +++ b/les/client.go @@ -97,6 +97,9 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { if config.OverrideCancun != nil { overrides.OverrideCancun = config.OverrideCancun } + if config.OverrideVerkle != nil { + overrides.OverrideVerkle = config.OverrideVerkle + } chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, trie.NewDatabase(chainDb), config.Genesis, &overrides) if _, isCompat := genesisErr.(*params.ConfigCompatError); genesisErr != nil && !isCompat { return nil, genesisErr diff --git a/params/config.go b/params/config.go index 455abe2062..8e6704bf1c 100644 --- a/params/config.go +++ b/params/config.go @@ -128,6 +128,7 @@ var ( ShanghaiTime: nil, CancunTime: nil, PragueTime: nil, + VerkleTime: nil, TerminalTotalDifficulty: nil, TerminalTotalDifficultyPassed: true, Ethash: new(EthashConfig), @@ -157,6 +158,7 @@ var ( ShanghaiTime: nil, CancunTime: nil, PragueTime: nil, + VerkleTime: nil, TerminalTotalDifficulty: nil, TerminalTotalDifficultyPassed: false, Ethash: nil, @@ -186,6 +188,7 @@ var ( ShanghaiTime: nil, CancunTime: nil, PragueTime: nil, + VerkleTime: nil, TerminalTotalDifficulty: nil, TerminalTotalDifficultyPassed: false, Ethash: new(EthashConfig), @@ -215,6 +218,7 @@ var ( ShanghaiTime: nil, CancunTime: nil, PragueTime: nil, + VerkleTime: nil, TerminalTotalDifficulty: nil, TerminalTotalDifficultyPassed: false, Ethash: new(EthashConfig), @@ -264,6 +268,7 @@ type ChainConfig struct { ShanghaiTime *uint64 `json:"shanghaiTime,omitempty"` // Shanghai switch time (nil = no fork, 0 = already on shanghai) CancunTime *uint64 `json:"cancunTime,omitempty"` // Cancun switch time (nil = no fork, 0 = already on cancun) PragueTime *uint64 `json:"pragueTime,omitempty"` // Prague switch time (nil = no fork, 0 = already on prague) + VerkleTime *uint64 `json:"verkleTime,omitempty"` // Verkle switch time (nil = no fork, 0 = already on verkle) // TerminalTotalDifficulty is the amount of total difficulty reached by // the network that triggers the consensus upgrade. @@ -384,6 +389,9 @@ func (c *ChainConfig) Description() string { if c.PragueTime != nil { banner += fmt.Sprintf(" - Prague: @%-10v\n", *c.PragueTime) } + if c.VerkleTime != nil { + banner += fmt.Sprintf(" - Verkle: @%-10v\n", *c.VerkleTime) + } return banner } @@ -482,6 +490,11 @@ func (c *ChainConfig) IsPrague(num *big.Int, time uint64) bool { return c.IsLondon(num) && isTimestampForked(c.PragueTime, time) } +// IsVerkle returns whether num is either equal to the Verkle fork time or greater. +func (c *ChainConfig) IsVerkle(num *big.Int, time uint64) bool { + return c.IsLondon(num) && isTimestampForked(c.VerkleTime, time) +} + // CheckCompatible checks whether scheduled fork transitions have been imported // with a mismatching chain configuration. func (c *ChainConfig) CheckCompatible(newcfg *ChainConfig, height uint64, time uint64) *ConfigCompatError { @@ -536,6 +549,7 @@ func (c *ChainConfig) CheckConfigForkOrder() error { {name: "shanghaiTime", timestamp: c.ShanghaiTime}, {name: "cancunTime", timestamp: c.CancunTime, optional: true}, {name: "pragueTime", timestamp: c.PragueTime, optional: true}, + {name: "verkleTime", timestamp: c.VerkleTime, optional: true}, } { if lastFork.name != "" { switch { @@ -639,6 +653,9 @@ func (c *ChainConfig) checkCompatible(newcfg *ChainConfig, headNumber *big.Int, if isForkTimestampIncompatible(c.PragueTime, newcfg.PragueTime, headTimestamp) { return newTimestampCompatError("Prague fork timestamp", c.PragueTime, newcfg.PragueTime) } + if isForkTimestampIncompatible(c.VerkleTime, newcfg.VerkleTime, headTimestamp) { + return newTimestampCompatError("Verkle fork timestamp", c.VerkleTime, newcfg.VerkleTime) + } return nil } @@ -784,6 +801,7 @@ type Rules struct { IsByzantium, IsConstantinople, IsPetersburg, IsIstanbul bool IsBerlin, IsLondon bool IsMerge, IsShanghai, IsCancun, IsPrague bool + IsVerkle bool } // Rules ensures c's ChainID is not nil. @@ -808,5 +826,6 @@ func (c *ChainConfig) Rules(num *big.Int, isMerge bool, timestamp uint64) Rules IsShanghai: c.IsShanghai(num, timestamp), IsCancun: c.IsCancun(num, timestamp), IsPrague: c.IsPrague(num, timestamp), + IsVerkle: c.IsVerkle(num, timestamp), } } From 8bbb16b70ef4a9e732745aac104525a6f16306d6 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Wed, 28 Jun 2023 11:11:02 +0200 Subject: [PATCH 44/90] core/state, light, les: make signature of ContractCode hash-independent (#27209) * core/state, light, les: make signature of ContractCode hash-independent * push current state for feedback * les: fix unit test * core, les, light: fix les unittests * core/state, trie, les, light: fix state iterator * core, les: address comments * les: fix lint --------- Co-authored-by: Gary Rong --- core/blockchain_reader.go | 6 ++- core/state/database.go | 19 ++++---- core/state/iterator.go | 17 +++++-- core/state/state_object.go | 6 +-- core/state/sync_test.go | 83 +++++++++++++++++---------------- core/state/trie_prefetcher.go | 2 +- les/benchmark.go | 2 +- les/handler_test.go | 8 ++-- les/odr_requests.go | 20 ++++---- les/request_test.go | 3 +- les/server_handler.go | 15 +++--- les/server_requests.go | 21 +++++---- light/odr.go | 32 ++++++------- light/odr_test.go | 4 +- light/trie.go | 20 ++++---- tests/fuzzers/les/les-fuzzer.go | 41 ++++++++-------- trie/database_wrap.go | 7 ++- 17 files changed, 164 insertions(+), 142 deletions(-) diff --git a/core/blockchain_reader.go b/core/blockchain_reader.go index fd65cb2db3..f9d0e3531f 100644 --- a/core/blockchain_reader.go +++ b/core/blockchain_reader.go @@ -306,9 +306,11 @@ func (bc *BlockChain) TrieNode(hash common.Hash) ([]byte, error) { // new code scheme. func (bc *BlockChain) ContractCodeWithPrefix(hash common.Hash) ([]byte, error) { type codeReader interface { - ContractCodeWithPrefix(addrHash, codeHash common.Hash) ([]byte, error) + ContractCodeWithPrefix(address common.Address, codeHash common.Hash) ([]byte, error) } - return bc.stateCache.(codeReader).ContractCodeWithPrefix(common.Hash{}, hash) + // TODO(rjl493456442) The associated account address is also required + // in Verkle scheme. Fix it once snap-sync is supported for Verkle. + return bc.stateCache.(codeReader).ContractCodeWithPrefix(common.Address{}, hash) } // State returns a new mutable state based on the current HEAD block. diff --git a/core/state/database.go b/core/state/database.go index 5ca2366360..a3b6322ae3 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common/lru" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/trienode" @@ -43,16 +44,16 @@ type Database interface { OpenTrie(root common.Hash) (Trie, error) // OpenStorageTrie opens the storage trie of an account. - OpenStorageTrie(stateRoot common.Hash, addrHash, root common.Hash) (Trie, error) + OpenStorageTrie(stateRoot common.Hash, address common.Address, root common.Hash) (Trie, error) // CopyTrie returns an independent copy of the given trie. CopyTrie(Trie) Trie // ContractCode retrieves a particular contract's code. - ContractCode(addrHash, codeHash common.Hash) ([]byte, error) + ContractCode(addr common.Address, codeHash common.Hash) ([]byte, error) // ContractCodeSize retrieves a particular contracts code's size. - ContractCodeSize(addrHash, codeHash common.Hash) (int, error) + ContractCodeSize(addr common.Address, codeHash common.Hash) (int, error) // DiskDB returns the underlying key-value disk database. DiskDB() ethdb.KeyValueStore @@ -177,8 +178,8 @@ func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) { } // OpenStorageTrie opens the storage trie of an account. -func (db *cachingDB) OpenStorageTrie(stateRoot common.Hash, addrHash, root common.Hash) (Trie, error) { - tr, err := trie.NewStateTrie(trie.StorageTrieID(stateRoot, addrHash, root), db.triedb) +func (db *cachingDB) OpenStorageTrie(stateRoot common.Hash, address common.Address, root common.Hash) (Trie, error) { + tr, err := trie.NewStateTrie(trie.StorageTrieID(stateRoot, crypto.Keccak256Hash(address.Bytes()), root), db.triedb) if err != nil { return nil, err } @@ -196,7 +197,7 @@ func (db *cachingDB) CopyTrie(t Trie) Trie { } // ContractCode retrieves a particular contract's code. -func (db *cachingDB) ContractCode(addrHash, codeHash common.Hash) ([]byte, error) { +func (db *cachingDB) ContractCode(address common.Address, codeHash common.Hash) ([]byte, error) { code, _ := db.codeCache.Get(codeHash) if len(code) > 0 { return code, nil @@ -213,7 +214,7 @@ func (db *cachingDB) ContractCode(addrHash, codeHash common.Hash) ([]byte, error // ContractCodeWithPrefix retrieves a particular contract's code. If the // code can't be found in the cache, then check the existence with **new** // db scheme. -func (db *cachingDB) ContractCodeWithPrefix(addrHash, codeHash common.Hash) ([]byte, error) { +func (db *cachingDB) ContractCodeWithPrefix(address common.Address, codeHash common.Hash) ([]byte, error) { code, _ := db.codeCache.Get(codeHash) if len(code) > 0 { return code, nil @@ -228,11 +229,11 @@ func (db *cachingDB) ContractCodeWithPrefix(addrHash, codeHash common.Hash) ([]b } // ContractCodeSize retrieves a particular contracts code's size. -func (db *cachingDB) ContractCodeSize(addrHash, codeHash common.Hash) (int, error) { +func (db *cachingDB) ContractCodeSize(addr common.Address, codeHash common.Hash) (int, error) { if cached, ok := db.codeSizeCache.Get(codeHash); ok { return cached, nil } - code, err := db.ContractCode(addrHash, codeHash) + code, err := db.ContractCode(addr, codeHash) return len(code), err } diff --git a/core/state/iterator.go b/core/state/iterator.go index 7802263b75..bb9af08206 100644 --- a/core/state/iterator.go +++ b/core/state/iterator.go @@ -18,6 +18,7 @@ package state import ( "bytes" + "errors" "fmt" "github.com/ethereum/go-ethereum/common" @@ -27,7 +28,8 @@ import ( ) // nodeIterator is an iterator to traverse the entire state trie post-order, -// including all of the contract code and contract state tries. +// including all of the contract code and contract state tries. Preimage is +// required in order to resolve the contract address. type nodeIterator struct { state *StateDB // State being iterated @@ -113,7 +115,15 @@ func (it *nodeIterator) step() error { if err := rlp.Decode(bytes.NewReader(it.stateIt.LeafBlob()), &account); err != nil { return err } - dataTrie, err := it.state.db.OpenStorageTrie(it.state.originalRoot, common.BytesToHash(it.stateIt.LeafKey()), account.Root) + // Lookup the preimage of account hash + preimage := it.state.trie.GetKey(it.stateIt.LeafKey()) + if preimage == nil { + return errors.New("account address is not available") + } + address := common.BytesToAddress(preimage) + + // Traverse the storage slots belong to the account + dataTrie, err := it.state.db.OpenStorageTrie(it.state.originalRoot, address, account.Root) if err != nil { return err } @@ -126,8 +136,7 @@ func (it *nodeIterator) step() error { } if !bytes.Equal(account.CodeHash, types.EmptyCodeHash.Bytes()) { it.codeHash = common.BytesToHash(account.CodeHash) - addrHash := common.BytesToHash(it.stateIt.LeafKey()) - it.code, err = it.state.db.ContractCode(addrHash, common.BytesToHash(account.CodeHash)) + it.code, err = it.state.db.ContractCode(address, common.BytesToHash(account.CodeHash)) if err != nil { return fmt.Errorf("code %x: %v", account.CodeHash, err) } diff --git a/core/state/state_object.go b/core/state/state_object.go index 148136fc2c..b443d12f12 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -142,7 +142,7 @@ func (s *stateObject) getTrie(db Database) (Trie, error) { s.trie = s.db.prefetcher.trie(s.addrHash, s.data.Root) } if s.trie == nil { - tr, err := db.OpenStorageTrie(s.db.originalRoot, s.addrHash, s.data.Root) + tr, err := db.OpenStorageTrie(s.db.originalRoot, s.address, s.data.Root) if err != nil { return nil, err } @@ -441,7 +441,7 @@ func (s *stateObject) Code(db Database) []byte { if bytes.Equal(s.CodeHash(), types.EmptyCodeHash.Bytes()) { return nil } - code, err := db.ContractCode(s.addrHash, common.BytesToHash(s.CodeHash())) + code, err := db.ContractCode(s.address, common.BytesToHash(s.CodeHash())) if err != nil { s.db.setError(fmt.Errorf("can't load code hash %x: %v", s.CodeHash(), err)) } @@ -459,7 +459,7 @@ func (s *stateObject) CodeSize(db Database) int { if bytes.Equal(s.CodeHash(), types.EmptyCodeHash.Bytes()) { return 0 } - size, err := db.ContractCodeSize(s.addrHash, common.BytesToHash(s.CodeHash())) + size, err := db.ContractCodeSize(s.address, common.BytesToHash(s.CodeHash())) if err != nil { s.db.setError(fmt.Errorf("can't load code size %x: %v", s.CodeHash(), err)) } diff --git a/core/state/sync_test.go b/core/state/sync_test.go index cac0ee6e48..7de69e0aa8 100644 --- a/core/state/sync_test.go +++ b/core/state/sync_test.go @@ -42,7 +42,7 @@ type testAccount struct { func makeTestState() (ethdb.Database, Database, common.Hash, []*testAccount) { // Create an empty state db := rawdb.NewMemoryDatabase() - sdb := NewDatabase(db) + sdb := NewDatabaseWithConfig(db, &trie.Config{Preimages: true}) state, _ := New(types.EmptyRootHash, sdb, nil) // Fill it with some arbitrary data @@ -100,28 +100,9 @@ func checkStateAccounts(t *testing.T, db ethdb.Database, root common.Hash, accou } } -// checkTrieConsistency checks that all nodes in a (sub-)trie are indeed present. -func checkTrieConsistency(db ethdb.Database, root common.Hash) error { - if v, _ := db.Get(root[:]); v == nil { - return nil // Consider a non existent state consistent. - } - trie, err := trie.New(trie.StateTrieID(root), trie.NewDatabase(db)) - if err != nil { - return err - } - it := trie.MustNodeIterator(nil) - for it.Next(true) { - } - return it.Error() -} - // checkStateConsistency checks that all data of a state root is present. func checkStateConsistency(db ethdb.Database, root common.Hash) error { - // Create and iterate a state trie rooted in a sub-node - if _, err := db.Get(root.Bytes()); err != nil { - return nil // Consider a non existent state consistent. - } - state, err := New(root, NewDatabase(db), nil) + state, err := New(root, NewDatabaseWithConfig(db, &trie.Config{Preimages: true}), nil) if err != nil { return err } @@ -171,7 +152,7 @@ type stateElement struct { func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) { // Create a random state to copy - _, srcDb, srcRoot, srcAccounts := makeTestState() + srcDisk, srcDb, srcRoot, srcAccounts := makeTestState() if commit { srcDb.TrieDB().Commit(srcRoot, false) } @@ -204,7 +185,7 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) { codeResults = make([]trie.CodeSyncResult, len(codeElements)) ) for i, element := range codeElements { - data, err := srcDb.ContractCode(common.Hash{}, element.code) + data, err := srcDb.ContractCode(common.Address{}, element.code) if err != nil { t.Fatalf("failed to retrieve contract bytecode for hash %x", element.code) } @@ -274,6 +255,10 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) { }) } } + // Copy the preimages from source db in order to traverse the state. + srcDb.TrieDB().WritePreimages() + copyPreimages(srcDisk, dstDb) + // Cross check that the two states are in sync checkStateAccounts(t, dstDb, srcRoot, srcAccounts) } @@ -282,7 +267,7 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) { // partial results are returned, and the others sent only later. func TestIterativeDelayedStateSync(t *testing.T) { // Create a random state to copy - _, srcDb, srcRoot, srcAccounts := makeTestState() + srcDisk, srcDb, srcRoot, srcAccounts := makeTestState() // Create a destination state and sync with the scheduler dstDb := rawdb.NewMemoryDatabase() @@ -312,7 +297,7 @@ func TestIterativeDelayedStateSync(t *testing.T) { if len(codeElements) > 0 { codeResults := make([]trie.CodeSyncResult, len(codeElements)/2+1) for i, element := range codeElements[:len(codeResults)] { - data, err := srcDb.ContractCode(common.Hash{}, element.code) + data, err := srcDb.ContractCode(common.Address{}, element.code) if err != nil { t.Fatalf("failed to retrieve contract bytecode for %x", element.code) } @@ -363,6 +348,10 @@ func TestIterativeDelayedStateSync(t *testing.T) { }) } } + // Copy the preimages from source db in order to traverse the state. + srcDb.TrieDB().WritePreimages() + copyPreimages(srcDisk, dstDb) + // Cross check that the two states are in sync checkStateAccounts(t, dstDb, srcRoot, srcAccounts) } @@ -375,7 +364,7 @@ func TestIterativeRandomStateSyncBatched(t *testing.T) { testIterativeRandomS func testIterativeRandomStateSync(t *testing.T, count int) { // Create a random state to copy - _, srcDb, srcRoot, srcAccounts := makeTestState() + srcDisk, srcDb, srcRoot, srcAccounts := makeTestState() // Create a destination state and sync with the scheduler dstDb := rawdb.NewMemoryDatabase() @@ -399,7 +388,7 @@ func testIterativeRandomStateSync(t *testing.T, count int) { if len(codeQueue) > 0 { results := make([]trie.CodeSyncResult, 0, len(codeQueue)) for hash := range codeQueue { - data, err := srcDb.ContractCode(common.Hash{}, hash) + data, err := srcDb.ContractCode(common.Address{}, hash) if err != nil { t.Fatalf("failed to retrieve node data for %x", hash) } @@ -447,6 +436,10 @@ func testIterativeRandomStateSync(t *testing.T, count int) { codeQueue[hash] = struct{}{} } } + // Copy the preimages from source db in order to traverse the state. + srcDb.TrieDB().WritePreimages() + copyPreimages(srcDisk, dstDb) + // Cross check that the two states are in sync checkStateAccounts(t, dstDb, srcRoot, srcAccounts) } @@ -455,7 +448,7 @@ func testIterativeRandomStateSync(t *testing.T, count int) { // partial results are returned (Even those randomly), others sent only later. func TestIterativeRandomDelayedStateSync(t *testing.T) { // Create a random state to copy - _, srcDb, srcRoot, srcAccounts := makeTestState() + srcDisk, srcDb, srcRoot, srcAccounts := makeTestState() // Create a destination state and sync with the scheduler dstDb := rawdb.NewMemoryDatabase() @@ -481,7 +474,7 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) { for hash := range codeQueue { delete(codeQueue, hash) - data, err := srcDb.ContractCode(common.Hash{}, hash) + data, err := srcDb.ContractCode(common.Address{}, hash) if err != nil { t.Fatalf("failed to retrieve node data for %x", hash) } @@ -537,6 +530,10 @@ func TestIterativeRandomDelayedStateSync(t *testing.T) { codeQueue[hash] = struct{}{} } } + // Copy the preimages from source db in order to traverse the state. + srcDb.TrieDB().WritePreimages() + copyPreimages(srcDisk, dstDb) + // Cross check that the two states are in sync checkStateAccounts(t, dstDb, srcRoot, srcAccounts) } @@ -555,7 +552,6 @@ func TestIncompleteStateSync(t *testing.T) { } } isCode[types.EmptyCodeHash] = struct{}{} - checkTrieConsistency(db, srcRoot) // Create a destination state and sync with the scheduler dstDb := rawdb.NewMemoryDatabase() @@ -588,7 +584,7 @@ func TestIncompleteStateSync(t *testing.T) { if len(codeQueue) > 0 { results := make([]trie.CodeSyncResult, 0, len(codeQueue)) for hash := range codeQueue { - data, err := srcDb.ContractCode(common.Hash{}, hash) + data, err := srcDb.ContractCode(common.Address{}, hash) if err != nil { t.Fatalf("failed to retrieve node data for %x", hash) } @@ -602,7 +598,6 @@ func TestIncompleteStateSync(t *testing.T) { } } } - var nodehashes []common.Hash if len(nodeQueue) > 0 { results := make([]trie.NodeSyncResult, 0, len(nodeQueue)) for path, element := range nodeQueue { @@ -617,7 +612,6 @@ func TestIncompleteStateSync(t *testing.T) { addedPaths = append(addedPaths, element.path) addedHashes = append(addedHashes, element.hash) } - nodehashes = append(nodehashes, element.hash) } // Process each of the state nodes for _, result := range results { @@ -632,13 +626,6 @@ func TestIncompleteStateSync(t *testing.T) { } batch.Write() - for _, root := range nodehashes { - // Can't use checkStateConsistency here because subtrie keys may have odd - // length and crash in LeafKey. - if err := checkTrieConsistency(dstDb, root); err != nil { - t.Fatalf("state inconsistent: %v", err) - } - } // Fetch the next batch to retrieve nodeQueue = make(map[string]stateElement) codeQueue = make(map[common.Hash]struct{}) @@ -654,6 +641,10 @@ func TestIncompleteStateSync(t *testing.T) { codeQueue[hash] = struct{}{} } } + // Copy the preimages from source db in order to traverse the state. + srcDb.TrieDB().WritePreimages() + copyPreimages(db, dstDb) + // Sanity check that removing any node from the database is detected for _, node := range addedCodes { val := rawdb.ReadCode(dstDb, node) @@ -678,3 +669,15 @@ func TestIncompleteStateSync(t *testing.T) { rawdb.WriteTrieNode(dstDb, owner, inner, hash, val, scheme) } } + +func copyPreimages(srcDb, dstDb ethdb.Database) { + it := srcDb.NewIterator(rawdb.PreimagePrefix, nil) + defer it.Release() + + preimages := make(map[common.Hash][]byte) + for it.Next() { + hash := it.Key()[len(rawdb.PreimagePrefix):] + preimages[common.BytesToHash(hash)] = common.CopyBytes(it.Value()) + } + rawdb.WritePreimages(dstDb, preimages) +} diff --git a/core/state/trie_prefetcher.go b/core/state/trie_prefetcher.go index 844f72fc10..4e8fd1e10f 100644 --- a/core/state/trie_prefetcher.go +++ b/core/state/trie_prefetcher.go @@ -302,7 +302,7 @@ func (sf *subfetcher) loop() { } sf.trie = trie } else { - trie, err := sf.db.OpenStorageTrie(sf.state, sf.owner, sf.root) + trie, err := sf.db.OpenStorageTrie(sf.state, sf.addr, sf.root) if err != nil { log.Warn("Trie prefetcher failed opening trie", "root", sf.root, "err", err) return diff --git a/les/benchmark.go b/les/benchmark.go index c63a2564d5..ab93518349 100644 --- a/les/benchmark.go +++ b/les/benchmark.go @@ -117,7 +117,7 @@ func (b *benchmarkProofsOrCode) request(peer *serverPeer, index int) error { key := make([]byte, 32) crand.Read(key) if b.code { - return peer.requestCode(0, []CodeReq{{BHash: b.headHash, AccKey: key}}) + return peer.requestCode(0, []CodeReq{{BHash: b.headHash, AccountAddress: key}}) } return peer.requestProofs(0, []ProofReq{{BHash: b.headHash, Key: key}}) } diff --git a/les/handler_test.go b/les/handler_test.go index d0878dcd99..31681fe368 100644 --- a/les/handler_test.go +++ b/les/handler_test.go @@ -295,8 +295,8 @@ func testGetCode(t *testing.T, protocol int) { for i := uint64(0); i <= bc.CurrentBlock().Number.Uint64(); i++ { header := bc.GetHeaderByNumber(i) req := &CodeReq{ - BHash: header.Hash(), - AccKey: crypto.Keccak256(testContractAddr[:]), + BHash: header.Hash(), + AccountAddress: testContractAddr[:], } codereqs = append(codereqs, req) if i >= testContractDeployed { @@ -331,8 +331,8 @@ func testGetStaleCode(t *testing.T, protocol int) { check := func(number uint64, expected [][]byte) { req := &CodeReq{ - BHash: bc.GetHeaderByNumber(number).Hash(), - AccKey: crypto.Keccak256(testContractAddr[:]), + BHash: bc.GetHeaderByNumber(number).Hash(), + AccountAddress: testContractAddr[:], } sendRequest(rawPeer.app, GetCodeMsg, 42, []*CodeReq{req}) if err := expectResponse(rawPeer.app, CodeMsg, 42, testBufLimit, expected); err != nil { diff --git a/les/odr_requests.go b/les/odr_requests.go index d8b094b727..2b23e0540c 100644 --- a/les/odr_requests.go +++ b/les/odr_requests.go @@ -183,9 +183,9 @@ func (r *ReceiptsRequest) Validate(db ethdb.Database, msg *Msg) error { } type ProofReq struct { - BHash common.Hash - AccKey, Key []byte - FromLevel uint + BHash common.Hash + AccountAddress, Key []byte + FromLevel uint } // ODR request type for state/storage trie entries, see LesOdrRequest interface @@ -206,9 +206,9 @@ func (r *TrieRequest) CanSend(peer *serverPeer) bool { func (r *TrieRequest) Request(reqID uint64, peer *serverPeer) error { peer.Log().Debug("Requesting trie proof", "root", r.Id.Root, "key", r.Key) req := ProofReq{ - BHash: r.Id.BlockHash, - AccKey: r.Id.AccKey, - Key: r.Key, + BHash: r.Id.BlockHash, + AccountAddress: r.Id.AccountAddress, + Key: r.Key, } return peer.requestProofs(reqID, []ProofReq{req}) } @@ -238,8 +238,8 @@ func (r *TrieRequest) Validate(db ethdb.Database, msg *Msg) error { } type CodeReq struct { - BHash common.Hash - AccKey []byte + BHash common.Hash + AccountAddress []byte } // CodeRequest is the ODR request type for node data (used for retrieving contract code), see LesOdrRequest interface @@ -260,8 +260,8 @@ func (r *CodeRequest) CanSend(peer *serverPeer) bool { func (r *CodeRequest) Request(reqID uint64, peer *serverPeer) error { peer.Log().Debug("Requesting code data", "hash", r.Hash) req := CodeReq{ - BHash: r.Id.BlockHash, - AccKey: r.Id.AccKey, + BHash: r.Id.BlockHash, + AccountAddress: r.Id.AccountAddress, } return peer.requestCode(reqID, []CodeReq{req}) } diff --git a/les/request_test.go b/les/request_test.go index 9b52e6bd86..30ae2c2bee 100644 --- a/les/request_test.go +++ b/les/request_test.go @@ -23,6 +23,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/light" @@ -77,7 +78,7 @@ func tfCodeAccess(db ethdb.Database, bhash common.Hash, num uint64) light.OdrReq return nil } sti := light.StateTrieID(header) - ci := light.StorageTrieID(sti, crypto.Keccak256Hash(testContractAddr[:]), common.Hash{}) + ci := light.StorageTrieID(sti, testContractAddr, types.EmptyRootHash) return &light.CodeRequest{Id: ci, Hash: crypto.Keccak256Hash(testContractCodeDeployed)} } diff --git a/les/server_handler.go b/les/server_handler.go index 418974c37c..ad18d3f4e4 100644 --- a/les/server_handler.go +++ b/les/server_handler.go @@ -18,6 +18,7 @@ package les import ( "errors" + "fmt" "sync" "time" @@ -34,7 +35,6 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" ) @@ -358,20 +358,19 @@ func (h *serverHandler) AddTxsSync() bool { } // getAccount retrieves an account from the state based on root. -func getAccount(triedb *trie.Database, root, hash common.Hash) (types.StateAccount, error) { - trie, err := trie.New(trie.StateTrieID(root), triedb) +func getAccount(triedb *trie.Database, root common.Hash, addr common.Address) (types.StateAccount, error) { + trie, err := trie.NewStateTrie(trie.StateTrieID(root), triedb) if err != nil { return types.StateAccount{}, err } - blob, err := trie.Get(hash[:]) + acc, err := trie.GetAccount(addr) if err != nil { return types.StateAccount{}, err } - var acc types.StateAccount - if err = rlp.DecodeBytes(blob, &acc); err != nil { - return types.StateAccount{}, err + if acc == nil { + return types.StateAccount{}, fmt.Errorf("account %#x is not present", addr) } - return acc, nil + return *acc, nil } // GetHelperTrie returns the post-processed trie root for the given trie ID and section index diff --git a/les/server_requests.go b/les/server_requests.go index f6208db443..30ff2cd05f 100644 --- a/les/server_requests.go +++ b/les/server_requests.go @@ -304,16 +304,16 @@ func handleGetCode(msg Decoder) (serveRequestFn, uint64, uint64, error) { continue } triedb := bc.StateCache().TrieDB() - - account, err := getAccount(triedb, header.Root, common.BytesToHash(request.AccKey)) + address := common.BytesToAddress(request.AccountAddress) + account, err := getAccount(triedb, header.Root, address) if err != nil { - p.Log().Warn("Failed to retrieve account for code", "block", header.Number, "hash", header.Hash(), "account", common.BytesToHash(request.AccKey), "err", err) + p.Log().Warn("Failed to retrieve account for code", "block", header.Number, "hash", header.Hash(), "account", address, "err", err) p.bumpInvalid() continue } - code, err := bc.StateCache().ContractCode(common.BytesToHash(request.AccKey), common.BytesToHash(account.CodeHash)) + code, err := bc.StateCache().ContractCode(address, common.BytesToHash(account.CodeHash)) if err != nil { - p.Log().Warn("Failed to retrieve account code", "block", header.Number, "hash", header.Hash(), "account", common.BytesToHash(request.AccKey), "codehash", common.BytesToHash(account.CodeHash), "err", err) + p.Log().Warn("Failed to retrieve account code", "block", header.Number, "hash", header.Hash(), "account", address, "codehash", common.BytesToHash(account.CodeHash), "err", err) continue } // Accumulate the code and abort if enough data was retrieved @@ -413,7 +413,7 @@ func handleGetProofs(msg Decoder) (serveRequestFn, uint64, uint64, error) { statedb := bc.StateCache() var trie state.Trie - switch len(request.AccKey) { + switch len(request.AccountAddress) { case 0: // No account key specified, open an account trie trie, err = statedb.OpenTrie(root) @@ -423,15 +423,16 @@ func handleGetProofs(msg Decoder) (serveRequestFn, uint64, uint64, error) { } default: // Account key specified, open a storage trie - account, err := getAccount(statedb.TrieDB(), root, common.BytesToHash(request.AccKey)) + address := common.BytesToAddress(request.AccountAddress) + account, err := getAccount(statedb.TrieDB(), root, address) if err != nil { - p.Log().Warn("Failed to retrieve account for proof", "block", header.Number, "hash", header.Hash(), "account", common.BytesToHash(request.AccKey), "err", err) + p.Log().Warn("Failed to retrieve account for proof", "block", header.Number, "hash", header.Hash(), "account", address, "err", err) p.bumpInvalid() continue } - trie, err = statedb.OpenStorageTrie(root, common.BytesToHash(request.AccKey), account.Root) + trie, err = statedb.OpenStorageTrie(root, address, account.Root) if trie == nil || err != nil { - p.Log().Warn("Failed to open storage trie for proof", "block", header.Number, "hash", header.Hash(), "account", common.BytesToHash(request.AccKey), "root", account.Root, "err", err) + p.Log().Warn("Failed to open storage trie for proof", "block", header.Number, "hash", header.Hash(), "account", address, "root", account.Root, "err", err) continue } } diff --git a/light/odr.go b/light/odr.go index d331d3c9b5..2597027435 100644 --- a/light/odr.go +++ b/light/odr.go @@ -54,35 +54,35 @@ type OdrRequest interface { // TrieID identifies a state or account storage trie type TrieID struct { - BlockHash common.Hash - BlockNumber uint64 - StateRoot common.Hash - Root common.Hash - AccKey []byte + BlockHash common.Hash + BlockNumber uint64 + StateRoot common.Hash + Root common.Hash + AccountAddress []byte } // StateTrieID returns a TrieID for a state trie belonging to a certain block // header. func StateTrieID(header *types.Header) *TrieID { return &TrieID{ - BlockHash: header.Hash(), - BlockNumber: header.Number.Uint64(), - StateRoot: header.Root, - Root: header.Root, - AccKey: nil, + BlockHash: header.Hash(), + BlockNumber: header.Number.Uint64(), + StateRoot: header.Root, + Root: header.Root, + AccountAddress: nil, } } // StorageTrieID returns a TrieID for a contract storage trie at a given account // of a given state trie. It also requires the root hash of the trie for // checking Merkle proofs. -func StorageTrieID(state *TrieID, addrHash, root common.Hash) *TrieID { +func StorageTrieID(state *TrieID, address common.Address, root common.Hash) *TrieID { return &TrieID{ - BlockHash: state.BlockHash, - BlockNumber: state.BlockNumber, - StateRoot: state.StateRoot, - AccKey: addrHash[:], - Root: root, + BlockHash: state.BlockHash, + BlockNumber: state.BlockNumber, + StateRoot: state.StateRoot, + AccountAddress: address[:], + Root: root, } } diff --git a/light/odr_test.go b/light/odr_test.go index 90fdea81dc..79f901bbdb 100644 --- a/light/odr_test.go +++ b/light/odr_test.go @@ -86,8 +86,8 @@ func (odr *testOdr) Retrieve(ctx context.Context, req OdrRequest) error { err error t state.Trie ) - if len(req.Id.AccKey) > 0 { - t, err = odr.serverState.OpenStorageTrie(req.Id.StateRoot, common.BytesToHash(req.Id.AccKey), req.Id.Root) + if len(req.Id.AccountAddress) > 0 { + t, err = odr.serverState.OpenStorageTrie(req.Id.StateRoot, common.BytesToAddress(req.Id.AccountAddress), req.Id.Root) } else { t, err = odr.serverState.OpenTrie(req.Id.Root) } diff --git a/light/trie.go b/light/trie.go index b411c299f2..43fccd5e36 100644 --- a/light/trie.go +++ b/light/trie.go @@ -55,8 +55,8 @@ func (db *odrDatabase) OpenTrie(root common.Hash) (state.Trie, error) { return &odrTrie{db: db, id: db.id}, nil } -func (db *odrDatabase) OpenStorageTrie(state, addrHash, root common.Hash) (state.Trie, error) { - return &odrTrie{db: db, id: StorageTrieID(db.id, addrHash, root)}, nil +func (db *odrDatabase) OpenStorageTrie(stateRoot common.Hash, address common.Address, root common.Hash) (state.Trie, error) { + return &odrTrie{db: db, id: StorageTrieID(db.id, address, root)}, nil } func (db *odrDatabase) CopyTrie(t state.Trie) state.Trie { @@ -72,7 +72,7 @@ func (db *odrDatabase) CopyTrie(t state.Trie) state.Trie { } } -func (db *odrDatabase) ContractCode(addrHash, codeHash common.Hash) ([]byte, error) { +func (db *odrDatabase) ContractCode(addr common.Address, codeHash common.Hash) ([]byte, error) { if codeHash == sha3Nil { return nil, nil } @@ -81,14 +81,14 @@ func (db *odrDatabase) ContractCode(addrHash, codeHash common.Hash) ([]byte, err return code, nil } id := *db.id - id.AccKey = addrHash[:] + id.AccountAddress = addr[:] req := &CodeRequest{Id: &id, Hash: codeHash} err := db.backend.Retrieve(db.ctx, req) return req.Data, err } -func (db *odrDatabase) ContractCodeSize(addrHash, codeHash common.Hash) (int, error) { - code, err := db.ContractCode(addrHash, codeHash) +func (db *odrDatabase) ContractCodeSize(addr common.Address, codeHash common.Hash) (int, error) { + code, err := db.ContractCode(addr, codeHash) return len(code), err } @@ -207,8 +207,8 @@ func (t *odrTrie) do(key []byte, fn func() error) error { var err error if t.trie == nil { var id *trie.ID - if len(t.id.AccKey) > 0 { - id = trie.StorageTrieID(t.id.StateRoot, common.BytesToHash(t.id.AccKey), t.id.Root) + if len(t.id.AccountAddress) > 0 { + id = trie.StorageTrieID(t.id.StateRoot, crypto.Keccak256Hash(t.id.AccountAddress), t.id.Root) } else { id = trie.StateTrieID(t.id.StateRoot) } @@ -239,8 +239,8 @@ func newNodeIterator(t *odrTrie, startkey []byte) trie.NodeIterator { if t.trie == nil { it.do(func() error { var id *trie.ID - if len(t.id.AccKey) > 0 { - id = trie.StorageTrieID(t.id.StateRoot, common.BytesToHash(t.id.AccKey), t.id.Root) + if len(t.id.AccountAddress) > 0 { + id = trie.StorageTrieID(t.id.StateRoot, crypto.Keccak256Hash(t.id.AccountAddress), t.id.Root) } else { id = trie.StateTrieID(t.id.StateRoot) } diff --git a/tests/fuzzers/les/les-fuzzer.go b/tests/fuzzers/les/les-fuzzer.go index b4599025e8..c62253a726 100644 --- a/tests/fuzzers/les/les-fuzzer.go +++ b/tests/fuzzers/les/les-fuzzer.go @@ -45,9 +45,9 @@ var ( testChainLen = 256 testContractCode = common.Hex2Bytes("606060405260cc8060106000396000f360606040526000357c01000000000000000000000000000000000000000000000000000000009004806360cd2685146041578063c16431b914606b57603f565b005b6055600480803590602001909190505060a9565b6040518082815260200191505060405180910390f35b60886004808035906020019091908035906020019091905050608a565b005b80600060005083606481101560025790900160005b50819055505b5050565b6000600060005082606481101560025790900160005b5054905060c7565b91905056") - chain *core.BlockChain - addrHashes []common.Hash - txHashes []common.Hash + chain *core.BlockChain + addresses []common.Address + txHashes []common.Hash chtTrie *trie.Trie bloomTrie *trie.Trie @@ -55,7 +55,7 @@ var ( bloomKeys [][]byte ) -func makechain() (bc *core.BlockChain, addrHashes, txHashes []common.Hash) { +func makechain() (bc *core.BlockChain, addresses []common.Address, txHashes []common.Hash) { gspec := &core.Genesis{ Config: params.TestChainConfig, Alloc: core.GenesisAlloc{bankAddr: {Balance: bankFunds}}, @@ -77,7 +77,7 @@ func makechain() (bc *core.BlockChain, addrHashes, txHashes []common.Hash) { tx, _ = types.SignTx(types.NewTransaction(nonce, addr, big.NewInt(10000), params.TxGas, big.NewInt(params.GWei), nil), signer, bankKey) } gen.AddTx(tx) - addrHashes = append(addrHashes, crypto.Keccak256Hash(addr[:])) + addresses = append(addresses, addr) txHashes = append(txHashes, tx.Hash()) }) bc, _ = core.NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) @@ -107,7 +107,7 @@ func makeTries() (chtTrie *trie.Trie, bloomTrie *trie.Trie, chtKeys, bloomKeys [ } func init() { - chain, addrHashes, txHashes = makechain() + chain, addresses, txHashes = makechain() chtTrie, bloomTrie, chtKeys, bloomKeys = makeTries() } @@ -116,7 +116,8 @@ type fuzzer struct { pool *txpool.TxPool chainLen int - addr, txs []common.Hash + addresses []common.Address + txs []common.Hash nonce uint64 chtKeys [][]byte @@ -135,7 +136,7 @@ func newFuzzer(input []byte) *fuzzer { return &fuzzer{ chain: chain, chainLen: testChainLen, - addr: addrHashes, + addresses: addresses, txs: txHashes, chtTrie: chtTrie, bloomTrie: bloomTrie, @@ -198,12 +199,12 @@ func (f *fuzzer) randomBlockHash() common.Hash { return common.BytesToHash(f.read(common.HashLength)) } -func (f *fuzzer) randomAddrHash() []byte { - i := f.randomInt(3 * len(f.addr)) - if i < len(f.addr) { - return f.addr[i].Bytes() +func (f *fuzzer) randomAddress() []byte { + i := f.randomInt(3 * len(f.addresses)) + if i < len(f.addresses) { + return f.addresses[i].Bytes() } - return f.read(common.HashLength) + return f.read(common.AddressLength) } func (f *fuzzer) randomCHTTrieKey() []byte { @@ -315,8 +316,8 @@ func Fuzz(input []byte) int { req := &l.GetCodePacket{Reqs: make([]l.CodeReq, f.randomInt(l.MaxCodeFetch+1))} for i := range req.Reqs { req.Reqs[i] = l.CodeReq{ - BHash: f.randomBlockHash(), - AccKey: f.randomAddrHash(), + BHash: f.randomBlockHash(), + AccountAddress: f.randomAddress(), } } f.doFuzz(l.GetCodeMsg, req) @@ -333,15 +334,15 @@ func Fuzz(input []byte) int { for i := range req.Reqs { if f.randomBool() { req.Reqs[i] = l.ProofReq{ - BHash: f.randomBlockHash(), - AccKey: f.randomAddrHash(), - Key: f.randomAddrHash(), - FromLevel: uint(f.randomX(3)), + BHash: f.randomBlockHash(), + AccountAddress: f.randomAddress(), + Key: f.randomAddress(), + FromLevel: uint(f.randomX(3)), } } else { req.Reqs[i] = l.ProofReq{ BHash: f.randomBlockHash(), - Key: f.randomAddrHash(), + Key: f.randomAddress(), FromLevel: uint(f.randomX(3)), } } diff --git a/trie/database_wrap.go b/trie/database_wrap.go index 30835b7d25..5ca013b38e 100644 --- a/trie/database_wrap.go +++ b/trie/database_wrap.go @@ -168,10 +168,15 @@ func (db *Database) Scheme() string { // It is meant to be called when closing the blockchain object, so that all // resources held can be released correctly. func (db *Database) Close() error { + db.WritePreimages() + return db.backend.Close() +} + +// WritePreimages flushes all accumulated preimages to disk forcibly. +func (db *Database) WritePreimages() { if db.preimages != nil { db.preimages.commit(true) } - return db.backend.Close() } // saveCache saves clean state cache to given directory path From 6ca3ef9a7b7ebf3ed0004cfdca0c7a27a38c28e7 Mon Sep 17 00:00:00 2001 From: Curith Date: Sun, 2 Jul 2023 19:21:16 +0800 Subject: [PATCH 45/90] node: fix listening on IPv6 address (#27628) (#27635) --- cmd/clef/main.go | 3 ++- cmd/utils/flags.go | 3 ++- internal/debug/flags.go | 3 ++- node/config.go | 5 +++-- node/rpcstack.go | 2 +- 5 files changed, 10 insertions(+), 6 deletions(-) diff --git a/cmd/clef/main.go b/cmd/clef/main.go index 14f09dc1a8..a8b0bf816a 100644 --- a/cmd/clef/main.go +++ b/cmd/clef/main.go @@ -27,6 +27,7 @@ import ( "fmt" "io" "math/big" + "net" "os" "os/signal" "path/filepath" @@ -743,7 +744,7 @@ func signer(c *cli.Context) error { port := c.Int(rpcPortFlag.Name) // start http server - httpEndpoint := fmt.Sprintf("%s:%d", c.String(utils.HTTPListenAddrFlag.Name), port) + httpEndpoint := net.JoinHostPort(c.String(utils.HTTPListenAddrFlag.Name), fmt.Sprintf("%d", port)) httpServer, addr, err := node.StartHTTPEndpoint(httpEndpoint, rpc.DefaultHTTPTimeouts, handler) if err != nil { utils.Fatalf("Could not start RPC api: %v", err) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 7bb47f1282..6f0e570250 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -26,6 +26,7 @@ import ( "fmt" "math" "math/big" + "net" "net/http" "os" "path/filepath" @@ -2014,7 +2015,7 @@ func SetupMetrics(ctx *cli.Context) { } if ctx.IsSet(MetricsHTTPFlag.Name) { - address := fmt.Sprintf("%s:%d", ctx.String(MetricsHTTPFlag.Name), ctx.Int(MetricsPortFlag.Name)) + address := net.JoinHostPort(ctx.String(MetricsHTTPFlag.Name), fmt.Sprintf("%d", ctx.Int(MetricsPortFlag.Name))) log.Info("Enabling stand-alone metrics HTTP endpoint", "address", address) exp.Setup(address) } else if ctx.IsSet(MetricsPortFlag.Name) { diff --git a/internal/debug/flags.go b/internal/debug/flags.go index d425a17db4..52a6342452 100644 --- a/internal/debug/flags.go +++ b/internal/debug/flags.go @@ -19,6 +19,7 @@ package debug import ( "fmt" "io" + "net" "net/http" _ "net/http/pprof" "os" @@ -308,7 +309,7 @@ func Setup(ctx *cli.Context) error { port := ctx.Int(pprofPortFlag.Name) - address := fmt.Sprintf("%s:%d", listenHost, port) + address := net.JoinHostPort(listenHost, fmt.Sprintf("%d", port)) // This context value ("metrics.addr") represents the utils.MetricsHTTPFlag.Name. // It cannot be imported because it will cause a cyclical dependency. StartPProf(address, !ctx.IsSet("metrics.addr")) diff --git a/node/config.go b/node/config.go index 37c1e4882b..949db887e4 100644 --- a/node/config.go +++ b/node/config.go @@ -19,6 +19,7 @@ package node import ( "crypto/ecdsa" "fmt" + "net" "os" "path/filepath" "runtime" @@ -263,7 +264,7 @@ func (c *Config) HTTPEndpoint() string { if c.HTTPHost == "" { return "" } - return fmt.Sprintf("%s:%d", c.HTTPHost, c.HTTPPort) + return net.JoinHostPort(c.HTTPHost, fmt.Sprintf("%d", c.HTTPPort)) } // DefaultHTTPEndpoint returns the HTTP endpoint used by default. @@ -278,7 +279,7 @@ func (c *Config) WSEndpoint() string { if c.WSHost == "" { return "" } - return fmt.Sprintf("%s:%d", c.WSHost, c.WSPort) + return net.JoinHostPort(c.WSHost, fmt.Sprintf("%d", c.WSPort)) } // DefaultWSEndpoint returns the websocket endpoint used by default. diff --git a/node/rpcstack.go b/node/rpcstack.go index e91585a2b6..b33c238051 100644 --- a/node/rpcstack.go +++ b/node/rpcstack.go @@ -112,7 +112,7 @@ func (h *httpServer) setListenAddr(host string, port int) error { } h.host, h.port = host, port - h.endpoint = fmt.Sprintf("%s:%d", host, port) + h.endpoint = net.JoinHostPort(host, fmt.Sprintf("%d", port)) return nil } From 59f7b289c329b5a56fa6f4e9acee64e504c4cc0d Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Tue, 4 Jul 2023 15:21:06 +0800 Subject: [PATCH 46/90] cmd, core, eth, graphql, trie: no persisted clean trie cache file (#27525) The clean trie cache is persisted periodically, therefore Geth can quickly warmup the cache in next restart. However it will reduce the robustness of system. The assumption is held in Geth that if the parent trie node is present, then the entire sub-trie associated with the parent are all prensent. Imagine the scenario that Geth rewinds itself to a past block and restart, but Geth finds the root node of "future state" in clean cache then regard this state is present in disk, while is not in fact. Another example is offline pruning tool. Whenever an offline pruning is performed, the clean cache file has to be removed to aviod hitting the root node of "deleted states" in clean cache. All in all, compare with the minor performance gain, system robustness is something we care more. --- cmd/devp2p/internal/ethtest/suite_test.go | 16 +++---- cmd/geth/config.go | 4 ++ cmd/geth/snapshot.go | 4 +- cmd/utils/flags.go | 18 -------- cmd/utils/flags_legacy.go | 13 ++++++ core/blockchain.go | 20 --------- core/state/pruner/pruner.go | 37 +--------------- eth/backend.go | 4 +- eth/ethconfig/config.go | 48 +++++++++----------- eth/ethconfig/gen_config.go | 12 ----- graphql/graphql_test.go | 14 +++--- tests/fuzzers/snap/fuzz_handler.go | 1 - trie/{database_wrap.go => database.go} | 54 ++--------------------- 13 files changed, 59 insertions(+), 186 deletions(-) rename trie/{database_wrap.go => database.go} (82%) diff --git a/cmd/devp2p/internal/ethtest/suite_test.go b/cmd/devp2p/internal/ethtest/suite_test.go index 8a2b132fa3..c5bcc3db1d 100644 --- a/cmd/devp2p/internal/ethtest/suite_test.go +++ b/cmd/devp2p/internal/ethtest/suite_test.go @@ -109,15 +109,13 @@ func setupGeth(stack *node.Node) error { } backend, err := eth.New(stack, ðconfig.Config{ - Genesis: &chain.genesis, - NetworkId: chain.genesis.Config.ChainID.Uint64(), // 19763 - DatabaseCache: 10, - TrieCleanCache: 10, - TrieCleanCacheJournal: "", - TrieCleanCacheRejournal: 60 * time.Minute, - TrieDirtyCache: 16, - TrieTimeout: 60 * time.Minute, - SnapshotCache: 10, + Genesis: &chain.genesis, + NetworkId: chain.genesis.Config.ChainID.Uint64(), // 19763 + DatabaseCache: 10, + TrieCleanCache: 10, + TrieDirtyCache: 16, + TrieTimeout: 60 * time.Minute, + SnapshotCache: 10, }) if err != nil { return err diff --git a/cmd/geth/config.go b/cmd/geth/config.go index 6edcd2f012..14906a860a 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -276,6 +276,10 @@ func deprecated(field string) bool { return true case "ethconfig.Config.EWASMInterpreter": return true + case "ethconfig.Config.TrieCleanCacheJournal": + return true + case "ethconfig.Config.TrieCleanCacheRejournal": + return true default: return false } diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 67837261c6..fafc827604 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -50,7 +50,6 @@ var ( ArgsUsage: "", Action: pruneState, Flags: flags.Merge([]cli.Flag{ - utils.CacheTrieJournalFlag, utils.BloomFilterSizeFlag, }, utils.NetworkFlags, utils.DatabasePathFlags), Description: ` @@ -160,7 +159,7 @@ block is used. // Deprecation: this command should be deprecated once the hash-based // scheme is deprecated. func pruneState(ctx *cli.Context) error { - stack, config := makeConfigNode(ctx) + stack, _ := makeConfigNode(ctx) defer stack.Close() chaindb := utils.MakeChainDatabase(ctx, stack, false) @@ -168,7 +167,6 @@ func pruneState(ctx *cli.Context) error { prunerconfig := pruner.Config{ Datadir: stack.ResolvePath(""), - Cachedir: stack.ResolvePath(config.Eth.TrieCleanCacheJournal), BloomSize: ctx.Uint64(utils.BloomFilterSizeFlag.Name), } pruner, err := pruner.NewPruner(chaindb, prunerconfig) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 6f0e570250..c8560868d7 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -410,18 +410,6 @@ var ( Value: 15, Category: flags.PerfCategory, } - CacheTrieJournalFlag = &cli.StringFlag{ - Name: "cache.trie.journal", - Usage: "Disk journal directory for trie cache to survive node restarts", - Value: ethconfig.Defaults.TrieCleanCacheJournal, - Category: flags.PerfCategory, - } - CacheTrieRejournalFlag = &cli.DurationFlag{ - Name: "cache.trie.rejournal", - Usage: "Time interval to regenerate the trie cache journal", - Value: ethconfig.Defaults.TrieCleanCacheRejournal, - Category: flags.PerfCategory, - } CacheGCFlag = &cli.IntFlag{ Name: "cache.gc", Usage: "Percentage of cache memory allowance to use for trie pruning (default = 25% full mode, 0% archive mode)", @@ -1710,12 +1698,6 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheTrieFlag.Name) { cfg.TrieCleanCache = ctx.Int(CacheFlag.Name) * ctx.Int(CacheTrieFlag.Name) / 100 } - if ctx.IsSet(CacheTrieJournalFlag.Name) { - cfg.TrieCleanCacheJournal = ctx.String(CacheTrieJournalFlag.Name) - } - if ctx.IsSet(CacheTrieRejournalFlag.Name) { - cfg.TrieCleanCacheRejournal = ctx.Duration(CacheTrieRejournalFlag.Name) - } if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheGCFlag.Name) { cfg.TrieDirtyCache = ctx.Int(CacheFlag.Name) * ctx.Int(CacheGCFlag.Name) / 100 } diff --git a/cmd/utils/flags_legacy.go b/cmd/utils/flags_legacy.go index 930b68fb91..b554deba8e 100644 --- a/cmd/utils/flags_legacy.go +++ b/cmd/utils/flags_legacy.go @@ -33,6 +33,8 @@ var ShowDeprecated = &cli.Command{ var DeprecatedFlags = []cli.Flag{ NoUSBFlag, + CacheTrieJournalFlag, + CacheTrieRejournalFlag, } var ( @@ -42,6 +44,17 @@ var ( Usage: "Disables monitoring for and managing USB hardware wallets (deprecated)", Category: flags.DeprecatedCategory, } + // (Deprecated June 2023, shown in aliased flags section) + CacheTrieJournalFlag = &cli.StringFlag{ + Name: "cache.trie.journal", + Usage: "Disk journal directory for trie cache to survive node restarts", + Category: flags.PerfCategory, + } + CacheTrieRejournalFlag = &cli.DurationFlag{ + Name: "cache.trie.rejournal", + Usage: "Time interval to regenerate the trie cache journal", + Category: flags.PerfCategory, + } ) // showDeprecated displays deprecated flags that will be soon removed from the codebase. diff --git a/core/blockchain.go b/core/blockchain.go index 1110e3e5f1..61e55d575d 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -130,8 +130,6 @@ const ( // that's resident in a blockchain. type CacheConfig struct { TrieCleanLimit int // Memory allowance (MB) to use for caching trie nodes in memory - TrieCleanJournal string // Disk journal for saving clean cache entries. - TrieCleanRejournal time.Duration // Time interval to dump clean cache to disk periodically TrieCleanNoPrefetch bool // Whether to disable heuristic state prefetching for followup blocks TrieDirtyLimit int // Memory limit (MB) at which to start flushing dirty trie nodes to disk TrieDirtyDisabled bool // Whether to disable trie write caching and GC altogether (archive node) @@ -238,7 +236,6 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis // Open trie database with provided config triedb := trie.NewDatabaseWithConfig(db, &trie.Config{ Cache: cacheConfig.TrieCleanLimit, - Journal: cacheConfig.TrieCleanJournal, Preimages: cacheConfig.Preimages, }) // Setup the genesis block, commit the provided genesis specification @@ -411,18 +408,6 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis bc.wg.Add(1) go bc.updateFutureBlocks() - // If periodic cache journal is required, spin it up. - if bc.cacheConfig.TrieCleanRejournal > 0 { - if bc.cacheConfig.TrieCleanRejournal < time.Minute { - log.Warn("Sanitizing invalid trie cache journal time", "provided", bc.cacheConfig.TrieCleanRejournal, "updated", time.Minute) - bc.cacheConfig.TrieCleanRejournal = time.Minute - } - bc.wg.Add(1) - go func() { - defer bc.wg.Done() - bc.triedb.SaveCachePeriodically(bc.cacheConfig.TrieCleanJournal, bc.cacheConfig.TrieCleanRejournal, bc.quit) - }() - } // Rewind the chain in case of an incompatible config upgrade. if compat, ok := genesisErr.(*params.ConfigCompatError); ok { log.Warn("Rewinding chain to upgrade configuration", "err", compat) @@ -987,11 +972,6 @@ func (bc *BlockChain) Stop() { if err := bc.stateCache.TrieDB().Close(); err != nil { log.Error("Failed to close trie db", "err", err) } - // Ensure all live cached entries be saved into disk, so that we can skip - // cache warmup when node restarts. - if bc.cacheConfig.TrieCleanJournal != "" { - bc.triedb.SaveCache(bc.cacheConfig.TrieCleanJournal) - } log.Info("Blockchain stopped") } diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index 1ee5a0e625..64c4a3a6eb 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -57,7 +57,6 @@ const ( // Config includes all the configurations for pruning. type Config struct { Datadir string // The directory of the state database - Cachedir string // The directory of state clean cache BloomSize uint64 // The Megabytes of memory allocated to bloom-filter } @@ -241,7 +240,7 @@ func (p *Pruner) Prune(root common.Hash) error { return err } if stateBloomRoot != (common.Hash{}) { - return RecoverPruning(p.config.Datadir, p.db, p.config.Cachedir) + return RecoverPruning(p.config.Datadir, p.db) } // If the target state root is not specified, use the HEAD-127 as the // target. The reason for picking it is: @@ -299,12 +298,6 @@ func (p *Pruner) Prune(root common.Hash) error { log.Info("Selecting user-specified state as the pruning target", "root", root) } } - // Before start the pruning, delete the clean trie cache first. - // It's necessary otherwise in the next restart we will hit the - // deleted state root in the "clean cache" so that the incomplete - // state is picked for usage. - deleteCleanTrieCache(p.config.Cachedir) - // All the state roots of the middle layer should be forcibly pruned, // otherwise the dangling state will be left. middleRoots := make(map[common.Hash]struct{}) @@ -342,7 +335,7 @@ func (p *Pruner) Prune(root common.Hash) error { // pruning can be resumed. What's more if the bloom filter is constructed, the // pruning **has to be resumed**. Otherwise a lot of dangling nodes may be left // in the disk. -func RecoverPruning(datadir string, db ethdb.Database, trieCachePath string) error { +func RecoverPruning(datadir string, db ethdb.Database) error { stateBloomPath, stateBloomRoot, err := findBloomFilter(datadir) if err != nil { return err @@ -378,12 +371,6 @@ func RecoverPruning(datadir string, db ethdb.Database, trieCachePath string) err } log.Info("Loaded state bloom filter", "path", stateBloomPath) - // Before start the pruning, delete the clean trie cache first. - // It's necessary otherwise in the next restart we will hit the - // deleted state root in the "clean cache" so that the incomplete - // state is picked for usage. - deleteCleanTrieCache(trieCachePath) - // All the state roots of the middle layers should be forcibly pruned, // otherwise the dangling state will be left. var ( @@ -497,23 +484,3 @@ func findBloomFilter(datadir string) (string, common.Hash, error) { } return stateBloomPath, stateBloomRoot, nil } - -const warningLog = ` - -WARNING! - -The clean trie cache is not found. Please delete it by yourself after the -pruning. Remember don't start the Geth without deleting the clean trie cache -otherwise the entire database may be damaged! - -Check the command description "geth snapshot prune-state --help" for more details. -` - -func deleteCleanTrieCache(path string) { - if !common.FileExist(path) { - log.Warn(warningLog) - return - } - os.RemoveAll(path) - log.Info("Deleted trie clean cache", "path", path) -} diff --git a/eth/backend.go b/eth/backend.go index cc555f0c23..63bd864b21 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -132,7 +132,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { if err != nil { return nil, err } - if err := pruner.RecoverPruning(stack.ResolvePath(""), chainDb, stack.ResolvePath(config.TrieCleanCacheJournal)); err != nil { + if err := pruner.RecoverPruning(stack.ResolvePath(""), chainDb); err != nil { log.Error("Failed to recover state", "error", err) } // Transfer mining-related config to the ethash config. @@ -184,8 +184,6 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { } cacheConfig = &core.CacheConfig{ TrieCleanLimit: config.TrieCleanCache, - TrieCleanJournal: stack.ResolvePath(config.TrieCleanCacheJournal), - TrieCleanRejournal: config.TrieCleanCacheRejournal, TrieCleanNoPrefetch: config.NoPrefetch, TrieDirtyLimit: config.TrieDirtyCache, TrieDirtyDisabled: config.NoPruning, diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index b3f7932d7f..dc798ec196 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -57,25 +57,23 @@ var LightClientGPO = gasprice.Config{ // Defaults contains default settings for use on the Ethereum main net. var Defaults = Config{ - SyncMode: downloader.SnapSync, - NetworkId: 1, - TxLookupLimit: 2350000, - LightPeers: 100, - UltraLightFraction: 75, - DatabaseCache: 512, - TrieCleanCache: 154, - TrieCleanCacheJournal: "triecache", - TrieCleanCacheRejournal: 60 * time.Minute, - TrieDirtyCache: 256, - TrieTimeout: 60 * time.Minute, - SnapshotCache: 102, - FilterLogCacheSize: 32, - Miner: miner.DefaultConfig, - TxPool: legacypool.DefaultConfig, - RPCGasCap: 50000000, - RPCEVMTimeout: 5 * time.Second, - GPO: FullNodeGPO, - RPCTxFeeCap: 1, // 1 ether + SyncMode: downloader.SnapSync, + NetworkId: 1, + TxLookupLimit: 2350000, + LightPeers: 100, + UltraLightFraction: 75, + DatabaseCache: 512, + TrieCleanCache: 154, + TrieDirtyCache: 256, + TrieTimeout: 60 * time.Minute, + SnapshotCache: 102, + FilterLogCacheSize: 32, + Miner: miner.DefaultConfig, + TxPool: legacypool.DefaultConfig, + RPCGasCap: 50000000, + RPCEVMTimeout: 5 * time.Second, + GPO: FullNodeGPO, + RPCTxFeeCap: 1, // 1 ether } //go:generate go run github.com/fjl/gencodec -type Config -formats toml -out gen_config.go @@ -124,13 +122,11 @@ type Config struct { DatabaseCache int DatabaseFreezer string - TrieCleanCache int - TrieCleanCacheJournal string `toml:",omitempty"` // Disk journal directory for trie cache to survive node restarts - TrieCleanCacheRejournal time.Duration `toml:",omitempty"` // Time interval to regenerate the journal for clean cache - TrieDirtyCache int - TrieTimeout time.Duration - SnapshotCache int - Preimages bool + TrieCleanCache int + TrieDirtyCache int + TrieTimeout time.Duration + SnapshotCache int + Preimages bool // This is the number of blocks for which logs will be cached in the filter system. FilterLogCacheSize int diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index b41c445fe6..8f48467236 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -39,8 +39,6 @@ func (c Config) MarshalTOML() (interface{}, error) { DatabaseCache int DatabaseFreezer string TrieCleanCache int - TrieCleanCacheJournal string `toml:",omitempty"` - TrieCleanCacheRejournal time.Duration `toml:",omitempty"` TrieDirtyCache int TrieTimeout time.Duration SnapshotCache int @@ -81,8 +79,6 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.DatabaseCache = c.DatabaseCache enc.DatabaseFreezer = c.DatabaseFreezer enc.TrieCleanCache = c.TrieCleanCache - enc.TrieCleanCacheJournal = c.TrieCleanCacheJournal - enc.TrieCleanCacheRejournal = c.TrieCleanCacheRejournal enc.TrieDirtyCache = c.TrieDirtyCache enc.TrieTimeout = c.TrieTimeout enc.SnapshotCache = c.SnapshotCache @@ -127,8 +123,6 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { DatabaseCache *int DatabaseFreezer *string TrieCleanCache *int - TrieCleanCacheJournal *string `toml:",omitempty"` - TrieCleanCacheRejournal *time.Duration `toml:",omitempty"` TrieDirtyCache *int TrieTimeout *time.Duration SnapshotCache *int @@ -218,12 +212,6 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.TrieCleanCache != nil { c.TrieCleanCache = *dec.TrieCleanCache } - if dec.TrieCleanCacheJournal != nil { - c.TrieCleanCacheJournal = *dec.TrieCleanCacheJournal - } - if dec.TrieCleanCacheRejournal != nil { - c.TrieCleanCacheRejournal = *dec.TrieCleanCacheRejournal - } if dec.TrieDirtyCache != nil { c.TrieDirtyCache = *dec.TrieDirtyCache } diff --git a/graphql/graphql_test.go b/graphql/graphql_test.go index 88a767e334..22d7181b0d 100644 --- a/graphql/graphql_test.go +++ b/graphql/graphql_test.go @@ -438,14 +438,12 @@ func createNode(t *testing.T) *node.Node { func newGQLService(t *testing.T, stack *node.Node, shanghai bool, gspec *core.Genesis, genBlocks int, genfunc func(i int, gen *core.BlockGen)) (*handler, []*types.Block) { ethConf := ðconfig.Config{ - Genesis: gspec, - NetworkId: 1337, - TrieCleanCache: 5, - TrieCleanCacheJournal: "triecache", - TrieCleanCacheRejournal: 60 * time.Minute, - TrieDirtyCache: 5, - TrieTimeout: 60 * time.Minute, - SnapshotCache: 5, + Genesis: gspec, + NetworkId: 1337, + TrieCleanCache: 5, + TrieDirtyCache: 5, + TrieTimeout: 60 * time.Minute, + SnapshotCache: 5, } ethBackend, err := eth.New(stack, ethConf) if err != nil { diff --git a/tests/fuzzers/snap/fuzz_handler.go b/tests/fuzzers/snap/fuzz_handler.go index 2e5dcd6e29..784b526dc0 100644 --- a/tests/fuzzers/snap/fuzz_handler.go +++ b/tests/fuzzers/snap/fuzz_handler.go @@ -71,7 +71,6 @@ func getChain() *core.BlockChain { TrieDirtyLimit: 0, TrieTimeLimit: 5 * time.Minute, TrieCleanNoPrefetch: true, - TrieCleanRejournal: 0, SnapshotLimit: 100, SnapshotWait: true, } diff --git a/trie/database_wrap.go b/trie/database.go similarity index 82% rename from trie/database_wrap.go rename to trie/database.go index 5ca013b38e..faaa1550e0 100644 --- a/trie/database_wrap.go +++ b/trie/database.go @@ -18,22 +18,18 @@ package trie import ( "errors" - "runtime" - "time" "github.com/VictoriaMetrics/fastcache" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/trie/triedb/hashdb" "github.com/ethereum/go-ethereum/trie/trienode" ) // Config defines all necessary options for database. type Config struct { - Cache int // Memory allowance (MB) to use for caching trie nodes in memory - Journal string // Journal of clean cache to survive node restarts - Preimages bool // Flag whether the preimage of trie key is recorded + Cache int // Memory allowance (MB) to use for caching trie nodes in memory + Preimages bool // Flag whether the preimage of trie key is recorded } // backend defines the methods needed to access/update trie nodes in different @@ -79,11 +75,7 @@ type Database struct { func prepare(diskdb ethdb.Database, config *Config) *Database { var cleans *fastcache.Cache if config != nil && config.Cache > 0 { - if config.Journal == "" { - cleans = fastcache.New(config.Cache * 1024 * 1024) - } else { - cleans = fastcache.LoadFromFileOrNew(config.Journal, config.Cache*1024*1024) - } + cleans = fastcache.New(config.Cache * 1024 * 1024) } var preimages *preimageStore if config != nil && config.Preimages { @@ -179,46 +171,6 @@ func (db *Database) WritePreimages() { } } -// saveCache saves clean state cache to given directory path -// using specified CPU cores. -func (db *Database) saveCache(dir string, threads int) error { - if db.cleans == nil { - return nil - } - log.Info("Writing clean trie cache to disk", "path", dir, "threads", threads) - - start := time.Now() - err := db.cleans.SaveToFileConcurrent(dir, threads) - if err != nil { - log.Error("Failed to persist clean trie cache", "error", err) - return err - } - log.Info("Persisted the clean trie cache", "path", dir, "elapsed", common.PrettyDuration(time.Since(start))) - return nil -} - -// SaveCache atomically saves fast cache data to the given dir using all -// available CPU cores. -func (db *Database) SaveCache(dir string) error { - return db.saveCache(dir, runtime.GOMAXPROCS(0)) -} - -// SaveCachePeriodically atomically saves fast cache data to the given dir with -// the specified interval. All dump operation will only use a single CPU core. -func (db *Database) SaveCachePeriodically(dir string, interval time.Duration, stopCh <-chan struct{}) { - ticker := time.NewTicker(interval) - defer ticker.Stop() - - for { - select { - case <-ticker.C: - db.saveCache(dir, 1) - case <-stopCh: - return - } - } -} - // Cap iteratively flushes old but still referenced trie nodes until the total // memory usage goes below the given threshold. The held pre-images accumulated // up to this point will be flushed in case the size exceeds the threshold. From 863f6dac194b2313c1806343cae1982845b9a9bb Mon Sep 17 00:00:00 2001 From: Exca-DK <85954505+Exca-DK@users.noreply.github.com> Date: Thu, 6 Jul 2023 00:04:07 +0200 Subject: [PATCH 47/90] ethdb/leveldb: update leveldb metrics collection (#27643) This removes text parsing in leveldb metrics collection code. All metrics can now be accessed through the stats API provided by leveldb. We also add new gauge-typed metrics that count the number of tables at each level. --------- Co-authored-by: Exca-DK Co-authored-by: Gary Rong Co-authored-by: Felix Lange --- ethdb/leveldb/leveldb.go | 171 ++++++++++----------------------------- 1 file changed, 42 insertions(+), 129 deletions(-) diff --git a/ethdb/leveldb/leveldb.go b/ethdb/leveldb/leveldb.go index ce13659d9d..190327accb 100644 --- a/ethdb/leveldb/leveldb.go +++ b/ethdb/leveldb/leveldb.go @@ -22,8 +22,6 @@ package leveldb import ( "fmt" - "strconv" - "strings" "sync" "time" @@ -77,6 +75,8 @@ type Database struct { seekCompGauge metrics.Gauge // Gauge for tracking the number of table compaction caused by read opt manualMemAllocGauge metrics.Gauge // Gauge to track the amount of memory that has been manually allocated (not a part of runtime/GC) + levelsGauge [7]metrics.Gauge // Gauge for tracking the number of tables in levels + quitLock sync.Mutex // Mutex protecting the quit channel access quitChan chan chan error // Quit channel to stop the metrics collection before closing the database @@ -146,6 +146,11 @@ func NewCustom(file string, namespace string, customize func(options *opt.Option ldb.seekCompGauge = metrics.NewRegisteredGauge(namespace+"compact/seek", nil) ldb.manualMemAllocGauge = metrics.NewRegisteredGauge(namespace+"memory/manualalloc", nil) + // leveldb has only up to 7 levels + for i := range ldb.levelsGauge { + ldb.levelsGauge[i] = metrics.NewRegisteredGauge(namespace+fmt.Sprintf("tables/level%v", i), nil) + } + // Start up the metrics gathering and return go ldb.meter(metricsGatheringInterval) return ldb, nil @@ -266,122 +271,63 @@ func (db *Database) Path() string { // meter periodically retrieves internal leveldb counters and reports them to // the metrics subsystem. -// -// This is how a LevelDB stats table looks like (currently): -// -// Compactions -// Level | Tables | Size(MB) | Time(sec) | Read(MB) | Write(MB) -// -------+------------+---------------+---------------+---------------+--------------- -// 0 | 0 | 0.00000 | 1.27969 | 0.00000 | 12.31098 -// 1 | 85 | 109.27913 | 28.09293 | 213.92493 | 214.26294 -// 2 | 523 | 1000.37159 | 7.26059 | 66.86342 | 66.77884 -// 3 | 570 | 1113.18458 | 0.00000 | 0.00000 | 0.00000 -// -// This is how the write delay look like (currently): -// DelayN:5 Delay:406.604657ms Paused: false -// -// This is how the iostats look like (currently): -// Read(MB):3895.04860 Write(MB):3654.64712 func (db *Database) meter(refresh time.Duration) { // Create the counters to store current and previous compaction values - compactions := make([][]float64, 2) + compactions := make([][]int64, 2) for i := 0; i < 2; i++ { - compactions[i] = make([]float64, 4) + compactions[i] = make([]int64, 4) } - // Create storage for iostats. - var iostats [2]float64 - - // Create storage and warning log tracer for write delay. - var ( - delaystats [2]int64 - lastWritePaused time.Time - ) - + // Create storages for states and warning log tracer. var ( errc chan error merr error - ) + stats leveldb.DBStats + iostats [2]int64 + delaystats [2]int64 + lastWritePaused time.Time + ) timer := time.NewTimer(refresh) defer timer.Stop() // Iterate ad infinitum and collect the stats for i := 1; errc == nil && merr == nil; i++ { // Retrieve the database stats - stats, err := db.db.GetProperty("leveldb.stats") + // Stats method resets buffers inside therefore it's okay to just pass the struct. + err := db.db.Stats(&stats) if err != nil { db.log.Error("Failed to read database stats", "err", err) merr = err continue } - // Find the compaction table, skip the header - lines := strings.Split(stats, "\n") - for len(lines) > 0 && strings.TrimSpace(lines[0]) != "Compactions" { - lines = lines[1:] - } - if len(lines) <= 3 { - db.log.Error("Compaction leveldbTable not found") - merr = errors.New("compaction leveldbTable not found") - continue - } - lines = lines[3:] - // Iterate over all the leveldbTable rows, and accumulate the entries for j := 0; j < len(compactions[i%2]); j++ { compactions[i%2][j] = 0 } - for _, line := range lines { - parts := strings.Split(line, "|") - if len(parts) != 6 { - break - } - for idx, counter := range parts[2:] { - value, err := strconv.ParseFloat(strings.TrimSpace(counter), 64) - if err != nil { - db.log.Error("Compaction entry parsing failed", "err", err) - merr = err - continue - } - compactions[i%2][idx] += value - } + compactions[i%2][0] = stats.LevelSizes.Sum() + for _, t := range stats.LevelDurations { + compactions[i%2][1] += t.Nanoseconds() } + compactions[i%2][2] = stats.LevelRead.Sum() + compactions[i%2][3] = stats.LevelWrite.Sum() // Update all the requested meters if db.diskSizeGauge != nil { - db.diskSizeGauge.Update(int64(compactions[i%2][0] * 1024 * 1024)) + db.diskSizeGauge.Update(compactions[i%2][0]) } if db.compTimeMeter != nil { - db.compTimeMeter.Mark(int64((compactions[i%2][1] - compactions[(i-1)%2][1]) * 1000 * 1000 * 1000)) + db.compTimeMeter.Mark(compactions[i%2][1] - compactions[(i-1)%2][1]) } if db.compReadMeter != nil { - db.compReadMeter.Mark(int64((compactions[i%2][2] - compactions[(i-1)%2][2]) * 1024 * 1024)) + db.compReadMeter.Mark(compactions[i%2][2] - compactions[(i-1)%2][2]) } if db.compWriteMeter != nil { - db.compWriteMeter.Mark(int64((compactions[i%2][3] - compactions[(i-1)%2][3]) * 1024 * 1024)) - } - // Retrieve the write delay statistic - writedelay, err := db.db.GetProperty("leveldb.writedelay") - if err != nil { - db.log.Error("Failed to read database write delay statistic", "err", err) - merr = err - continue + db.compWriteMeter.Mark(compactions[i%2][3] - compactions[(i-1)%2][3]) } var ( - delayN int64 - delayDuration string - duration time.Duration - paused bool + delayN = int64(stats.WriteDelayCount) + duration = stats.WriteDelayDuration + paused = stats.WritePaused ) - if n, err := fmt.Sscanf(writedelay, "DelayN:%d Delay:%s Paused:%t", &delayN, &delayDuration, &paused); n != 3 || err != nil { - db.log.Error("Write delay statistic not found") - merr = err - continue - } - duration, err = time.ParseDuration(delayDuration) - if err != nil { - db.log.Error("Failed to parse delay duration", "err", err) - merr = err - continue - } if db.writeDelayNMeter != nil { db.writeDelayNMeter.Mark(delayN - delaystats[0]) } @@ -397,60 +343,27 @@ func (db *Database) meter(refresh time.Duration) { } delaystats[0], delaystats[1] = delayN, duration.Nanoseconds() - // Retrieve the database iostats. - ioStats, err := db.db.GetProperty("leveldb.iostats") - if err != nil { - db.log.Error("Failed to read database iostats", "err", err) - merr = err - continue - } - var nRead, nWrite float64 - parts := strings.Split(ioStats, " ") - if len(parts) < 2 { - db.log.Error("Bad syntax of ioStats", "ioStats", ioStats) - merr = fmt.Errorf("bad syntax of ioStats %s", ioStats) - continue - } - if n, err := fmt.Sscanf(parts[0], "Read(MB):%f", &nRead); n != 1 || err != nil { - db.log.Error("Bad syntax of read entry", "entry", parts[0]) - merr = err - continue - } - if n, err := fmt.Sscanf(parts[1], "Write(MB):%f", &nWrite); n != 1 || err != nil { - db.log.Error("Bad syntax of write entry", "entry", parts[1]) - merr = err - continue - } + var ( + nRead = int64(stats.IORead) + nWrite = int64(stats.IOWrite) + ) if db.diskReadMeter != nil { - db.diskReadMeter.Mark(int64((nRead - iostats[0]) * 1024 * 1024)) + db.diskReadMeter.Mark(nRead - iostats[0]) } if db.diskWriteMeter != nil { - db.diskWriteMeter.Mark(int64((nWrite - iostats[1]) * 1024 * 1024)) + db.diskWriteMeter.Mark(nWrite - iostats[1]) } iostats[0], iostats[1] = nRead, nWrite - compCount, err := db.db.GetProperty("leveldb.compcount") - if err != nil { - db.log.Error("Failed to read database iostats", "err", err) - merr = err - continue - } + db.memCompGauge.Update(int64(stats.MemComp)) + db.level0CompGauge.Update(int64(stats.Level0Comp)) + db.nonlevel0CompGauge.Update(int64(stats.NonLevel0Comp)) + db.seekCompGauge.Update(int64(stats.SeekComp)) - var ( - memComp uint32 - level0Comp uint32 - nonLevel0Comp uint32 - seekComp uint32 - ) - if n, err := fmt.Sscanf(compCount, "MemComp:%d Level0Comp:%d NonLevel0Comp:%d SeekComp:%d", &memComp, &level0Comp, &nonLevel0Comp, &seekComp); n != 4 || err != nil { - db.log.Error("Compaction count statistic not found") - merr = err - continue + // update tables amount + for i, tables := range stats.LevelTablesCounts { + db.levelsGauge[i].Update(int64(tables)) } - db.memCompGauge.Update(int64(memComp)) - db.level0CompGauge.Update(int64(level0Comp)) - db.nonlevel0CompGauge.Update(int64(nonLevel0Comp)) - db.seekCompGauge.Update(int64(seekComp)) // Sleep a bit, then repeat the stats collection select { From 83d7f426d1652b30855a61aba7fd86f5e294d279 Mon Sep 17 00:00:00 2001 From: Darioush Jalali Date: Wed, 5 Jul 2023 23:56:42 -0700 Subject: [PATCH 48/90] core: stop chain when tests are finished (#27660) core (blockchain_test): add chain.Stop() to tests --- core/blockchain_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 5ec685c99f..e626dbb5f7 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -4129,6 +4129,7 @@ func testCreateThenDelete(t *testing.T, config *params.ChainConfig) { if err != nil { t.Fatalf("failed to create tester chain: %v", err) } + defer chain.Stop() // Import the blocks for _, block := range blocks { if _, err := chain.InsertChain([]*types.Block{block}); err != nil { @@ -4215,6 +4216,7 @@ func TestTransientStorageReset(t *testing.T) { if err != nil { t.Fatalf("failed to create tester chain: %v", err) } + defer chain.Stop() // Import the blocks if _, err := chain.InsertChain(blocks); err != nil { t.Fatalf("failed to insert into chain: %v", err) @@ -4309,6 +4311,7 @@ func TestEIP3651(t *testing.T) { if err != nil { t.Fatalf("failed to create tester chain: %v", err) } + defer chain.Stop() if n, err := chain.InsertChain(blocks); err != nil { t.Fatalf("block %d: failed to insert into chain: %v", n, err) } From ab0e0f3517ab089c0446e6f8d06bb275ba5c4a9a Mon Sep 17 00:00:00 2001 From: John Chase <68833933+joohhnnn@users.noreply.github.com> Date: Thu, 6 Jul 2023 15:50:44 +0800 Subject: [PATCH 49/90] signer/core: show helptext when clef rejects signing (#27538) Co-authored-by: Martin Holst Swende --- signer/core/api.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/signer/core/api.go b/signer/core/api.go index 3c1c94801b..43eb89ee00 100644 --- a/signer/core/api.go +++ b/signer/core/api.go @@ -553,6 +553,7 @@ func (api *SignerAPI) SignTransaction(ctx context.Context, args apitypes.SendTxA // If we are in 'rejectMode', then reject rather than show the user warnings if api.rejectMode { if err := msgs.GetWarnings(); err != nil { + log.Info("Signing aborted due to warnings. In order to continue despite warnings, please use the flag '--advanced'.") return nil, err } } @@ -625,6 +626,7 @@ func (api *SignerAPI) SignGnosisSafeTx(ctx context.Context, signerAddress common // If we are in 'rejectMode', then reject rather than show the user warnings if api.rejectMode { if err := msgs.GetWarnings(); err != nil { + log.Info("Signing aborted due to warnings. In order to continue despite warnings, please use the flag '--advanced'.") return nil, err } } From ea782809f732da63f893f9ad6f104227fd1e606f Mon Sep 17 00:00:00 2001 From: jwasinger Date: Thu, 6 Jul 2023 10:42:34 +0200 Subject: [PATCH 50/90] cmd/geth: implement dev mode for post-merge (#27327) This change adds back the 'geth --dev' mode of operation, using a cl-mocker. --------- Co-authored-by: Martin Holst Swende Co-authored-by: rjl493456442 Co-authored-by: lightclient <14004106+lightclient@users.noreply.github.com> --- cmd/geth/config.go | 18 ++ cmd/geth/main.go | 4 +- cmd/utils/flags.go | 7 +- console/console_test.go | 2 +- core/genesis.go | 11 +- eth/catalyst/api.go | 29 +++- eth/catalyst/queue.go | 7 +- eth/catalyst/simulated_beacon.go | 231 ++++++++++++++++++++++++++ eth/catalyst/simulated_beacon_api.go | 36 ++++ eth/catalyst/simulated_beacon_test.go | 141 ++++++++++++++++ internal/web3ext/web3ext.go | 20 +++ miner/miner_test.go | 31 +++- miner/payload_building.go | 11 +- params/config.go | 26 ++- 14 files changed, 543 insertions(+), 31 deletions(-) create mode 100644 eth/catalyst/simulated_beacon.go create mode 100644 eth/catalyst/simulated_beacon_api.go create mode 100644 eth/catalyst/simulated_beacon_test.go diff --git a/cmd/geth/config.go b/cmd/geth/config.go index 14906a860a..b6c8004665 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -32,6 +32,8 @@ import ( "github.com/ethereum/go-ethereum/accounts/scwallet" "github.com/ethereum/go-ethereum/accounts/usbwallet" "github.com/ethereum/go-ethereum/cmd/utils" + "github.com/ethereum/go-ethereum/eth/catalyst" + ethcatalyst "github.com/ethereum/go-ethereum/eth/catalyst" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/internal/ethapi" @@ -193,6 +195,22 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { if ctx.IsSet(utils.SyncTargetFlag.Name) && cfg.Eth.SyncMode == downloader.FullSync { utils.RegisterFullSyncTester(stack, eth, ctx.Path(utils.SyncTargetFlag.Name)) } + + // Start the dev mode if requested, or launch the engine API for + // interacting with external consensus client. + if ctx.IsSet(utils.DeveloperFlag.Name) { + simBeacon, err := catalyst.NewSimulatedBeacon(ctx.Uint64(utils.DeveloperPeriodFlag.Name), eth) + if err != nil { + utils.Fatalf("failed to register dev mode catalyst service: %v", err) + } + catalyst.RegisterSimulatedBeaconAPIs(stack, simBeacon) + stack.RegisterLifecycle(simBeacon) + } else if cfg.Eth.SyncMode != downloader.LightSync { + err := ethcatalyst.Register(stack, eth) + if err != nil { + utils.Fatalf("failed to register catalyst service: %v", err) + } + } return stack, backend } diff --git a/cmd/geth/main.go b/cmd/geth/main.go index ede10ed909..42a05f4d1a 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -128,8 +128,8 @@ var ( utils.NodeKeyHexFlag, utils.DNSDiscoveryFlag, utils.DeveloperFlag, - utils.DeveloperPeriodFlag, utils.DeveloperGasLimitFlag, + utils.DeveloperPeriodFlag, utils.VMEnableDebugFlag, utils.NetworkIdFlag, utils.EthStatsURLFlag, @@ -408,7 +408,7 @@ func startNode(ctx *cli.Context, stack *node.Node, backend ethapi.Backend, isCon } // Start auxiliary services if enabled - if ctx.Bool(utils.MiningEnabledFlag.Name) || ctx.Bool(utils.DeveloperFlag.Name) { + if ctx.Bool(utils.MiningEnabledFlag.Name) { // Mining only makes sense if a full Ethereum node is running if ctx.String(utils.SyncModeFlag.Name) == "light" { utils.Fatalf("Light clients do not support mining") diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index c8560868d7..525abf042d 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -160,7 +160,7 @@ var ( Usage: "Ephemeral proof-of-authority network with a pre-funded developer account, mining enabled", Category: flags.DevCategory, } - DeveloperPeriodFlag = &cli.IntFlag{ + DeveloperPeriodFlag = &cli.Uint64Flag{ Name: "dev.period", Usage: "Block period to use in developer mode (0 = mine only if transaction pending)", Category: flags.DevCategory, @@ -1817,7 +1817,7 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { log.Info("Using developer account", "address", developer.Address) // Create a new developer genesis block or reuse existing one - cfg.Genesis = core.DeveloperGenesisBlock(uint64(ctx.Int(DeveloperPeriodFlag.Name)), ctx.Uint64(DeveloperGasLimitFlag.Name), developer.Address) + cfg.Genesis = core.DeveloperGenesisBlock(ctx.Uint64(DeveloperGasLimitFlag.Name), developer.Address) if ctx.IsSet(DataDirFlag.Name) { // If datadir doesn't exist we need to open db in write-mode // so leveldb can create files. @@ -1892,9 +1892,6 @@ func RegisterEthService(stack *node.Node, cfg *ethconfig.Config) (ethapi.Backend Fatalf("Failed to create the LES server: %v", err) } } - if err := ethcatalyst.Register(stack, backend); err != nil { - Fatalf("Failed to register the Engine API service: %v", err) - } stack.RegisterAPIs(tracers.APIs(backend.APIBackend)) return backend.APIBackend, backend } diff --git a/console/console_test.go b/console/console_test.go index a2550d8ca8..4b8f943c9e 100644 --- a/console/console_test.go +++ b/console/console_test.go @@ -94,7 +94,7 @@ func newTester(t *testing.T, confOverride func(*ethconfig.Config)) *tester { t.Fatalf("failed to create node: %v", err) } ethConf := ðconfig.Config{ - Genesis: core.DeveloperGenesisBlock(15, 11_500_000, common.Address{}), + Genesis: core.DeveloperGenesisBlock(11_500_000, common.Address{}), Miner: miner.Config{ Etherbase: common.HexToAddress(testAddress), }, diff --git a/core/genesis.go b/core/genesis.go index ab5623ad00..013ef78601 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -554,21 +554,16 @@ func DefaultSepoliaGenesisBlock() *Genesis { } // DeveloperGenesisBlock returns the 'geth --dev' genesis block. -func DeveloperGenesisBlock(period uint64, gasLimit uint64, faucet common.Address) *Genesis { +func DeveloperGenesisBlock(gasLimit uint64, faucet common.Address) *Genesis { // Override the default period to the user requested one - config := *params.AllCliqueProtocolChanges - config.Clique = ¶ms.CliqueConfig{ - Period: period, - Epoch: config.Clique.Epoch, - } + config := *params.AllDevChainProtocolChanges // Assemble and return the genesis with the precompiles and faucet pre-funded return &Genesis{ Config: &config, - ExtraData: append(append(make([]byte, 32), faucet[:]...), make([]byte, crypto.SignatureLength)...), GasLimit: gasLimit, BaseFee: big.NewInt(params.InitialBaseFee), - Difficulty: big.NewInt(1), + Difficulty: big.NewInt(0), Alloc: map[common.Address]GenesisAccount{ common.BytesToAddress([]byte{1}): {Balance: big.NewInt(1)}, // ECRecover common.BytesToAddress([]byte{2}): {Balance: big.NewInt(1)}, // SHA256 diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index 456bd77413..e4b5533b75 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -407,7 +407,22 @@ func (api *ConsensusAPI) GetPayloadV2(payloadID engine.PayloadID) (*engine.Execu func (api *ConsensusAPI) getPayload(payloadID engine.PayloadID) (*engine.ExecutionPayloadEnvelope, error) { log.Trace("Engine API request received", "method", "GetPayload", "id", payloadID) - data := api.localBlocks.get(payloadID) + data := api.localBlocks.get(payloadID, false) + if data == nil { + return nil, engine.UnknownPayload + } + return data, nil +} + +// getFullPayload returns a cached payload by it. The difference is that this +// function always expects a non-empty payload, but can also return empty one +// if no transaction is executable. +// +// Note, this function is not a part of standard engine API, meant to be used +// by consensus client mock in dev mode. +func (api *ConsensusAPI) getFullPayload(payloadID engine.PayloadID) (*engine.ExecutionPayloadEnvelope, error) { + log.Trace("Engine API request received", "method", "GetFullPayload", "id", payloadID) + data := api.localBlocks.get(payloadID, true) if data == nil { return nil, engine.UnknownPayload } @@ -715,8 +730,8 @@ func (api *ConsensusAPI) ExchangeCapabilities([]string) []string { return caps } -// GetPayloadBodiesV1 implements engine_getPayloadBodiesByHashV1 which allows for retrieval of a list -// of block bodies by the engine api. +// GetPayloadBodiesByHashV1 implements engine_getPayloadBodiesByHashV1 which +// allows for retrieval of a list of block bodies by the engine api. func (api *ConsensusAPI) GetPayloadBodiesByHashV1(hashes []common.Hash) []*engine.ExecutionPayloadBodyV1 { var bodies = make([]*engine.ExecutionPayloadBodyV1, len(hashes)) for i, hash := range hashes { @@ -726,8 +741,8 @@ func (api *ConsensusAPI) GetPayloadBodiesByHashV1(hashes []common.Hash) []*engin return bodies } -// GetPayloadBodiesByRangeV1 implements engine_getPayloadBodiesByRangeV1 which allows for retrieval of a range -// of block bodies by the engine api. +// GetPayloadBodiesByRangeV1 implements engine_getPayloadBodiesByRangeV1 which +// allows for retrieval of a range of block bodies by the engine api. func (api *ConsensusAPI) GetPayloadBodiesByRangeV1(start, count hexutil.Uint64) ([]*engine.ExecutionPayloadBodyV1, error) { if start == 0 || count == 0 { return nil, engine.InvalidParams.With(fmt.Errorf("invalid start or count, start: %v count: %v", start, count)) @@ -753,23 +768,19 @@ func getBody(block *types.Block) *engine.ExecutionPayloadBodyV1 { if block == nil { return nil } - var ( body = block.Body() txs = make([]hexutil.Bytes, len(body.Transactions)) withdrawals = body.Withdrawals ) - for j, tx := range body.Transactions { data, _ := tx.MarshalBinary() txs[j] = hexutil.Bytes(data) } - // Post-shanghai withdrawals MUST be set to empty slice instead of nil if withdrawals == nil && block.Header().WithdrawalsHash != nil { withdrawals = make([]*types.Withdrawal, 0) } - return &engine.ExecutionPayloadBodyV1{ TransactionData: txs, Withdrawals: withdrawals, diff --git a/eth/catalyst/queue.go b/eth/catalyst/queue.go index e8037aacad..634dc1b2e6 100644 --- a/eth/catalyst/queue.go +++ b/eth/catalyst/queue.go @@ -73,7 +73,7 @@ func (q *payloadQueue) put(id engine.PayloadID, payload *miner.Payload) { } // get retrieves a previously stored payload item or nil if it does not exist. -func (q *payloadQueue) get(id engine.PayloadID) *engine.ExecutionPayloadEnvelope { +func (q *payloadQueue) get(id engine.PayloadID, full bool) *engine.ExecutionPayloadEnvelope { q.lock.RLock() defer q.lock.RUnlock() @@ -82,7 +82,10 @@ func (q *payloadQueue) get(id engine.PayloadID) *engine.ExecutionPayloadEnvelope return nil // no more items } if item.id == id { - return item.payload.Resolve() + if !full { + return item.payload.Resolve() + } + return item.payload.ResolveFull() } } return nil diff --git a/eth/catalyst/simulated_beacon.go b/eth/catalyst/simulated_beacon.go new file mode 100644 index 0000000000..f0956d262c --- /dev/null +++ b/eth/catalyst/simulated_beacon.go @@ -0,0 +1,231 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package catalyst + +import ( + "errors" + "fmt" + "sync" + "time" + + "github.com/ethereum/go-ethereum/beacon/engine" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/rpc" +) + +// withdrawalQueue implements a FIFO queue which holds withdrawals that are +// pending inclusion. +type withdrawalQueue struct { + pending chan *types.Withdrawal +} + +// add queues a withdrawal for future inclusion. +func (w *withdrawalQueue) add(withdrawal *types.Withdrawal) error { + select { + case w.pending <- withdrawal: + break + default: + return errors.New("withdrawal queue full") + } + return nil +} + +// gatherPending returns a number of queued withdrawals up to a maximum count. +func (w *withdrawalQueue) gatherPending(maxCount int) []*types.Withdrawal { + withdrawals := []*types.Withdrawal{} + for { + select { + case withdrawal := <-w.pending: + withdrawals = append(withdrawals, withdrawal) + if len(withdrawals) == maxCount { + break + } + default: + return withdrawals + } + } +} + +type SimulatedBeacon struct { + shutdownCh chan struct{} + eth *eth.Ethereum + period uint64 + withdrawals withdrawalQueue + + feeRecipient common.Address + feeRecipientLock sync.Mutex // lock gates concurrent access to the feeRecipient + + engineAPI *ConsensusAPI + curForkchoiceState engine.ForkchoiceStateV1 + lastBlockTime uint64 +} + +func NewSimulatedBeacon(period uint64, eth *eth.Ethereum) (*SimulatedBeacon, error) { + chainConfig := eth.APIBackend.ChainConfig() + if !chainConfig.IsDevMode { + return nil, errors.New("incompatible pre-existing chain configuration") + } + block := eth.BlockChain().CurrentBlock() + current := engine.ForkchoiceStateV1{ + HeadBlockHash: block.Hash(), + SafeBlockHash: block.Hash(), + FinalizedBlockHash: block.Hash(), + } + engineAPI := NewConsensusAPI(eth) + + // if genesis block, send forkchoiceUpdated to trigger transition to PoS + if block.Number.Sign() == 0 { + if _, err := engineAPI.ForkchoiceUpdatedV2(current, nil); err != nil { + return nil, err + } + } + return &SimulatedBeacon{ + eth: eth, + period: period, + shutdownCh: make(chan struct{}), + engineAPI: engineAPI, + lastBlockTime: block.Time, + curForkchoiceState: current, + withdrawals: withdrawalQueue{make(chan *types.Withdrawal, 20)}, + }, nil +} + +func (c *SimulatedBeacon) setFeeRecipient(feeRecipient common.Address) { + c.feeRecipientLock.Lock() + c.feeRecipient = feeRecipient + c.feeRecipientLock.Unlock() +} + +// Start invokes the SimulatedBeacon life-cycle function in a goroutine. +func (c *SimulatedBeacon) Start() error { + if c.period == 0 { + go c.loopOnDemand() + } else { + go c.loop() + } + return nil +} + +// Stop halts the SimulatedBeacon service. +func (c *SimulatedBeacon) Stop() error { + close(c.shutdownCh) + return nil +} + +// sealBlock initiates payload building for a new block and creates a new block +// with the completed payload. +func (c *SimulatedBeacon) sealBlock(withdrawals []*types.Withdrawal) error { + tstamp := uint64(time.Now().Unix()) + if tstamp <= c.lastBlockTime { + tstamp = c.lastBlockTime + 1 + } + c.feeRecipientLock.Lock() + feeRecipient := c.feeRecipient + c.feeRecipientLock.Unlock() + + fcResponse, err := c.engineAPI.ForkchoiceUpdatedV2(c.curForkchoiceState, &engine.PayloadAttributes{ + Timestamp: tstamp, + SuggestedFeeRecipient: feeRecipient, + Withdrawals: withdrawals, + }) + if err != nil { + return fmt.Errorf("error calling forkchoice update: %v", err) + } + + envelope, err := c.engineAPI.getFullPayload(*fcResponse.PayloadID) + if err != nil { + return fmt.Errorf("error retrieving payload: %v", err) + } + payload := envelope.ExecutionPayload + + // mark the payload as canon + if _, err = c.engineAPI.NewPayloadV2(*payload); err != nil { + return fmt.Errorf("failed to mark payload as canonical: %v", err) + } + c.curForkchoiceState = engine.ForkchoiceStateV1{ + HeadBlockHash: payload.BlockHash, + SafeBlockHash: payload.BlockHash, + FinalizedBlockHash: payload.BlockHash, + } + // mark the block containing the payload as canonical + if _, err = c.engineAPI.ForkchoiceUpdatedV2(c.curForkchoiceState, nil); err != nil { + return fmt.Errorf("failed to mark block as canonical: %v", err) + } + c.lastBlockTime = payload.Timestamp + return nil +} + +// loopOnDemand runs the block production loop for "on-demand" configuration (period = 0) +func (c *SimulatedBeacon) loopOnDemand() { + var ( + newTxs = make(chan core.NewTxsEvent) + sub = c.eth.TxPool().SubscribeNewTxsEvent(newTxs) + ) + defer sub.Unsubscribe() + + for { + select { + case <-c.shutdownCh: + return + case w := <-c.withdrawals.pending: + withdrawals := append(c.withdrawals.gatherPending(9), w) + if err := c.sealBlock(withdrawals); err != nil { + log.Error("Error performing sealing-work", "err", err) + return + } + case <-newTxs: + withdrawals := c.withdrawals.gatherPending(10) + if err := c.sealBlock(withdrawals); err != nil { + log.Error("Error performing sealing-work", "err", err) + return + } + } + } +} + +// loopOnDemand runs the block production loop for non-zero period configuration +func (c *SimulatedBeacon) loop() { + timer := time.NewTimer(0) + for { + select { + case <-c.shutdownCh: + return + case <-timer.C: + withdrawals := c.withdrawals.gatherPending(10) + if err := c.sealBlock(withdrawals); err != nil { + log.Error("Error performing sealing-work", "err", err) + return + } + timer.Reset(time.Second * time.Duration(c.period)) + } + } +} + +func RegisterSimulatedBeaconAPIs(stack *node.Node, sim *SimulatedBeacon) { + stack.RegisterAPIs([]rpc.API{ + { + Namespace: "dev", + Service: &api{sim}, + Version: "1.0", + }, + }) +} diff --git a/eth/catalyst/simulated_beacon_api.go b/eth/catalyst/simulated_beacon_api.go new file mode 100644 index 0000000000..93670257f6 --- /dev/null +++ b/eth/catalyst/simulated_beacon_api.go @@ -0,0 +1,36 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package catalyst + +import ( + "context" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" +) + +type api struct { + simBeacon *SimulatedBeacon +} + +func (a *api) AddWithdrawal(ctx context.Context, withdrawal *types.Withdrawal) error { + return a.simBeacon.withdrawals.add(withdrawal) +} + +func (a *api) SetFeeRecipient(ctx context.Context, feeRecipient common.Address) { + a.simBeacon.setFeeRecipient(feeRecipient) +} diff --git a/eth/catalyst/simulated_beacon_test.go b/eth/catalyst/simulated_beacon_test.go new file mode 100644 index 0000000000..0df195fb9d --- /dev/null +++ b/eth/catalyst/simulated_beacon_test.go @@ -0,0 +1,141 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package catalyst + +import ( + "context" + "math/big" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/node" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/params" +) + +func startSimulatedBeaconEthService(t *testing.T, genesis *core.Genesis) (*node.Node, *eth.Ethereum, *SimulatedBeacon) { + t.Helper() + + n, err := node.New(&node.Config{ + P2P: p2p.Config{ + ListenAddr: "127.0.0.1:8545", + NoDiscovery: true, + MaxPeers: 0, + }, + }) + if err != nil { + t.Fatal("can't create node:", err) + } + + ethcfg := ðconfig.Config{Genesis: genesis, SyncMode: downloader.FullSync, TrieTimeout: time.Minute, TrieDirtyCache: 256, TrieCleanCache: 256} + ethservice, err := eth.New(n, ethcfg) + if err != nil { + t.Fatal("can't create eth service:", err) + } + + simBeacon, err := NewSimulatedBeacon(1, ethservice) + if err != nil { + t.Fatal("can't create simulated beacon:", err) + } + + n.RegisterLifecycle(simBeacon) + + if err := n.Start(); err != nil { + t.Fatal("can't start node:", err) + } + + ethservice.SetSynced() + return n, ethservice, simBeacon +} + +// send 20 transactions, >10 withdrawals and ensure they are included in order +// send enough transactions to fill multiple blocks +func TestSimulatedBeaconSendWithdrawals(t *testing.T) { + var withdrawals []types.Withdrawal + txs := make(map[common.Hash]types.Transaction) + + var ( + // testKey is a private key to use for funding a tester account. + testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + + // testAddr is the Ethereum address of the tester account. + testAddr = crypto.PubkeyToAddress(testKey.PublicKey) + ) + + // short period (1 second) for testing purposes + var gasLimit uint64 = 10_000_000 + genesis := core.DeveloperGenesisBlock(gasLimit, testAddr) + node, ethService, mock := startSimulatedBeaconEthService(t, genesis) + _ = mock + defer node.Close() + + chainHeadCh := make(chan core.ChainHeadEvent, 10) + subscription := ethService.BlockChain().SubscribeChainHeadEvent(chainHeadCh) + defer subscription.Unsubscribe() + + // generate some withdrawals + for i := 0; i < 20; i++ { + withdrawals = append(withdrawals, types.Withdrawal{Index: uint64(i)}) + if err := mock.withdrawals.add(&withdrawals[i]); err != nil { + t.Fatal("addWithdrawal failed", err) + } + } + + // generate a bunch of transactions + signer := types.NewEIP155Signer(ethService.BlockChain().Config().ChainID) + for i := 0; i < 20; i++ { + tx, err := types.SignTx(types.NewTransaction(uint64(i), common.Address{}, big.NewInt(1000), params.TxGas, big.NewInt(params.InitialBaseFee), nil), signer, testKey) + if err != nil { + t.Fatalf("error signing transaction, err=%v", err) + } + txs[tx.Hash()] = *tx + + if err := ethService.APIBackend.SendTx(context.Background(), tx); err != nil { + t.Fatal("SendTx failed", err) + } + } + + includedTxs := make(map[common.Hash]struct{}) + var includedWithdrawals []uint64 + + timer := time.NewTimer(12 * time.Second) + for { + select { + case evt := <-chainHeadCh: + for _, includedTx := range evt.Block.Transactions() { + includedTxs[includedTx.Hash()] = struct{}{} + } + for _, includedWithdrawal := range evt.Block.Withdrawals() { + includedWithdrawals = append(includedWithdrawals, includedWithdrawal.Index) + } + + // ensure all withdrawals/txs included. this will take two blocks b/c number of withdrawals > 10 + if len(includedTxs) == len(txs) && len(includedWithdrawals) == len(withdrawals) && evt.Block.Number().Cmp(big.NewInt(2)) == 0 { + return + } + case <-timer.C: + t.Fatal("timed out without including all withdrawals/txs") + } + } +} diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index c171709586..f47564dc85 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -30,6 +30,7 @@ var Modules = map[string]string{ "txpool": TxpoolJs, "les": LESJs, "vflux": VfluxJs, + "dev": DevJs, } const CliqueJs = ` @@ -886,3 +887,22 @@ web3._extend({ ] }); ` + +const DevJs = ` +web3._extend({ + property: 'dev', + methods: + [ + new web3._extend.Method({ + name: 'addWithdrawal', + call: 'dev_addWithdrawal', + params: 1 + }), + new web3._extend.Method({ + name: 'setFeeRecipient', + call: 'dev_setFeeRecipient', + params: 1 + }), + ], +}); +` diff --git a/miner/miner_test.go b/miner/miner_test.go index 3f84446ff6..21db1ce4a9 100644 --- a/miner/miner_test.go +++ b/miner/miner_test.go @@ -32,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/core/txpool/legacypool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/params" @@ -252,6 +253,34 @@ func waitForMiningState(t *testing.T, m *Miner, mining bool) { t.Fatalf("Mining() == %t, want %t", state, mining) } +func minerTestGenesisBlock(period uint64, gasLimit uint64, faucet common.Address) *core.Genesis { + config := *params.AllCliqueProtocolChanges + config.Clique = ¶ms.CliqueConfig{ + Period: period, + Epoch: config.Clique.Epoch, + } + + // Assemble and return the genesis with the precompiles and faucet pre-funded + return &core.Genesis{ + Config: &config, + ExtraData: append(append(make([]byte, 32), faucet[:]...), make([]byte, crypto.SignatureLength)...), + GasLimit: gasLimit, + BaseFee: big.NewInt(params.InitialBaseFee), + Difficulty: big.NewInt(1), + Alloc: map[common.Address]core.GenesisAccount{ + common.BytesToAddress([]byte{1}): {Balance: big.NewInt(1)}, // ECRecover + common.BytesToAddress([]byte{2}): {Balance: big.NewInt(1)}, // SHA256 + common.BytesToAddress([]byte{3}): {Balance: big.NewInt(1)}, // RIPEMD + common.BytesToAddress([]byte{4}): {Balance: big.NewInt(1)}, // Identity + common.BytesToAddress([]byte{5}): {Balance: big.NewInt(1)}, // ModExp + common.BytesToAddress([]byte{6}): {Balance: big.NewInt(1)}, // ECAdd + common.BytesToAddress([]byte{7}): {Balance: big.NewInt(1)}, // ECScalarMul + common.BytesToAddress([]byte{8}): {Balance: big.NewInt(1)}, // ECPairing + common.BytesToAddress([]byte{9}): {Balance: big.NewInt(1)}, // BLAKE2b + faucet: {Balance: new(big.Int).Sub(new(big.Int).Lsh(big.NewInt(1), 256), big.NewInt(9))}, + }, + } +} func createMiner(t *testing.T) (*Miner, *event.TypeMux, func(skipMiner bool)) { // Create Ethash config config := Config{ @@ -259,7 +288,7 @@ func createMiner(t *testing.T) (*Miner, *event.TypeMux, func(skipMiner bool)) { } // Create chainConfig chainDB := rawdb.NewMemoryDatabase() - genesis := core.DeveloperGenesisBlock(15, 11_500_000, common.HexToAddress("12345")) + genesis := minerTestGenesisBlock(15, 11_500_000, common.HexToAddress("12345")) chainConfig, _, err := core.SetupGenesisBlock(chainDB, trie.NewDatabase(chainDB), genesis) if err != nil { t.Fatalf("can't create new chain config: %v", err) diff --git a/miner/payload_building.go b/miner/payload_building.go index 819937ba32..9b13a4a695 100644 --- a/miner/payload_building.go +++ b/miner/payload_building.go @@ -135,7 +135,7 @@ func (payload *Payload) ResolveEmpty() *engine.ExecutionPayloadEnvelope { } // ResolveFull is basically identical to Resolve, but it expects full block only. -// It's only used in tests. +// Don't call Resolve until ResolveFull returns, otherwise it might block forever. func (payload *Payload) ResolveFull() *engine.ExecutionPayloadEnvelope { payload.lock.Lock() defer payload.lock.Unlock() @@ -146,8 +146,17 @@ func (payload *Payload) ResolveFull() *engine.ExecutionPayloadEnvelope { return nil default: } + // Wait the full payload construction. Note it might block + // forever if Resolve is called in the meantime which + // terminates the background construction process. payload.cond.Wait() } + // Terminate the background payload construction + select { + case <-payload.stop: + default: + close(payload.stop) + } return engine.BlockToExecutableData(payload.full, payload.fullFees) } diff --git a/params/config.go b/params/config.go index 8e6704bf1c..75c8fd89d0 100644 --- a/params/config.go +++ b/params/config.go @@ -135,6 +135,27 @@ var ( Clique: nil, } + AllDevChainProtocolChanges = &ChainConfig{ + ChainID: big.NewInt(1337), + HomesteadBlock: big.NewInt(0), + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), + ConstantinopleBlock: big.NewInt(0), + PetersburgBlock: big.NewInt(0), + IstanbulBlock: big.NewInt(0), + MuirGlacierBlock: big.NewInt(0), + BerlinBlock: big.NewInt(0), + LondonBlock: big.NewInt(0), + ArrowGlacierBlock: big.NewInt(0), + GrayGlacierBlock: big.NewInt(0), + ShanghaiTime: newUint64(0), + TerminalTotalDifficulty: big.NewInt(0), + TerminalTotalDifficultyPassed: true, + IsDevMode: true, + } + // AllCliqueProtocolChanges contains every protocol change (EIPs) introduced // and accepted by the Ethereum core developers into the Clique consensus. AllCliqueProtocolChanges = &ChainConfig{ @@ -280,8 +301,9 @@ type ChainConfig struct { TerminalTotalDifficultyPassed bool `json:"terminalTotalDifficultyPassed,omitempty"` // Various consensus engines - Ethash *EthashConfig `json:"ethash,omitempty"` - Clique *CliqueConfig `json:"clique,omitempty"` + Ethash *EthashConfig `json:"ethash,omitempty"` + Clique *CliqueConfig `json:"clique,omitempty"` + IsDevMode bool `json:"isDev,omitempty"` } // EthashConfig is the consensus engine configs for proof-of-work based sealing. From cbf2579691092499ba3387fba1c0502af1592fcf Mon Sep 17 00:00:00 2001 From: lightclient <14004106+lightclient@users.noreply.github.com> Date: Thu, 6 Jul 2023 08:20:31 -0600 Subject: [PATCH 51/90] p2p, p2p/discover: add dial metrics (#27621) This PR adds metrics for p2p dialing, which gives us visibility into the quality of the dial candidates returned by our discovery methods. --- eth/handler.go | 8 +++ eth/protocols/eth/handshake.go | 26 ++++++++++ eth/protocols/eth/metrics.go | 81 ++++++++++++++++++++++++++++++ eth/protocols/snap/metrics.go | 29 +++++++++++ p2p/dial.go | 5 +- p2p/discover/metrics.go | 8 +++ p2p/discover/table.go | 36 +++++++++++-- p2p/discover/v4_udp.go | 2 +- p2p/discover/v4_udp_test.go | 2 +- p2p/discover/v5_udp.go | 2 +- p2p/metrics.go | 92 ++++++++++++++++++++++++---------- p2p/server.go | 27 ++++++---- p2p/server_test.go | 10 ++-- 13 files changed, 277 insertions(+), 51 deletions(-) create mode 100644 eth/protocols/eth/metrics.go create mode 100644 eth/protocols/snap/metrics.go diff --git a/eth/handler.go b/eth/handler.go index 6a8487649e..9d6a8c4225 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -38,6 +38,7 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/p2p" ) @@ -424,6 +425,13 @@ func (h *handler) runSnapExtension(peer *snap.Peer, handler snap.Handler) error defer h.peerWG.Done() if err := h.peers.registerSnapExtension(peer); err != nil { + if metrics.Enabled { + if peer.Inbound() { + snap.IngressRegistrationErrorMeter.Mark(1) + } else { + snap.EgressRegistrationErrorMeter.Mark(1) + } + } peer.Log().Warn("Snapshot extension registration failed", "err", err) return err } diff --git a/eth/protocols/eth/handshake.go b/eth/protocols/eth/handshake.go index 9a2769fa0d..ea16a85b1e 100644 --- a/eth/protocols/eth/handshake.go +++ b/eth/protocols/eth/handshake.go @@ -17,12 +17,14 @@ package eth import ( + "errors" "fmt" "math/big" "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/forkid" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/p2p" ) @@ -59,9 +61,11 @@ func (p *Peer) Handshake(network uint64, td *big.Int, head common.Hash, genesis select { case err := <-errc: if err != nil { + markError(p, err) return err } case <-timeout.C: + markError(p, p2p.DiscReadTimeout) return p2p.DiscReadTimeout } } @@ -105,3 +109,25 @@ func (p *Peer) readStatus(network uint64, status *StatusPacket, genesis common.H } return nil } + +// markError registers the error with the corresponding metric. +func markError(p *Peer, err error) { + if !metrics.Enabled { + return + } + m := meters.get(p.Inbound()) + switch errors.Unwrap(err) { + case errNetworkIDMismatch: + m.networkIDMismatch.Mark(1) + case errProtocolVersionMismatch: + m.protocolVersionMismatch.Mark(1) + case errGenesisMismatch: + m.genesisMismatch.Mark(1) + case errForkIDRejected: + m.forkidRejected.Mark(1) + case p2p.DiscReadTimeout: + m.timeoutError.Mark(1) + default: + m.peerError.Mark(1) + } +} diff --git a/eth/protocols/eth/metrics.go b/eth/protocols/eth/metrics.go new file mode 100644 index 0000000000..5e0aee39f8 --- /dev/null +++ b/eth/protocols/eth/metrics.go @@ -0,0 +1,81 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package eth + +import "github.com/ethereum/go-ethereum/metrics" + +// meters stores ingress and egress handshake meters. +var meters bidirectionalMeters + +// bidirectionalMeters stores ingress and egress handshake meters. +type bidirectionalMeters struct { + ingress *hsMeters + egress *hsMeters +} + +// get returns the corresponding meter depending if ingress or egress is +// desired. +func (h *bidirectionalMeters) get(ingress bool) *hsMeters { + if ingress { + return h.ingress + } + return h.egress +} + +// hsMeters is a collection of meters which track metrics related to the +// eth subprotocol handshake. +type hsMeters struct { + // peerError measures the number of errors related to incorrect peer + // behaviour, such as invalid message code, size, encoding, etc. + peerError metrics.Meter + + // timeoutError measures the number of timeouts. + timeoutError metrics.Meter + + // networkIDMismatch measures the number of network id mismatch errors. + networkIDMismatch metrics.Meter + + // protocolVersionMismatch measures the number of differing protocol + // versions. + protocolVersionMismatch metrics.Meter + + // genesisMismatch measures the number of differing genesises. + genesisMismatch metrics.Meter + + // forkidRejected measures the number of differing forkids. + forkidRejected metrics.Meter +} + +// newHandshakeMeters registers and returns handshake meters for the given +// base. +func newHandshakeMeters(base string) *hsMeters { + return &hsMeters{ + peerError: metrics.NewRegisteredMeter(base+"error/peer", nil), + timeoutError: metrics.NewRegisteredMeter(base+"error/timeout", nil), + networkIDMismatch: metrics.NewRegisteredMeter(base+"error/network", nil), + protocolVersionMismatch: metrics.NewRegisteredMeter(base+"error/version", nil), + genesisMismatch: metrics.NewRegisteredMeter(base+"error/genesis", nil), + forkidRejected: metrics.NewRegisteredMeter(base+"error/forkid", nil), + } +} + +func init() { + meters = bidirectionalMeters{ + ingress: newHandshakeMeters("eth/protocols/eth/ingress/handshake/"), + egress: newHandshakeMeters("eth/protocols/eth/egress/handshake/"), + } +} diff --git a/eth/protocols/snap/metrics.go b/eth/protocols/snap/metrics.go new file mode 100644 index 0000000000..a9f35ca447 --- /dev/null +++ b/eth/protocols/snap/metrics.go @@ -0,0 +1,29 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package snap + +import ( + metrics "github.com/ethereum/go-ethereum/metrics" +) + +var ( + ingressRegistrationErrorName = "eth/protocols/snap/ingress/registration/error" + egressRegistrationErrorName = "eth/protocols/snap/egress/registration/error" + + IngressRegistrationErrorMeter = metrics.NewRegisteredMeter(ingressRegistrationErrorName, nil) + EgressRegistrationErrorMeter = metrics.NewRegisteredMeter(egressRegistrationErrorName, nil) +) diff --git a/p2p/dial.go b/p2p/dial.go index 134e6e2eae..5e4ab1d50d 100644 --- a/p2p/dial.go +++ b/p2p/dial.go @@ -521,13 +521,14 @@ func (t *dialTask) resolve(d *dialScheduler) bool { // dial performs the actual connection attempt. func (t *dialTask) dial(d *dialScheduler, dest *enode.Node) error { + dialMeter.Mark(1) fd, err := d.dialer.Dial(d.ctx, t.dest) if err != nil { d.log.Trace("Dial error", "id", t.dest.ID(), "addr", nodeAddr(t.dest), "conn", t.flags, "err", cleanupDialErr(err)) + dialConnectionError.Mark(1) return &dialError{err} } - mfd := newMeteredConn(fd, false, &net.TCPAddr{IP: dest.IP(), Port: dest.TCP()}) - return d.setupFunc(mfd, t.flags, dest) + return d.setupFunc(newMeteredConn(fd), t.flags, dest) } func (t *dialTask) String() string { diff --git a/p2p/discover/metrics.go b/p2p/discover/metrics.go index bf1a2fa2b8..da8e9cb817 100644 --- a/p2p/discover/metrics.go +++ b/p2p/discover/metrics.go @@ -17,6 +17,7 @@ package discover import ( + "fmt" "net" "github.com/ethereum/go-ethereum/metrics" @@ -32,10 +33,17 @@ const ( ) var ( + bucketsCounter []metrics.Counter ingressTrafficMeter = metrics.NewRegisteredMeter(ingressMeterName, nil) egressTrafficMeter = metrics.NewRegisteredMeter(egressMeterName, nil) ) +func init() { + for i := 0; i < nBuckets; i++ { + bucketsCounter = append(bucketsCounter, metrics.NewRegisteredCounter(fmt.Sprintf("%s/bucket/%d/count", moduleName, i), nil)) + } +} + // meteredConn is a wrapper around a net.UDPConn that meters both the // inbound and outbound network traffic. type meteredUdpConn struct { diff --git a/p2p/discover/table.go b/p2p/discover/table.go index 1397348aec..b601f233d0 100644 --- a/p2p/discover/table.go +++ b/p2p/discover/table.go @@ -34,6 +34,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/netutil" ) @@ -80,7 +81,8 @@ type Table struct { closeReq chan struct{} closed chan struct{} - nodeAddedHook func(*node) // for testing + nodeAddedHook func(*bucket, *node) + nodeRemovedHook func(*bucket, *node) } // transport is implemented by the UDP transports. @@ -98,6 +100,7 @@ type bucket struct { entries []*node // live entries, sorted by time of last contact replacements []*node // recently seen nodes to be used if revalidation fails ips netutil.DistinctNetSet + index int } func newTable(t transport, db *enode.DB, cfg Config) (*Table, error) { @@ -119,7 +122,8 @@ func newTable(t transport, db *enode.DB, cfg Config) (*Table, error) { } for i := range tab.buckets { tab.buckets[i] = &bucket{ - ips: netutil.DistinctNetSet{Subnet: bucketSubnet, Limit: bucketIPLimit}, + index: i, + ips: netutil.DistinctNetSet{Subnet: bucketSubnet, Limit: bucketIPLimit}, } } tab.seedRand() @@ -128,6 +132,22 @@ func newTable(t transport, db *enode.DB, cfg Config) (*Table, error) { return tab, nil } +func newMeteredTable(t transport, db *enode.DB, cfg Config) (*Table, error) { + tab, err := newTable(t, db, cfg) + if err != nil { + return nil, err + } + if metrics.Enabled { + tab.nodeAddedHook = func(b *bucket, n *node) { + bucketsCounter[b.index].Inc(1) + } + tab.nodeRemovedHook = func(b *bucket, n *node) { + bucketsCounter[b.index].Dec(1) + } + } + return tab, nil +} + // Nodes returns all nodes contained in the table. func (tab *Table) Nodes() []*enode.Node { if !tab.isInitDone() { @@ -495,7 +515,7 @@ func (tab *Table) addSeenNode(n *node) { n.addedAt = time.Now() if tab.nodeAddedHook != nil { - tab.nodeAddedHook(n) + tab.nodeAddedHook(b, n) } } @@ -539,7 +559,7 @@ func (tab *Table) addVerifiedNode(n *node) { n.addedAt = time.Now() if tab.nodeAddedHook != nil { - tab.nodeAddedHook(n) + tab.nodeAddedHook(b, n) } } @@ -638,8 +658,16 @@ func (tab *Table) bumpInBucket(b *bucket, n *node) bool { } func (tab *Table) deleteInBucket(b *bucket, n *node) { + // Check if the node is actually in the bucket so the removed hook + // isn't called multiple times for the same node. + if !contains(b.entries, n.ID()) { + return + } b.entries = deleteNode(b.entries, n) tab.removeIP(b, n.IP()) + if tab.nodeRemovedHook != nil { + tab.nodeRemovedHook(b, n) + } } func contains(ns []*node, id enode.ID) bool { diff --git a/p2p/discover/v4_udp.go b/p2p/discover/v4_udp.go index ab335e2ced..988f16b01d 100644 --- a/p2p/discover/v4_udp.go +++ b/p2p/discover/v4_udp.go @@ -142,7 +142,7 @@ func ListenV4(c UDPConn, ln *enode.LocalNode, cfg Config) (*UDPv4, error) { log: cfg.Log, } - tab, err := newTable(t, ln.Database(), cfg) + tab, err := newMeteredTable(t, ln.Database(), cfg) if err != nil { return nil, err } diff --git a/p2p/discover/v4_udp_test.go b/p2p/discover/v4_udp_test.go index 21f0d75172..5add9cefa1 100644 --- a/p2p/discover/v4_udp_test.go +++ b/p2p/discover/v4_udp_test.go @@ -394,7 +394,7 @@ func TestUDPv4_pingMatchIP(t *testing.T) { func TestUDPv4_successfulPing(t *testing.T) { test := newUDPTest(t) added := make(chan *node, 1) - test.table.nodeAddedHook = func(n *node) { added <- n } + test.table.nodeAddedHook = func(b *bucket, n *node) { added <- n } defer test.close() // The remote side sends a ping packet to initiate the exchange. diff --git a/p2p/discover/v5_udp.go b/p2p/discover/v5_udp.go index 7bed9dbcfd..6ba7a90618 100644 --- a/p2p/discover/v5_udp.go +++ b/p2p/discover/v5_udp.go @@ -174,7 +174,7 @@ func newUDPv5(conn UDPConn, ln *enode.LocalNode, cfg Config) (*UDPv5, error) { cancelCloseCtx: cancelCloseCtx, } t.talk = newTalkSystem(t) - tab, err := newTable(t, t.db, cfg) + tab, err := newMeteredTable(t, t.db, cfg) if err != nil { return nil, err } diff --git a/p2p/metrics.go b/p2p/metrics.go index 1bb505cdfb..a6e36b91a8 100644 --- a/p2p/metrics.go +++ b/p2p/metrics.go @@ -19,30 +19,86 @@ package p2p import ( + "errors" "net" "github.com/ethereum/go-ethereum/metrics" ) const ( + // HandleHistName is the prefix of the per-packet serving time histograms. + HandleHistName = "p2p/handle" + // ingressMeterName is the prefix of the per-packet inbound metrics. ingressMeterName = "p2p/ingress" // egressMeterName is the prefix of the per-packet outbound metrics. egressMeterName = "p2p/egress" - - // HandleHistName is the prefix of the per-packet serving time histograms. - HandleHistName = "p2p/handle" ) var ( - ingressConnectMeter = metrics.NewRegisteredMeter("p2p/serves", nil) - ingressTrafficMeter = metrics.NewRegisteredMeter(ingressMeterName, nil) - egressConnectMeter = metrics.NewRegisteredMeter("p2p/dials", nil) - egressTrafficMeter = metrics.NewRegisteredMeter(egressMeterName, nil) - activePeerGauge = metrics.NewRegisteredGauge("p2p/peers", nil) + activePeerGauge metrics.Gauge = metrics.NilGauge{} + + ingressTrafficMeter = metrics.NewRegisteredMeter("p2p/ingress", nil) + egressTrafficMeter = metrics.NewRegisteredMeter("p2p/egress", nil) + + // general ingress/egress connection meters + serveMeter metrics.Meter = metrics.NilMeter{} + serveSuccessMeter metrics.Meter = metrics.NilMeter{} + dialMeter metrics.Meter = metrics.NilMeter{} + dialSuccessMeter metrics.Meter = metrics.NilMeter{} + dialConnectionError metrics.Meter = metrics.NilMeter{} + + // handshake error meters + dialTooManyPeers = metrics.NewRegisteredMeter("p2p/dials/error/saturated", nil) + dialAlreadyConnected = metrics.NewRegisteredMeter("p2p/dials/error/known", nil) + dialSelf = metrics.NewRegisteredMeter("p2p/dials/error/self", nil) + dialUselessPeer = metrics.NewRegisteredMeter("p2p/dials/error/useless", nil) + dialUnexpectedIdentity = metrics.NewRegisteredMeter("p2p/dials/error/id/unexpected", nil) + dialEncHandshakeError = metrics.NewRegisteredMeter("p2p/dials/error/rlpx/enc", nil) + dialProtoHandshakeError = metrics.NewRegisteredMeter("p2p/dials/error/rlpx/proto", nil) ) +func init() { + if !metrics.Enabled { + return + } + + activePeerGauge = metrics.NewRegisteredGauge("p2p/peers", nil) + serveMeter = metrics.NewRegisteredMeter("p2p/serves", nil) + serveSuccessMeter = metrics.NewRegisteredMeter("p2p/serves/success", nil) + dialMeter = metrics.NewRegisteredMeter("p2p/dials", nil) + dialSuccessMeter = metrics.NewRegisteredMeter("p2p/dials/success", nil) + dialConnectionError = metrics.NewRegisteredMeter("p2p/dials/error/connection", nil) +} + +// markDialError matches errors that occur while setting up a dial connection +// to the corresponding meter. +func markDialError(err error) { + if !metrics.Enabled { + return + } + if err2 := errors.Unwrap(err); err2 != nil { + err = err2 + } + switch err { + case DiscTooManyPeers: + dialTooManyPeers.Mark(1) + case DiscAlreadyConnected: + dialAlreadyConnected.Mark(1) + case DiscSelf: + dialSelf.Mark(1) + case DiscUselessPeer: + dialUselessPeer.Mark(1) + case DiscUnexpectedIdentity: + dialUnexpectedIdentity.Mark(1) + case errEncHandshakeError: + dialEncHandshakeError.Mark(1) + case errProtoHandshakeError: + dialProtoHandshakeError.Mark(1) + } +} + // meteredConn is a wrapper around a net.Conn that meters both the // inbound and outbound network traffic. type meteredConn struct { @@ -52,18 +108,10 @@ type meteredConn struct { // newMeteredConn creates a new metered connection, bumps the ingress or egress // connection meter and also increases the metered peer count. If the metrics // system is disabled, function returns the original connection. -func newMeteredConn(conn net.Conn, ingress bool, addr *net.TCPAddr) net.Conn { - // Short circuit if metrics are disabled +func newMeteredConn(conn net.Conn) net.Conn { if !metrics.Enabled { return conn } - // Bump the connection counters and wrap the connection - if ingress { - ingressConnectMeter.Mark(1) - } else { - egressConnectMeter.Mark(1) - } - activePeerGauge.Inc(1) return &meteredConn{Conn: conn} } @@ -82,13 +130,3 @@ func (c *meteredConn) Write(b []byte) (n int, err error) { egressTrafficMeter.Mark(int64(n)) return n, err } - -// Close delegates a close operation to the underlying connection, unregisters -// the peer from the traffic registries and emits close event. -func (c *meteredConn) Close() error { - err := c.Conn.Close() - if err == nil { - activePeerGauge.Dec(1) - } - return err -} diff --git a/p2p/server.go b/p2p/server.go index bdfeb74c67..8c417635e6 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -64,7 +64,11 @@ const ( frameWriteTimeout = 20 * time.Second ) -var errServerStopped = errors.New("server stopped") +var ( + errServerStopped = errors.New("server stopped") + errEncHandshakeError = errors.New("rlpx enc error") + errProtoHandshakeError = errors.New("rlpx proto error") +) // Config holds Server options. type Config struct { @@ -772,7 +776,11 @@ running: srv.dialsched.peerAdded(c) if p.Inbound() { inboundCount++ + serveSuccessMeter.Mark(1) + } else { + dialSuccessMeter.Mark(1) } + activePeerGauge.Inc(1) } c.cont <- err @@ -785,6 +793,7 @@ running: if pd.Inbound() { inboundCount-- } + activePeerGauge.Dec(1) } } @@ -894,11 +903,8 @@ func (srv *Server) listenLoop() { continue } if remoteIP != nil { - var addr *net.TCPAddr - if tcp, ok := fd.RemoteAddr().(*net.TCPAddr); ok { - addr = tcp - } - fd = newMeteredConn(fd, true, addr) + fd = newMeteredConn(fd) + serveMeter.Mark(1) srv.log.Trace("Accepted connection", "addr", fd.RemoteAddr()) } go func() { @@ -939,6 +945,9 @@ func (srv *Server) SetupConn(fd net.Conn, flags connFlag, dialDest *enode.Node) err := srv.setupConn(c, flags, dialDest) if err != nil { + if !c.is(inboundConn) { + markDialError(err) + } c.close(err) } return err @@ -957,7 +966,7 @@ func (srv *Server) setupConn(c *conn, flags connFlag, dialDest *enode.Node) erro if dialDest != nil { dialPubkey := new(ecdsa.PublicKey) if err := dialDest.Load((*enode.Secp256k1)(dialPubkey)); err != nil { - err = errors.New("dial destination doesn't have a secp256k1 public key") + err = fmt.Errorf("%w: dial destination doesn't have a secp256k1 public key", errEncHandshakeError) srv.log.Trace("Setting up connection failed", "addr", c.fd.RemoteAddr(), "conn", c.flags, "err", err) return err } @@ -967,7 +976,7 @@ func (srv *Server) setupConn(c *conn, flags connFlag, dialDest *enode.Node) erro remotePubkey, err := c.doEncHandshake(srv.PrivateKey) if err != nil { srv.log.Trace("Failed RLPx handshake", "addr", c.fd.RemoteAddr(), "conn", c.flags, "err", err) - return err + return fmt.Errorf("%w: %v", errEncHandshakeError, err) } if dialDest != nil { c.node = dialDest @@ -985,7 +994,7 @@ func (srv *Server) setupConn(c *conn, flags connFlag, dialDest *enode.Node) erro phs, err := c.doProtoHandshake(srv.ourHandshake) if err != nil { clog.Trace("Failed p2p handshake", "err", err) - return err + return fmt.Errorf("%w: %v", errProtoHandshakeError, err) } if id := c.node.ID(); !bytes.Equal(crypto.Keccak256(phs.ID), id[:]) { clog.Trace("Wrong devp2p handshake identity", "phsid", hex.EncodeToString(phs.ID)) diff --git a/p2p/server_test.go b/p2p/server_test.go index f6f5700c5e..c8bf4c941c 100644 --- a/p2p/server_test.go +++ b/p2p/server_test.go @@ -370,8 +370,6 @@ func TestServerSetupConn(t *testing.T) { clientkey, srvkey = newkey(), newkey() clientpub = &clientkey.PublicKey srvpub = &srvkey.PublicKey - fooErr = errors.New("foo") - readErr = errors.New("read error") ) tests := []struct { dontstart bool @@ -389,10 +387,10 @@ func TestServerSetupConn(t *testing.T) { wantCloseErr: errServerStopped, }, { - tt: &setupTransport{pubkey: clientpub, encHandshakeErr: readErr}, + tt: &setupTransport{pubkey: clientpub, encHandshakeErr: errEncHandshakeError}, flags: inboundConn, wantCalls: "doEncHandshake,close,", - wantCloseErr: readErr, + wantCloseErr: errEncHandshakeError, }, { tt: &setupTransport{pubkey: clientpub, phs: protoHandshake{ID: randomID().Bytes()}}, @@ -402,11 +400,11 @@ func TestServerSetupConn(t *testing.T) { wantCloseErr: DiscUnexpectedIdentity, }, { - tt: &setupTransport{pubkey: clientpub, protoHandshakeErr: fooErr}, + tt: &setupTransport{pubkey: clientpub, protoHandshakeErr: errProtoHandshakeError}, dialDest: enode.NewV4(clientpub, nil, 0, 0), flags: dynDialedConn, wantCalls: "doEncHandshake,doProtoHandshake,close,", - wantCloseErr: fooErr, + wantCloseErr: errProtoHandshakeError, }, { tt: &setupTransport{pubkey: srvpub, phs: protoHandshake{ID: crypto.FromECDSAPub(srvpub)[1:]}}, From c866dfdc78fda6a9cd940970aa8c787dd4762205 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 6 Jul 2023 12:05:34 -0400 Subject: [PATCH 52/90] core: remove outdated tests (#27662) Back before #27178 , we spun up a number of ethash verifiers to verify headers. So we also had tests to ensure that we were indeed able to abort verification even if we had multiple workers running. With PR #27178, we removed the parallelism in verification, and these tests are now failing, since we now just sequentially fire away the results as fast as possible on one routine. This change removes the (sometimes failing) tests --- core/block_validator_test.go | 113 ---------------------- core/txpool/legacypool/legacypool_test.go | 18 ++-- 2 files changed, 9 insertions(+), 122 deletions(-) diff --git a/core/block_validator_test.go b/core/block_validator_test.go index d189545166..460eaad21c 100644 --- a/core/block_validator_test.go +++ b/core/block_validator_test.go @@ -18,7 +18,6 @@ package core import ( "math/big" - "runtime" "testing" "time" @@ -235,118 +234,6 @@ func testHeaderVerificationForMerging(t *testing.T, isClique bool) { } } -// Tests that concurrent header verification works, for both good and bad blocks. -func TestHeaderConcurrentVerification2(t *testing.T) { testHeaderConcurrentVerification(t, 2) } -func TestHeaderConcurrentVerification8(t *testing.T) { testHeaderConcurrentVerification(t, 8) } -func TestHeaderConcurrentVerification32(t *testing.T) { testHeaderConcurrentVerification(t, 32) } - -func testHeaderConcurrentVerification(t *testing.T, threads int) { - // Create a simple chain to verify - var ( - gspec = &Genesis{Config: params.TestChainConfig} - _, blocks, _ = GenerateChainWithGenesis(gspec, ethash.NewFaker(), 8, nil) - ) - headers := make([]*types.Header, len(blocks)) - for i, block := range blocks { - headers[i] = block.Header() - } - // Set the number of threads to verify on - old := runtime.GOMAXPROCS(threads) - defer runtime.GOMAXPROCS(old) - - // Run the header checker for the entire block chain at once both for a valid and - // also an invalid chain (enough if one arbitrary block is invalid). - for i, valid := range []bool{true, false} { - var results <-chan error - - if valid { - chain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) - _, results = chain.engine.VerifyHeaders(chain, headers) - chain.Stop() - } else { - chain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, ethash.NewFakeFailer(uint64(len(headers)-1)), vm.Config{}, nil, nil) - _, results = chain.engine.VerifyHeaders(chain, headers) - chain.Stop() - } - // Wait for all the verification results - checks := make(map[int]error) - for j := 0; j < len(blocks); j++ { - select { - case result := <-results: - checks[j] = result - - case <-time.After(time.Second): - t.Fatalf("test %d.%d: verification timeout", i, j) - } - } - // Check nonce check validity - for j := 0; j < len(blocks); j++ { - want := valid || (j < len(blocks)-2) // We chose the last-but-one nonce in the chain to fail - if (checks[j] == nil) != want { - t.Errorf("test %d.%d: validity mismatch: have %v, want %v", i, j, checks[j], want) - } - if !want { - // A few blocks after the first error may pass verification due to concurrent - // workers. We don't care about those in this test, just that the correct block - // errors out. - break - } - } - // Make sure no more data is returned - select { - case result := <-results: - t.Fatalf("test %d: unexpected result returned: %v", i, result) - case <-time.After(25 * time.Millisecond): - } - } -} - -// Tests that aborting a header validation indeed prevents further checks from being -// run, as well as checks that no left-over goroutines are leaked. -func TestHeaderConcurrentAbortion2(t *testing.T) { testHeaderConcurrentAbortion(t, 2) } -func TestHeaderConcurrentAbortion8(t *testing.T) { testHeaderConcurrentAbortion(t, 8) } -func TestHeaderConcurrentAbortion32(t *testing.T) { testHeaderConcurrentAbortion(t, 32) } - -func testHeaderConcurrentAbortion(t *testing.T, threads int) { - // Create a simple chain to verify - var ( - gspec = &Genesis{Config: params.TestChainConfig} - _, blocks, _ = GenerateChainWithGenesis(gspec, ethash.NewFaker(), 1024, nil) - ) - headers := make([]*types.Header, len(blocks)) - for i, block := range blocks { - headers[i] = block.Header() - } - // Set the number of threads to verify on - old := runtime.GOMAXPROCS(threads) - defer runtime.GOMAXPROCS(old) - - // Start the verifications and immediately abort - chain, _ := NewBlockChain(rawdb.NewMemoryDatabase(), nil, gspec, nil, ethash.NewFakeDelayer(time.Millisecond), vm.Config{}, nil, nil) - defer chain.Stop() - - abort, results := chain.engine.VerifyHeaders(chain, headers) - close(abort) - - // Deplete the results channel - verified := 0 - for depleted := false; !depleted; { - select { - case result := <-results: - if result != nil { - t.Errorf("header %d: validation failed: %v", verified, result) - } - verified++ - case <-time.After(50 * time.Millisecond): - depleted = true - } - } - // Check that abortion was honored by not processing too many POWs - if verified > 2*threads { - t.Errorf("verification count too large: have %d, want below %d", verified, 2*threads) - } -} - func TestCalcGasLimit(t *testing.T) { for i, tc := range []struct { pGasLimit uint64 diff --git a/core/txpool/legacypool/legacypool_test.go b/core/txpool/legacypool/legacypool_test.go index e3302487b3..14612a686c 100644 --- a/core/txpool/legacypool/legacypool_test.go +++ b/core/txpool/legacypool/legacypool_test.go @@ -1090,7 +1090,7 @@ func testQueueTimeLimiting(t *testing.T, nolocals bool) { t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 2) } if queued != 2 { - t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 3) + t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 2) } if err := validatePoolInternals(pool); err != nil { t.Fatalf("pool internal state corrupted: %v", err) @@ -1455,7 +1455,7 @@ func TestRepricing(t *testing.T) { if err := pool.addRemote(pricedTransaction(0, 100000, big.NewInt(2), keys[1])); err != nil { t.Fatalf("failed to add pending transaction: %v", err) } - if err := pool.addRemote(pricedTransaction(2, 100000, big.NewInt(2), keys[2])); err != nil { + if err := pool.addRemoteSync(pricedTransaction(2, 100000, big.NewInt(2), keys[2])); err != nil { t.Fatalf("failed to add queued transaction: %v", err) } if err := validateEvents(events, 5); err != nil { @@ -1582,7 +1582,7 @@ func TestRepricingDynamicFee(t *testing.T) { t.Fatalf("failed to add pending transaction: %v", err) } tx = dynamicFeeTx(2, 100000, big.NewInt(2), big.NewInt(2), keys[2]) - if err := pool.addRemote(tx); err != nil { + if err := pool.addRemoteSync(tx); err != nil { t.Fatalf("failed to add queued transaction: %v", err) } if err := validateEvents(events, 5); err != nil { @@ -1723,18 +1723,18 @@ func TestUnderpricing(t *testing.T) { t.Fatalf("pool internal state corrupted: %v", err) } // Ensure that adding an underpriced transaction on block limit fails - if err := pool.addRemote(pricedTransaction(0, 100000, big.NewInt(1), keys[1])); !errors.Is(err, txpool.ErrUnderpriced) { + if err := pool.addRemoteSync(pricedTransaction(0, 100000, big.NewInt(1), keys[1])); !errors.Is(err, txpool.ErrUnderpriced) { t.Fatalf("adding underpriced pending transaction error mismatch: have %v, want %v", err, txpool.ErrUnderpriced) } // Replace a future transaction with a future transaction - if err := pool.addRemote(pricedTransaction(1, 100000, big.NewInt(2), keys[1])); err != nil { // +K1:1 => -K1:1 => Pend K0:0, K0:1, K2:0; Que K1:1 + if err := pool.addRemoteSync(pricedTransaction(1, 100000, big.NewInt(2), keys[1])); err != nil { // +K1:1 => -K1:1 => Pend K0:0, K0:1, K2:0; Que K1:1 t.Fatalf("failed to add well priced transaction: %v", err) } // Ensure that adding high priced transactions drops cheap ones, but not own - if err := pool.addRemote(pricedTransaction(0, 100000, big.NewInt(3), keys[1])); err != nil { // +K1:0 => -K1:1 => Pend K0:0, K0:1, K1:0, K2:0; Que - + if err := pool.addRemoteSync(pricedTransaction(0, 100000, big.NewInt(3), keys[1])); err != nil { // +K1:0 => -K1:1 => Pend K0:0, K0:1, K1:0, K2:0; Que - t.Fatalf("failed to add well priced transaction: %v", err) } - if err := pool.addRemote(pricedTransaction(2, 100000, big.NewInt(4), keys[1])); err != nil { // +K1:2 => -K0:0 => Pend K1:0, K2:0; Que K0:1 K1:2 + if err := pool.addRemoteSync(pricedTransaction(2, 100000, big.NewInt(4), keys[1])); err != nil { // +K1:2 => -K0:0 => Pend K1:0, K2:0; Que K0:1 K1:2 t.Fatalf("failed to add well priced transaction: %v", err) } if err := pool.addRemote(pricedTransaction(3, 100000, big.NewInt(5), keys[1])); err != nil { // +K1:3 => -K0:1 => Pend K1:0, K2:0; Que K1:2 K1:3 @@ -1915,11 +1915,11 @@ func TestUnderpricingDynamicFee(t *testing.T) { } tx = pricedTransaction(1, 100000, big.NewInt(3), keys[1]) - if err := pool.addRemote(tx); err != nil { // +K1:2, -K0:1 => Pend K0:0 K1:0, K2:0; Que K1:2 + if err := pool.addRemoteSync(tx); err != nil { // +K1:2, -K0:1 => Pend K0:0 K1:0, K2:0; Que K1:2 t.Fatalf("failed to add well priced transaction: %v", err) } tx = dynamicFeeTx(2, 100000, big.NewInt(4), big.NewInt(1), keys[1]) - if err := pool.addRemote(tx); err != nil { // +K1:3, -K1:0 => Pend K0:0 K2:0; Que K1:2 K1:3 + if err := pool.addRemoteSync(tx); err != nil { // +K1:3, -K1:0 => Pend K0:0 K2:0; Que K1:2 K1:3 t.Fatalf("failed to add well priced transaction: %v", err) } pending, queued = pool.Stats() From c2db667c8f824ec0d025cbde95e6aaad67fc10ba Mon Sep 17 00:00:00 2001 From: jwasinger Date: Fri, 7 Jul 2023 10:14:24 +0200 Subject: [PATCH 53/90] miner: remove unused commented code (#27664) --- miner/worker.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/miner/worker.go b/miner/worker.go index fb30142aa6..d524bb133d 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -904,11 +904,6 @@ func (w *worker) fillTransactions(interrupt *atomic.Int32, env *environment) err // Split the pending transactions into locals and remotes // Fill the block with all available pending transactions. pending := w.eth.TxPool().Pending(true) - /*blobtxs := w.eth.BlobPool().Pending( - uint256.MustFromBig(env.header.BaseFee), - uint256.MustFromBig(misc.CalcBlobFee(*env.header.ExcessDataGas)), - ) - log.Trace("Side-effect log, much wow", "blobs", len(blobtxs))*/ localTxs, remoteTxs := make(map[common.Address][]*types.Transaction), pending for _, account := range w.eth.TxPool().Locals() { From af8b138c1a4362bd3c9892ceefec8e4ca2128f58 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Mon, 10 Jul 2023 14:40:31 -0400 Subject: [PATCH 54/90] core/vm: define cancun + enable 1153 (tstore/tload) in cancun (#27663) --- core/vm/jump_table.go | 1 + core/vm/jump_table_export.go | 6 +++--- tests/init.go | 19 +++++++++++++++++++ 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/core/vm/jump_table.go b/core/vm/jump_table.go index 41a89ec6b6..6f30c65143 100644 --- a/core/vm/jump_table.go +++ b/core/vm/jump_table.go @@ -83,6 +83,7 @@ func validate(jt JumpTable) JumpTable { func newCancunInstructionSet() JumpTable { instructionSet := newShanghaiInstructionSet() enable4844(&instructionSet) // BLOBHASH opcode + enable1153(&instructionSet) // EIP-1153 "Transient Storage" return validate(instructionSet) } diff --git a/core/vm/jump_table_export.go b/core/vm/jump_table_export.go index 9b160ad18b..6ea47d63a2 100644 --- a/core/vm/jump_table_export.go +++ b/core/vm/jump_table_export.go @@ -27,11 +27,11 @@ import ( func LookupInstructionSet(rules params.Rules) (JumpTable, error) { switch { case rules.IsVerkle: - return newShanghaiInstructionSet(), errors.New("verkle-fork not defined yet") + return newCancunInstructionSet(), errors.New("verkle-fork not defined yet") case rules.IsPrague: - return newShanghaiInstructionSet(), errors.New("prague-fork not defined yet") + return newCancunInstructionSet(), errors.New("prague-fork not defined yet") case rules.IsCancun: - return newShanghaiInstructionSet(), errors.New("cancun-fork not defined yet") + return newCancunInstructionSet(), nil case rules.IsShanghai: return newShanghaiInstructionSet(), nil case rules.IsMerge: diff --git a/tests/init.go b/tests/init.go index 869f1bfcdd..a04e227dc7 100644 --- a/tests/init.go +++ b/tests/init.go @@ -299,6 +299,25 @@ var Forks = map[string]*params.ChainConfig{ TerminalTotalDifficulty: big.NewInt(0), ShanghaiTime: u64(15_000), }, + "Cancun": { + ChainID: big.NewInt(1), + HomesteadBlock: big.NewInt(0), + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), + ConstantinopleBlock: big.NewInt(0), + PetersburgBlock: big.NewInt(0), + IstanbulBlock: big.NewInt(0), + MuirGlacierBlock: big.NewInt(0), + BerlinBlock: big.NewInt(0), + LondonBlock: big.NewInt(0), + ArrowGlacierBlock: big.NewInt(0), + MergeNetsplitBlock: big.NewInt(0), + TerminalTotalDifficulty: big.NewInt(0), + ShanghaiTime: u64(0), + CancunTime: u64(0), + }, } // AvailableForks returns the set of defined fork names From 5c9cbc218a67ea6d71652f0d93f4c354a687a965 Mon Sep 17 00:00:00 2001 From: Charles Cooper Date: Tue, 11 Jul 2023 03:55:34 -0400 Subject: [PATCH 55/90] core/vm: implement EIP-5656, mcopy instruction (#26181) Implements [EIP 5656](https://eips.ethereum.org/EIPS/eip-5656), MCOPY instruction, and enables it for Cancun. --------- Co-authored-by: Martin Holst Swende --- core/vm/eips.go | 27 +++++++ core/vm/gas_table.go | 2 + core/vm/instructions_test.go | 140 +++++++++++++++++++++++++++++++++++ core/vm/jump_table.go | 3 +- core/vm/memory.go | 11 +++ core/vm/memory_table.go | 8 ++ core/vm/memory_test.go | 69 +++++++++++++++++ core/vm/opcodes.go | 3 + 8 files changed, 262 insertions(+), 1 deletion(-) create mode 100644 core/vm/memory_test.go diff --git a/core/vm/eips.go b/core/vm/eips.go index ff1f132cb3..0daa122ca0 100644 --- a/core/vm/eips.go +++ b/core/vm/eips.go @@ -26,6 +26,7 @@ import ( ) var activators = map[int]func(*JumpTable){ + 5656: enable5656, 3855: enable3855, 3860: enable3860, 3529: enable3529, @@ -242,6 +243,32 @@ func enable3860(jt *JumpTable) { jt[CREATE2].dynamicGas = gasCreate2Eip3860 } +// enable5656 enables EIP-5656 (MCOPY opcode) +// https://eips.ethereum.org/EIPS/eip-5656 +func enable5656(jt *JumpTable) { + jt[MCOPY] = &operation{ + execute: opMcopy, + constantGas: GasFastestStep, + dynamicGas: gasMcopy, + minStack: minStack(3, 0), + maxStack: maxStack(3, 0), + memorySize: memoryMcopy, + } +} + +// opMcopy implements the MCOPY opcode (https://eips.ethereum.org/EIPS/eip-5656) +func opMcopy(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + var ( + dst = scope.Stack.pop() + src = scope.Stack.pop() + length = scope.Stack.pop() + ) + // These values are checked for overflow during memory expansion calculation + // (the memorySize function on the opcode). + scope.Memory.Copy(dst.Uint64(), src.Uint64(), length.Uint64()) + return nil, nil +} + // opBlobHash implements the BLOBHASH opcode func opBlobHash(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { index := scope.Stack.peek() diff --git a/core/vm/gas_table.go b/core/vm/gas_table.go index 4f961ef4db..79aba9d4ff 100644 --- a/core/vm/gas_table.go +++ b/core/vm/gas_table.go @@ -60,6 +60,7 @@ func memoryGasCost(mem *Memory, newMemSize uint64) (uint64, error) { // as argument: // CALLDATACOPY (stack position 2) // CODECOPY (stack position 2) +// MCOPY (stack position 2) // EXTCODECOPY (stack position 3) // RETURNDATACOPY (stack position 2) func memoryCopierGas(stackpos int) gasFunc { @@ -89,6 +90,7 @@ func memoryCopierGas(stackpos int) gasFunc { var ( gasCallDataCopy = memoryCopierGas(2) gasCodeCopy = memoryCopierGas(2) + gasMcopy = memoryCopierGas(2) gasExtCodeCopy = memoryCopierGas(3) gasReturnDataCopy = memoryCopierGas(2) ) diff --git a/core/vm/instructions_test.go b/core/vm/instructions_test.go index 6fa0737e82..36886cb9ed 100644 --- a/core/vm/instructions_test.go +++ b/core/vm/instructions_test.go @@ -22,9 +22,11 @@ import ( "fmt" "math/big" "os" + "strings" "testing" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" @@ -788,3 +790,141 @@ func TestBlobHash(t *testing.T) { } } } + +func TestOpMCopy(t *testing.T) { + // Test cases from https://eips.ethereum.org/EIPS/eip-5656#test-cases + for i, tc := range []struct { + dst, src, len string + pre string + want string + wantGas uint64 + }{ + { // MCOPY 0 32 32 - copy 32 bytes from offset 32 to offset 0. + dst: "0x0", src: "0x20", len: "0x20", + pre: "0000000000000000000000000000000000000000000000000000000000000000 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + want: "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + wantGas: 6, + }, + + { // MCOPY 0 0 32 - copy 32 bytes from offset 0 to offset 0. + dst: "0x0", src: "0x0", len: "0x20", + pre: "0101010101010101010101010101010101010101010101010101010101010101", + want: "0101010101010101010101010101010101010101010101010101010101010101", + wantGas: 6, + }, + { // MCOPY 0 1 8 - copy 8 bytes from offset 1 to offset 0 (overlapping). + dst: "0x0", src: "0x1", len: "0x8", + pre: "000102030405060708 000000000000000000000000000000000000000000000000", + want: "010203040506070808 000000000000000000000000000000000000000000000000", + wantGas: 6, + }, + { // MCOPY 1 0 8 - copy 8 bytes from offset 0 to offset 1 (overlapping). + dst: "0x1", src: "0x0", len: "0x8", + pre: "000102030405060708 000000000000000000000000000000000000000000000000", + want: "000001020304050607 000000000000000000000000000000000000000000000000", + wantGas: 6, + }, + // Tests below are not in the EIP, but maybe should be added + { // MCOPY 0xFFFFFFFFFFFF 0xFFFFFFFFFFFF 0 - copy zero bytes from out-of-bounds index(overlapping). + dst: "0xFFFFFFFFFFFF", src: "0xFFFFFFFFFFFF", len: "0x0", + pre: "11", + want: "11", + wantGas: 3, + }, + { // MCOPY 0xFFFFFFFFFFFF 0 0 - copy zero bytes from start of mem to out-of-bounds. + dst: "0xFFFFFFFFFFFF", src: "0x0", len: "0x0", + pre: "11", + want: "11", + wantGas: 3, + }, + { // MCOPY 0 0xFFFFFFFFFFFF 0 - copy zero bytes from out-of-bounds to start of mem + dst: "0x0", src: "0xFFFFFFFFFFFF", len: "0x0", + pre: "11", + want: "11", + wantGas: 3, + }, + { // MCOPY - copy 1 from space outside of uint64 space + dst: "0x0", src: "0x10000000000000000", len: "0x1", + pre: "0", + }, + { // MCOPY - copy 1 from 0 to space outside of uint64 + dst: "0x10000000000000000", src: "0x0", len: "0x1", + pre: "0", + }, + { // MCOPY - copy nothing from 0 to space outside of uint64 + dst: "0x10000000000000000", src: "0x0", len: "0x0", + pre: "", + want: "", + wantGas: 3, + }, + { // MCOPY - copy 1 from 0x20 to 0x10, with no prior allocated mem + dst: "0x10", src: "0x20", len: "0x1", + pre: "", + // 64 bytes + want: "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + wantGas: 12, + }, + { // MCOPY - copy 1 from 0x19 to 0x10, with no prior allocated mem + dst: "0x10", src: "0x19", len: "0x1", + pre: "", + // 32 bytes + want: "0x0000000000000000000000000000000000000000000000000000000000000000", + wantGas: 9, + }, + } { + var ( + env = NewEVM(BlockContext{}, TxContext{}, nil, params.TestChainConfig, Config{}) + stack = newstack() + pc = uint64(0) + evmInterpreter = env.interpreter + ) + data := common.FromHex(strings.ReplaceAll(tc.pre, " ", "")) + // Set pre + mem := NewMemory() + mem.Resize(uint64(len(data))) + mem.Set(0, uint64(len(data)), data) + // Push stack args + len, _ := uint256.FromHex(tc.len) + src, _ := uint256.FromHex(tc.src) + dst, _ := uint256.FromHex(tc.dst) + + stack.push(len) + stack.push(src) + stack.push(dst) + wantErr := (tc.wantGas == 0) + // Calc mem expansion + var memorySize uint64 + if memSize, overflow := memoryMcopy(stack); overflow { + if wantErr { + continue + } + t.Errorf("overflow") + } else { + var overflow bool + if memorySize, overflow = math.SafeMul(toWordSize(memSize), 32); overflow { + t.Error(ErrGasUintOverflow) + } + } + // and the dynamic cost + var haveGas uint64 + if dynamicCost, err := gasMcopy(env, nil, stack, mem, memorySize); err != nil { + t.Error(err) + } else { + haveGas = GasFastestStep + dynamicCost + } + // Expand mem + if memorySize > 0 { + mem.Resize(memorySize) + } + // Do the copy + opMcopy(&pc, evmInterpreter, &ScopeContext{mem, stack, nil}) + want := common.FromHex(strings.ReplaceAll(tc.want, " ", "")) + if have := mem.store; !bytes.Equal(want, have) { + t.Errorf("case %d: \nwant: %#x\nhave: %#x\n", i, want, have) + } + wantGas := tc.wantGas + if haveGas != wantGas { + t.Errorf("case %d: gas wrong, want %d have %d\n", i, wantGas, haveGas) + } + } +} diff --git a/core/vm/jump_table.go b/core/vm/jump_table.go index 6f30c65143..a59fa87473 100644 --- a/core/vm/jump_table.go +++ b/core/vm/jump_table.go @@ -82,8 +82,9 @@ func validate(jt JumpTable) JumpTable { func newCancunInstructionSet() JumpTable { instructionSet := newShanghaiInstructionSet() - enable4844(&instructionSet) // BLOBHASH opcode + enable4844(&instructionSet) // EIP-4844 (DATAHASH opcode) enable1153(&instructionSet) // EIP-1153 "Transient Storage" + enable5656(&instructionSet) // EIP-5656 (MCOPY opcode) return validate(instructionSet) } diff --git a/core/vm/memory.go b/core/vm/memory.go index 35b7299960..e0202fd7c0 100644 --- a/core/vm/memory.go +++ b/core/vm/memory.go @@ -103,3 +103,14 @@ func (m *Memory) Len() int { func (m *Memory) Data() []byte { return m.store } + +// Copy copies data from the src position slice into the dst position. +// The source and destination may overlap. +// OBS: This operation assumes that any necessary memory expansion has already been performed, +// and this method may panic otherwise. +func (m *Memory) Copy(dst, src, len uint64) { + if len == 0 { + return + } + copy(m.store[dst:], m.store[src:src+len]) +} diff --git a/core/vm/memory_table.go b/core/vm/memory_table.go index e35ca84e0e..61a910a03d 100644 --- a/core/vm/memory_table.go +++ b/core/vm/memory_table.go @@ -48,6 +48,14 @@ func memoryMStore(stack *Stack) (uint64, bool) { return calcMemSize64WithUint(stack.Back(0), 32) } +func memoryMcopy(stack *Stack) (uint64, bool) { + mStart := stack.Back(0) // stack[0]: dest + if stack.Back(1).Gt(mStart) { + mStart = stack.Back(1) // stack[1]: source + } + return calcMemSize64(mStart, stack.Back(2)) // stack[2]: length +} + func memoryCreate(stack *Stack) (uint64, bool) { return calcMemSize64(stack.Back(1), stack.Back(2)) } diff --git a/core/vm/memory_test.go b/core/vm/memory_test.go new file mode 100644 index 0000000000..ba36f8023c --- /dev/null +++ b/core/vm/memory_test.go @@ -0,0 +1,69 @@ +package vm + +import ( + "bytes" + "strings" + "testing" + + "github.com/ethereum/go-ethereum/common" +) + +func TestMemoryCopy(t *testing.T) { + // Test cases from https://eips.ethereum.org/EIPS/eip-5656#test-cases + for i, tc := range []struct { + dst, src, len uint64 + pre string + want string + }{ + { // MCOPY 0 32 32 - copy 32 bytes from offset 32 to offset 0. + 0, 32, 32, + "0000000000000000000000000000000000000000000000000000000000000000 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + }, + + { // MCOPY 0 0 32 - copy 32 bytes from offset 0 to offset 0. + 0, 0, 32, + "0101010101010101010101010101010101010101010101010101010101010101", + "0101010101010101010101010101010101010101010101010101010101010101", + }, + { // MCOPY 0 1 8 - copy 8 bytes from offset 1 to offset 0 (overlapping). + 0, 1, 8, + "000102030405060708 000000000000000000000000000000000000000000000000", + "010203040506070808 000000000000000000000000000000000000000000000000", + }, + { // MCOPY 1 0 8 - copy 8 bytes from offset 0 to offset 1 (overlapping). + 1, 0, 8, + "000102030405060708 000000000000000000000000000000000000000000000000", + "000001020304050607 000000000000000000000000000000000000000000000000", + }, + // Tests below are not in the EIP, but maybe should be added + { // MCOPY 0xFFFFFFFFFFFF 0xFFFFFFFFFFFF 0 - copy zero bytes from out-of-bounds index(overlapping). + 0xFFFFFFFFFFFF, 0xFFFFFFFFFFFF, 0, + "11", + "11", + }, + { // MCOPY 0xFFFFFFFFFFFF 0 0 - copy zero bytes from start of mem to out-of-bounds. + 0xFFFFFFFFFFFF, 0, 0, + "11", + "11", + }, + { // MCOPY 0 0xFFFFFFFFFFFF 0 - copy zero bytes from out-of-bounds to start of mem + 0, 0xFFFFFFFFFFFF, 0, + "11", + "11", + }, + } { + m := NewMemory() + // Clean spaces + data := common.FromHex(strings.ReplaceAll(tc.pre, " ", "")) + // Set pre + m.Resize(uint64(len(data))) + m.Set(0, uint64(len(data)), data) + // Do the copy + m.Copy(tc.dst, tc.src, tc.len) + want := common.FromHex(strings.ReplaceAll(tc.want, " ", "")) + if have := m.store; !bytes.Equal(want, have) { + t.Errorf("case %d: want: %#x\nhave: %#x\n", i, want, have) + } + } +} diff --git a/core/vm/opcodes.go b/core/vm/opcodes.go index 3cdeab8ff9..2929b8ce92 100644 --- a/core/vm/opcodes.go +++ b/core/vm/opcodes.go @@ -119,6 +119,7 @@ const ( JUMPDEST OpCode = 0x5b TLOAD OpCode = 0x5c TSTORE OpCode = 0x5d + MCOPY OpCode = 0x5e PUSH0 OpCode = 0x5f ) @@ -302,6 +303,7 @@ var opCodeToString = map[OpCode]string{ JUMPDEST: "JUMPDEST", TLOAD: "TLOAD", TSTORE: "TSTORE", + MCOPY: "MCOPY", PUSH0: "PUSH0", // 0x60 range - pushes. @@ -473,6 +475,7 @@ var stringToOp = map[string]OpCode{ "JUMPDEST": JUMPDEST, "TLOAD": TLOAD, "TSTORE": TSTORE, + "MCOPY": MCOPY, "PUSH0": PUSH0, "PUSH1": PUSH1, "PUSH2": PUSH2, From e1fd3d67e5ec0ea193706fa0cbe6535c81c7399c Mon Sep 17 00:00:00 2001 From: lightclient <14004106+lightclient@users.noreply.github.com> Date: Tue, 11 Jul 2023 01:57:42 -0600 Subject: [PATCH 56/90] eth: better active protocol handler tracking (#27665) Fixes an issue where waitgroups were used erroneously, which could lead to waitgroup being added to while wait was already invoked. --- eth/handler.go | 68 ++++++++++++++++++++++++++++++++++++++-------- eth/handler_eth.go | 2 +- eth/sync.go | 2 +- 3 files changed, 58 insertions(+), 14 deletions(-) diff --git a/eth/handler.go b/eth/handler.go index 9d6a8c4225..f0b8d65a12 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -124,7 +124,9 @@ type handler struct { chainSync *chainSyncer wg sync.WaitGroup - peerWG sync.WaitGroup + + handlerStartCh chan struct{} + handlerDoneCh chan struct{} } // newHandler returns a handler for all Ethereum chain management protocol. @@ -144,6 +146,8 @@ func newHandler(config *handlerConfig) (*handler, error) { merger: config.Merger, requiredBlocks: config.RequiredBlocks, quitSync: make(chan struct{}), + handlerDoneCh: make(chan struct{}), + handlerStartCh: make(chan struct{}), } if config.Sync == downloader.FullSync { // The database seems empty as the current block is the genesis. Yet the snap @@ -289,9 +293,50 @@ func newHandler(config *handlerConfig) (*handler, error) { return h, nil } +// protoTracker tracks the number of active protocol handlers. +func (h *handler) protoTracker() { + defer h.wg.Done() + var active int + for { + select { + case <-h.handlerStartCh: + active++ + case <-h.handlerDoneCh: + active-- + case <-h.quitSync: + // Wait for all active handlers to finish. + for ; active > 0; active-- { + <-h.handlerDoneCh + } + return + } + } +} + +// incHandlers signals to increment the number of active handlers if not +// quitting. +func (h *handler) incHandlers() bool { + select { + case h.handlerStartCh <- struct{}{}: + return true + case <-h.quitSync: + return false + } +} + +// decHandlers signals to decrement the number of active handlers. +func (h *handler) decHandlers() { + h.handlerDoneCh <- struct{}{} +} + // runEthPeer registers an eth peer into the joint eth/snap peerset, adds it to // various subsystems and starts handling messages. func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { + if !h.incHandlers() { + return p2p.DiscQuitting + } + defer h.decHandlers() + // If the peer has a `snap` extension, wait for it to connect so we can have // a uniform initialization/teardown mechanism snap, err := h.peers.waitSnapExtension(peer) @@ -299,12 +344,6 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { peer.Log().Error("Snapshot extension barrier failed", "err", err) return err } - // TODO(karalabe): Not sure why this is needed - if !h.chainSync.handlePeerEvent(peer) { - return p2p.DiscQuitting - } - h.peerWG.Add(1) - defer h.peerWG.Done() // Execute the Ethereum handshake var ( @@ -360,7 +399,7 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { return err } } - h.chainSync.handlePeerEvent(peer) + h.chainSync.handlePeerEvent() // Propagate existing transactions. new transactions appearing // after this will be sent via broadcasts. @@ -421,8 +460,10 @@ func (h *handler) runEthPeer(peer *eth.Peer, handler eth.Handler) error { // `eth`, all subsystem registrations and lifecycle management will be done by // the main `eth` handler to prevent strange races. func (h *handler) runSnapExtension(peer *snap.Peer, handler snap.Handler) error { - h.peerWG.Add(1) - defer h.peerWG.Done() + if !h.incHandlers() { + return p2p.DiscQuitting + } + defer h.decHandlers() if err := h.peers.registerSnapExtension(peer); err != nil { if metrics.Enabled { @@ -494,6 +535,10 @@ func (h *handler) Start(maxPeers int) { // start sync handlers h.wg.Add(1) go h.chainSync.loop() + + // start peer handler tracker + h.wg.Add(1) + go h.protoTracker() } func (h *handler) Stop() { @@ -503,14 +548,13 @@ func (h *handler) Stop() { // Quit chainSync and txsync64. // After this is done, no new peers will be accepted. close(h.quitSync) - h.wg.Wait() // Disconnect existing sessions. // This also closes the gate for any new registrations on the peer set. // sessions which are already established but not added to h.peers yet // will exit when they try to register. h.peers.close() - h.peerWG.Wait() + h.wg.Wait() log.Info("Ethereum protocol stopped") } diff --git a/eth/handler_eth.go b/eth/handler_eth.go index 00be022d9e..3a5e6608bb 100644 --- a/eth/handler_eth.go +++ b/eth/handler_eth.go @@ -134,7 +134,7 @@ func (h *ethHandler) handleBlockBroadcast(peer *eth.Peer, block *types.Block, td // Update the peer's total difficulty if better than the previous if _, td := peer.Head(); trueTD.Cmp(td) > 0 { peer.SetHead(trueHead, trueTD) - h.chainSync.handlePeerEvent(peer) + h.chainSync.handlePeerEvent() } return nil } diff --git a/eth/sync.go b/eth/sync.go index ace1071143..89bffbe653 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -89,7 +89,7 @@ func newChainSyncer(handler *handler) *chainSyncer { // handlePeerEvent notifies the syncer about a change in the peer set. // This is called for new peers and every time a peer announces a new // chain head. -func (cs *chainSyncer) handlePeerEvent(peer *eth.Peer) bool { +func (cs *chainSyncer) handlePeerEvent() bool { select { case cs.peerEventCh <- struct{}{}: return true From aecf3f9579b151cf6d73217dd4749b8078c44d4a Mon Sep 17 00:00:00 2001 From: Delweng Date: Tue, 11 Jul 2023 20:57:02 +0800 Subject: [PATCH 57/90] internal/blocktest: add package for shared test code (#27270) --- core/rawdb/accessors_indexes_test.go | 28 +------------ core/types/block_test.go | 30 +------------- internal/blocktest/test_hash.go | 59 ++++++++++++++++++++++++++++ internal/ethapi/api_test.go | 32 ++------------- 4 files changed, 66 insertions(+), 83 deletions(-) create mode 100644 internal/blocktest/test_hash.go diff --git a/core/rawdb/accessors_indexes_test.go b/core/rawdb/accessors_indexes_test.go index 37b0f79b39..0200755251 100644 --- a/core/rawdb/accessors_indexes_test.go +++ b/core/rawdb/accessors_indexes_test.go @@ -18,42 +18,18 @@ package rawdb import ( "bytes" - "hash" "math/big" "testing" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/internal/blocktest" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" - "golang.org/x/crypto/sha3" ) -// testHasher is the helper tool for transaction/receipt list hashing. -// The original hasher is trie, in order to get rid of import cycle, -// use the testing hasher instead. -type testHasher struct { - hasher hash.Hash -} - -func newHasher() *testHasher { - return &testHasher{hasher: sha3.NewLegacyKeccak256()} -} - -func (h *testHasher) Reset() { - h.hasher.Reset() -} - -func (h *testHasher) Update(key, val []byte) error { - h.hasher.Write(key) - h.hasher.Write(val) - return nil -} - -func (h *testHasher) Hash() common.Hash { - return common.BytesToHash(h.hasher.Sum(nil)) -} +var newHasher = blocktest.NewHasher // Tests that positional lookup metadata can be stored and retrieved. func TestLookupStorage(t *testing.T) { diff --git a/core/types/block_test.go b/core/types/block_test.go index 966015eb01..cf0b1dd85c 100644 --- a/core/types/block_test.go +++ b/core/types/block_test.go @@ -18,7 +18,6 @@ package types import ( "bytes" - "hash" "math/big" "reflect" "testing" @@ -26,9 +25,9 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/internal/blocktest" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" - "golang.org/x/crypto/sha3" ) // from bcValidBlockTest.json, "SimpleTx" @@ -217,31 +216,6 @@ func BenchmarkEncodeBlock(b *testing.B) { } } -// testHasher is the helper tool for transaction/receipt list hashing. -// The original hasher is trie, in order to get rid of import cycle, -// use the testing hasher instead. -type testHasher struct { - hasher hash.Hash -} - -func newHasher() *testHasher { - return &testHasher{hasher: sha3.NewLegacyKeccak256()} -} - -func (h *testHasher) Reset() { - h.hasher.Reset() -} - -func (h *testHasher) Update(key, val []byte) error { - h.hasher.Write(key) - h.hasher.Write(val) - return nil -} - -func (h *testHasher) Hash() common.Hash { - return common.BytesToHash(h.hasher.Sum(nil)) -} - func makeBenchBlock() *Block { var ( key, _ = crypto.GenerateKey() @@ -280,7 +254,7 @@ func makeBenchBlock() *Block { Extra: []byte("benchmark uncle"), } } - return NewBlock(header, txs, uncles, receipts, newHasher()) + return NewBlock(header, txs, uncles, receipts, blocktest.NewHasher()) } func TestRlpDecodeParentHash(t *testing.T) { diff --git a/internal/blocktest/test_hash.go b/internal/blocktest/test_hash.go new file mode 100644 index 0000000000..4d2b077e89 --- /dev/null +++ b/internal/blocktest/test_hash.go @@ -0,0 +1,59 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package utesting provides a standalone replacement for package testing. +// +// This package exists because package testing cannot easily be embedded into a +// standalone go program. It provides an API that mirrors the standard library +// testing API. + +package blocktest + +import ( + "hash" + + "github.com/ethereum/go-ethereum/common" + "golang.org/x/crypto/sha3" +) + +// testHasher is the helper tool for transaction/receipt list hashing. +// The original hasher is trie, in order to get rid of import cycle, +// use the testing hasher instead. +type testHasher struct { + hasher hash.Hash +} + +// NewHasher returns a new testHasher instance. +func NewHasher() *testHasher { + return &testHasher{hasher: sha3.NewLegacyKeccak256()} +} + +// Reset resets the hash state. +func (h *testHasher) Reset() { + h.hasher.Reset() +} + +// Update updates the hash state with the given key and value. +func (h *testHasher) Update(key, val []byte) error { + h.hasher.Write(key) + h.hasher.Write(val) + return nil +} + +// Hash returns the hash value. +func (h *testHasher) Hash() common.Hash { + return common.BytesToHash(h.hasher.Sum(nil)) +} diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go index 3e58e54f97..fd662b5812 100644 --- a/internal/ethapi/api_test.go +++ b/internal/ethapi/api_test.go @@ -21,7 +21,6 @@ import ( "crypto/ecdsa" "encoding/json" "errors" - "hash" "math/big" "reflect" "testing" @@ -42,10 +41,10 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/internal/blocktest" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" "github.com/stretchr/testify/require" - "golang.org/x/crypto/sha3" "golang.org/x/exp/slices" ) @@ -668,31 +667,6 @@ func hex2Bytes(str string) *hexutil.Bytes { return &rpcBytes } -// testHasher is the helper tool for transaction/receipt list hashing. -// The original hasher is trie, in order to get rid of import cycle, -// use the testing hasher instead. -type testHasher struct { - hasher hash.Hash -} - -func newHasher() *testHasher { - return &testHasher{hasher: sha3.NewLegacyKeccak256()} -} - -func (h *testHasher) Reset() { - h.hasher.Reset() -} - -func (h *testHasher) Update(key, val []byte) error { - h.hasher.Write(key) - h.hasher.Write(val) - return nil -} - -func (h *testHasher) Hash() common.Hash { - return common.BytesToHash(h.hasher.Sum(nil)) -} - func TestRPCMarshalBlock(t *testing.T) { t.Parallel() var ( @@ -723,7 +697,7 @@ func TestRPCMarshalBlock(t *testing.T) { } txs = append(txs, tx) } - block := types.NewBlock(&types.Header{Number: big.NewInt(100)}, txs, nil, nil, newHasher()) + block := types.NewBlock(&types.Header{Number: big.NewInt(100)}, txs, nil, nil, blocktest.NewHasher()) var testSuite = []struct { inclTx bool @@ -795,7 +769,7 @@ func TestRPCGetBlockOrHeader(t *testing.T) { Address: common.Address{0x12, 0x34}, Amount: 10, } - pending = types.NewBlockWithWithdrawals(&types.Header{Number: big.NewInt(11), Time: 42}, []*types.Transaction{tx}, nil, nil, []*types.Withdrawal{withdrawal}, newHasher()) + pending = types.NewBlockWithWithdrawals(&types.Header{Number: big.NewInt(11), Time: 42}, []*types.Transaction{tx}, nil, nil, []*types.Withdrawal{withdrawal}, blocktest.NewHasher()) ) backend := newTestBackend(t, genBlocks, genesis, func(i int, b *core.BlockGen) { // Transfer from account[0] to account[1] From 4b06e4f25e9a97880ddf834b571b05a59fa1290c Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Tue, 11 Jul 2023 21:43:23 +0800 Subject: [PATCH 58/90] core/state: value diff tracking in StateDB (#27349) This change makes the StateDB track the state key value diff of a block transition. We already tracked current account and storage values for the purpose of updating the state snapshot. With this PR, we now also track the original (pre-transition) values of accounts and storage slots. --- core/state/dump.go | 2 +- core/state/journal.go | 14 +- core/state/metrics.go | 7 + core/state/snapshot/generate.go | 2 +- core/state/snapshot/generate_test.go | 2 +- core/state/state_object.go | 124 +++++--- core/state/state_test.go | 16 +- core/state/statedb.go | 403 ++++++++++++++++++------- core/state/statedb_fuzz_test.go | 373 +++++++++++++++++++++++ core/state/statedb_test.go | 2 +- core/types/state_account.go | 23 ++ eth/protocols/snap/sync_test.go | 8 +- light/postprocess.go | 4 +- light/trie.go | 25 +- tests/fuzzers/stacktrie/trie_fuzzer.go | 2 +- tests/fuzzers/trie/trie-fuzzer.go | 2 +- trie/database.go | 9 +- trie/iterator_test.go | 18 +- trie/secure_trie_test.go | 2 +- trie/sync_test.go | 6 +- trie/tracer_test.go | 18 +- trie/trie_test.go | 20 +- trie/trienode/node.go | 24 +- trie/triestate/state.go | 28 ++ 24 files changed, 907 insertions(+), 227 deletions(-) create mode 100644 core/state/statedb_fuzz_test.go create mode 100644 trie/triestate/state.go diff --git a/core/state/dump.go b/core/state/dump.go index 2d9c65e473..3ea987731f 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -171,7 +171,7 @@ func (s *StateDB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey [] } else { address = &addr } - obj := newObject(s, addr, data) + obj := newObject(s, addr, &data) if !conf.SkipCode { account.Code = obj.Code(s.db) } diff --git a/core/state/journal.go b/core/state/journal.go index cf8ef3e387..95a218b3a3 100644 --- a/core/state/journal.go +++ b/core/state/journal.go @@ -95,6 +95,10 @@ type ( prevdestruct bool prevAccount []byte prevStorage map[common.Hash][]byte + + prevAccountOriginExist bool + prevAccountOrigin []byte + prevStorageOrigin map[common.Hash][]byte } suicideChange struct { account *common.Address @@ -163,10 +167,16 @@ func (ch resetObjectChange) revert(s *StateDB) { delete(s.stateObjectsDestruct, ch.prev.address) } if ch.prevAccount != nil { - s.snapAccounts[ch.prev.addrHash] = ch.prevAccount + s.accounts[ch.prev.addrHash] = ch.prevAccount } if ch.prevStorage != nil { - s.snapStorage[ch.prev.addrHash] = ch.prevStorage + s.storages[ch.prev.addrHash] = ch.prevStorage + } + if ch.prevAccountOriginExist { + s.accountsOrigin[ch.prev.addrHash] = ch.prevAccountOrigin + } + if ch.prevStorageOrigin != nil { + s.storagesOrigin[ch.prev.addrHash] = ch.prevStorageOrigin } } diff --git a/core/state/metrics.go b/core/state/metrics.go index e702ef3a81..64c651461e 100644 --- a/core/state/metrics.go +++ b/core/state/metrics.go @@ -27,4 +27,11 @@ var ( storageTriesUpdatedMeter = metrics.NewRegisteredMeter("state/update/storagenodes", nil) accountTrieDeletedMeter = metrics.NewRegisteredMeter("state/delete/accountnodes", nil) storageTriesDeletedMeter = metrics.NewRegisteredMeter("state/delete/storagenodes", nil) + + slotDeletionMaxCount = metrics.NewRegisteredGauge("state/delete/storage/max/slot", nil) + slotDeletionMaxSize = metrics.NewRegisteredGauge("state/delete/storage/max/size", nil) + slotDeletionTimer = metrics.NewRegisteredResettingTimer("state/delete/storage/timer", nil) + slotDeletionCount = metrics.NewRegisteredMeter("state/delete/storage/slot", nil) + slotDeletionSize = metrics.NewRegisteredMeter("state/delete/storage/size", nil) + slotDeletionSkip = metrics.NewRegisteredGauge("state/delete/storage/skip", nil) ) diff --git a/core/state/snapshot/generate.go b/core/state/snapshot/generate.go index f5db0ff921..f80525fcd2 100644 --- a/core/state/snapshot/generate.go +++ b/core/state/snapshot/generate.go @@ -366,7 +366,7 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, trieId *trie.ID, prefi return false, nil, err } if nodes != nil { - tdb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + tdb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) tdb.Commit(root, false) } resolver = func(owner common.Hash, path []byte, hash common.Hash) []byte { diff --git a/core/state/snapshot/generate_test.go b/core/state/snapshot/generate_test.go index 20a389e477..8996877ceb 100644 --- a/core/state/snapshot/generate_test.go +++ b/core/state/snapshot/generate_test.go @@ -203,7 +203,7 @@ func (t *testHelper) Commit() common.Hash { if nodes != nil { t.nodes.Merge(nodes) } - t.triedb.Update(root, types.EmptyRootHash, t.nodes) + t.triedb.Update(root, types.EmptyRootHash, t.nodes, nil) t.triedb.Commit(root, false) return root } diff --git a/core/state/state_object.go b/core/state/state_object.go index b443d12f12..2529ec6929 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -57,29 +57,35 @@ func (s Storage) Copy() Storage { // stateObject represents an Ethereum account which is being modified. // // The usage pattern is as follows: -// First you need to obtain a state object. -// Account values can be accessed and modified through the object. -// Finally, call commitTrie to write the modified storage trie into a database. +// - First you need to obtain a state object. +// - Account values as well as storages can be accessed and modified through the object. +// - Finally, call commit to return the changes of storage trie and update account data. type stateObject struct { - address common.Address - addrHash common.Hash // hash of ethereum address of the account - data types.StateAccount db *StateDB + address common.Address // address of ethereum account + addrHash common.Hash // hash of ethereum address of the account + origin *types.StateAccount // Account original data without any change applied, nil means it was not existent + data types.StateAccount // Account data with all mutations applied in the scope of block // Write caches. trie Trie // storage trie, which becomes non-nil on first access code Code // contract bytecode, which gets set when code is loaded - originStorage Storage // Storage cache of original entries to dedup rewrites, reset for every transaction + originStorage Storage // Storage cache of original entries to dedup rewrites pendingStorage Storage // Storage entries that need to be flushed to disk, at the end of an entire block - dirtyStorage Storage // Storage entries that have been modified in the current transaction execution + dirtyStorage Storage // Storage entries that have been modified in the current transaction execution, reset for every transaction // Cache flags. - // When an object is marked suicided it will be deleted from the trie - // during the "update" phase of the state transition. dirtyCode bool // true if the code was updated - suicided bool - deleted bool + + // Flag whether the account was marked as suicided. The suicided account + // is still accessible in the scope of same transaction. + suicided bool + + // Flag whether the account was marked as deleted. The suicided account + // or the account is considered as empty will be marked as deleted at + // the end of transaction and no longer accessible anymore. + deleted bool } // empty returns whether the account is considered empty. @@ -88,21 +94,17 @@ func (s *stateObject) empty() bool { } // newObject creates a state object. -func newObject(db *StateDB, address common.Address, data types.StateAccount) *stateObject { - if data.Balance == nil { - data.Balance = new(big.Int) - } - if data.CodeHash == nil { - data.CodeHash = types.EmptyCodeHash.Bytes() - } - if data.Root == (common.Hash{}) { - data.Root = types.EmptyRootHash +func newObject(db *StateDB, address common.Address, acct *types.StateAccount) *stateObject { + origin := acct + if acct == nil { + acct = types.NewEmptyStateAccount() } return &stateObject{ db: db, address: address, addrHash: crypto.Keccak256Hash(address[:]), - data: data, + origin: origin, + data: *acct, originStorage: make(Storage), pendingStorage: make(Storage), dirtyStorage: make(Storage), @@ -135,10 +137,8 @@ func (s *stateObject) touch() { func (s *stateObject) getTrie(db Database) (Trie, error) { if s.trie == nil { // Try fetching from prefetcher first - // We don't prefetch empty tries if s.data.Root != types.EmptyRootHash && s.db.prefetcher != nil { - // When the miner is creating the pending state, there is no - // prefetcher + // When the miner is creating the pending state, there is no prefetcher s.trie = s.db.prefetcher.trie(s.addrHash, s.data.Root) } if s.trie == nil { @@ -277,6 +277,7 @@ func (s *stateObject) updateTrie(db Database) (Trie, error) { // The snapshot storage map for the object var ( storage map[common.Hash][]byte + origin map[common.Hash][]byte hasher = s.db.hasher ) tr, err := s.getTrie(db) @@ -291,6 +292,7 @@ func (s *stateObject) updateTrie(db Database) (Trie, error) { if value == s.originStorage[key] { continue } + prev := s.originStorage[key] s.originStorage[key] = value // rlp-encoded value to be used by the snapshot @@ -311,17 +313,34 @@ func (s *stateObject) updateTrie(db Database) (Trie, error) { } s.db.StorageUpdated += 1 } - // If state snapshotting is active, cache the data til commit - if s.db.snap != nil { - if storage == nil { - // Retrieve the old storage map, if available, create a new one otherwise - if storage = s.db.snapStorage[s.addrHash]; storage == nil { - storage = make(map[common.Hash][]byte) - s.db.snapStorage[s.addrHash] = storage - } + // Cache the mutated storage slots until commit + if storage == nil { + if storage = s.db.storages[s.addrHash]; storage == nil { + storage = make(map[common.Hash][]byte) + s.db.storages[s.addrHash] = storage } - storage[crypto.HashData(hasher, key[:])] = snapshotVal // will be nil if it's deleted } + khash := crypto.HashData(hasher, key[:]) + storage[khash] = snapshotVal // snapshotVal will be nil if it's deleted + + // Cache the original value of mutated storage slots + if origin == nil { + if origin = s.db.storagesOrigin[s.addrHash]; origin == nil { + origin = make(map[common.Hash][]byte) + s.db.storagesOrigin[s.addrHash] = origin + } + } + // Track the original value of slot only if it's mutated first time + if _, ok := origin[khash]; !ok { + if prev == (common.Hash{}) { + origin[khash] = nil // nil if it was not present previously + } else { + // Encoding []byte cannot fail, ok to ignore the error. + b, _ := rlp.EncodeToBytes(common.TrimLeftZeroes(prev[:])) + origin[khash] = b + } + } + // Cache the items for preloading usedStorage = append(usedStorage, common.CopyBytes(key[:])) // Copy needed for closure } if s.db.prefetcher != nil { @@ -351,15 +370,15 @@ func (s *stateObject) updateRoot(db Database) { s.data.Root = tr.Hash() } -// commitTrie submits the storage changes into the storage trie and re-computes -// the root. Besides, all trie changes will be collected in a nodeset and returned. -func (s *stateObject) commitTrie(db Database) (*trienode.NodeSet, error) { +// commit returns the changes made in storage trie and updates the account data. +func (s *stateObject) commit(db Database) (*trienode.NodeSet, error) { tr, err := s.updateTrie(db) if err != nil { return nil, err } // If nothing changed, don't bother with committing anything if tr == nil { + s.origin = s.data.Copy() return nil, nil } // Track the amount of time wasted on committing the storage trie @@ -371,6 +390,9 @@ func (s *stateObject) commitTrie(db Database) (*trienode.NodeSet, error) { return nil, err } s.data.Root = root + + // Update original account data after commit + s.origin = s.data.Copy() return nodes, nil } @@ -410,18 +432,24 @@ func (s *stateObject) setBalance(amount *big.Int) { } func (s *stateObject) deepCopy(db *StateDB) *stateObject { - stateObject := newObject(db, s.address, s.data) - if s.trie != nil { - stateObject.trie = db.db.CopyTrie(s.trie) + obj := &stateObject{ + db: db, + address: s.address, + addrHash: s.addrHash, + origin: s.origin, + data: s.data, } - stateObject.code = s.code - stateObject.dirtyStorage = s.dirtyStorage.Copy() - stateObject.originStorage = s.originStorage.Copy() - stateObject.pendingStorage = s.pendingStorage.Copy() - stateObject.suicided = s.suicided - stateObject.dirtyCode = s.dirtyCode - stateObject.deleted = s.deleted - return stateObject + if s.trie != nil { + obj.trie = db.db.CopyTrie(s.trie) + } + obj.code = s.code + obj.dirtyStorage = s.dirtyStorage.Copy() + obj.originStorage = s.originStorage.Copy() + obj.pendingStorage = s.pendingStorage.Copy() + obj.suicided = s.suicided + obj.dirtyCode = s.dirtyCode + obj.deleted = s.deleted + return obj } // diff --git a/core/state/state_test.go b/core/state/state_test.go index 555e1f7454..b6c11728be 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -30,22 +30,22 @@ import ( "github.com/ethereum/go-ethereum/trie" ) -type stateTest struct { +type stateEnv struct { db ethdb.Database state *StateDB } -func newStateTest() *stateTest { +func newStateEnv() *stateEnv { db := rawdb.NewMemoryDatabase() sdb, _ := New(types.EmptyRootHash, NewDatabase(db), nil) - return &stateTest{db: db, state: sdb} + return &stateEnv{db: db, state: sdb} } func TestDump(t *testing.T) { db := rawdb.NewMemoryDatabase() tdb := NewDatabaseWithConfig(db, &trie.Config{Preimages: true}) sdb, _ := New(types.EmptyRootHash, tdb, nil) - s := &stateTest{db: db, state: sdb} + s := &stateEnv{db: db, state: sdb} // generate a few entries obj1 := s.state.GetOrNewStateObject(common.BytesToAddress([]byte{0x01})) @@ -99,7 +99,7 @@ func TestIterativeDump(t *testing.T) { db := rawdb.NewMemoryDatabase() tdb := NewDatabaseWithConfig(db, &trie.Config{Preimages: true}) sdb, _ := New(types.EmptyRootHash, tdb, nil) - s := &stateTest{db: db, state: sdb} + s := &stateEnv{db: db, state: sdb} // generate a few entries obj1 := s.state.GetOrNewStateObject(common.BytesToAddress([]byte{0x01})) @@ -133,7 +133,7 @@ func TestIterativeDump(t *testing.T) { } func TestNull(t *testing.T) { - s := newStateTest() + s := newStateEnv() address := common.HexToAddress("0x823140710bf13990e4500136726d8b55") s.state.CreateAccount(address) //value := common.FromHex("0x823140710bf13990e4500136726d8b55") @@ -155,7 +155,7 @@ func TestSnapshot(t *testing.T) { var storageaddr common.Hash data1 := common.BytesToHash([]byte{42}) data2 := common.BytesToHash([]byte{43}) - s := newStateTest() + s := newStateEnv() // snapshot the genesis state genesis := s.state.Snapshot() @@ -186,7 +186,7 @@ func TestSnapshot(t *testing.T) { } func TestSnapshotEmpty(t *testing.T) { - s := newStateTest() + s := newStateEnv() s.state.RevertToSnapshot(s.state.Snapshot()) } diff --git a/core/state/statedb.go b/core/state/statedb.go index 998fb176df..71a863add7 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -35,6 +35,7 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/trienode" + "github.com/ethereum/go-ethereum/trie/triestate" ) type revision struct { @@ -69,21 +70,26 @@ type StateDB struct { prefetcher *triePrefetcher trie Trie hasher crypto.KeccakState + snaps *snapshot.Tree // Nil if snapshot is not available + snap snapshot.Snapshot // Nil if snapshot is not available // originalRoot is the pre-state root, before any changes were made. // It will be updated when the Commit is called. originalRoot common.Hash - snaps *snapshot.Tree - snap snapshot.Snapshot - snapAccounts map[common.Hash][]byte - snapStorage map[common.Hash]map[common.Hash][]byte + // These maps hold the state changes (including the corresponding + // original value) that occurred in this **block**. + accounts map[common.Hash][]byte // The mutated accounts in 'slim RLP' encoding + storages map[common.Hash]map[common.Hash][]byte // The mutated slots in prefix-zero trimmed rlp format + accountsOrigin map[common.Hash][]byte // The original value of mutated accounts in 'slim RLP' encoding + storagesOrigin map[common.Hash]map[common.Hash][]byte // The original value of mutated slots in prefix-zero trimmed rlp format - // This map holds 'live' objects, which will get modified while processing a state transition. + // This map holds 'live' objects, which will get modified while processing + // a state transition. stateObjects map[common.Address]*stateObject - stateObjectsPending map[common.Address]struct{} // State objects finalized but not yet written to the trie - stateObjectsDirty map[common.Address]struct{} // State objects modified in the current execution - stateObjectsDestruct map[common.Address]struct{} // State objects destructed in the block + stateObjectsPending map[common.Address]struct{} // State objects finalized but not yet written to the trie + stateObjectsDirty map[common.Address]struct{} // State objects modified in the current execution + stateObjectsDestruct map[common.Address]*types.StateAccount // State objects destructed in the block along with its previous value // DB error. // State objects are used by the consensus core and VM which are @@ -97,11 +103,13 @@ type StateDB struct { // The refund counter, also used by state transitioning. refund uint64 + // The tx context and all occurred logs in the scope of transaction. thash common.Hash txIndex int logs map[common.Hash][]*types.Log logSize uint + // Preimages occurred seen by VM in the scope of block. preimages map[common.Hash][]byte // Per-transaction access list @@ -147,10 +155,14 @@ func New(root common.Hash, db Database, snaps *snapshot.Tree) (*StateDB, error) trie: tr, originalRoot: root, snaps: snaps, + accounts: make(map[common.Hash][]byte), + storages: make(map[common.Hash]map[common.Hash][]byte), + accountsOrigin: make(map[common.Hash][]byte), + storagesOrigin: make(map[common.Hash]map[common.Hash][]byte), stateObjects: make(map[common.Address]*stateObject), stateObjectsPending: make(map[common.Address]struct{}), stateObjectsDirty: make(map[common.Address]struct{}), - stateObjectsDestruct: make(map[common.Address]struct{}), + stateObjectsDestruct: make(map[common.Address]*types.StateAccount), logs: make(map[common.Hash][]*types.Log), preimages: make(map[common.Hash][]byte), journal: newJournal(), @@ -159,10 +171,7 @@ func New(root common.Hash, db Database, snaps *snapshot.Tree) (*StateDB, error) hasher: crypto.NewKeccakState(), } if sdb.snaps != nil { - if sdb.snap = sdb.snaps.Snapshot(root); sdb.snap != nil { - sdb.snapAccounts = make(map[common.Hash][]byte) - sdb.snapStorage = make(map[common.Hash]map[common.Hash][]byte) - } + sdb.snap = sdb.snaps.Snapshot(root) } return sdb, nil } @@ -445,14 +454,20 @@ func (s *StateDB) SetState(addr common.Address, key, value common.Hash) { } // SetStorage replaces the entire storage for the specified account with given -// storage. This function should only be used for debugging. +// storage. This function should only be used for debugging and the mutations +// must be discarded afterwards. func (s *StateDB) SetStorage(addr common.Address, storage map[common.Hash]common.Hash) { // SetStorage needs to wipe existing storage. We achieve this by pretending // that the account self-destructed earlier in this block, by flagging // it in stateObjectsDestruct. The effect of doing so is that storage lookups // will not hit disk, since it is assumed that the disk-data is belonging // to a previous incarnation of the object. - s.stateObjectsDestruct[addr] = struct{}{} + // + // TODO(rjl493456442) this function should only be supported by 'unwritable' + // state and all mutations made should all be discarded afterwards. + if _, ok := s.stateObjectsDestruct[addr]; !ok { + s.stateObjectsDestruct[addr] = nil + } stateObject := s.GetOrNewStateObject(addr) for k, v := range storage { stateObject.SetState(s.db, k, v) @@ -476,7 +491,6 @@ func (s *StateDB) Suicide(addr common.Address) bool { }) stateObject.markSuicided() stateObject.data.Balance = new(big.Int) - return true } @@ -522,13 +536,21 @@ func (s *StateDB) updateStateObject(obj *stateObject) { if err := s.trie.UpdateAccount(addr, &obj.data); err != nil { s.setError(fmt.Errorf("updateStateObject (%x) error: %v", addr[:], err)) } + // Cache the data until commit. Note, this update mechanism is not symmetric + // to the deletion, because whereas it is enough to track account updates + // at commit time, deletions need tracking at transaction boundary level to + // ensure we capture state clearing. + s.accounts[obj.addrHash] = types.SlimAccountRLP(obj.data) - // If state snapshotting is active, cache the data til commit. Note, this - // update mechanism is not symmetric to the deletion, because whereas it is - // enough to track account updates at commit time, deletions need tracking - // at transaction boundary level to ensure we capture state clearing. - if s.snap != nil { - s.snapAccounts[obj.addrHash] = types.SlimAccountRLP(obj.data) + // Track the original value of mutated account, nil means it was not present. + // Skip if it has been tracked (because updateStateObject may be called + // multiple times in a block). + if _, ok := s.accountsOrigin[obj.addrHash]; !ok { + if obj.origin == nil { + s.accountsOrigin[obj.addrHash] = nil + } else { + s.accountsOrigin[obj.addrHash] = types.SlimAccountRLP(*obj.origin) + } } } @@ -607,7 +629,7 @@ func (s *StateDB) getDeletedStateObject(addr common.Address) *stateObject { } } // Insert into the live set - obj := newObject(s, addr, *data) + obj := newObject(s, addr, data) s.setStateObject(obj) return obj } @@ -629,38 +651,36 @@ func (s *StateDB) GetOrNewStateObject(addr common.Address) *stateObject { // the given address, it is overwritten and returned as the second return value. func (s *StateDB) createObject(addr common.Address) (newobj, prev *stateObject) { prev = s.getDeletedStateObject(addr) // Note, prev might have been deleted, we need that! - newobj = newObject(s, addr, types.StateAccount{}) + newobj = newObject(s, addr, nil) if prev == nil { s.journal.append(createObjectChange{account: &addr}) } else { // The original account should be marked as destructed and all cached // account and storage data should be cleared as well. Note, it must - // be done here, otherwise the destruction event of original one will - // be lost. + // be done here, otherwise the destruction event of "original account" + // will be lost. _, prevdestruct := s.stateObjectsDestruct[prev.address] if !prevdestruct { - s.stateObjectsDestruct[prev.address] = struct{}{} + s.stateObjectsDestruct[prev.address] = prev.origin } - var ( - account []byte - storage map[common.Hash][]byte - ) // There may be some cached account/storage data already since IntermediateRoot // will be called for each transaction before byzantium fork which will always // cache the latest account/storage data. - if s.snap != nil { - account = s.snapAccounts[prev.addrHash] - storage = s.snapStorage[prev.addrHash] - delete(s.snapAccounts, prev.addrHash) - delete(s.snapStorage, prev.addrHash) - } + prevAccount, ok := s.accountsOrigin[prev.addrHash] s.journal.append(resetObjectChange{ - account: &addr, - prev: prev, - prevdestruct: prevdestruct, - prevAccount: account, - prevStorage: storage, + account: &addr, + prev: prev, + prevdestruct: prevdestruct, + prevAccount: s.accounts[prev.addrHash], + prevStorage: s.storages[prev.addrHash], + prevAccountOriginExist: ok, + prevAccountOrigin: prevAccount, + prevStorageOrigin: s.storagesOrigin[prev.addrHash], }) + delete(s.accounts, prev.addrHash) + delete(s.storages, prev.addrHash) + delete(s.accountsOrigin, prev.addrHash) + delete(s.storagesOrigin, prev.addrHash) } s.setStateObject(newobj) if prev != nil && !prev.deleted { @@ -731,16 +751,27 @@ func (s *StateDB) Copy() *StateDB { db: s.db, trie: s.db.CopyTrie(s.trie), originalRoot: s.originalRoot, + accounts: make(map[common.Hash][]byte), + storages: make(map[common.Hash]map[common.Hash][]byte), + accountsOrigin: make(map[common.Hash][]byte), + storagesOrigin: make(map[common.Hash]map[common.Hash][]byte), stateObjects: make(map[common.Address]*stateObject, len(s.journal.dirties)), stateObjectsPending: make(map[common.Address]struct{}, len(s.stateObjectsPending)), stateObjectsDirty: make(map[common.Address]struct{}, len(s.journal.dirties)), - stateObjectsDestruct: make(map[common.Address]struct{}, len(s.stateObjectsDestruct)), + stateObjectsDestruct: make(map[common.Address]*types.StateAccount, len(s.stateObjectsDestruct)), refund: s.refund, logs: make(map[common.Hash][]*types.Log, len(s.logs)), logSize: s.logSize, preimages: make(map[common.Hash][]byte, len(s.preimages)), journal: newJournal(), hasher: crypto.NewKeccakState(), + + // In order for the block producer to be able to use and make additions + // to the snapshot tree, we need to copy that as well. Otherwise, any + // block mined by ourselves will cause gaps in the tree, and force the + // miner to operate trie-backed only. + snaps: s.snaps, + snap: s.snap, } // Copy the dirty states, logs, and preimages for addr := range s.journal.dirties { @@ -774,10 +805,18 @@ func (s *StateDB) Copy() *StateDB { } state.stateObjectsDirty[addr] = struct{}{} } - // Deep copy the destruction flag. - for addr := range s.stateObjectsDestruct { - state.stateObjectsDestruct[addr] = struct{}{} + // Deep copy the destruction markers. + for addr, value := range s.stateObjectsDestruct { + state.stateObjectsDestruct[addr] = value } + // Deep copy the state changes made in the scope of block + // along with their original values. + state.accounts = copyAccounts(s.accounts) + state.storages = copyStorages(s.storages) + state.accountsOrigin = copyAccounts(state.accountsOrigin) + state.storagesOrigin = copyStorages(state.storagesOrigin) + + // Deep copy the logs occurred in the scope of block for hash, logs := range s.logs { cpy := make([]*types.Log, len(logs)) for i, l := range logs { @@ -786,6 +825,7 @@ func (s *StateDB) Copy() *StateDB { } state.logs[hash] = cpy } + // Deep copy the preimages occurred in the scope of block for hash, preimage := range s.preimages { state.preimages[hash] = preimage } @@ -804,28 +844,6 @@ func (s *StateDB) Copy() *StateDB { if s.prefetcher != nil { state.prefetcher = s.prefetcher.copy() } - if s.snaps != nil { - // In order for the miner to be able to use and make additions - // to the snapshot tree, we need to copy that as well. - // Otherwise, any block mined by ourselves will cause gaps in the tree, - // and force the miner to operate trie-backed only - state.snaps = s.snaps - state.snap = s.snap - - // deep copy needed - state.snapAccounts = make(map[common.Hash][]byte, len(s.snapAccounts)) - for k, v := range s.snapAccounts { - state.snapAccounts[k] = v - } - state.snapStorage = make(map[common.Hash]map[common.Hash][]byte, len(s.snapStorage)) - for k, v := range s.snapStorage { - temp := make(map[common.Hash][]byte, len(v)) - for kk, vv := range v { - temp[kk] = vv - } - state.snapStorage[k] = temp - } - } return state } @@ -878,17 +896,18 @@ func (s *StateDB) Finalise(deleteEmptyObjects bool) { obj.deleted = true // We need to maintain account deletions explicitly (will remain - // set indefinitely). - s.stateObjectsDestruct[obj.address] = struct{}{} - - // If state snapshotting is active, also mark the destruction there. + // set indefinitely). Note only the first occurred self-destruct + // event is tracked. + if _, ok := s.stateObjectsDestruct[obj.address]; !ok { + s.stateObjectsDestruct[obj.address] = obj.origin + } // Note, we can't do this only at the end of a block because multiple // transactions within the same block might self destruct and then // resurrect an account; but the snapshotter needs both events. - if s.snap != nil { - delete(s.snapAccounts, obj.addrHash) // Clear out any previously updated account data (may be recreated via a resurrect) - delete(s.snapStorage, obj.addrHash) // Clear out any previously updated storage data (may be recreated via a resurrect) - } + delete(s.accounts, obj.addrHash) // Clear out any previously updated account data (may be recreated via a resurrect) + delete(s.storages, obj.addrHash) // Clear out any previously updated storage data (may be recreated via a resurrect) + delete(s.accountsOrigin, obj.addrHash) // Clear out any previously updated account data (may be recreated via a resurrect) + delete(s.storagesOrigin, obj.addrHash) // Clear out any previously updated storage data (may be recreated via a resurrect) } else { obj.finalise(true) // Prefetch slots in the background } @@ -986,6 +1005,137 @@ func (s *StateDB) clearJournalAndRefund() { s.validRevisions = s.validRevisions[:0] // Snapshots can be created without journal entries } +// deleteStorage iterates the storage trie belongs to the account and mark all +// slots inside as deleted. +func (s *StateDB) deleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (bool, map[common.Hash][]byte, *trienode.NodeSet, error) { + start := time.Now() + tr, err := s.db.OpenStorageTrie(s.originalRoot, addr, root) + if err != nil { + return false, nil, nil, fmt.Errorf("failed to open storage trie, err: %w", err) + } + it, err := tr.NodeIterator(nil) + if err != nil { + return false, nil, nil, fmt.Errorf("failed to open storage iterator, err: %w", err) + } + var ( + set = trienode.NewNodeSet(addrHash) + slots = make(map[common.Hash][]byte) + stateSize common.StorageSize + nodeSize common.StorageSize + ) + for it.Next(true) { + // arbitrary stateSize limit, make it configurable + if stateSize+nodeSize > 512*1024*1024 { + log.Info("Skip large storage deletion", "address", addr.Hex(), "states", stateSize, "nodes", nodeSize) + if metrics.EnabledExpensive { + slotDeletionSkip.Inc(1) + } + return true, nil, nil, nil + } + if it.Leaf() { + slots[common.BytesToHash(it.LeafKey())] = common.CopyBytes(it.LeafBlob()) + stateSize += common.StorageSize(common.HashLength + len(it.LeafBlob())) + continue + } + if it.Hash() == (common.Hash{}) { + continue + } + nodeSize += common.StorageSize(len(it.Path()) + len(it.NodeBlob())) + set.AddNode(it.Path(), trienode.NewWithPrev(common.Hash{}, nil, it.NodeBlob())) + } + if err := it.Error(); err != nil { + return false, nil, nil, err + } + if metrics.EnabledExpensive { + if int64(len(slots)) > slotDeletionMaxCount.Value() { + slotDeletionMaxCount.Update(int64(len(slots))) + } + if int64(stateSize+nodeSize) > slotDeletionMaxSize.Value() { + slotDeletionMaxSize.Update(int64(stateSize + nodeSize)) + } + slotDeletionTimer.UpdateSince(start) + slotDeletionCount.Mark(int64(len(slots))) + slotDeletionSize.Mark(int64(stateSize + nodeSize)) + } + return false, slots, set, nil +} + +// handleDestruction processes all destruction markers and deletes the account +// and associated storage slots if necessary. There are four possible situations +// here: +// +// - the account was not existent and be marked as destructed +// +// - the account was not existent and be marked as destructed, +// however, it's resurrected later in the same block. +// +// - the account was existent and be marked as destructed +// +// - the account was existent and be marked as destructed, +// however it's resurrected later in the same block. +// +// In case (a), nothing needs be deleted, nil to nil transition can be ignored. +// +// In case (b), nothing needs be deleted, nil is used as the original value for +// newly created account and storages +// +// In case (c), **original** account along with its storages should be deleted, +// with their values be tracked as original value. +// +// In case (d), **original** account along with its storages should be deleted, +// with their values be tracked as original value. +func (s *StateDB) handleDestruction(nodes *trienode.MergedNodeSet) (map[common.Hash]struct{}, error) { + incomplete := make(map[common.Hash]struct{}) + for addr, prev := range s.stateObjectsDestruct { + // The original account was non-existing, and it's marked as destructed + // in the scope of block. It can be case (a) or (b). + // - for (a), skip it without doing anything. + // - for (b), track account's original value as nil. It may overwrite + // the data cached in s.accountsOrigin set by 'updateStateObject'. + addrHash := crypto.Keccak256Hash(addr[:]) + if prev == nil { + if _, ok := s.accounts[addrHash]; ok { + s.accountsOrigin[addrHash] = nil // case (b) + } + continue + } + // It can overwrite the data in s.accountsOrigin set by 'updateStateObject'. + s.accountsOrigin[addrHash] = types.SlimAccountRLP(*prev) // case (c) or (d) + + // Short circuit if the storage was empty. + if prev.Root == types.EmptyRootHash { + continue + } + // Remove storage slots belong to the account. + aborted, slots, set, err := s.deleteStorage(addr, addrHash, prev.Root) + if err != nil { + return nil, fmt.Errorf("failed to delete storage, err: %w", err) + } + // The storage is too huge to handle, skip it but mark as incomplete. + // For case (d), the account is resurrected might with a few slots + // created. In this case, wipe the entire storage state diff because + // of aborted deletion. + if aborted { + incomplete[addrHash] = struct{}{} + delete(s.storagesOrigin, addrHash) + continue + } + if s.storagesOrigin[addrHash] == nil { + s.storagesOrigin[addrHash] = slots + } else { + // It can overwrite the data in s.storagesOrigin[addrHash] set by + // 'object.updateTrie'. + for key, val := range slots { + s.storagesOrigin[addrHash][key] = val + } + } + if err := nodes.Merge(set); err != nil { + return nil, err + } + } + return incomplete, nil +} + // Commit writes the state to the underlying in-memory trie database. // Once the state is committed, tries cached in stateDB (including account // trie, storage tries) will no longer be functional. A new state instance @@ -1008,38 +1158,39 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { nodes = trienode.NewMergedNodeSet() codeWriter = s.db.DiskDB().NewBatch() ) + // Handle all state deletions first + incomplete, err := s.handleDestruction(nodes) + if err != nil { + return common.Hash{}, err + } + // Handle all state updates afterwards for addr := range s.stateObjectsDirty { - if obj := s.stateObjects[addr]; !obj.deleted { - // Write any contract code associated with the state object - if obj.code != nil && obj.dirtyCode { - s.trie.UpdateContractCode(obj.Address(), common.BytesToHash(obj.CodeHash()), obj.code) - rawdb.WriteCode(codeWriter, common.BytesToHash(obj.CodeHash()), obj.code) - obj.dirtyCode = false - } - // Write any storage changes in the state object to its storage trie - set, err := obj.commitTrie(s.db) - if err != nil { + obj := s.stateObjects[addr] + if obj.deleted { + continue + } + // Write any contract code associated with the state object + if obj.code != nil && obj.dirtyCode { + rawdb.WriteCode(codeWriter, common.BytesToHash(obj.CodeHash()), obj.code) + s.trie.UpdateContractCode(obj.Address(), common.BytesToHash(obj.CodeHash()), obj.code) + obj.dirtyCode = false + } + // Write any storage changes in the state object to its storage trie + set, err := obj.commit(s.db) + if err != nil { + return common.Hash{}, err + } + // Merge the dirty nodes of storage trie into global set. It is possible + // that the account was destructed and then resurrected in the same block. + // In this case, the node set is shared by both accounts. + if set != nil { + if err := nodes.Merge(set); err != nil { return common.Hash{}, err } - // Merge the dirty nodes of storage trie into global set. - if set != nil { - if err := nodes.Merge(set); err != nil { - return common.Hash{}, err - } - updates, deleted := set.Size() - storageTrieNodesUpdated += updates - storageTrieNodesDeleted += deleted - } + updates, deleted := set.Size() + storageTrieNodesUpdated += updates + storageTrieNodesDeleted += deleted } - // If the contract is destructed, the storage is still left in the - // database as dangling data. Theoretically it's should be wiped from - // database as well, but in hash-based-scheme it's extremely hard to - // determine that if the trie nodes are also referenced by other storage, - // and in path-based-scheme some technical challenges are still unsolved. - // Although it won't affect the correctness but please fix it TODO(rjl493456442). - } - if len(s.stateObjectsDirty) > 0 { - s.stateObjectsDirty = make(map[common.Address]struct{}) } if codeWriter.ValueSize() > 0 { if err := codeWriter.Write(); err != nil { @@ -1081,7 +1232,7 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { start := time.Now() // Only update if there's a state transition (skip empty Clique blocks) if parent := s.snap.Root(); parent != root { - if err := s.snaps.Update(root, parent, s.convertAccountSet(s.stateObjectsDestruct), s.snapAccounts, s.snapStorage); err != nil { + if err := s.snaps.Update(root, parent, s.convertAccountSet(s.stateObjectsDestruct), s.accounts, s.storages); err != nil { log.Warn("Failed to update snapshot tree", "from", parent, "to", root, "err", err) } // Keep 128 diff layers in the memory, persistent layer is 129th. @@ -1095,10 +1246,7 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { if metrics.EnabledExpensive { s.SnapshotCommits += time.Since(start) } - s.snap, s.snapAccounts, s.snapStorage = nil, nil, nil - } - if len(s.stateObjectsDestruct) > 0 { - s.stateObjectsDestruct = make(map[common.Address]struct{}) + s.snap = nil } if root == (common.Hash{}) { root = types.EmptyRootHash @@ -1109,7 +1257,12 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { } if root != origin { start := time.Now() - if err := s.db.TrieDB().Update(root, origin, nodes); err != nil { + set := &triestate.Set{ + Accounts: s.accountsOrigin, + Storages: s.storagesOrigin, + Incomplete: incomplete, + } + if err := s.db.TrieDB().Update(root, origin, nodes, set); err != nil { return common.Hash{}, err } s.originalRoot = root @@ -1117,6 +1270,13 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { s.TrieDBCommits += time.Since(start) } } + // Clear all internal flags at the end of commit operation. + s.accounts = make(map[common.Hash][]byte) + s.storages = make(map[common.Hash]map[common.Hash][]byte) + s.accountsOrigin = make(map[common.Hash][]byte) + s.storagesOrigin = make(map[common.Hash]map[common.Hash][]byte) + s.stateObjectsDirty = make(map[common.Address]struct{}) + s.stateObjectsDestruct = make(map[common.Address]*types.StateAccount) return root, nil } @@ -1197,7 +1357,7 @@ func (s *StateDB) SlotInAccessList(addr common.Address, slot common.Hash) (addre } // convertAccountSet converts a provided account set from address keyed to hash keyed. -func (s *StateDB) convertAccountSet(set map[common.Address]struct{}) map[common.Hash]struct{} { +func (s *StateDB) convertAccountSet(set map[common.Address]*types.StateAccount) map[common.Hash]struct{} { ret := make(map[common.Hash]struct{}, len(set)) for addr := range set { obj, exist := s.stateObjects[addr] @@ -1209,3 +1369,24 @@ func (s *StateDB) convertAccountSet(set map[common.Address]struct{}) map[common. } return ret } + +// copyAccounts returns a deep-copied account set of the provided one. +func copyAccounts(set map[common.Hash][]byte) map[common.Hash][]byte { + copied := make(map[common.Hash][]byte, len(set)) + for key, val := range set { + copied[key] = common.CopyBytes(val) + } + return copied +} + +// copyStorages returns a deep-copied storage set of the provided one. +func copyStorages(set map[common.Hash]map[common.Hash][]byte) map[common.Hash]map[common.Hash][]byte { + copied := make(map[common.Hash]map[common.Hash][]byte, len(set)) + for addr, subset := range set { + copied[addr] = make(map[common.Hash][]byte, len(subset)) + for key, val := range subset { + copied[addr][key] = common.CopyBytes(val) + } + } + return copied +} diff --git a/core/state/statedb_fuzz_test.go b/core/state/statedb_fuzz_test.go new file mode 100644 index 0000000000..d014b2b3cc --- /dev/null +++ b/core/state/statedb_fuzz_test.go @@ -0,0 +1,373 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see + +package state + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "math" + "math/big" + "math/rand" + "reflect" + "strings" + "testing" + "testing/quick" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/triestate" +) + +// A stateTest checks that the state changes are correctly captured. Instances +// of this test with pseudorandom content are created by Generate. +// +// The test works as follows: +// +// A list of states are created by applying actions. The state changes between +// each state instance are tracked and be verified. +type stateTest struct { + addrs []common.Address // all account addresses + actions [][]testAction // modifications to the state, grouped by block + chunk int // The number of actions per chunk + err error // failure details are reported through this field +} + +// newStateTestAction creates a random action that changes state. +func newStateTestAction(addr common.Address, r *rand.Rand, index int) testAction { + actions := []testAction{ + { + name: "SetBalance", + fn: func(a testAction, s *StateDB) { + s.SetBalance(addr, big.NewInt(a.args[0])) + }, + args: make([]int64, 1), + }, + { + name: "SetNonce", + fn: func(a testAction, s *StateDB) { + s.SetNonce(addr, uint64(a.args[0])) + }, + args: make([]int64, 1), + }, + { + name: "SetState", + fn: func(a testAction, s *StateDB) { + var key, val common.Hash + binary.BigEndian.PutUint16(key[:], uint16(a.args[0])) + binary.BigEndian.PutUint16(val[:], uint16(a.args[1])) + s.SetState(addr, key, val) + }, + args: make([]int64, 2), + }, + { + name: "SetCode", + fn: func(a testAction, s *StateDB) { + code := make([]byte, 16) + binary.BigEndian.PutUint64(code, uint64(a.args[0])) + binary.BigEndian.PutUint64(code[8:], uint64(a.args[1])) + s.SetCode(addr, code) + }, + args: make([]int64, 2), + }, + { + name: "CreateAccount", + fn: func(a testAction, s *StateDB) { + s.CreateAccount(addr) + }, + }, + { + name: "Suicide", + fn: func(a testAction, s *StateDB) { + s.Suicide(addr) + }, + }, + } + var nonRandom = index != -1 + if index == -1 { + index = r.Intn(len(actions)) + } + action := actions[index] + var names []string + if !action.noAddr { + names = append(names, addr.Hex()) + } + for i := range action.args { + if nonRandom { + action.args[i] = rand.Int63n(10000) + 1 // set balance to non-zero + } else { + action.args[i] = rand.Int63n(10000) + } + names = append(names, fmt.Sprint(action.args[i])) + } + action.name += " " + strings.Join(names, ", ") + return action +} + +// Generate returns a new snapshot test of the given size. All randomness is +// derived from r. +func (*stateTest) Generate(r *rand.Rand, size int) reflect.Value { + addrs := make([]common.Address, 5) + for i := range addrs { + addrs[i][0] = byte(i) + } + actions := make([][]testAction, rand.Intn(5)+1) + + for i := 0; i < len(actions); i++ { + actions[i] = make([]testAction, size) + for j := range actions[i] { + if j == 0 { + // Always include a set balance action to make sure + // the state changes are not empty. + actions[i][j] = newStateTestAction(common.HexToAddress("0xdeadbeef"), r, 0) + continue + } + actions[i][j] = newStateTestAction(addrs[r.Intn(len(addrs))], r, -1) + } + } + chunk := int(math.Sqrt(float64(size))) + if size > 0 && chunk == 0 { + chunk = 1 + } + return reflect.ValueOf(&stateTest{ + addrs: addrs, + actions: actions, + chunk: chunk, + }) +} + +func (test *stateTest) String() string { + out := new(bytes.Buffer) + for i, actions := range test.actions { + fmt.Fprintf(out, "---- block %d ----\n", i) + for j, action := range actions { + if j%test.chunk == 0 { + fmt.Fprintf(out, "---- transaction %d ----\n", j/test.chunk) + } + fmt.Fprintf(out, "%4d: %s\n", j%test.chunk, action.name) + } + } + return out.String() +} + +func (test *stateTest) run() bool { + var ( + roots []common.Hash + accountList []map[common.Hash][]byte + storageList []map[common.Hash]map[common.Hash][]byte + onCommit = func(states *triestate.Set) { + accountList = append(accountList, copyAccounts(states.Accounts)) + storageList = append(storageList, copyStorages(states.Storages)) + } + disk = rawdb.NewMemoryDatabase() + tdb = trie.NewDatabaseWithConfig(disk, &trie.Config{OnCommit: onCommit}) + sdb = NewDatabaseWithNodeDB(disk, tdb) + byzantium = rand.Intn(2) == 0 + ) + for i, actions := range test.actions { + root := types.EmptyRootHash + if i != 0 { + root = roots[len(roots)-1] + } + state, err := New(root, sdb, nil) + if err != nil { + panic(err) + } + for i, action := range actions { + if i%test.chunk == 0 && i != 0 { + if byzantium { + state.Finalise(true) // call finalise at the transaction boundary + } else { + state.IntermediateRoot(true) // call intermediateRoot at the transaction boundary + } + } + action.fn(action, state) + } + if byzantium { + state.Finalise(true) // call finalise at the transaction boundary + } else { + state.IntermediateRoot(true) // call intermediateRoot at the transaction boundary + } + nroot, err := state.Commit(true) // call commit at the block boundary + if err != nil { + panic(err) + } + if nroot == root { + return true // filter out non-change state transition + } + roots = append(roots, nroot) + } + for i := 0; i < len(test.actions); i++ { + root := types.EmptyRootHash + if i != 0 { + root = roots[i-1] + } + test.err = test.verify(root, roots[i], tdb, accountList[i], storageList[i]) + if test.err != nil { + return false + } + } + return true +} + +// verifyAccountCreation this function is called once the state diff says that +// specific account was not present. A serial of checks will be performed to +// ensure the state diff is correct, includes: +// +// - the account was indeed not present in trie +// - the account is present in new trie, nil->nil is regarded as invalid +// - the slots transition is correct +func (test *stateTest) verifyAccountCreation(next common.Hash, db *trie.Database, otr, ntr *trie.Trie, addrHash common.Hash, slots map[common.Hash][]byte) error { + // Verify account change + oBlob, err := otr.Get(addrHash.Bytes()) + if err != nil { + return err + } + nBlob, err := ntr.Get(addrHash.Bytes()) + if err != nil { + return err + } + if len(oBlob) != 0 { + return fmt.Errorf("unexpected account in old trie, %x", addrHash) + } + if len(nBlob) == 0 { + return fmt.Errorf("missing account in new trie, %x", addrHash) + } + + // Verify storage changes + var nAcct types.StateAccount + if err := rlp.DecodeBytes(nBlob, &nAcct); err != nil { + return err + } + // Account has no slot, empty slot set is expected + if nAcct.Root == types.EmptyRootHash { + if len(slots) != 0 { + return fmt.Errorf("unexpected slot changes %x", addrHash) + } + return nil + } + // Account has slots, ensure all new slots are contained + st, err := trie.New(trie.StorageTrieID(next, addrHash, nAcct.Root), db) + if err != nil { + return err + } + for key, val := range slots { + st.Update(key.Bytes(), val) + } + if st.Hash() != types.EmptyRootHash { + return errors.New("invalid slot changes") + } + return nil +} + +// verifyAccountUpdate this function is called once the state diff says that +// specific account was present. A serial of checks will be performed to +// ensure the state diff is correct, includes: +// +// - the account was indeed present in trie +// - the account in old trie matches the provided value +// - the slots transition is correct +func (test *stateTest) verifyAccountUpdate(next common.Hash, db *trie.Database, otr, ntr *trie.Trie, addrHash common.Hash, origin []byte, slots map[common.Hash][]byte) error { + // Verify account change + oBlob, err := otr.Get(addrHash.Bytes()) + if err != nil { + return err + } + nBlob, err := ntr.Get(addrHash.Bytes()) + if err != nil { + return err + } + if len(oBlob) == 0 { + return fmt.Errorf("missing account in old trie, %x", addrHash) + } + full, err := types.FullAccountRLP(origin) + if err != nil { + return err + } + if !bytes.Equal(full, oBlob) { + return fmt.Errorf("account value is not matched, %x", addrHash) + } + + // Decode accounts + var ( + oAcct types.StateAccount + nAcct types.StateAccount + nRoot common.Hash + ) + if err := rlp.DecodeBytes(oBlob, &oAcct); err != nil { + return err + } + if len(nBlob) == 0 { + nRoot = types.EmptyRootHash + } else { + if err := rlp.DecodeBytes(nBlob, &nAcct); err != nil { + return err + } + nRoot = nAcct.Root + } + + // Verify storage + st, err := trie.New(trie.StorageTrieID(next, addrHash, nRoot), db) + if err != nil { + return err + } + for key, val := range slots { + st.Update(key.Bytes(), val) + } + if st.Hash() != oAcct.Root { + return errors.New("invalid slot changes") + } + return nil +} + +func (test *stateTest) verify(root common.Hash, next common.Hash, db *trie.Database, accountsOrigin map[common.Hash][]byte, storagesOrigin map[common.Hash]map[common.Hash][]byte) error { + otr, err := trie.New(trie.StateTrieID(root), db) + if err != nil { + return err + } + ntr, err := trie.New(trie.StateTrieID(next), db) + if err != nil { + return err + } + for addrHash, account := range accountsOrigin { + var err error + if len(account) == 0 { + err = test.verifyAccountCreation(next, db, otr, ntr, addrHash, storagesOrigin[addrHash]) + } else { + err = test.verifyAccountUpdate(next, db, otr, ntr, addrHash, accountsOrigin[addrHash], storagesOrigin[addrHash]) + } + if err != nil { + return err + } + } + return nil +} + +func TestStateChanges(t *testing.T) { + config := &quick.Config{MaxCount: 1000} + err := quick.Check((*stateTest).run, config) + if cerr, ok := err.(*quick.CheckError); ok { + test := cerr.In[0].(*stateTest) + t.Errorf("%v:\n%s", test.err, test) + } else if err != nil { + t.Error(err) + } +} diff --git a/core/state/statedb_test.go b/core/state/statedb_test.go index 44eace01d8..8290a154ff 100644 --- a/core/state/statedb_test.go +++ b/core/state/statedb_test.go @@ -485,7 +485,7 @@ func (test *snapshotTest) checkEqual(state, checkstate *StateDB) error { } func TestTouchDelete(t *testing.T) { - s := newStateTest() + s := newStateEnv() s.state.GetOrNewStateObject(common.Address{}) root, _ := s.state.Commit(false) s.state, _ = New(root, s.state.db, s.state.snaps) diff --git a/core/types/state_account.go b/core/types/state_account.go index b721929e42..314f4943ec 100644 --- a/core/types/state_account.go +++ b/core/types/state_account.go @@ -35,6 +35,29 @@ type StateAccount struct { CodeHash []byte } +// NewEmptyStateAccount constructs an empty state account. +func NewEmptyStateAccount() *StateAccount { + return &StateAccount{ + Balance: new(big.Int), + Root: EmptyRootHash, + CodeHash: EmptyCodeHash.Bytes(), + } +} + +// Copy returns a deep-copied state account object. +func (acct *StateAccount) Copy() *StateAccount { + var balance *big.Int + if acct.Balance != nil { + balance = new(big.Int).Set(acct.Balance) + } + return &StateAccount{ + Nonce: acct.Nonce, + Balance: balance, + Root: acct.Root, + CodeHash: common.CopyBytes(acct.CodeHash), + } +} + // SlimAccount is a modified version of an Account, where the root is replaced // with a byte slice. This format can be used to represent full-consensus format // or slim format which replaces the empty root and code hash as nil byte slice. diff --git a/eth/protocols/snap/sync_test.go b/eth/protocols/snap/sync_test.go index c1875cc721..1c5f103cb5 100644 --- a/eth/protocols/snap/sync_test.go +++ b/eth/protocols/snap/sync_test.go @@ -1387,7 +1387,7 @@ func makeAccountTrieNoStorage(n int) (string, *trie.Trie, []*kv) { // Commit the state changes into db and re-create the trie // for accessing later. root, nodes, _ := accTrie.Commit(false) - db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) accTrie, _ = trie.New(trie.StateTrieID(root), db) return db.Scheme(), accTrie, entries @@ -1449,7 +1449,7 @@ func makeBoundaryAccountTrie(n int) (string, *trie.Trie, []*kv) { // Commit the state changes into db and re-create the trie // for accessing later. root, nodes, _ := accTrie.Commit(false) - db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) accTrie, _ = trie.New(trie.StateTrieID(root), db) return db.Scheme(), accTrie, entries @@ -1498,7 +1498,7 @@ func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) nodes.Merge(set) // Commit gathered dirty nodes into database - db.Update(root, types.EmptyRootHash, nodes) + db.Update(root, types.EmptyRootHash, nodes, nil) // Re-create tries with new root accTrie, _ = trie.New(trie.StateTrieID(root), db) @@ -1563,7 +1563,7 @@ func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (strin nodes.Merge(set) // Commit gathered dirty nodes into database - db.Update(root, types.EmptyRootHash, nodes) + db.Update(root, types.EmptyRootHash, nodes, nil) // Re-create tries with new root accTrie, err := trie.New(trie.StateTrieID(root), db) diff --git a/light/postprocess.go b/light/postprocess.go index 7ef797ff47..8905694211 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -220,7 +220,7 @@ func (c *ChtIndexerBackend) Commit() error { } // Commit trie changes into trie database in case it's not nil. if nodes != nil { - if err := c.triedb.Update(root, c.originRoot, trienode.NewWithNodeSet(nodes)); err != nil { + if err := c.triedb.Update(root, c.originRoot, trienode.NewWithNodeSet(nodes), nil); err != nil { return err } if err := c.triedb.Commit(root, false); err != nil { @@ -473,7 +473,7 @@ func (b *BloomTrieIndexerBackend) Commit() error { } // Commit trie changes into trie database in case it's not nil. if nodes != nil { - if err := b.triedb.Update(root, b.originRoot, trienode.NewWithNodeSet(nodes)); err != nil { + if err := b.triedb.Update(root, b.originRoot, trienode.NewWithNodeSet(nodes), nil); err != nil { return err } if err := b.triedb.Commit(root, false); err != nil { diff --git a/light/trie.go b/light/trie.go index 43fccd5e36..4967cc74e5 100644 --- a/light/trie.go +++ b/light/trie.go @@ -121,19 +121,22 @@ func (t *odrTrie) GetStorage(_ common.Address, key []byte) ([]byte, error) { } func (t *odrTrie) GetAccount(address common.Address) (*types.StateAccount, error) { - var res types.StateAccount - key := crypto.Keccak256(address.Bytes()) + var ( + enc []byte + key = crypto.Keccak256(address.Bytes()) + ) err := t.do(key, func() (err error) { - value, err := t.trie.Get(key) - if err != nil { - return err - } - if value == nil { - return nil - } - return rlp.DecodeBytes(value, &res) + enc, err = t.trie.Get(key) + return err }) - return &res, err + if err != nil || len(enc) == 0 { + return nil, err + } + acct := new(types.StateAccount) + if err := rlp.DecodeBytes(enc, acct); err != nil { + return nil, err + } + return acct, nil } func (t *odrTrie) UpdateAccount(address common.Address, acc *types.StateAccount) error { diff --git a/tests/fuzzers/stacktrie/trie_fuzzer.go b/tests/fuzzers/stacktrie/trie_fuzzer.go index a6b1b3bc79..deb0c92c27 100644 --- a/tests/fuzzers/stacktrie/trie_fuzzer.go +++ b/tests/fuzzers/stacktrie/trie_fuzzer.go @@ -176,7 +176,7 @@ func (f *fuzzer) fuzz() int { panic(err) } if nodes != nil { - dbA.Update(rootA, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + dbA.Update(rootA, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) } // Flush memdb -> disk (sponge) dbA.Commit(rootA, false) diff --git a/tests/fuzzers/trie/trie-fuzzer.go b/tests/fuzzers/trie/trie-fuzzer.go index 004fb64372..2f9b97390b 100644 --- a/tests/fuzzers/trie/trie-fuzzer.go +++ b/tests/fuzzers/trie/trie-fuzzer.go @@ -170,7 +170,7 @@ func runRandTest(rt randTest) error { return err } if nodes != nil { - if err := triedb.Update(hash, origin, trienode.NewWithNodeSet(nodes)); err != nil { + if err := triedb.Update(hash, origin, trienode.NewWithNodeSet(nodes), nil); err != nil { return err } } diff --git a/trie/database.go b/trie/database.go index faaa1550e0..85681d7597 100644 --- a/trie/database.go +++ b/trie/database.go @@ -24,12 +24,16 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/trie/triedb/hashdb" "github.com/ethereum/go-ethereum/trie/trienode" + "github.com/ethereum/go-ethereum/trie/triestate" ) // Config defines all necessary options for database. type Config struct { Cache int // Memory allowance (MB) to use for caching trie nodes in memory Preimages bool // Flag whether the preimage of trie key is recorded + + // Testing hooks + OnCommit func(states *triestate.Set) // Hook invoked when commit is performed } // backend defines the methods needed to access/update trie nodes in different @@ -114,7 +118,10 @@ func (db *Database) Reader(blockRoot common.Hash) (Reader, error) { // given set in order to update state from the specified parent to the specified // root. The held pre-images accumulated up to this point will be flushed in case // the size exceeds the threshold. -func (db *Database) Update(root common.Hash, parent common.Hash, nodes *trienode.MergedNodeSet) error { +func (db *Database) Update(root common.Hash, parent common.Hash, nodes *trienode.MergedNodeSet, states *triestate.Set) error { + if db.config != nil && db.config.OnCommit != nil { + db.config.OnCommit(states) + } if db.preimages != nil { db.preimages.commit(false) } diff --git a/trie/iterator_test.go b/trie/iterator_test.go index 8f9849abbd..1ffb4bfbe8 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -63,7 +63,7 @@ func TestIterator(t *testing.T) { trie.MustUpdate([]byte(val.k), []byte(val.v)) } root, nodes, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) trie, _ = New(TrieID(root), db) found := make(map[string]string) @@ -255,7 +255,7 @@ func TestDifferenceIterator(t *testing.T) { triea.MustUpdate([]byte(val.k), []byte(val.v)) } rootA, nodesA, _ := triea.Commit(false) - dba.Update(rootA, types.EmptyRootHash, trienode.NewWithNodeSet(nodesA)) + dba.Update(rootA, types.EmptyRootHash, trienode.NewWithNodeSet(nodesA), nil) triea, _ = New(TrieID(rootA), dba) dbb := NewDatabase(rawdb.NewMemoryDatabase()) @@ -264,7 +264,7 @@ func TestDifferenceIterator(t *testing.T) { trieb.MustUpdate([]byte(val.k), []byte(val.v)) } rootB, nodesB, _ := trieb.Commit(false) - dbb.Update(rootB, types.EmptyRootHash, trienode.NewWithNodeSet(nodesB)) + dbb.Update(rootB, types.EmptyRootHash, trienode.NewWithNodeSet(nodesB), nil) trieb, _ = New(TrieID(rootB), dbb) found := make(map[string]string) @@ -297,7 +297,7 @@ func TestUnionIterator(t *testing.T) { triea.MustUpdate([]byte(val.k), []byte(val.v)) } rootA, nodesA, _ := triea.Commit(false) - dba.Update(rootA, types.EmptyRootHash, trienode.NewWithNodeSet(nodesA)) + dba.Update(rootA, types.EmptyRootHash, trienode.NewWithNodeSet(nodesA), nil) triea, _ = New(TrieID(rootA), dba) dbb := NewDatabase(rawdb.NewMemoryDatabase()) @@ -306,7 +306,7 @@ func TestUnionIterator(t *testing.T) { trieb.MustUpdate([]byte(val.k), []byte(val.v)) } rootB, nodesB, _ := trieb.Commit(false) - dbb.Update(rootB, types.EmptyRootHash, trienode.NewWithNodeSet(nodesB)) + dbb.Update(rootB, types.EmptyRootHash, trienode.NewWithNodeSet(nodesB), nil) trieb, _ = New(TrieID(rootB), dbb) di, _ := NewUnionIterator([]NodeIterator{triea.MustNodeIterator(nil), trieb.MustNodeIterator(nil)}) @@ -368,7 +368,7 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool, scheme string) { tr.MustUpdate([]byte(val.k), []byte(val.v)) } root, nodes, _ := tr.Commit(false) - tdb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + tdb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) if !memonly { tdb.Commit(root, false) } @@ -484,7 +484,7 @@ func testIteratorContinueAfterSeekError(t *testing.T, memonly bool, scheme strin break } } - triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) if !memonly { triedb.Commit(root, false) } @@ -605,7 +605,7 @@ func makeLargeTestTrie() (*Database, *StateTrie, *loggingDb) { trie.MustUpdate(key, val) } root, nodes, _ := trie.Commit(false) - triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) triedb.Commit(root, false) // Return the generated trie @@ -648,7 +648,7 @@ func testIteratorNodeBlob(t *testing.T, scheme string) { trie.MustUpdate([]byte(val.k), []byte(val.v)) } root, nodes, _ := trie.Commit(false) - triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) triedb.Commit(root, false) var found = make(map[common.Hash][]byte) diff --git a/trie/secure_trie_test.go b/trie/secure_trie_test.go index 4c70cbaa44..d756cb5223 100644 --- a/trie/secure_trie_test.go +++ b/trie/secure_trie_test.go @@ -61,7 +61,7 @@ func makeTestStateTrie() (*Database, *StateTrie, map[string][]byte) { } } root, nodes, _ := trie.Commit(false) - if err := triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)); err != nil { + if err := triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil); err != nil { panic(fmt.Errorf("failed to commit db %v", err)) } // Re-create the trie based on the new state diff --git a/trie/sync_test.go b/trie/sync_test.go index cfb7261a04..437cca7312 100644 --- a/trie/sync_test.go +++ b/trie/sync_test.go @@ -57,7 +57,7 @@ func makeTestTrie(scheme string) (ethdb.Database, *Database, *StateTrie, map[str } } root, nodes, _ := trie.Commit(false) - if err := triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)); err != nil { + if err := triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil); err != nil { panic(fmt.Errorf("failed to commit db %v", err)) } if err := triedb.Commit(root, false); err != nil { @@ -740,7 +740,7 @@ func testSyncMovingTarget(t *testing.T, scheme string) { diff[string(key)] = val } root, nodes, _ := srcTrie.Commit(false) - if err := srcDb.Update(root, preRoot, trienode.NewWithNodeSet(nodes)); err != nil { + if err := srcDb.Update(root, preRoot, trienode.NewWithNodeSet(nodes), nil); err != nil { panic(err) } if err := srcDb.Commit(root, false); err != nil { @@ -765,7 +765,7 @@ func testSyncMovingTarget(t *testing.T, scheme string) { reverted[k] = val } root, nodes, _ = srcTrie.Commit(false) - if err := srcDb.Update(root, preRoot, trienode.NewWithNodeSet(nodes)); err != nil { + if err := srcDb.Update(root, preRoot, trienode.NewWithNodeSet(nodes), nil); err != nil { panic(err) } if err := srcDb.Commit(root, false); err != nil { diff --git a/trie/tracer_test.go b/trie/tracer_test.go index ca9d64766b..a68cf3abaf 100644 --- a/trie/tracer_test.go +++ b/trie/tracer_test.go @@ -71,7 +71,7 @@ func testTrieTracer(t *testing.T, vals []struct{ k, v string }) { insertSet := copySet(trie.tracer.inserts) // copy before commit deleteSet := copySet(trie.tracer.deletes) // copy before commit root, nodes, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) seen := setKeys(iterNodes(db, root)) if !compareSet(insertSet, seen) { @@ -137,7 +137,7 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) { trie.MustUpdate([]byte(val.k), []byte(val.v)) } root, nodes, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) trie, _ = New(TrieID(root), db) if err := verifyAccessList(orig, trie, nodes); err != nil { @@ -152,7 +152,7 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) { trie.MustUpdate([]byte(val.k), randBytes(32)) } root, nodes, _ = trie.Commit(false) - db.Update(root, parent, trienode.NewWithNodeSet(nodes)) + db.Update(root, parent, trienode.NewWithNodeSet(nodes), nil) trie, _ = New(TrieID(root), db) if err := verifyAccessList(orig, trie, nodes); err != nil { @@ -170,7 +170,7 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) { trie.MustUpdate(key, randBytes(32)) } root, nodes, _ = trie.Commit(false) - db.Update(root, parent, trienode.NewWithNodeSet(nodes)) + db.Update(root, parent, trienode.NewWithNodeSet(nodes), nil) trie, _ = New(TrieID(root), db) if err := verifyAccessList(orig, trie, nodes); err != nil { @@ -185,7 +185,7 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) { trie.MustUpdate([]byte(key), nil) } root, nodes, _ = trie.Commit(false) - db.Update(root, parent, trienode.NewWithNodeSet(nodes)) + db.Update(root, parent, trienode.NewWithNodeSet(nodes), nil) trie, _ = New(TrieID(root), db) if err := verifyAccessList(orig, trie, nodes); err != nil { @@ -200,7 +200,7 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) { trie.MustUpdate([]byte(val.k), nil) } root, nodes, _ = trie.Commit(false) - db.Update(root, parent, trienode.NewWithNodeSet(nodes)) + db.Update(root, parent, trienode.NewWithNodeSet(nodes), nil) trie, _ = New(TrieID(root), db) if err := verifyAccessList(orig, trie, nodes); err != nil { @@ -219,7 +219,7 @@ func TestAccessListLeak(t *testing.T) { trie.MustUpdate([]byte(val.k), []byte(val.v)) } root, nodes, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) var cases = []struct { op func(tr *Trie) @@ -269,7 +269,7 @@ func TestTinyTree(t *testing.T) { trie.MustUpdate([]byte(val.k), randBytes(32)) } root, set, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(set)) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(set), nil) parent := root trie, _ = New(TrieID(root), db) @@ -278,7 +278,7 @@ func TestTinyTree(t *testing.T) { trie.MustUpdate([]byte(val.k), []byte(val.v)) } root, set, _ = trie.Commit(false) - db.Update(root, parent, trienode.NewWithNodeSet(set)) + db.Update(root, parent, trienode.NewWithNodeSet(set), nil) trie, _ = New(TrieID(root), db) if err := verifyAccessList(orig, trie, set); err != nil { diff --git a/trie/trie_test.go b/trie/trie_test.go index 3997c4565d..8e7066eade 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -89,7 +89,7 @@ func testMissingNode(t *testing.T, memonly bool, scheme string) { updateString(trie, "120000", "qwerqwerqwerqwerqwerqwerqwerqwer") updateString(trie, "123456", "asdfasdfasdfasdfasdfasdfasdfasdf") root, nodes, _ := trie.Commit(false) - triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) if !memonly { triedb.Commit(root, false) @@ -203,7 +203,7 @@ func TestGet(t *testing.T) { return } root, nodes, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) trie, _ = New(TrieID(root), db) } } @@ -275,7 +275,7 @@ func TestReplication(t *testing.T) { updateString(trie, val.k, val.v) } root, nodes, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) // create a new trie on top of the database and check that lookups work. trie2, err := New(TrieID(root), db) @@ -294,7 +294,7 @@ func TestReplication(t *testing.T) { // recreate the trie after commit if nodes != nil { - db.Update(hash, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + db.Update(hash, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) } trie2, err = New(TrieID(hash), db) if err != nil { @@ -503,7 +503,7 @@ func runRandTest(rt randTest) bool { case opCommit: root, nodes, _ := tr.Commit(true) if nodes != nil { - triedb.Update(root, origin, trienode.NewWithNodeSet(nodes)) + triedb.Update(root, origin, trienode.NewWithNodeSet(nodes), nil) } newtr, err := New(TrieID(root), triedb) if err != nil { @@ -838,7 +838,7 @@ func TestCommitSequence(t *testing.T) { } // Flush trie -> database root, nodes, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) // Flush memdb -> disk (sponge) db.Commit(root, false) if got, exp := s.sponge.Sum(nil), tc.expWriteSeqHash; !bytes.Equal(got, exp) { @@ -879,7 +879,7 @@ func TestCommitSequenceRandomBlobs(t *testing.T) { } // Flush trie -> database root, nodes, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) // Flush memdb -> disk (sponge) db.Commit(root, false) if got, exp := s.sponge.Sum(nil), tc.expWriteSeqHash; !bytes.Equal(got, exp) { @@ -919,7 +919,7 @@ func TestCommitSequenceStackTrie(t *testing.T) { // Flush trie -> database root, nodes, _ := trie.Commit(false) // Flush memdb -> disk (sponge) - db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) db.Commit(root, false) // And flush stacktrie -> disk stRoot, err := stTrie.Commit() @@ -967,7 +967,7 @@ func TestCommitSequenceSmallRoot(t *testing.T) { // Flush trie -> database root, nodes, _ := trie.Commit(false) // Flush memdb -> disk (sponge) - db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) db.Commit(root, false) // And flush stacktrie -> disk stRoot, err := stTrie.Commit() @@ -1139,7 +1139,7 @@ func benchmarkDerefRootFixedSize(b *testing.B, addresses [][20]byte, accounts [] } h := trie.Hash() root, nodes, _ := trie.Commit(false) - triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) + triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) b.StartTimer() triedb.Dereference(h) b.StopTimer() diff --git a/trie/trienode/node.go b/trie/trienode/node.go index b233f517f0..d99f04bd5e 100644 --- a/trie/trienode/node.go +++ b/trie/trienode/node.go @@ -123,6 +123,26 @@ func (set *NodeSet) AddNode(path []byte, n *WithPrev) { set.Nodes[string(path)] = n } +// Merge adds a set of nodes into the set. +func (set *NodeSet) Merge(owner common.Hash, nodes map[string]*WithPrev) error { + if set.Owner != owner { + return fmt.Errorf("nodesets belong to different owner are not mergeable %x-%x", set.Owner, owner) + } + for path, node := range nodes { + prev, ok := set.Nodes[path] + if ok { + // overwrite happens, revoke the counter + if prev.IsDeleted() { + set.deletes -= 1 + } else { + set.updates -= 1 + } + } + set.AddNode([]byte(path), node) + } + return nil +} + // AddLeaf adds the provided leaf node into set. TODO(rjl493456442) how can // we get rid of it? func (set *NodeSet) AddLeaf(parent common.Hash, blob []byte) { @@ -190,9 +210,9 @@ func NewWithNodeSet(set *NodeSet) *MergedNodeSet { // Merge merges the provided dirty nodes of a trie into the set. The assumption // is held that no duplicated set belonging to the same trie will be merged twice. func (set *MergedNodeSet) Merge(other *NodeSet) error { - _, present := set.Sets[other.Owner] + subset, present := set.Sets[other.Owner] if present { - return fmt.Errorf("duplicate trie for owner %#x", other.Owner) + return subset.Merge(other.Owner, other.Nodes) } set.Sets[other.Owner] = other return nil diff --git a/trie/triestate/state.go b/trie/triestate/state.go new file mode 100644 index 0000000000..e5d0b87cb7 --- /dev/null +++ b/trie/triestate/state.go @@ -0,0 +1,28 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see + +package triestate + +import "github.com/ethereum/go-ethereum/common" + +// Set represents a collection of mutated states during a state transition. +// The value refers to the original content of state before the transition +// is made. Nil means that the state was not present previously. +type Set struct { + Accounts map[common.Hash][]byte // Mutated account set, nil means the account was not present + Storages map[common.Hash]map[common.Hash][]byte // Mutated storage set, nil means the slot was not present + Incomplete map[common.Hash]struct{} // Indicator whether the storage slot is incomplete due to large deletion +} From 645b0db98e5d264cd0b33b6a2711ca4413ae9172 Mon Sep 17 00:00:00 2001 From: lightclient <14004106+lightclient@users.noreply.github.com> Date: Tue, 11 Jul 2023 13:21:32 -0600 Subject: [PATCH 59/90] cmd/utils, p2p: clean up discovery setup (#27518) This simplifies the code that initializes the discovery a bit, and adds new flags for enabling/disabling discv4 and discv5 separately. --------- Co-authored-by: Felix Lange --- cmd/geth/les_test.go | 4 +- cmd/geth/main.go | 2 + cmd/utils/flags.go | 33 +++++++----- cmd/utils/flags_legacy.go | 21 ++++++-- p2p/server.go | 111 ++++++++++++++++++++------------------ 5 files changed, 100 insertions(+), 71 deletions(-) diff --git a/cmd/geth/les_test.go b/cmd/geth/les_test.go index 607b454ead..746f12c765 100644 --- a/cmd/geth/les_test.go +++ b/cmd/geth/les_test.go @@ -146,13 +146,13 @@ func startLightServer(t *testing.T) *gethrpc { t.Logf("Importing keys to geth") runGeth(t, "account", "import", "--datadir", datadir, "--password", "./testdata/password.txt", "--lightkdf", "./testdata/key.prv").WaitExit() account := "0x02f0d131f1f97aef08aec6e3291b957d9efe7105" - server := startGethWithIpc(t, "lightserver", "--allow-insecure-unlock", "--datadir", datadir, "--password", "./testdata/password.txt", "--unlock", account, "--miner.etherbase=0x02f0d131f1f97aef08aec6e3291b957d9efe7105", "--mine", "--light.serve=100", "--light.maxpeers=1", "--nodiscover", "--nat=extip:127.0.0.1", "--verbosity=4") + server := startGethWithIpc(t, "lightserver", "--allow-insecure-unlock", "--datadir", datadir, "--password", "./testdata/password.txt", "--unlock", account, "--miner.etherbase=0x02f0d131f1f97aef08aec6e3291b957d9efe7105", "--mine", "--light.serve=100", "--light.maxpeers=1", "--discv4=false", "--nat=extip:127.0.0.1", "--verbosity=4") return server } func startClient(t *testing.T, name string) *gethrpc { datadir := initGeth(t) - return startGethWithIpc(t, name, "--datadir", datadir, "--nodiscover", "--syncmode=light", "--nat=extip:127.0.0.1", "--verbosity=4") + return startGethWithIpc(t, name, "--datadir", datadir, "--discv4=false", "--syncmode=light", "--nat=extip:127.0.0.1", "--verbosity=4") } func TestPriorityClient(t *testing.T) { diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 42a05f4d1a..b9584e70b3 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -122,7 +122,9 @@ var ( utils.MinerNewPayloadTimeout, utils.NATFlag, utils.NoDiscoverFlag, + utils.DiscoveryV4Flag, utils.DiscoveryV5Flag, + utils.LegacyDiscoveryV5Flag, utils.NetrestrictFlag, utils.NodeKeyFileFlag, utils.NodeKeyHexFlag, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 525abf042d..feceb99d3e 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -253,11 +253,6 @@ var ( Usage: "Comma separated block number-to-hash mappings to require for peering (=)", Category: flags.EthCategory, } - LegacyWhitelistFlag = &cli.StringFlag{ - Name: "whitelist", - Usage: "Comma separated block number-to-hash mappings to enforce (=) (deprecated in favor of --eth.requiredblocks)", - Category: flags.DeprecatedCategory, - } BloomFilterSizeFlag = &cli.Uint64Flag{ Name: "bloomfilter.size", Usage: "Megabytes of memory allocated to bloom-filter for pruning", @@ -770,8 +765,16 @@ var ( Usage: "Disables the peer discovery mechanism (manual peer addition)", Category: flags.NetworkingCategory, } + DiscoveryV4Flag = &cli.BoolFlag{ + Name: "discovery.v4", + Aliases: []string{"discv4"}, + Usage: "Enables the V4 discovery mechanism", + Category: flags.NetworkingCategory, + Value: true, + } DiscoveryV5Flag = &cli.BoolFlag{ - Name: "v5disc", + Name: "discovery.v5", + Aliases: []string{"discv5"}, Usage: "Enables the experimental RLPx V5 (Topic Discovery) mechanism", Category: flags.NetworkingCategory, } @@ -1361,13 +1364,17 @@ func SetP2PConfig(ctx *cli.Context, cfg *p2p.Config) { cfg.NoDiscovery = true } - // if we're running a light client or server, force enable the v5 peer discovery - // unless it is explicitly disabled with --nodiscover note that explicitly specifying - // --v5disc overrides --nodiscover, in which case the later only disables v4 discovery - forceV5Discovery := (lightClient || lightServer) && !ctx.Bool(NoDiscoverFlag.Name) - if ctx.IsSet(DiscoveryV5Flag.Name) { - cfg.DiscoveryV5 = ctx.Bool(DiscoveryV5Flag.Name) - } else if forceV5Discovery { + // Disallow --nodiscover when used in conjunction with light mode. + if (lightClient || lightServer) && ctx.Bool(NoDiscoverFlag.Name) { + Fatalf("Cannot use --" + NoDiscoverFlag.Name + " in light client or light server mode") + } + CheckExclusive(ctx, DiscoveryV4Flag, NoDiscoverFlag) + CheckExclusive(ctx, DiscoveryV5Flag, NoDiscoverFlag) + cfg.DiscoveryV4 = ctx.Bool(DiscoveryV4Flag.Name) + cfg.DiscoveryV5 = ctx.Bool(DiscoveryV5Flag.Name) + + // If we're running a light client or server, force enable the v5 peer discovery. + if lightClient || lightServer { cfg.DiscoveryV5 = true } diff --git a/cmd/utils/flags_legacy.go b/cmd/utils/flags_legacy.go index b554deba8e..cf51b51936 100644 --- a/cmd/utils/flags_legacy.go +++ b/cmd/utils/flags_legacy.go @@ -33,27 +33,40 @@ var ShowDeprecated = &cli.Command{ var DeprecatedFlags = []cli.Flag{ NoUSBFlag, + LegacyWhitelistFlag, CacheTrieJournalFlag, CacheTrieRejournalFlag, + LegacyDiscoveryV5Flag, } var ( - // (Deprecated May 2020, shown in aliased flags section) + // Deprecated May 2020, shown in aliased flags section NoUSBFlag = &cli.BoolFlag{ Name: "nousb", Usage: "Disables monitoring for and managing USB hardware wallets (deprecated)", Category: flags.DeprecatedCategory, } - // (Deprecated June 2023, shown in aliased flags section) + // Deprecated March 2022 + LegacyWhitelistFlag = &cli.StringFlag{ + Name: "whitelist", + Usage: "Comma separated block number-to-hash mappings to enforce (=) (deprecated in favor of --eth.requiredblocks)", + Category: flags.DeprecatedCategory, + } + // Deprecated July 2023 CacheTrieJournalFlag = &cli.StringFlag{ Name: "cache.trie.journal", Usage: "Disk journal directory for trie cache to survive node restarts", - Category: flags.PerfCategory, + Category: flags.DeprecatedCategory, } CacheTrieRejournalFlag = &cli.DurationFlag{ Name: "cache.trie.rejournal", Usage: "Time interval to regenerate the trie cache journal", - Category: flags.PerfCategory, + Category: flags.DeprecatedCategory, + } + LegacyDiscoveryV5Flag = &cli.BoolFlag{ + Name: "v5disc", + Usage: "Enables the experimental RLPx V5 (Topic Discovery) mechanism (deprecated, use --discv5 instead)", + Category: flags.DeprecatedCategory, } ) diff --git a/p2p/server.go b/p2p/server.go index 8c417635e6..bd0fd70950 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -93,6 +93,9 @@ type Config struct { // Disabling is useful for protocol debugging (manual topology). NoDiscovery bool + // DiscoveryV4 specifies whether V4 discovery should be started. + DiscoveryV4 bool `toml:",omitempty"` + // DiscoveryV5 specifies whether the new topic-discovery based V5 discovery // protocol should be started or not. DiscoveryV5 bool `toml:",omitempty"` @@ -542,57 +545,28 @@ func (srv *Server) setupLocalNode() error { func (srv *Server) setupDiscovery() error { srv.discmix = enode.NewFairMix(discmixTimeout) - // Add protocol-specific discovery sources. - added := make(map[string]bool) - for _, proto := range srv.Protocols { - if proto.DialCandidates != nil && !added[proto.Name] { - srv.discmix.AddSource(proto.DialCandidates) - added[proto.Name] = true - } - } - // Don't listen on UDP endpoint if DHT is disabled. - if srv.NoDiscovery && !srv.DiscoveryV5 { + if srv.NoDiscovery { return nil } - - listenAddr := srv.ListenAddr - - // Use an alternate listening address for UDP if - // a custom discovery address is configured. - if srv.DiscAddr != "" { - listenAddr = srv.DiscAddr - } - - addr, err := net.ResolveUDPAddr("udp", listenAddr) + conn, err := srv.setupUDPListening() if err != nil { return err } - conn, err := net.ListenUDP("udp", addr) - if err != nil { - return err - } - realaddr := conn.LocalAddr().(*net.UDPAddr) - srv.log.Debug("UDP listener up", "addr", realaddr) - if srv.NAT != nil { - if !realaddr.IP.IsLoopback() { - srv.loopWG.Add(1) - go func() { - nat.Map(srv.NAT, srv.quit, "udp", realaddr.Port, realaddr.Port, "ethereum discovery") - srv.loopWG.Done() - }() - } - } - srv.localnode.SetFallbackUDP(realaddr.Port) - // Discovery V4 - var unhandled chan discover.ReadPacket - var sconn *sharedUDPConn - if !srv.NoDiscovery { - if srv.DiscoveryV5 { - unhandled = make(chan discover.ReadPacket, 100) - sconn = &sharedUDPConn{conn, unhandled} - } + var ( + sconn discover.UDPConn = conn + unhandled chan discover.ReadPacket + ) + // If both versions of discovery are running, setup a shared + // connection, so v5 can read unhandled messages from v4. + if srv.DiscoveryV4 && srv.DiscoveryV5 { + unhandled = make(chan discover.ReadPacket, 100) + sconn = &sharedUDPConn{conn, unhandled} + } + + // Start discovery services. + if srv.DiscoveryV4 { cfg := discover.Config{ PrivateKey: srv.PrivateKey, NetRestrict: srv.NetRestrict, @@ -607,8 +581,6 @@ func (srv *Server) setupDiscovery() error { srv.ntab = ntab srv.discmix.AddSource(ntab.RandomNodes()) } - - // Discovery V5 if srv.DiscoveryV5 { cfg := discover.Config{ PrivateKey: srv.PrivateKey, @@ -616,16 +588,20 @@ func (srv *Server) setupDiscovery() error { Bootnodes: srv.BootstrapNodesV5, Log: srv.log, } - var err error - if sconn != nil { - srv.DiscV5, err = discover.ListenV5(sconn, srv.localnode, cfg) - } else { - srv.DiscV5, err = discover.ListenV5(conn, srv.localnode, cfg) - } + srv.DiscV5, err = discover.ListenV5(sconn, srv.localnode, cfg) if err != nil { return err } } + + // Add protocol-specific discovery sources. + added := make(map[string]bool) + for _, proto := range srv.Protocols { + if proto.DialCandidates != nil && !added[proto.Name] { + srv.discmix.AddSource(proto.DialCandidates) + added[proto.Name] = true + } + } return nil } @@ -696,6 +672,37 @@ func (srv *Server) setupListening() error { return nil } +func (srv *Server) setupUDPListening() (*net.UDPConn, error) { + listenAddr := srv.ListenAddr + + // Use an alternate listening address for UDP if + // a custom discovery address is configured. + if srv.DiscAddr != "" { + listenAddr = srv.DiscAddr + } + addr, err := net.ResolveUDPAddr("udp", listenAddr) + if err != nil { + return nil, err + } + conn, err := net.ListenUDP("udp", addr) + if err != nil { + return nil, err + } + realaddr := conn.LocalAddr().(*net.UDPAddr) + srv.log.Debug("UDP listener up", "addr", realaddr) + if srv.NAT != nil { + if !realaddr.IP.IsLoopback() { + srv.loopWG.Add(1) + go func() { + nat.Map(srv.NAT, srv.quit, "udp", realaddr.Port, realaddr.Port, "ethereum discovery") + srv.loopWG.Done() + }() + } + } + srv.localnode.SetFallbackUDP(realaddr.Port) + return conn, nil +} + // doPeerOp runs fn on the main loop. func (srv *Server) doPeerOp(fn peerOpFunc) { select { From 7a565fa4fe2b3a093a7f99f3524972353f66932d Mon Sep 17 00:00:00 2001 From: Mskxn <118117161+Mskxn@users.noreply.github.com> Date: Wed, 12 Jul 2023 03:27:48 +0800 Subject: [PATCH 60/90] console: add cleanup to avoid leaks in newTester (#27695) --- console/console_test.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/console/console_test.go b/console/console_test.go index 4b8f943c9e..ee5c36be4a 100644 --- a/console/console_test.go +++ b/console/console_test.go @@ -111,6 +111,10 @@ func newTester(t *testing.T, confOverride func(*ethconfig.Config)) *tester { t.Fatalf("failed to start test stack: %v", err) } client := stack.Attach() + t.Cleanup(func() { + client.Close() + }) + prompter := &hookedPrompter{scheduler: make(chan string)} printer := new(bytes.Buffer) From cecd22143b9f7e8ef5279d76e5f60cf87ac67e21 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 11 Jul 2023 22:34:22 +0200 Subject: [PATCH 61/90] go.mod: upgrade github.com/karalabe/usb to fix build warning (#27698) --- go.mod | 2 +- go.sum | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 5090cf1a98..0b79746f57 100644 --- a/go.mod +++ b/go.mod @@ -44,7 +44,7 @@ require ( github.com/jackpal/go-nat-pmp v1.0.2 github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e github.com/julienschmidt/httprouter v1.3.0 - github.com/karalabe/usb v0.0.2 + github.com/karalabe/usb v0.0.3-0.20230711191512-61db3e06439c github.com/kylelemons/godebug v1.1.0 github.com/mattn/go-colorable v0.1.13 github.com/mattn/go-isatty v0.0.16 diff --git a/go.sum b/go.sum index 58b40cdb01..b1ef5999b2 100644 --- a/go.sum +++ b/go.sum @@ -19,6 +19,8 @@ github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrU github.com/VictoriaMetrics/fastcache v1.6.0 h1:C/3Oi3EiBCqufydp1neRZkqcwmEiuRT9c3fqvvgKm5o= github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= @@ -72,6 +74,7 @@ github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811 h1:ytcWPaNPhNoG github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811/go.mod h1:Nb5lgvnQ2+oGlE/EyZy4+2/CxRh9KfvCXnag1vtpxVM= github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ= github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ= github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= @@ -162,6 +165,7 @@ github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclK github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJpoZOs= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= @@ -267,14 +271,18 @@ github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e h1:UvSe12bq+U github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= github.com/karalabe/usb v0.0.2 h1:M6QQBNxF+CQ8OFvxrT90BA0qBOXymndZnk5q235mFc4= github.com/karalabe/usb v0.0.2/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= +github.com/karalabe/usb v0.0.3-0.20230711191512-61db3e06439c h1:AqsttAyEyIEsNz5WLRwuRwjiT5CMDUfLk6cFJDVPebs= +github.com/karalabe/usb v0.0.3-0.20230711191512-61db3e06439c/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= github.com/kataras/golog v0.0.10/go.mod h1:yJ8YKCmyL+nWjERB90Qwn+bdyBZsaQwU3bTVFgkFIp8= github.com/kataras/iris/v12 v12.1.8/go.mod h1:LMYy4VlP67TQ3Zgriz8RE2h2kMZV2SgMYbq3UhfoFmE= github.com/kataras/neffos v0.0.14/go.mod h1:8lqADm8PnbeFfL7CLXh1WHw53dG27MC3pgi2R1rmoTE= @@ -343,8 +351,10 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hzifhks= github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 h1:shk/vn9oCoOTmwcouEdwIeOtOGA/ELRUw/GwvxwfT+0= @@ -643,6 +653,7 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 0b1f97e151e8b34a0a0d528a3472e27de1d12a9c Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Wed, 12 Jul 2023 15:19:01 +0800 Subject: [PATCH 62/90] core/rawdb: support freezer batch read with no size limit (#27687) This change adds the ability to perform reads from freezer without size limitation. This can be useful in cases where callers are certain that out-of-memory will not happen (e.g. reading only a few elements). The previous API was designed to behave both optimally and secure while servicing a request from a peer, whereas this change should _not_ be used when an untrusted peer can influence the query size. --- core/rawdb/freezer.go | 7 ++--- core/rawdb/freezer_table.go | 33 +++++++++++------------ core/rawdb/freezer_table_test.go | 46 ++++++++++++++++++++++++++++++++ core/rawdb/freezer_utils.go | 16 +++++++++++ ethdb/database.go | 7 ++--- 5 files changed, 86 insertions(+), 23 deletions(-) diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index a08f126d6b..2846858e0b 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -197,9 +197,10 @@ func (f *Freezer) Ancient(kind string, number uint64) ([]byte, error) { // AncientRange retrieves multiple items in sequence, starting from the index 'start'. // It will return -// - at most 'max' items, -// - at least 1 item (even if exceeding the maxByteSize), but will otherwise -// return as many items as fit into maxByteSize. +// - at most 'count' items, +// - if maxBytes is specified: at least 1 item (even if exceeding the maxByteSize), +// but will otherwise return as many items as fit into maxByteSize. +// - if maxBytes is not specified, 'count' items will be returned if they are present. func (f *Freezer) AncientRange(kind string, start, count, maxBytes uint64) ([][]byte, error) { if table := f.tables[kind]; table != nil { return table.RetrieveItems(start, count, maxBytes) diff --git a/core/rawdb/freezer_table.go b/core/rawdb/freezer_table.go index 928b37d70b..fc6316c953 100644 --- a/core/rawdb/freezer_table.go +++ b/core/rawdb/freezer_table.go @@ -712,7 +712,7 @@ func (t *freezerTable) RetrieveItems(start, count, maxBytes uint64) ([][]byte, e if !t.noCompression { decompressedSize, _ = snappy.DecodedLen(item) } - if i > 0 && uint64(outputSize+decompressedSize) > maxBytes { + if i > 0 && maxBytes != 0 && uint64(outputSize+decompressedSize) > maxBytes { break } if !t.noCompression { @@ -730,8 +730,10 @@ func (t *freezerTable) RetrieveItems(start, count, maxBytes uint64) ([][]byte, e } // retrieveItems reads up to 'count' items from the table. It reads at least -// one item, but otherwise avoids reading more than maxBytes bytes. -// It returns the (potentially compressed) data, and the sizes. +// one item, but otherwise avoids reading more than maxBytes bytes. Freezer +// will ignore the size limitation and continuously allocate memory to store +// data if maxBytes is 0. It returns the (potentially compressed) data, and +// the sizes. func (t *freezerTable) retrieveItems(start, count, maxBytes uint64) ([]byte, []int, error) { t.lock.RLock() defer t.lock.RUnlock() @@ -752,25 +754,22 @@ func (t *freezerTable) retrieveItems(start, count, maxBytes uint64) ([]byte, []i if start+count > items { count = items - start } - var ( - output = make([]byte, maxBytes) // Buffer to read data into - outputSize int // Used size of that buffer - ) + var output []byte // Buffer to read data into + if maxBytes != 0 { + output = make([]byte, 0, maxBytes) + } else { + output = make([]byte, 0, 1024) // initial buffer cap + } // readData is a helper method to read a single data item from disk. readData := func(fileId, start uint32, length int) error { - // In case a small limit is used, and the elements are large, may need to - // realloc the read-buffer when reading the first (and only) item. - if len(output) < length { - output = make([]byte, length) - } + output = grow(output, length) dataFile, exist := t.files[fileId] if !exist { return fmt.Errorf("missing data file %d", fileId) } - if _, err := dataFile.ReadAt(output[outputSize:outputSize+length], int64(start)); err != nil { + if _, err := dataFile.ReadAt(output[len(output)-length:], int64(start)); err != nil { return err } - outputSize += length return nil } // Read all the indexes in one go @@ -801,7 +800,7 @@ func (t *freezerTable) retrieveItems(start, count, maxBytes uint64) ([]byte, []i } readStart = 0 } - if i > 0 && uint64(totalSize+size) > maxBytes { + if i > 0 && uint64(totalSize+size) > maxBytes && maxBytes != 0 { // About to break out due to byte limit being exceeded. We don't // read this last item, but we need to do the deferred reads now. if unreadSize > 0 { @@ -815,7 +814,7 @@ func (t *freezerTable) retrieveItems(start, count, maxBytes uint64) ([]byte, []i unreadSize += size totalSize += size sizes = append(sizes, size) - if i == len(indices)-2 || uint64(totalSize) > maxBytes { + if i == len(indices)-2 || (uint64(totalSize) > maxBytes && maxBytes != 0) { // Last item, need to do the read now if err := readData(secondIndex.filenum, readStart, unreadSize); err != nil { return nil, nil, err @@ -826,7 +825,7 @@ func (t *freezerTable) retrieveItems(start, count, maxBytes uint64) ([]byte, []i // Update metrics. t.readMeter.Mark(int64(totalSize)) - return output[:outputSize], sizes, nil + return output, sizes, nil } // has returns an indicator whether the specified number data is still accessible diff --git a/core/rawdb/freezer_table_test.go b/core/rawdb/freezer_table_test.go index 5c4cc40edd..939d093946 100644 --- a/core/rawdb/freezer_table_test.go +++ b/core/rawdb/freezer_table_test.go @@ -994,6 +994,52 @@ func TestSequentialReadByteLimit(t *testing.T) { } } +// TestSequentialReadNoByteLimit tests the batch-read if maxBytes is not specified. +// Freezer should return the requested items regardless the size limitation. +func TestSequentialReadNoByteLimit(t *testing.T) { + rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge() + fname := fmt.Sprintf("batchread-3-%d", rand.Uint64()) + { // Fill table + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, true, false) + if err != nil { + t.Fatal(err) + } + // Write 10 bytes 30 times, + // Splitting it at every 100 bytes (10 items) + writeChunks(t, f, 30, 10) + f.Close() + } + for i, tc := range []struct { + items uint64 + want int + }{ + {1, 1}, + {30, 30}, + {31, 30}, + } { + { + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, true, false) + if err != nil { + t.Fatal(err) + } + items, err := f.RetrieveItems(0, tc.items, 0) + if err != nil { + t.Fatal(err) + } + if have, want := len(items), tc.want; have != want { + t.Fatalf("test %d: want %d items, have %d ", i, want, have) + } + for ii, have := range items { + want := getChunk(10, ii) + if !bytes.Equal(want, have) { + t.Fatalf("test %d: data corruption item %d: have\n%x\n, want \n%x\n", i, ii, have, want) + } + } + f.Close() + } + } +} + func TestFreezerReadonly(t *testing.T) { tmpdir := os.TempDir() // Case 1: Check it fails on non-existent file. diff --git a/core/rawdb/freezer_utils.go b/core/rawdb/freezer_utils.go index e7cce2920d..1bbb50c498 100644 --- a/core/rawdb/freezer_utils.go +++ b/core/rawdb/freezer_utils.go @@ -117,3 +117,19 @@ func truncateFreezerFile(file *os.File, size int64) error { } return nil } + +// grow prepares the slice space for new item, and doubles the slice capacity +// if space is not enough. +func grow(buf []byte, n int) []byte { + if cap(buf)-len(buf) < n { + newcap := 2 * cap(buf) + if newcap-len(buf) < n { + newcap = len(buf) + n + } + nbuf := make([]byte, len(buf), newcap) + copy(nbuf, buf) + buf = nbuf + } + buf = buf[:len(buf)+n] + return buf +} diff --git a/ethdb/database.go b/ethdb/database.go index 361218f247..3fe6dfa73c 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -79,9 +79,10 @@ type AncientReaderOp interface { // AncientRange retrieves multiple items in sequence, starting from the index 'start'. // It will return - // - at most 'count' items, - // - at least 1 item (even if exceeding the maxBytes), but will otherwise - // return as many items as fit into maxBytes. + // - at most 'count' items, + // - if maxBytes is specified: at least 1 item (even if exceeding the maxByteSize), + // but will otherwise return as many items as fit into maxByteSize. + // - if maxBytes is not specified, 'count' items will be returned if they are present AncientRange(kind string, start, count, maxBytes uint64) ([][]byte, error) // Ancients returns the ancient item numbers in the ancient store. From a426999fc95f02f71359161ab0a07a392fc4df95 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 12 Jul 2023 12:01:38 +0200 Subject: [PATCH 63/90] p2p/discover: filter bootnodes by netrestrict (#27701) This prevents an issue where the node would attempt to contact the bootstrap nodes even if they weren't contained in the netrestrict list. --- p2p/discover/table.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/p2p/discover/table.go b/p2p/discover/table.go index b601f233d0..f476d2079f 100644 --- a/p2p/discover/table.go +++ b/p2p/discover/table.go @@ -203,12 +203,18 @@ func (tab *Table) close() { // are used to connect to the network if the table is empty and there // are no known nodes in the database. func (tab *Table) setFallbackNodes(nodes []*enode.Node) error { + nursery := make([]*node, 0, len(nodes)) for _, n := range nodes { if err := n.ValidateComplete(); err != nil { return fmt.Errorf("bad bootstrap node %q: %v", n, err) } + if tab.cfg.NetRestrict != nil && !tab.cfg.NetRestrict.Contains(n.IP()) { + tab.log.Error("Bootstrap node filtered by netrestrict", "id", n.ID(), "ip", n.IP()) + continue + } + nursery = append(nursery, wrapNode(n)) } - tab.nursery = wrapNodes(nodes) + tab.nursery = nursery return nil } From 80b7bfe70de59356a547c5a5e1c64f0a69e3873e Mon Sep 17 00:00:00 2001 From: cui fliter Date: Wed, 12 Jul 2023 18:01:55 +0800 Subject: [PATCH 64/90] accounts, cmd/evm: fix docstrings (#27703) fix function name in comment Signed-off-by: cui fliter --- accounts/abi/type.go | 2 +- accounts/usbwallet/wallet.go | 2 +- cmd/evm/internal/t8ntool/block.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/accounts/abi/type.go b/accounts/abi/type.go index 3c05d774a6..2eee11787f 100644 --- a/accounts/abi/type.go +++ b/accounts/abi/type.go @@ -348,7 +348,7 @@ func (t Type) pack(v reflect.Value) ([]byte, error) { } } -// requireLengthPrefix returns whether the type requires any sort of length +// requiresLengthPrefix returns whether the type requires any sort of length // prefixing. func (t Type) requiresLengthPrefix() bool { return t.T == StringTy || t.T == BytesTy || t.T == SliceTy diff --git a/accounts/usbwallet/wallet.go b/accounts/usbwallet/wallet.go index 0e399a6d09..05add081ab 100644 --- a/accounts/usbwallet/wallet.go +++ b/accounts/usbwallet/wallet.go @@ -624,7 +624,7 @@ func (w *wallet) SignTx(account accounts.Account, tx *types.Transaction, chainID return signed, nil } -// SignHashWithPassphrase implements accounts.Wallet, however signing arbitrary +// SignTextWithPassphrase implements accounts.Wallet, however signing arbitrary // data is not supported for Ledger wallets, so this method will always return // an error. func (w *wallet) SignTextWithPassphrase(account accounts.Account, passphrase string, text []byte) ([]byte, error) { diff --git a/cmd/evm/internal/t8ntool/block.go b/cmd/evm/internal/t8ntool/block.go index 64804a17ff..09dca8984e 100644 --- a/cmd/evm/internal/t8ntool/block.go +++ b/cmd/evm/internal/t8ntool/block.go @@ -306,7 +306,7 @@ func readInput(ctx *cli.Context) (*bbInput, error) { return inputData, nil } -// dispatchOutput writes the output data to either stderr or stdout, or to the specified +// dispatchBlock writes the output data to either stderr or stdout, or to the specified // files func dispatchBlock(ctx *cli.Context, baseDir string, block *types.Block) error { raw, _ := rlp.EncodeToBytes(block) From 040a4a543b9b5355bcd997c37c4d7bd6b1047542 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Kj=C3=A6rstad?= Date: Wed, 12 Jul 2023 19:38:12 +0200 Subject: [PATCH 65/90] build: upgrade -dlgo version to Go 1.20.6 (#27708) --- build/checksums.txt | 28 ++++++++++++++-------------- build/ci.go | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/build/checksums.txt b/build/checksums.txt index 9b038582a8..1d215e5a5b 100644 --- a/build/checksums.txt +++ b/build/checksums.txt @@ -1,19 +1,19 @@ # This file contains sha256 checksums of optional build dependencies. -e447b498cde50215c4f7619e5124b0fc4e25fb5d16ea47271c47f278e7aa763a go1.20.3.src.tar.gz -c1e1161d6d859deb576e6cfabeb40e3d042ceb1c6f444f617c3c9d76269c3565 go1.20.3.darwin-amd64.tar.gz -86b0ed0f2b2df50fa8036eea875d1cf2d76cefdacf247c44639a1464b7e36b95 go1.20.3.darwin-arm64.tar.gz -340e80abd047c597fdc0f50a6cc59617f06c297d62f7fc77f4a0164e2da6f7aa go1.20.3.freebsd-386.tar.gz -2169fcd8b6c94c5fbe07c0b470ccfb6001d343f6548ad49f3d9ab78e3b5753c7 go1.20.3.freebsd-amd64.tar.gz -e12384311403f1389d14cc1c1295bfb4e0dd5ab919403b80da429f671a223507 go1.20.3.linux-386.tar.gz -979694c2c25c735755bf26f4f45e19e64e4811d661dd07b8c010f7a8e18adfca go1.20.3.linux-amd64.tar.gz -eb186529f13f901e7a2c4438a05c2cd90d74706aaa0a888469b2a4a617b6ee54 go1.20.3.linux-arm64.tar.gz -b421e90469a83671641f81b6e20df6500f033e9523e89cbe7b7223704dd1035c go1.20.3.linux-armv6l.tar.gz -943c89aa1624ea544a022b31e3d6e16a037200e436370bdd5fd67f3fa60be282 go1.20.3.linux-ppc64le.tar.gz -126cf823a5634ef2544b866db107b9d351d3ea70d9e240b0bdcfb46f4dcae54b go1.20.3.linux-s390x.tar.gz -37e9146e1f9d681cfcaa6fee6c7b890c44c64bc50228c9588f3c4231346d33bd go1.20.3.windows-386.zip -143a2837821c7dbacf7744cbb1a8421c1f48307c6fdfaeffc5f8c2f69e1b7932 go1.20.3.windows-amd64.zip -158cb159e00bc979f473e0f5b5a561613129c5e51067967b72b8e072e5a4db81 go1.20.3.windows-arm64.zip +62ee5bc6fb55b8bae8f705e0cb8df86d6453626b4ecf93279e2867092e0b7f70 go1.20.6.src.tar.gz +98a09c085b4c385abae7d35b9155195d5e584d14988347ac7f18e4cbe3b5ef3d go1.20.6.darwin-amd64.tar.gz +1163be1998835a13f00dfc869a8e3cdebf86984ad41ff2fff43e35ac2a0d8344 go1.20.6.darwin-arm64.tar.gz +3e6801d33a52a599af9c5258e8626e368d6c7ba958e1b6a468a9a3eecad25d8b go1.20.6.freebsd-386.tar.gz +e0d35bb22fa792448b675368189519ca636dedd7af7f8a2fafca344e497eb70e go1.20.6.freebsd-amd64.tar.gz +2e27c9db1defbf4d58e907f9843bf60a1ce229688f8463bf24d6a0a19dc949de go1.20.6.linux-386.tar.gz +b945ae2bb5db01a0fb4786afde64e6fbab50b67f6fa0eb6cfa4924f16a7ff1eb go1.20.6.linux-amd64.tar.gz +4e15ab37556e979181a1a1cc60f6d796932223a0f5351d7c83768b356f84429b go1.20.6.linux-arm64.tar.gz +669902f5c8efefbd5d5fd078db01e34355af3693e48659b89593da7db367c488 go1.20.6.linux-armv6l.tar.gz +a1b91a42a40bba54bfd5c96c23d72250e0c424038d0d2b5c7950b828b4905822 go1.20.6.linux-ppc64le.tar.gz +c5ec315cc57edd646f66d7079b51d3717db5bbeae4c83a771b709761db73688d go1.20.6.linux-s390x.tar.gz +315c49723f93295bfaff0c15179e2d6936fb1ffc6f92837321d0d608fe1b6b51 go1.20.6.windows-386.zip +b67dd7f2b4589701e53c98e348e1b4d9a7c3536dc316941172b2f0b60ae4ce5f go1.20.6.windows-amd64.zip +9027e52be386e779ef1a0c938994ee2361689496ac832100407238f5ed0fd82a go1.20.6.windows-arm64.zip fba08acc4027f69f07cef48fbff70b8a7ecdfaa1c2aba9ad3fb31d60d9f5d4bc golangci-lint-1.51.1-darwin-amd64.tar.gz 75b8f0ff3a4e68147156be4161a49d4576f1be37a0b506473f8c482140c1e7f2 golangci-lint-1.51.1-darwin-arm64.tar.gz diff --git a/build/ci.go b/build/ci.go index 7933724d74..e6293aef1d 100644 --- a/build/ci.go +++ b/build/ci.go @@ -139,7 +139,7 @@ var ( // This is the version of Go that will be downloaded by // // go run ci.go install -dlgo - dlgoVersion = "1.20.3" + dlgoVersion = "1.20.6" // This is the version of Go that will be used to bootstrap the PPA builder. // From 714f75943b96bace91cd3f1c51f592ba7863755a Mon Sep 17 00:00:00 2001 From: Delweng Date: Thu, 13 Jul 2023 14:55:31 +0800 Subject: [PATCH 66/90] eth, miner: rm redundant type declare (#27713) * eth: rm redundant type from array, slice, or map * miner: rm redundant type from array, slice, or map --- eth/api_backend.go | 2 +- miner/payload_building_test.go | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/eth/api_backend.go b/eth/api_backend.go index 02d1946e8e..80f5bcee61 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -294,7 +294,7 @@ func (b *EthAPIBackend) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscri } func (b *EthAPIBackend) SendTx(ctx context.Context, signedTx *types.Transaction) error { - return b.eth.txPool.Add([]*txpool.Transaction{&txpool.Transaction{Tx: signedTx}}, true, false)[0] + return b.eth.txPool.Add([]*txpool.Transaction{{Tx: signedTx}}, true, false)[0] } func (b *EthAPIBackend) GetPoolTransactions() (types.Transactions, error) { diff --git a/miner/payload_building_test.go b/miner/payload_building_test.go index b2f1d68f3c..6f57363441 100644 --- a/miner/payload_building_test.go +++ b/miner/payload_building_test.go @@ -84,48 +84,48 @@ func TestBuildPayload(t *testing.T) { func TestPayloadId(t *testing.T) { ids := make(map[string]int) for i, tt := range []*BuildPayloadArgs{ - &BuildPayloadArgs{ + { Parent: common.Hash{1}, Timestamp: 1, Random: common.Hash{0x1}, FeeRecipient: common.Address{0x1}, }, // Different parent - &BuildPayloadArgs{ + { Parent: common.Hash{2}, Timestamp: 1, Random: common.Hash{0x1}, FeeRecipient: common.Address{0x1}, }, // Different timestamp - &BuildPayloadArgs{ + { Parent: common.Hash{2}, Timestamp: 2, Random: common.Hash{0x1}, FeeRecipient: common.Address{0x1}, }, // Different Random - &BuildPayloadArgs{ + { Parent: common.Hash{2}, Timestamp: 2, Random: common.Hash{0x2}, FeeRecipient: common.Address{0x1}, }, // Different fee-recipient - &BuildPayloadArgs{ + { Parent: common.Hash{2}, Timestamp: 2, Random: common.Hash{0x2}, FeeRecipient: common.Address{0x2}, }, // Different withdrawals (non-empty) - &BuildPayloadArgs{ + { Parent: common.Hash{2}, Timestamp: 2, Random: common.Hash{0x2}, FeeRecipient: common.Address{0x2}, Withdrawals: []*types.Withdrawal{ - &types.Withdrawal{ + { Index: 0, Validator: 0, Address: common.Address{}, @@ -134,13 +134,13 @@ func TestPayloadId(t *testing.T) { }, }, // Different withdrawals (non-empty) - &BuildPayloadArgs{ + { Parent: common.Hash{2}, Timestamp: 2, Random: common.Hash{0x2}, FeeRecipient: common.Address{0x2}, Withdrawals: []*types.Withdrawal{ - &types.Withdrawal{ + { Index: 2, Validator: 0, Address: common.Address{}, From 4f0d8f0d15f06c70865cbfe1c993753ada979940 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi <1591639+s1na@users.noreply.github.com> Date: Thu, 13 Jul 2023 12:41:58 +0200 Subject: [PATCH 67/90] graphql: fix race in withdrawals test (#27706) graphql: fix race in test --- graphql/graphql_test.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/graphql/graphql_test.go b/graphql/graphql_test.go index 22d7181b0d..4bbfb7251d 100644 --- a/graphql/graphql_test.go +++ b/graphql/graphql_test.go @@ -445,19 +445,21 @@ func newGQLService(t *testing.T, stack *node.Node, shanghai bool, gspec *core.Ge TrieTimeout: 60 * time.Minute, SnapshotCache: 5, } - ethBackend, err := eth.New(stack, ethConf) - if err != nil { - t.Fatalf("could not create eth backend: %v", err) - } var engine consensus.Engine = ethash.NewFaker() if shanghai { engine = beacon.NewFaker() chainCfg := gspec.Config chainCfg.TerminalTotalDifficultyPassed = true chainCfg.TerminalTotalDifficulty = common.Big0 - shanghaiTime := uint64(0) + // GenerateChain will increment timestamps by 10. + // Shanghai upgrade at block 1. + shanghaiTime := uint64(5) chainCfg.ShanghaiTime = &shanghaiTime } + ethBackend, err := eth.New(stack, ethConf) + if err != nil { + t.Fatalf("could not create eth backend: %v", err) + } // Create some blocks and import them chain, _ := core.GenerateChain(params.AllEthashProtocolChanges, ethBackend.BlockChain().Genesis(), engine, ethBackend.ChainDb(), genBlocks, genfunc) From 10a136a4f99e80043a526edba4a432627f846d1a Mon Sep 17 00:00:00 2001 From: Delweng Date: Thu, 13 Jul 2023 18:42:48 +0800 Subject: [PATCH 68/90] internal: early exit if tx for getTxReceipt not found (#27712) internal/ethapi: fast exit if tx notfound --- internal/ethapi/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index ec29cf28f0..85df85d30a 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1681,7 +1681,7 @@ func (s *TransactionAPI) GetRawTransactionByHash(ctx context.Context, hash commo // GetTransactionReceipt returns the transaction receipt for the given transaction hash. func (s *TransactionAPI) GetTransactionReceipt(ctx context.Context, hash common.Hash) (map[string]interface{}, error) { tx, blockHash, blockNumber, index, err := s.b.GetTransaction(ctx, hash) - if err != nil { + if tx == nil || err != nil { // When the transaction doesn't exist, the RPC method should return JSON null // as per specification. return nil, nil From 5d035043ea53a083215735d1f1e3c9e9bdfd91a7 Mon Sep 17 00:00:00 2001 From: Delweng Date: Thu, 13 Jul 2023 18:47:32 +0800 Subject: [PATCH 69/90] internal/ethapi: add more testcase for GetTransactionReceipt (#27527) * internal/ethapi: testBackend reuse the same db Signed-off-by: jsvisa * internal/ethapi: implment GetTransaction Signed-off-by: jsvisa * internal/ethapi: implement GetReceipts Signed-off-by: jsvisa * internal/ethapi: insert receipts and setup txlookup Signed-off-by: jsvisa * internal/ethapi: add simple success tx receipt test Signed-off-by: jsvisa * internal/ethapi: add case create contract Signed-off-by: jsvisa * internal/ethapi: add contract call receipt Signed-off-by: jsvisa * internal/ethapi: add tx notfound Signed-off-by: jsvisa * internal/ethapi: add dynamic fee testcase * internal/ethapi: add accessList receipt * internal/ethapi: no need to insert receipt chain, no error * internal/ethapi: use HeaderByHash instead * internal/ethapi: add one more case --------- Signed-off-by: jsvisa --- internal/ethapi/api_test.go | 168 ++++++++++++++++++++++++++++++++++-- 1 file changed, 159 insertions(+), 9 deletions(-) diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go index fd662b5812..4d8c7e0707 100644 --- a/internal/ethapi/api_test.go +++ b/internal/ethapi/api_test.go @@ -21,6 +21,7 @@ import ( "crypto/ecdsa" "encoding/json" "errors" + "fmt" "math/big" "reflect" "testing" @@ -209,10 +210,7 @@ type testBackend struct { func newTestBackend(t *testing.T, n int, gspec *core.Genesis, generator func(i int, b *core.BlockGen)) *testBackend { var ( - engine = ethash.NewFaker() - backend = &testBackend{ - db: rawdb.NewMemoryDatabase(), - } + engine = ethash.NewFaker() cacheConfig = &core.CacheConfig{ TrieCleanLimit: 256, TrieDirtyLimit: 256, @@ -222,15 +220,17 @@ func newTestBackend(t *testing.T, n int, gspec *core.Genesis, generator func(i i } ) // Generate blocks for testing - _, blocks, _ := core.GenerateChainWithGenesis(gspec, engine, n, generator) - chain, err := core.NewBlockChain(backend.db, cacheConfig, gspec, nil, engine, vm.Config{}, nil, nil) + db, blocks, _ := core.GenerateChainWithGenesis(gspec, engine, n, generator) + txlookupLimit := uint64(0) + chain, err := core.NewBlockChain(db, cacheConfig, gspec, nil, engine, vm.Config{}, nil, &txlookupLimit) if err != nil { t.Fatalf("failed to create tester chain: %v", err) } if n, err := chain.InsertChain(blocks); err != nil { t.Fatalf("block %d: failed to insert into chain: %v", n, err) } - backend.chain = chain + + backend := &testBackend{db: db, chain: chain} return backend } @@ -323,7 +323,12 @@ func (b testBackend) StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOr } func (b testBackend) PendingBlockAndReceipts() (*types.Block, types.Receipts) { panic("implement me") } func (b testBackend) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) { - panic("implement me") + header, err := b.HeaderByHash(ctx, hash) + if header == nil || err != nil { + return nil, err + } + receipts := rawdb.ReadReceipts(b.db, hash, header.Number.Uint64(), header.Time, b.chain.Config()) + return receipts, nil } func (b testBackend) GetTd(ctx context.Context, hash common.Hash) *big.Int { if b.pending != nil && hash == b.pending.Hash() { @@ -356,7 +361,8 @@ func (b testBackend) SendTx(ctx context.Context, signedTx *types.Transaction) er panic("implement me") } func (b testBackend) GetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error) { - panic("implement me") + tx, blockHash, blockNumber, index := rawdb.ReadTransaction(b.db, txHash) + return tx, blockHash, blockNumber, index, nil } func (b testBackend) GetPoolTransactions() (types.Transactions, error) { panic("implement me") } func (b testBackend) GetPoolTransaction(txHash common.Hash) *types.Transaction { panic("implement me") } @@ -990,3 +996,147 @@ func TestRPCGetBlockOrHeader(t *testing.T) { require.JSONEqf(t, want, have, "test %d: json not match, want: %s, have: %s", i, want, have) } } + +func TestRPCGetTransactionReceipt(t *testing.T) { + t.Parallel() + + // Initialize test accounts + var ( + acc1Key, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") + acc2Key, _ = crypto.HexToECDSA("49a7b37aa6f6645917e7b807e9d1c00d4fa71f18343b0d4122a4d2df64dd6fee") + acc1Addr = crypto.PubkeyToAddress(acc1Key.PublicKey) + acc2Addr = crypto.PubkeyToAddress(acc2Key.PublicKey) + contract = common.HexToAddress("0000000000000000000000000000000000031ec7") + genesis = &core.Genesis{ + Config: params.TestChainConfig, + Alloc: core.GenesisAlloc{ + acc1Addr: {Balance: big.NewInt(params.Ether)}, + acc2Addr: {Balance: big.NewInt(params.Ether)}, + // // SPDX-License-Identifier: GPL-3.0 + // pragma solidity >=0.7.0 <0.9.0; + // + // contract Token { + // event Transfer(address indexed from, address indexed to, uint256 value); + // function transfer(address to, uint256 value) public returns (bool) { + // emit Transfer(msg.sender, to, value); + // return true; + // } + // } + contract: {Balance: big.NewInt(params.Ether), Code: common.FromHex("0x608060405234801561001057600080fd5b506004361061002b5760003560e01c8063a9059cbb14610030575b600080fd5b61004a6004803603810190610045919061016a565b610060565b60405161005791906101c5565b60405180910390f35b60008273ffffffffffffffffffffffffffffffffffffffff163373ffffffffffffffffffffffffffffffffffffffff167fddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef846040516100bf91906101ef565b60405180910390a36001905092915050565b600080fd5b600073ffffffffffffffffffffffffffffffffffffffff82169050919050565b6000610101826100d6565b9050919050565b610111816100f6565b811461011c57600080fd5b50565b60008135905061012e81610108565b92915050565b6000819050919050565b61014781610134565b811461015257600080fd5b50565b6000813590506101648161013e565b92915050565b60008060408385031215610181576101806100d1565b5b600061018f8582860161011f565b92505060206101a085828601610155565b9150509250929050565b60008115159050919050565b6101bf816101aa565b82525050565b60006020820190506101da60008301846101b6565b92915050565b6101e981610134565b82525050565b600060208201905061020460008301846101e0565b9291505056fea2646970667358221220b469033f4b77b9565ee84e0a2f04d496b18160d26034d54f9487e57788fd36d564736f6c63430008120033")}, + }, + } + genBlocks = 5 + signer = types.LatestSignerForChainID(params.TestChainConfig.ChainID) + txHashes = make([]common.Hash, genBlocks) + ) + backend := newTestBackend(t, genBlocks, genesis, func(i int, b *core.BlockGen) { + var ( + tx *types.Transaction + err error + ) + switch i { + case 0: + // transfer 1000wei + tx, err = types.SignTx(types.NewTx(&types.LegacyTx{Nonce: uint64(i), To: &acc2Addr, Value: big.NewInt(1000), Gas: params.TxGas, GasPrice: b.BaseFee(), Data: nil}), types.HomesteadSigner{}, acc1Key) + case 1: + // create contract + tx, err = types.SignTx(types.NewTx(&types.LegacyTx{Nonce: uint64(i), To: nil, Gas: 53100, GasPrice: b.BaseFee(), Data: common.FromHex("0x60806040")}), signer, acc1Key) + case 2: + // with logs + // transfer(address to, uint256 value) + data := fmt.Sprintf("0xa9059cbb%s%s", common.HexToHash(common.BigToAddress(big.NewInt(int64(i + 1))).Hex()).String()[2:], common.BytesToHash([]byte{byte(i + 11)}).String()[2:]) + tx, err = types.SignTx(types.NewTx(&types.LegacyTx{Nonce: uint64(i), To: &contract, Gas: 60000, GasPrice: b.BaseFee(), Data: common.FromHex(data)}), signer, acc1Key) + case 3: + // dynamic fee with logs + // transfer(address to, uint256 value) + data := fmt.Sprintf("0xa9059cbb%s%s", common.HexToHash(common.BigToAddress(big.NewInt(int64(i + 1))).Hex()).String()[2:], common.BytesToHash([]byte{byte(i + 11)}).String()[2:]) + fee := big.NewInt(500) + fee.Add(fee, b.BaseFee()) + tx, err = types.SignTx(types.NewTx(&types.DynamicFeeTx{Nonce: uint64(i), To: &contract, Gas: 60000, Value: big.NewInt(1), GasTipCap: big.NewInt(500), GasFeeCap: fee, Data: common.FromHex(data)}), signer, acc1Key) + case 4: + // access list with contract create + accessList := types.AccessList{{ + Address: contract, + StorageKeys: []common.Hash{{0}}, + }} + tx, err = types.SignTx(types.NewTx(&types.AccessListTx{Nonce: uint64(i), To: nil, Gas: 58100, GasPrice: b.BaseFee(), Data: common.FromHex("0x60806040"), AccessList: accessList}), signer, acc1Key) + } + if err != nil { + t.Errorf("failed to sign tx: %v", err) + } + if tx != nil { + b.AddTx(tx) + txHashes[i] = tx.Hash() + } + }) + api := NewTransactionAPI(backend, new(AddrLocker)) + blockHashes := make([]common.Hash, genBlocks+1) + ctx := context.Background() + for i := 0; i <= genBlocks; i++ { + header, err := backend.HeaderByNumber(ctx, rpc.BlockNumber(i)) + if err != nil { + t.Errorf("failed to get block: %d err: %v", i, err) + } + blockHashes[i] = header.Hash() + } + + var testSuite = []struct { + txHash common.Hash + want string + }{ + // 0. normal success + { + txHash: txHashes[0], + want: `{"blockHash":"0x1356e49a24d4504e450b303aa770f4ae13c29b9ffacaea1d7dd4043396229dd9","blockNumber":"0x1","contractAddress":null,"cumulativeGasUsed":"0x5208","effectiveGasPrice":"0x342770c0","from":"0x703c4b2bd70c169f5717101caee543299fc946c7","gasUsed":"0x5208","logs":[],"logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","status":"0x1","to":"0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e","transactionHash":"0x644a31c354391520d00e95b9affbbb010fc79ac268144ab8e28207f4cf51097e","transactionIndex":"0x0","type":"0x0"}`, + }, + // 1. create contract + { + txHash: txHashes[1], + want: `{"blockHash":"0x4fc27a4efa7fb8faa04b12b53ec8c8424ab4c21aab1323846365f000e8b4a594","blockNumber":"0x2","contractAddress":"0xae9bea628c4ce503dcfd7e305cab4e29e7476592","cumulativeGasUsed":"0xcf4e","effectiveGasPrice":"0x2db16291","from":"0x703c4b2bd70c169f5717101caee543299fc946c7","gasUsed":"0xcf4e","logs":[],"logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","status":"0x1","to":null,"transactionHash":"0x340e58cda5086495010b571fe25067fecc9954dc4ee3cedece00691fa3f5904a","transactionIndex":"0x0","type":"0x0"}`, + }, + // 2. with logs success + { + txHash: txHashes[2], + want: `{"blockHash":"0x73385c190219326907524b0020ef453ebc450eaa971ebce16f79e2d23e7e8d4d","blockNumber":"0x3","contractAddress":null,"cumulativeGasUsed":"0x5e28","effectiveGasPrice":"0x281c2534","from":"0x703c4b2bd70c169f5717101caee543299fc946c7","gasUsed":"0x5e28","logs":[{"address":"0x0000000000000000000000000000000000031ec7","topics":["0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef","0x000000000000000000000000703c4b2bd70c169f5717101caee543299fc946c7","0x0000000000000000000000000000000000000000000000000000000000000003"],"data":"0x000000000000000000000000000000000000000000000000000000000000000d","blockNumber":"0x3","transactionHash":"0x9dbf43ec9afc8d711932618616471088f66ba4f25fd5c672d97473d02dae967f","transactionIndex":"0x0","blockHash":"0x73385c190219326907524b0020ef453ebc450eaa971ebce16f79e2d23e7e8d4d","logIndex":"0x0","removed":false}],"logsBloom":"0x00000000000000000000008000000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000000000000800000000000000008000000000000000000000000000000000020000000080000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800000000000000000400000000002000000000000800000000000000000000000000000000000000000000000000000000000000000000000000000000020000000000000000000000000","status":"0x1","to":"0x0000000000000000000000000000000000031ec7","transactionHash":"0x9dbf43ec9afc8d711932618616471088f66ba4f25fd5c672d97473d02dae967f","transactionIndex":"0x0","type":"0x0"}`, + }, + // 3. dynamic tx with logs success + { + txHash: txHashes[3], + want: `{"blockHash":"0x77c3f8919590e0e68db4ce74a3da3140ac3e96dd3d078a48db1da4c08b07503d","blockNumber":"0x4","contractAddress":null,"cumulativeGasUsed":"0x538d","effectiveGasPrice":"0x2325c3e8","from":"0x703c4b2bd70c169f5717101caee543299fc946c7","gasUsed":"0x538d","logs":[],"logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","status":"0x0","to":"0x0000000000000000000000000000000000031ec7","transactionHash":"0x672e3e39adf23b5656989b7a36e54d54004b1866f53871113bc52e137edb9faf","transactionIndex":"0x0","type":"0x2"}`, + }, + // 4. access list tx with create contract + { + txHash: txHashes[4], + want: `{"blockHash":"0x08e23d8e3711a21fbb8becd7de22fda8fb0a49fba14e1be763d00f99063627e1","blockNumber":"0x5","contractAddress":"0xfdaa97661a584d977b4d3abb5370766ff5b86a18","cumulativeGasUsed":"0xe01a","effectiveGasPrice":"0x1ecb3f75","from":"0x703c4b2bd70c169f5717101caee543299fc946c7","gasUsed":"0xe01a","logs":[],"logsBloom":"0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000","status":"0x1","to":null,"transactionHash":"0x8f3c4e2663af0312d508ebd8587f0c88dccbbc8a9bcc322421ff4bc28c456a92","transactionIndex":"0x0","type":"0x1"}`, + }, + // 5. txhash empty + { + txHash: common.Hash{}, + want: `null`, + }, + // 6. txhash not found + { + txHash: common.HexToHash("deadbeef"), + want: `null`, + }, + } + + for i, tt := range testSuite { + var ( + result interface{} + err error + ) + result, err = api.GetTransactionReceipt(context.Background(), tt.txHash) + if err != nil { + t.Errorf("test %d: want no error, have %v", i, err) + continue + } + data, err := json.Marshal(result) + if err != nil { + t.Errorf("test %d: json marshal error", i) + continue + } + want, have := tt.want, string(data) + require.JSONEqf(t, want, have, "test %d: json not match, want: %s, have: %s", i, want, have) + } +} From 1e069cf8026a9f71b5f7e80959465e4b273d5806 Mon Sep 17 00:00:00 2001 From: Delweng Date: Thu, 13 Jul 2023 20:43:25 +0800 Subject: [PATCH 70/90] eth/tracers: add ReturnData in the tracer's response (#27704) The struct logger (or opcode tracer) was missing the return data field even if this was explicitly enabled by user via `"enableReturnData": true` in the config. This PR fixes this issue. Co-authored-by: Sina Mahmoodi --- eth/tracers/logger/logger.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/eth/tracers/logger/logger.go b/eth/tracers/logger/logger.go index c7f171c5bd..4c9b910a27 100644 --- a/eth/tracers/logger/logger.go +++ b/eth/tracers/logger/logger.go @@ -418,6 +418,7 @@ type StructLogRes struct { Depth int `json:"depth"` Error string `json:"error,omitempty"` Stack *[]string `json:"stack,omitempty"` + ReturnData string `json:"returnData,omitempty"` Memory *[]string `json:"memory,omitempty"` Storage *map[string]string `json:"storage,omitempty"` RefundCounter uint64 `json:"refund,omitempty"` @@ -443,6 +444,9 @@ func formatLogs(logs []StructLog) []StructLogRes { } formatted[index].Stack = &stack } + if trace.ReturnData != nil && len(trace.ReturnData) > 0 { + formatted[index].ReturnData = hexutil.Bytes(trace.ReturnData).String() + } if trace.Memory != nil { memory := make([]string, 0, (len(trace.Memory)+31)/32) for i := 0; i+32 <= len(trace.Memory); i += 32 { From 517ac886d45fc282f0f925f2c1a9d76eaadfcdfd Mon Sep 17 00:00:00 2001 From: Ryan Schneider Date: Thu, 13 Jul 2023 05:44:42 -0700 Subject: [PATCH 71/90] eth/tracers/native: panic on memory read in prestateTracer (#27691) Co-authored-by: Sina Mahmoodi --- .../internal/tracetest/calltrace_test.go | 97 +++++++++++-------- eth/tracers/native/call.go | 2 + eth/tracers/native/prestate.go | 7 +- 3 files changed, 67 insertions(+), 39 deletions(-) diff --git a/eth/tracers/internal/tracetest/calltrace_test.go b/eth/tracers/internal/tracetest/calltrace_test.go index bf1c31c08d..26c4455e30 100644 --- a/eth/tracers/internal/tracetest/calltrace_test.go +++ b/eth/tracers/internal/tracetest/calltrace_test.go @@ -302,13 +302,13 @@ func TestInternals(t *testing.T) { byte(vm.CALL), }, tracer: mkTracer("callTracer", nil), - want: `{"from":"0x000000000000000000000000000000000000feed","gas":"0xc350","gasUsed":"0x54d8","to":"0x00000000000000000000000000000000deadbeef","input":"0x","calls":[{"from":"0x00000000000000000000000000000000deadbeef","gas":"0x6cbf","gasUsed":"0x0","to":"0x00000000000000000000000000000000000000ff","input":"0x","value":"0x0","type":"CALL"}],"value":"0x0","type":"CALL"}`, + want: `{"from":"0x000000000000000000000000000000000000feed","gas":"0x13880","gasUsed":"0x54d8","to":"0x00000000000000000000000000000000deadbeef","input":"0x","calls":[{"from":"0x00000000000000000000000000000000deadbeef","gas":"0xe01a","gasUsed":"0x0","to":"0x00000000000000000000000000000000000000ff","input":"0x","value":"0x0","type":"CALL"}],"value":"0x0","type":"CALL"}`, }, { name: "Stack depletion in LOG0", code: []byte{byte(vm.LOG3)}, tracer: mkTracer("callTracer", json.RawMessage(`{ "withLog": true }`)), - want: `{"from":"0x000000000000000000000000000000000000feed","gas":"0xc350","gasUsed":"0xc350","to":"0x00000000000000000000000000000000deadbeef","input":"0x","error":"stack underflow (0 \u003c=\u003e 5)","value":"0x0","type":"CALL"}`, + want: `{"from":"0x000000000000000000000000000000000000feed","gas":"0x13880","gasUsed":"0x13880","to":"0x00000000000000000000000000000000deadbeef","input":"0x","error":"stack underflow (0 \u003c=\u003e 5)","value":"0x0","type":"CALL"}`, }, { name: "Mem expansion in LOG0", @@ -321,11 +321,11 @@ func TestInternals(t *testing.T) { byte(vm.LOG0), }, tracer: mkTracer("callTracer", json.RawMessage(`{ "withLog": true }`)), - want: `{"from":"0x000000000000000000000000000000000000feed","gas":"0xc350","gasUsed":"0x5b9e","to":"0x00000000000000000000000000000000deadbeef","input":"0x","logs":[{"address":"0x00000000000000000000000000000000deadbeef","topics":[],"data":"0x000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"}],"value":"0x0","type":"CALL"}`, + want: `{"from":"0x000000000000000000000000000000000000feed","gas":"0x13880","gasUsed":"0x5b9e","to":"0x00000000000000000000000000000000deadbeef","input":"0x","logs":[{"address":"0x00000000000000000000000000000000deadbeef","topics":[],"data":"0x000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"}],"value":"0x0","type":"CALL"}`, }, { // Leads to OOM on the prestate tracer - name: "Prestate-tracer - mem expansion in CREATE2", + name: "Prestate-tracer - CREATE2 OOM", code: []byte{ byte(vm.PUSH1), 0x1, byte(vm.PUSH1), 0x0, @@ -339,41 +339,62 @@ func TestInternals(t *testing.T) { byte(vm.PUSH1), 0x0, byte(vm.LOG0), }, - tracer: mkTracer("prestateTracer", json.RawMessage(`{ "withLog": true }`)), - want: `{"0x0000000000000000000000000000000000000000":{"balance":"0x0"},"0x000000000000000000000000000000000000feed":{"balance":"0x1c6bf52640350"},"0x00000000000000000000000000000000deadbeef":{"balance":"0x0","code":"0x6001600052600164ffffffffff60016000f560ff6000a0"}}`, + tracer: mkTracer("prestateTracer", nil), + want: `{"0x0000000000000000000000000000000000000000":{"balance":"0x0"},"0x000000000000000000000000000000000000feed":{"balance":"0x1c6bf52647880"},"0x00000000000000000000000000000000deadbeef":{"balance":"0x0","code":"0x6001600052600164ffffffffff60016000f560ff6000a0"}}`, + }, + { + // CREATE2 which requires padding memory by prestate tracer + name: "Prestate-tracer - CREATE2 Memory padding", + code: []byte{ + byte(vm.PUSH1), 0x1, + byte(vm.PUSH1), 0x0, + byte(vm.MSTORE), + byte(vm.PUSH1), 0x1, + byte(vm.PUSH1), 0xff, + byte(vm.PUSH1), 0x1, + byte(vm.PUSH1), 0x0, + byte(vm.CREATE2), + byte(vm.PUSH1), 0xff, + byte(vm.PUSH1), 0x0, + byte(vm.LOG0), + }, + tracer: mkTracer("prestateTracer", nil), + want: `{"0x0000000000000000000000000000000000000000":{"balance":"0x0"},"0x000000000000000000000000000000000000feed":{"balance":"0x1c6bf52647880"},"0x00000000000000000000000000000000deadbeef":{"balance":"0x0","code":"0x6001600052600160ff60016000f560ff6000a0"},"0x91ff9a805d36f54e3e272e230f3e3f5c1b330804":{"balance":"0x0"}}`, }, } { - _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), - core.GenesisAlloc{ - to: core.GenesisAccount{ - Code: tc.code, - }, - origin: core.GenesisAccount{ - Balance: big.NewInt(500000000000000), - }, - }, false) - evm := vm.NewEVM(context, txContext, statedb, params.MainnetChainConfig, vm.Config{Tracer: tc.tracer}) - msg := &core.Message{ - To: &to, - From: origin, - Value: big.NewInt(0), - GasLimit: 50000, - GasPrice: big.NewInt(0), - GasFeeCap: big.NewInt(0), - GasTipCap: big.NewInt(0), - SkipAccountChecks: false, - } - st := core.NewStateTransition(evm, msg, new(core.GasPool).AddGas(msg.GasLimit)) - if _, err := st.TransitionDb(); err != nil { - t.Fatalf("test %v: failed to execute transaction: %v", tc.name, err) - } - // Retrieve the trace result and compare against the expected - res, err := tc.tracer.GetResult() - if err != nil { - t.Fatalf("test %v: failed to retrieve trace result: %v", tc.name, err) - } - if string(res) != tc.want { - t.Fatalf("test %v: trace mismatch\n have: %v\n want: %v\n", tc.name, string(res), tc.want) - } + t.Run(tc.name, func(t *testing.T) { + _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), + core.GenesisAlloc{ + to: core.GenesisAccount{ + Code: tc.code, + }, + origin: core.GenesisAccount{ + Balance: big.NewInt(500000000000000), + }, + }, false) + evm := vm.NewEVM(context, txContext, statedb, params.MainnetChainConfig, vm.Config{Tracer: tc.tracer}) + msg := &core.Message{ + To: &to, + From: origin, + Value: big.NewInt(0), + GasLimit: 80000, + GasPrice: big.NewInt(0), + GasFeeCap: big.NewInt(0), + GasTipCap: big.NewInt(0), + SkipAccountChecks: false, + } + st := core.NewStateTransition(evm, msg, new(core.GasPool).AddGas(msg.GasLimit)) + if _, err := st.TransitionDb(); err != nil { + t.Fatalf("test %v: failed to execute transaction: %v", tc.name, err) + } + // Retrieve the trace result and compare against the expected + res, err := tc.tracer.GetResult() + if err != nil { + t.Fatalf("test %v: failed to retrieve trace result: %v", tc.name, err) + } + if string(res) != tc.want { + t.Errorf("test %v: trace mismatch\n have: %v\n want: %v\n", tc.name, string(res), tc.want) + } + }) } } diff --git a/eth/tracers/native/call.go b/eth/tracers/native/call.go index 4ac03e512f..34cf027aca 100644 --- a/eth/tracers/native/call.go +++ b/eth/tracers/native/call.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/tracers" + "github.com/ethereum/go-ethereum/log" ) //go:generate go run github.com/fjl/gencodec -type callFrame -field-override callFrameMarshaling -out gen_callframe_json.go @@ -183,6 +184,7 @@ func (t *callTracer) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, sco data, err := tracers.GetMemoryCopyPadded(scope.Memory, int64(mStart.Uint64()), int64(mSize.Uint64())) if err != nil { // mSize was unrealistically large + log.Warn("failed to copy CREATE2 input", "err", err, "tracer", "callTracer", "offset", mStart, "size", mSize) return } diff --git a/eth/tracers/native/prestate.go b/eth/tracers/native/prestate.go index b71d5d6215..82451c40a6 100644 --- a/eth/tracers/native/prestate.go +++ b/eth/tracers/native/prestate.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/tracers" + "github.com/ethereum/go-ethereum/log" ) //go:generate go run github.com/fjl/gencodec -type account -field-override accountMarshaling -out gen_account_json.go @@ -165,7 +166,11 @@ func (t *prestateTracer) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, case stackLen >= 4 && op == vm.CREATE2: offset := stackData[stackLen-2] size := stackData[stackLen-3] - init := scope.Memory.GetCopy(int64(offset.Uint64()), int64(size.Uint64())) + init, err := tracers.GetMemoryCopyPadded(scope.Memory, int64(offset.Uint64()), int64(size.Uint64())) + if err != nil { + log.Warn("failed to copy CREATE2 input", "err", err, "tracer", "prestateTracer", "offset", offset, "size", size) + return + } inithash := crypto.Keccak256(init) salt := stackData[stackLen-4] addr := crypto.CreateAddress2(caller, salt.Bytes32(), inithash) From e1fe6bc8469c626afaa86b1dfb819737e980a574 Mon Sep 17 00:00:00 2001 From: ucwong Date: Thu, 13 Jul 2023 15:34:29 +0100 Subject: [PATCH 72/90] go.sum: go mod tidy (#27717) --- go.sum | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/go.sum b/go.sum index b1ef5999b2..b7a3d1df40 100644 --- a/go.sum +++ b/go.sum @@ -19,8 +19,6 @@ github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrU github.com/VictoriaMetrics/fastcache v1.6.0 h1:C/3Oi3EiBCqufydp1neRZkqcwmEiuRT9c3fqvvgKm5o= github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= @@ -74,7 +72,6 @@ github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811 h1:ytcWPaNPhNoG github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811/go.mod h1:Nb5lgvnQ2+oGlE/EyZy4+2/CxRh9KfvCXnag1vtpxVM= github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ= github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= -github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ= github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= @@ -87,8 +84,6 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/crate-crypto/go-ipa v0.0.0-20220523130400-f11357ae11c7 h1:6IrxszG5G+O7zhtkWxq6+unVvnrm1fqV2Pe+T95DUzw= -github.com/crate-crypto/go-ipa v0.0.0-20220523130400-f11357ae11c7/go.mod h1:gFnFS95y8HstDP6P9pPwzrxOOC5TRDkwbM+ao15ChAI= github.com/crate-crypto/go-ipa v0.0.0-20230601170251-1830d0757c80 h1:DuBDHVjgGMPki7bAyh91+3cF1Vh34sAEdH8JQgbc2R0= github.com/crate-crypto/go-ipa v0.0.0-20230601170251-1830d0757c80/go.mod h1:gzbVz57IDJgQ9rLQwfSk696JGWof8ftznEL9GoAv3NI= github.com/crate-crypto/go-kzg-4844 v0.3.0 h1:UBlWE0CgyFqqzTI+IFyCzA7A3Zw4iip6uzRv5NIXG0A= @@ -149,8 +144,6 @@ github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61/go.mod h1:Q0X6pkwTILD github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= -github.com/gballet/go-verkle v0.0.0-20220902153445-097bd83b7732 h1:AB7YjNrzlVHsYz06zCULVV2zYCEft82P86dSmtwxKL0= -github.com/gballet/go-verkle v0.0.0-20220902153445-097bd83b7732/go.mod h1:o/XfIXWi4/GqbQirfRm5uTbXMG5NpqxkxblnbZ+QM9I= github.com/gballet/go-verkle v0.0.0-20230607174250-df487255f46b h1:vMT47RYsrftsHSTQhqXwC3BYflo38OLC3Y4LtXtLyU0= github.com/gballet/go-verkle v0.0.0-20230607174250-df487255f46b/go.mod h1:CDncRYVRSDqwakm282WEkjfaAj1hxU/v5RXxk5nXOiI= github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= @@ -165,7 +158,6 @@ github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclK github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJpoZOs= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= @@ -271,16 +263,12 @@ github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e h1:UvSe12bq+U github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= -github.com/karalabe/usb v0.0.2 h1:M6QQBNxF+CQ8OFvxrT90BA0qBOXymndZnk5q235mFc4= -github.com/karalabe/usb v0.0.2/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= github.com/karalabe/usb v0.0.3-0.20230711191512-61db3e06439c h1:AqsttAyEyIEsNz5WLRwuRwjiT5CMDUfLk6cFJDVPebs= github.com/karalabe/usb v0.0.3-0.20230711191512-61db3e06439c/go.mod h1:Od972xHfMJowv7NGVDiWVxk2zxnWgjLlJzE+F4F7AGU= github.com/kataras/golog v0.0.10/go.mod h1:yJ8YKCmyL+nWjERB90Qwn+bdyBZsaQwU3bTVFgkFIp8= @@ -351,10 +339,8 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hzifhks= github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 h1:shk/vn9oCoOTmwcouEdwIeOtOGA/ELRUw/GwvxwfT+0= @@ -540,8 +526,6 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= -golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -583,8 +567,6 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -653,7 +635,6 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From c40ab6af72ce282020d03c33e8273ea9b03d58f6 Mon Sep 17 00:00:00 2001 From: ucwong Date: Fri, 14 Jul 2023 13:06:51 +0100 Subject: [PATCH 73/90] core/blockchain: fast to snap in comments (#27722) --- core/blockchain.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 61e55d575d..b760a301fb 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -344,7 +344,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis needRewind = true low = fullBlock.Number.Uint64() } - // In fast sync, it may happen that ancient data has been written to the + // In snap sync, it may happen that ancient data has been written to the // ancient store, but the LastFastBlock has not been updated, truncate the // extra data here. snapBlock := bc.CurrentSnapBlock() @@ -472,7 +472,7 @@ func (bc *BlockChain) loadLastState() error { } bc.hc.SetCurrentHeader(headHeader) - // Restore the last known head fast block + // Restore the last known head snap block bc.currentSnapBlock.Store(headBlock.Header()) headFastBlockGauge.Update(int64(headBlock.NumberU64())) @@ -507,21 +507,21 @@ func (bc *BlockChain) loadLastState() error { } log.Info("Loaded most recent local block", "number", headBlock.Number(), "hash", headBlock.Hash(), "td", blockTd, "age", common.PrettyAge(time.Unix(int64(headBlock.Time()), 0))) if headBlock.Hash() != currentSnapBlock.Hash() { - fastTd := bc.GetTd(currentSnapBlock.Hash(), currentSnapBlock.Number.Uint64()) - log.Info("Loaded most recent local snap block", "number", currentSnapBlock.Number, "hash", currentSnapBlock.Hash(), "td", fastTd, "age", common.PrettyAge(time.Unix(int64(currentSnapBlock.Time), 0))) + snapTd := bc.GetTd(currentSnapBlock.Hash(), currentSnapBlock.Number.Uint64()) + log.Info("Loaded most recent local snap block", "number", currentSnapBlock.Number, "hash", currentSnapBlock.Hash(), "td", snapTd, "age", common.PrettyAge(time.Unix(int64(currentSnapBlock.Time), 0))) } if currentFinalBlock != nil { finalTd := bc.GetTd(currentFinalBlock.Hash(), currentFinalBlock.Number.Uint64()) log.Info("Loaded most recent local finalized block", "number", currentFinalBlock.Number, "hash", currentFinalBlock.Hash(), "td", finalTd, "age", common.PrettyAge(time.Unix(int64(currentFinalBlock.Time), 0))) } if pivot := rawdb.ReadLastPivotNumber(bc.db); pivot != nil { - log.Info("Loaded last fast-sync pivot marker", "number", *pivot) + log.Info("Loaded last snap-sync pivot marker", "number", *pivot) } return nil } // SetHead rewinds the local chain to a new head. Depending on whether the node -// was fast synced or full synced and in which state, the method will try to +// was snap synced or full synced and in which state, the method will try to // delete minimal data from disk whilst retaining chain consistency. func (bc *BlockChain) SetHead(head uint64) error { if _, err := bc.setHeadBeyondRoot(head, 0, common.Hash{}, false); err != nil { @@ -542,7 +542,7 @@ func (bc *BlockChain) SetHead(head uint64) error { } // SetHeadWithTimestamp rewinds the local chain to a new head that has at max -// the given timestamp. Depending on whether the node was fast synced or full +// the given timestamp. Depending on whether the node was snap synced or full // synced and in which state, the method will try to delete minimal data from // disk whilst retaining chain consistency. func (bc *BlockChain) SetHeadWithTimestamp(timestamp uint64) error { @@ -588,7 +588,7 @@ func (bc *BlockChain) SetSafe(header *types.Header) { // setHeadBeyondRoot rewinds the local chain to a new head with the extra condition // that the rewind must pass the specified state root. This method is meant to be // used when rewinding with snapshots enabled to ensure that we go back further than -// persistent disk layer. Depending on whether the node was fast synced or full, and +// persistent disk layer. Depending on whether the node was snap synced or full, and // in which state, the method will try to delete minimal data from disk whilst // retaining chain consistency. // @@ -676,7 +676,7 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha bc.currentBlock.Store(newHeadBlock.Header()) headBlockGauge.Update(int64(newHeadBlock.NumberU64())) } - // Rewind the fast block in a simpleton way to the target head + // Rewind the snap block in a simpleton way to the target head if currentSnapBlock := bc.CurrentSnapBlock(); currentSnapBlock != nil && header.Number.Uint64() < currentSnapBlock.Number.Uint64() { newHeadSnapBlock := bc.GetBlock(header.Hash(), header.Number.Uint64()) // If either blocks reached nil, reset to the genesis state @@ -734,7 +734,7 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha } } else { // Rewind the chain to the requested head and keep going backwards until a - // block with a state is found or fast sync pivot is passed + // block with a state is found or snap sync pivot is passed if time > 0 { log.Warn("Rewinding blockchain to timestamp", "target", time) bc.hc.SetHeadWithTimestamp(time, updateFn, delFn) @@ -868,7 +868,7 @@ func (bc *BlockChain) ExportN(w io.Writer, first uint64, last uint64) error { // writeHeadBlock injects a new head block into the current block chain. This method // assumes that the block is indeed a true head. It will also reset the head -// header and the head fast sync block to this very same block if they are older +// header and the head snap sync block to this very same block if they are older // or if they are on a different side chain. // // Note, this function assumes that the `mu` mutex is held! @@ -1049,7 +1049,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ size = int64(0) ) - // updateHead updates the head fast sync block if the inserted blocks are better + // updateHead updates the head snap sync block if the inserted blocks are better // and returns an indicator whether the inserted blocks are canonical. updateHead := func(head *types.Block) bool { if !bc.chainmu.TryLock() { @@ -1148,7 +1148,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ if err := bc.db.Sync(); err != nil { return 0, err } - // Update the current fast block because all block data is now present in DB. + // Update the current snap block because all block data is now present in DB. previousSnapBlock := bc.CurrentSnapBlock().Number.Uint64() if !updateHead(blockChain[len(blockChain)-1]) { // We end up here if the header chain has reorg'ed, and the blocks/receipts @@ -1578,11 +1578,11 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool) (int, error) block, err = it.next() } // The remaining blocks are still known blocks, the only scenario here is: - // During the fast sync, the pivot point is already submitted but rollback + // During the snap sync, the pivot point is already submitted but rollback // happens. Then node resets the head full block to a lower height via `rollback` // and leaves a few known blocks in the database. // - // When node runs a fast sync again, it can re-import a batch of known blocks via + // When node runs a snap sync again, it can re-import a batch of known blocks via // `insertChain` while a part of them have higher total difficulty than current // head full block(new pivot point). for block != nil && bc.skipBlock(err, it) { From 60ecf48dd4496ad3576f25b616a091dee2ca4916 Mon Sep 17 00:00:00 2001 From: Seungbae Yu Date: Fri, 14 Jul 2023 21:58:27 +0900 Subject: [PATCH 74/90] cmd/bootnode, p2p: support for alternate mapped ports (#26359) This changes the port mapping procedure such that, when the requested port is unavailable an alternative port suggested by the router is used instead. We now also repeatedly request the external IP from the router in order to catch any IP changes. Co-authored-by: Felix Lange --- cmd/bootnode/main.go | 78 ++++++++++++++--- p2p/nat/nat.go | 23 ++--- p2p/nat/natpmp.go | 21 ++--- p2p/nat/natupnp.go | 41 ++++++++- p2p/server.go | 57 +++++-------- p2p/server_nat.go | 187 +++++++++++++++++++++++++++++++++++++++++ p2p/server_nat_test.go | 102 ++++++++++++++++++++++ p2p/server_test.go | 10 +++ 8 files changed, 446 insertions(+), 73 deletions(-) create mode 100644 p2p/server_nat.go create mode 100644 p2p/server_nat_test.go diff --git a/cmd/bootnode/main.go b/cmd/bootnode/main.go index 748113aa48..bddfcbe2a5 100644 --- a/cmd/bootnode/main.go +++ b/cmd/bootnode/main.go @@ -23,6 +23,7 @@ import ( "fmt" "net" "os" + "time" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/crypto" @@ -108,20 +109,18 @@ func main() { utils.Fatalf("-ListenUDP: %v", err) } - realaddr := conn.LocalAddr().(*net.UDPAddr) - if natm != nil { - if !realaddr.IP.IsLoopback() { - go nat.Map(natm, nil, "udp", realaddr.Port, realaddr.Port, "ethereum discovery") - } - if ext, err := natm.ExternalIP(); err == nil { - realaddr = &net.UDPAddr{IP: ext, Port: realaddr.Port} + db, _ := enode.OpenDB("") + ln := enode.NewLocalNode(db, nodeKey) + + listenerAddr := conn.LocalAddr().(*net.UDPAddr) + if natm != nil && !listenerAddr.IP.IsLoopback() { + natAddr := doPortMapping(natm, ln, listenerAddr) + if natAddr != nil { + listenerAddr = natAddr } } - printNotice(&nodeKey.PublicKey, *realaddr) - - db, _ := enode.OpenDB("") - ln := enode.NewLocalNode(db, nodeKey) + printNotice(&nodeKey.PublicKey, *listenerAddr) cfg := discover.Config{ PrivateKey: nodeKey, NetRestrict: restrictList, @@ -148,3 +147,60 @@ func printNotice(nodeKey *ecdsa.PublicKey, addr net.UDPAddr) { fmt.Println("Note: you're using cmd/bootnode, a developer tool.") fmt.Println("We recommend using a regular node as bootstrap node for production deployments.") } + +func doPortMapping(natm nat.Interface, ln *enode.LocalNode, addr *net.UDPAddr) *net.UDPAddr { + const ( + protocol = "udp" + name = "ethereum discovery" + ) + newLogger := func(external int, internal int) log.Logger { + return log.New("proto", protocol, "extport", external, "intport", internal, "interface", natm) + } + + var ( + intport = addr.Port + extaddr = &net.UDPAddr{IP: addr.IP, Port: addr.Port} + mapTimeout = nat.DefaultMapTimeout + log = newLogger(addr.Port, intport) + ) + addMapping := func() { + // Get the external address. + var err error + extaddr.IP, err = natm.ExternalIP() + if err != nil { + log.Debug("Couldn't get external IP", "err", err) + return + } + // Create the mapping. + p, err := natm.AddMapping(protocol, extaddr.Port, intport, name, mapTimeout) + if err != nil { + log.Debug("Couldn't add port mapping", "err", err) + return + } + if p != uint16(extaddr.Port) { + extaddr.Port = int(p) + log = newLogger(extaddr.Port, intport) + log.Info("NAT mapped alternative port") + } else { + log.Info("NAT mapped port") + } + // Update IP/port information of the local node. + ln.SetStaticIP(extaddr.IP) + ln.SetFallbackUDP(extaddr.Port) + } + + // Perform mapping once, synchronously. + log.Info("Attempting port mapping") + addMapping() + + // Refresh the mapping periodically. + go func() { + refresh := time.NewTimer(mapTimeout) + for range refresh.C { + addMapping() + refresh.Reset(mapTimeout) + } + }() + + return extaddr +} diff --git a/p2p/nat/nat.go b/p2p/nat/nat.go index ad4c36582a..61b6922988 100644 --- a/p2p/nat/nat.go +++ b/p2p/nat/nat.go @@ -38,7 +38,7 @@ type Interface interface { // protocol is "UDP" or "TCP". Some implementations allow setting // a display name for the mapping. The mapping may be removed by // the gateway when its lifetime ends. - AddMapping(protocol string, extport, intport int, name string, lifetime time.Duration) error + AddMapping(protocol string, extport, intport int, name string, lifetime time.Duration) (uint16, error) DeleteMapping(protocol string, extport, intport int) error // ExternalIP should return the external (Internet-facing) @@ -91,20 +91,23 @@ func Parse(spec string) (Interface, error) { } const ( - mapTimeout = 10 * time.Minute + DefaultMapTimeout = 10 * time.Minute ) // Map adds a port mapping on m and keeps it alive until c is closed. // This function is typically invoked in its own goroutine. +// +// Note that Map does not handle the situation where the NAT interface assigns a different +// external port than the requested one. func Map(m Interface, c <-chan struct{}, protocol string, extport, intport int, name string) { log := log.New("proto", protocol, "extport", extport, "intport", intport, "interface", m) - refresh := time.NewTimer(mapTimeout) + refresh := time.NewTimer(DefaultMapTimeout) defer func() { refresh.Stop() log.Debug("Deleting port mapping") m.DeleteMapping(protocol, extport, intport) }() - if err := m.AddMapping(protocol, extport, intport, name, mapTimeout); err != nil { + if _, err := m.AddMapping(protocol, extport, intport, name, DefaultMapTimeout); err != nil { log.Debug("Couldn't add port mapping", "err", err) } else { log.Info("Mapped network port") @@ -117,10 +120,10 @@ func Map(m Interface, c <-chan struct{}, protocol string, extport, intport int, } case <-refresh.C: log.Trace("Refreshing port mapping") - if err := m.AddMapping(protocol, extport, intport, name, mapTimeout); err != nil { + if _, err := m.AddMapping(protocol, extport, intport, name, DefaultMapTimeout); err != nil { log.Debug("Couldn't add port mapping", "err", err) } - refresh.Reset(mapTimeout) + refresh.Reset(DefaultMapTimeout) } } } @@ -135,8 +138,8 @@ func (n ExtIP) String() string { return fmt.Sprintf("ExtIP(%v)", ne // These do nothing. -func (ExtIP) AddMapping(string, int, int, string, time.Duration) error { return nil } -func (ExtIP) DeleteMapping(string, int, int) error { return nil } +func (ExtIP) AddMapping(string, int, int, string, time.Duration) (uint16, error) { return 0, nil } +func (ExtIP) DeleteMapping(string, int, int) error { return nil } // Any returns a port mapper that tries to discover any supported // mechanism on the local network. @@ -193,9 +196,9 @@ func startautodisc(what string, doit func() Interface) Interface { return &autodisc{what: what, doit: doit} } -func (n *autodisc) AddMapping(protocol string, extport, intport int, name string, lifetime time.Duration) error { +func (n *autodisc) AddMapping(protocol string, extport, intport int, name string, lifetime time.Duration) (uint16, error) { if err := n.wait(); err != nil { - return err + return 0, err } return n.found.AddMapping(protocol, extport, intport, name, lifetime) } diff --git a/p2p/nat/natpmp.go b/p2p/nat/natpmp.go index 40f2aff44e..97601c99dc 100644 --- a/p2p/nat/natpmp.go +++ b/p2p/nat/natpmp.go @@ -44,28 +44,21 @@ func (n *pmp) ExternalIP() (net.IP, error) { return response.ExternalIPAddress[:], nil } -func (n *pmp) AddMapping(protocol string, extport, intport int, name string, lifetime time.Duration) error { +func (n *pmp) AddMapping(protocol string, extport, intport int, name string, lifetime time.Duration) (uint16, error) { if lifetime <= 0 { - return fmt.Errorf("lifetime must not be <= 0") + return 0, fmt.Errorf("lifetime must not be <= 0") } // Note order of port arguments is switched between our // AddMapping and the client's AddPortMapping. res, err := n.c.AddPortMapping(strings.ToLower(protocol), intport, extport, int(lifetime/time.Second)) if err != nil { - return err + return 0, err } - // NAT-PMP maps an alternative available port number if the requested - // port is already mapped to another address and returns success. In this - // case, we return an error because there is no way to return the new port - // to the caller. - if uint16(extport) != res.MappedExternalPort { - // Destroy the mapping in NAT device. - n.c.AddPortMapping(strings.ToLower(protocol), intport, 0, 0) - return fmt.Errorf("port %d already mapped to another address (%s)", extport, protocol) - } - - return nil + // NAT-PMP maps an alternative available port number if the requested port + // is already mapped to another address and returns success. Handling of + // alternate port numbers is done by the caller. + return res.MappedExternalPort, nil } func (n *pmp) DeleteMapping(protocol string, extport, intport int) (err error) { diff --git a/p2p/nat/natupnp.go b/p2p/nat/natupnp.go index a8de00e978..c90c4f3de8 100644 --- a/p2p/nat/natupnp.go +++ b/p2p/nat/natupnp.go @@ -19,6 +19,8 @@ package nat import ( "errors" "fmt" + "math" + "math/rand" "net" "strings" "sync" @@ -40,6 +42,7 @@ type upnp struct { client upnpClient mu sync.Mutex lastReqTime time.Time + rand *rand.Rand } type upnpClient interface { @@ -76,18 +79,50 @@ func (n *upnp) ExternalIP() (addr net.IP, err error) { return ip, nil } -func (n *upnp) AddMapping(protocol string, extport, intport int, desc string, lifetime time.Duration) error { +func (n *upnp) AddMapping(protocol string, extport, intport int, desc string, lifetime time.Duration) (uint16, error) { ip, err := n.internalAddress() if err != nil { - return nil // TODO: Shouldn't we return the error? + return 0, nil // TODO: Shouldn't we return the error? } protocol = strings.ToUpper(protocol) lifetimeS := uint32(lifetime / time.Second) n.DeleteMapping(protocol, extport, intport) - return n.withRateLimit(func() error { + err = n.withRateLimit(func() error { return n.client.AddPortMapping("", uint16(extport), protocol, uint16(intport), ip.String(), true, desc, lifetimeS) }) + if err == nil { + return uint16(extport), nil + } + + return uint16(extport), n.withRateLimit(func() error { + p, err := n.addAnyPortMapping(protocol, extport, intport, ip, desc, lifetimeS) + if err == nil { + extport = int(p) + } + return err + }) +} + +func (n *upnp) addAnyPortMapping(protocol string, extport, intport int, ip net.IP, desc string, lifetimeS uint32) (uint16, error) { + if client, ok := n.client.(*internetgateway2.WANIPConnection2); ok { + return client.AddAnyPortMapping("", uint16(extport), protocol, uint16(intport), ip.String(), true, desc, lifetimeS) + } + // It will retry with a random port number if the client does + // not support AddAnyPortMapping. + extport = n.randomPort() + err := n.client.AddPortMapping("", uint16(extport), protocol, uint16(intport), ip.String(), true, desc, lifetimeS) + if err != nil { + return 0, err + } + return uint16(extport), nil +} + +func (n *upnp) randomPort() int { + if n.rand == nil { + n.rand = rand.New(rand.NewSource(time.Now().UnixNano())) + } + return n.rand.Intn(math.MaxUint16-10000) + 10000 } func (n *upnp) internalAddress() (net.IP, error) { diff --git a/p2p/server.go b/p2p/server.go index bd0fd70950..d4e2be6783 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -195,6 +195,9 @@ type Server struct { discmix *enode.FairMix dialsched *dialScheduler + // This is read by the NAT port mapping loop. + portMappingRegister chan *portMapping + // Channels into the run loop. quit chan struct{} addtrusted chan *enode.Node @@ -483,6 +486,8 @@ func (srv *Server) Start() (err error) { if err := srv.setupLocalNode(); err != nil { return err } + srv.setupPortMapping() + if srv.ListenAddr != "" { if err := srv.setupListening(); err != nil { return err @@ -521,24 +526,6 @@ func (srv *Server) setupLocalNode() error { srv.localnode.Set(e) } } - switch srv.NAT.(type) { - case nil: - // No NAT interface, do nothing. - case nat.ExtIP: - // ExtIP doesn't block, set the IP right away. - ip, _ := srv.NAT.ExternalIP() - srv.localnode.SetStaticIP(ip) - default: - // Ask the router about the IP. This takes a while and blocks startup, - // do it in the background. - srv.loopWG.Add(1) - go func() { - defer srv.loopWG.Done() - if ip, err := srv.NAT.ExternalIP(); err == nil { - srv.localnode.SetStaticIP(ip) - } - }() - } return nil } @@ -656,14 +643,15 @@ func (srv *Server) setupListening() error { srv.ListenAddr = listener.Addr().String() // Update the local node record and map the TCP listening port if NAT is configured. - if tcp, ok := listener.Addr().(*net.TCPAddr); ok { + tcp, isTCP := listener.Addr().(*net.TCPAddr) + if isTCP { srv.localnode.Set(enr.TCP(tcp.Port)) - if !tcp.IP.IsLoopback() && srv.NAT != nil { - srv.loopWG.Add(1) - go func() { - nat.Map(srv.NAT, srv.quit, "tcp", tcp.Port, tcp.Port, "ethereum p2p") - srv.loopWG.Done() - }() + if !tcp.IP.IsLoopback() && !tcp.IP.IsPrivate() { + srv.portMappingRegister <- &portMapping{ + protocol: "TCP", + name: "ethereum p2p", + port: tcp.Port, + } } } @@ -688,18 +676,17 @@ func (srv *Server) setupUDPListening() (*net.UDPConn, error) { if err != nil { return nil, err } - realaddr := conn.LocalAddr().(*net.UDPAddr) - srv.log.Debug("UDP listener up", "addr", realaddr) - if srv.NAT != nil { - if !realaddr.IP.IsLoopback() { - srv.loopWG.Add(1) - go func() { - nat.Map(srv.NAT, srv.quit, "udp", realaddr.Port, realaddr.Port, "ethereum discovery") - srv.loopWG.Done() - }() + laddr := conn.LocalAddr().(*net.UDPAddr) + srv.localnode.SetFallbackUDP(laddr.Port) + srv.log.Debug("UDP listener up", "addr", laddr) + if !laddr.IP.IsLoopback() && !laddr.IP.IsPrivate() { + srv.portMappingRegister <- &portMapping{ + protocol: "UDP", + name: "ethereum peer discovery", + port: laddr.Port, } } - srv.localnode.SetFallbackUDP(realaddr.Port) + return conn, nil } diff --git a/p2p/server_nat.go b/p2p/server_nat.go new file mode 100644 index 0000000000..354597cc7a --- /dev/null +++ b/p2p/server_nat.go @@ -0,0 +1,187 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package p2p + +import ( + "net" + "time" + + "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p/enr" + "github.com/ethereum/go-ethereum/p2p/nat" +) + +const ( + portMapDuration = 10 * time.Minute + portMapRefreshInterval = 8 * time.Minute + portMapRetryInterval = 5 * time.Minute + extipRetryInterval = 2 * time.Minute +) + +type portMapping struct { + protocol string + name string + port int + + // for use by the portMappingLoop goroutine: + extPort int // the mapped port returned by the NAT interface + nextTime mclock.AbsTime +} + +// setupPortMapping starts the port mapping loop if necessary. +// Note: this needs to be called after the LocalNode instance has been set on the server. +func (srv *Server) setupPortMapping() { + // portMappingRegister will receive up to two values: one for the TCP port if + // listening is enabled, and one more for enabling UDP port mapping if discovery is + // enabled. We make it buffered to avoid blocking setup while a mapping request is in + // progress. + srv.portMappingRegister = make(chan *portMapping, 2) + + switch srv.NAT.(type) { + case nil: + // No NAT interface configured. + srv.loopWG.Add(1) + go srv.consumePortMappingRequests() + + case nat.ExtIP: + // ExtIP doesn't block, set the IP right away. + ip, _ := srv.NAT.ExternalIP() + srv.localnode.SetStaticIP(ip) + srv.loopWG.Add(1) + go srv.consumePortMappingRequests() + + default: + srv.loopWG.Add(1) + go srv.portMappingLoop() + } +} + +func (srv *Server) consumePortMappingRequests() { + defer srv.loopWG.Done() + for { + select { + case <-srv.quit: + return + case <-srv.portMappingRegister: + } + } +} + +// portMappingLoop manages port mappings for UDP and TCP. +func (srv *Server) portMappingLoop() { + defer srv.loopWG.Done() + + newLogger := func(p string, e int, i int) log.Logger { + return log.New("proto", p, "extport", e, "intport", i, "interface", srv.NAT) + } + + var ( + mappings = make(map[string]*portMapping, 2) + refresh = mclock.NewAlarm(srv.clock) + extip = mclock.NewAlarm(srv.clock) + lastExtIP net.IP + ) + extip.Schedule(srv.clock.Now()) + defer func() { + refresh.Stop() + extip.Stop() + for _, m := range mappings { + if m.extPort != 0 { + log := newLogger(m.protocol, m.extPort, m.port) + log.Debug("Deleting port mapping") + srv.NAT.DeleteMapping(m.protocol, m.extPort, m.port) + } + } + }() + + for { + // Schedule refresh of existing mappings. + for _, m := range mappings { + refresh.Schedule(m.nextTime) + } + + select { + case <-srv.quit: + return + + case <-extip.C(): + extip.Schedule(srv.clock.Now().Add(extipRetryInterval)) + ip, err := srv.NAT.ExternalIP() + if err != nil { + log.Debug("Couldn't get external IP", "err", err, "interface", srv.NAT) + } else if !ip.Equal(lastExtIP) { + log.Debug("External IP changed", "ip", extip, "interface", srv.NAT) + } else { + return + } + // Here, we either failed to get the external IP, or it has changed. + lastExtIP = ip + srv.localnode.SetStaticIP(ip) + // Ensure port mappings are refreshed in case we have moved to a new network. + for _, m := range mappings { + m.nextTime = srv.clock.Now() + } + + case m := <-srv.portMappingRegister: + if m.protocol != "TCP" && m.protocol != "UDP" { + panic("unknown NAT protocol name: " + m.protocol) + } + mappings[m.protocol] = m + m.nextTime = srv.clock.Now() + + case <-refresh.C(): + for _, m := range mappings { + if srv.clock.Now() < m.nextTime { + continue + } + + external := m.port + if m.extPort != 0 { + external = m.extPort + } + log := newLogger(m.protocol, external, m.port) + + log.Trace("Attempting port mapping") + p, err := srv.NAT.AddMapping(m.protocol, external, m.port, m.name, portMapDuration) + if err != nil { + log.Debug("Couldn't add port mapping", "err", err) + m.extPort = 0 + m.nextTime = srv.clock.Now().Add(portMapRetryInterval) + continue + } + // It was mapped! + m.extPort = int(p) + m.nextTime = srv.clock.Now().Add(portMapRefreshInterval) + if external != m.extPort { + log = newLogger(m.protocol, m.extPort, m.port) + log.Info("NAT mapped alternative port") + } else { + log.Info("NAT mapped port") + } + + // Update port in local ENR. + switch m.protocol { + case "TCP": + srv.localnode.Set(enr.TCP(m.extPort)) + case "UDP": + srv.localnode.SetFallbackUDP(m.extPort) + } + } + } + } +} diff --git a/p2p/server_nat_test.go b/p2p/server_nat_test.go new file mode 100644 index 0000000000..de935fcfc5 --- /dev/null +++ b/p2p/server_nat_test.go @@ -0,0 +1,102 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package p2p + +import ( + "net" + "sync/atomic" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/internal/testlog" + "github.com/ethereum/go-ethereum/log" +) + +func TestServerPortMapping(t *testing.T) { + clock := new(mclock.Simulated) + mockNAT := &mockNAT{mappedPort: 30000} + srv := Server{ + Config: Config{ + PrivateKey: newkey(), + NoDial: true, + ListenAddr: ":0", + NAT: mockNAT, + Logger: testlog.Logger(t, log.LvlTrace), + clock: clock, + }, + } + err := srv.Start() + if err != nil { + t.Fatal(err) + } + defer srv.Stop() + + // Wait for the port mapping to be registered. Synchronization with the port mapping + // goroutine works like this: For each iteration, we allow other goroutines to run and + // also advance the virtual clock by 1 second. Waiting stops when the NAT interface + // has received some requests, or when the clock reaches a timeout. + deadline := clock.Now().Add(portMapRefreshInterval) + for clock.Now() < deadline && mockNAT.mapRequests.Load() < 2 { + time.Sleep(10 * time.Millisecond) + clock.Run(1 * time.Second) + } + + if mockNAT.ipRequests.Load() == 0 { + t.Fatal("external IP was never requested") + } + reqCount := mockNAT.mapRequests.Load() + if reqCount != 2 { + t.Error("wrong request count:", reqCount) + } + enr := srv.LocalNode().Node() + if enr.IP().String() != "192.0.2.0" { + t.Error("wrong IP in ENR:", enr.IP()) + } + if enr.TCP() != 30000 { + t.Error("wrong TCP port in ENR:", enr.TCP()) + } + if enr.UDP() != 30000 { + t.Error("wrong UDP port in ENR:", enr.UDP()) + } +} + +type mockNAT struct { + mappedPort uint16 + mapRequests atomic.Int32 + unmapRequests atomic.Int32 + ipRequests atomic.Int32 +} + +func (m *mockNAT) AddMapping(protocol string, extport, intport int, name string, lifetime time.Duration) (uint16, error) { + m.mapRequests.Add(1) + return m.mappedPort, nil +} + +func (m *mockNAT) DeleteMapping(protocol string, extport, intport int) error { + m.unmapRequests.Add(1) + return nil +} + +func (m *mockNAT) ExternalIP() (net.IP, error) { + m.ipRequests.Add(1) + return net.ParseIP("192.0.2.0"), nil +} + +func (m *mockNAT) String() string { + return "mockNAT" +} diff --git a/p2p/server_test.go b/p2p/server_test.go index c8bf4c941c..a0491e984a 100644 --- a/p2p/server_test.go +++ b/p2p/server_test.go @@ -24,6 +24,8 @@ import ( "math/rand" "net" "reflect" + "strconv" + "strings" "testing" "time" @@ -224,6 +226,14 @@ func TestServerRemovePeerDisconnect(t *testing.T) { srv2.Start() defer srv2.Stop() + s := strings.Split(srv2.ListenAddr, ":") + if len(s) != 2 { + t.Fatal("invalid ListenAddr") + } + if port, err := strconv.Atoi(s[1]); err == nil { + srv2.localnode.Set(enr.TCP(uint16(port))) + } + if !syncAddPeer(srv1, srv2.Self()) { t.Fatal("peer not connected") } From 13c0305106438656b57afdf6a10cf7904f7f9f15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96mer=20Faruk=20Irmak?= Date: Fri, 14 Jul 2023 19:10:16 +0300 Subject: [PATCH 75/90] metrics: NilTimer should still run the function to be timed (#27723) --- metrics/timer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metrics/timer.go b/metrics/timer.go index a63c9dfb6c..2e1a9be472 100644 --- a/metrics/timer.go +++ b/metrics/timer.go @@ -123,7 +123,7 @@ func (NilTimer) Stop() {} func (NilTimer) Sum() int64 { return 0 } // Time is a no-op. -func (NilTimer) Time(func()) {} +func (NilTimer) Time(f func()) { f() } // Update is a no-op. func (NilTimer) Update(time.Duration) {} From 47b9f1b4ae41f7e2974f61f12292d228af48a06d Mon Sep 17 00:00:00 2001 From: Delweng Date: Sat, 15 Jul 2023 01:14:47 +0800 Subject: [PATCH 76/90] cmd/geth: use automaxprocs to apply cpu quota correctly (#27506) It is usually best to set GOMAXPROCS to the number of available CPU cores. However, setting it like that does not work well when the process is quota-limited to a certain number of CPUs. The automaxprocs library configures GOMAXPROCS, taking such limits into account. --- cmd/geth/main.go | 3 +++ go.mod | 1 + go.sum | 3 +++ 3 files changed, 7 insertions(+) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index b9584e70b3..05e6b727d6 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -44,6 +44,9 @@ import ( _ "github.com/ethereum/go-ethereum/eth/tracers/js" _ "github.com/ethereum/go-ethereum/eth/tracers/native" + // Automatically set GOMAXPROCS to match Linux container CPU quota. + _ "go.uber.org/automaxprocs" + "github.com/urfave/cli/v2" ) diff --git a/go.mod b/go.mod index 0b79746f57..47535d7923 100644 --- a/go.mod +++ b/go.mod @@ -60,6 +60,7 @@ require ( github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 github.com/tyler-smith/go-bip39 v1.1.0 github.com/urfave/cli/v2 v2.17.2-0.20221006022127-8f469abc00aa + go.uber.org/automaxprocs v1.5.2 golang.org/x/crypto v0.9.0 golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc golang.org/x/sync v0.3.0 diff --git a/go.sum b/go.sum index b7a3d1df40..4fce0746cc 100644 --- a/go.sum +++ b/go.sum @@ -374,6 +374,7 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -460,6 +461,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.uber.org/automaxprocs v1.5.2 h1:2LxUOGiR3O6tw8ui5sZa2LAaHnsviZdVOUZw4fvbnME= +go.uber.org/automaxprocs v1.5.2/go.mod h1:eRbA25aqJrxAbsLO0xy5jVwPt7FQnRgjW+efnwa1WM0= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= From 34d507215951fb3f4a5983b65e127577989a6db8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=96mer=20Faruk=20Irmak?= Date: Fri, 14 Jul 2023 20:19:03 +0300 Subject: [PATCH 77/90] metrics: NilResettingTimer.Time should execute the timed function (#27724) --- metrics/resetting_timer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metrics/resetting_timer.go b/metrics/resetting_timer.go index 924ba11052..8e23c8eeea 100644 --- a/metrics/resetting_timer.go +++ b/metrics/resetting_timer.go @@ -66,7 +66,7 @@ func (NilResettingTimer) Snapshot() ResettingTimer { } // Time is a no-op. -func (NilResettingTimer) Time(func()) {} +func (NilResettingTimer) Time(f func()) { f() } // Update is a no-op. func (NilResettingTimer) Update(time.Duration) {} From 00408f74796ea1da2bba16ae30ecd5c322fa45ff Mon Sep 17 00:00:00 2001 From: Yurie <28754813+yuriechan@users.noreply.github.com> Date: Sat, 15 Jul 2023 23:33:46 +0900 Subject: [PATCH 78/90] accounts/abi/bind/backends: fix goroutine leak in unit test (#27705) --- accounts/abi/bind/backends/simulated_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/accounts/abi/bind/backends/simulated_test.go b/accounts/abi/bind/backends/simulated_test.go index 698bfc5765..11900a6cf5 100644 --- a/accounts/abi/bind/backends/simulated_test.go +++ b/accounts/abi/bind/backends/simulated_test.go @@ -161,6 +161,7 @@ func TestAdjustTime(t *testing.T) { func TestNewAdjustTimeFail(t *testing.T) { testAddr := crypto.PubkeyToAddress(testKey.PublicKey) sim := simTestBackend(testAddr) + defer sim.blockchain.Stop() // Create tx and send head, _ := sim.HeaderByNumber(context.Background(), nil) // Should be child's, good enough From d233b6b23af33dd64fda3237377130e3b5991ad8 Mon Sep 17 00:00:00 2001 From: jwasinger Date: Sat, 15 Jul 2023 16:35:30 +0200 Subject: [PATCH 79/90] core: replace instances of 'suicide' with 'selfdestruct' to improve code consistency. (#27716) --------- Co-authored-by: Martin Holst Swende Co-authored-by: lightclient <14004106+lightclient@users.noreply.github.com> --- accounts/abi/bind/backend.go | 2 +- cmd/evm/internal/t8ntool/execution.go | 2 +- core/state/journal.go | 10 +++++----- core/state/state_object.go | 14 +++++++------- core/state/state_test.go | 4 ++-- core/state/statedb.go | 23 +++++++++++------------ core/state/statedb_fuzz_test.go | 4 ++-- core/state/statedb_test.go | 8 ++++---- core/vm/gas_table.go | 2 +- core/vm/instructions.go | 4 ++-- core/vm/interface.go | 6 +++--- core/vm/operations_acl.go | 2 +- eth/tracers/native/call_flat.go | 4 ++-- tests/state_test_util.go | 4 ++-- 14 files changed, 44 insertions(+), 45 deletions(-) diff --git a/accounts/abi/bind/backend.go b/accounts/abi/bind/backend.go index c16990f395..34530a6455 100644 --- a/accounts/abi/bind/backend.go +++ b/accounts/abi/bind/backend.go @@ -29,7 +29,7 @@ import ( var ( // ErrNoCode is returned by call and transact operations for which the requested // recipient contract to operate on does not exist in the state db or does not - // have any code associated with it (i.e. suicided). + // have any code associated with it (i.e. self-destructed). ErrNoCode = errors.New("no contract code at given address") // ErrNoPendingState is raised when attempting to perform a pending state action diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index fb85794b76..338427bd46 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -239,7 +239,7 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, if miningReward >= 0 { // Add mining reward. The mining reward may be `0`, which only makes a difference in the cases // where - // - the coinbase suicided, or + // - the coinbase self-destructed, or // - there are only 'bad' transactions, which aren't executed. In those cases, // the coinbase gets no txfee, so isn't created, and thus needs to be touched var ( diff --git a/core/state/journal.go b/core/state/journal.go index 95a218b3a3..f7ec26625b 100644 --- a/core/state/journal.go +++ b/core/state/journal.go @@ -100,9 +100,9 @@ type ( prevAccountOrigin []byte prevStorageOrigin map[common.Hash][]byte } - suicideChange struct { + selfDestructChange struct { account *common.Address - prev bool // whether account had already suicided + prev bool // whether account had already self-destructed prevbalance *big.Int } @@ -184,15 +184,15 @@ func (ch resetObjectChange) dirtied() *common.Address { return ch.account } -func (ch suicideChange) revert(s *StateDB) { +func (ch selfDestructChange) revert(s *StateDB) { obj := s.getStateObject(*ch.account) if obj != nil { - obj.suicided = ch.prev + obj.selfDestructed = ch.prev obj.setBalance(ch.prevbalance) } } -func (ch suicideChange) dirtied() *common.Address { +func (ch selfDestructChange) dirtied() *common.Address { return ch.account } diff --git a/core/state/state_object.go b/core/state/state_object.go index 2529ec6929..ed5f2b2945 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -78,12 +78,12 @@ type stateObject struct { // Cache flags. dirtyCode bool // true if the code was updated - // Flag whether the account was marked as suicided. The suicided account + // Flag whether the account was marked as self-destructed. The self-destructed account // is still accessible in the scope of same transaction. - suicided bool + selfDestructed bool - // Flag whether the account was marked as deleted. The suicided account - // or the account is considered as empty will be marked as deleted at + // Flag whether the account was marked as deleted. A self-destructed account + // or an account that is considered as empty will be marked as deleted at // the end of transaction and no longer accessible anymore. deleted bool } @@ -116,8 +116,8 @@ func (s *stateObject) EncodeRLP(w io.Writer) error { return rlp.Encode(w, &s.data) } -func (s *stateObject) markSuicided() { - s.suicided = true +func (s *stateObject) markSelfdestructed() { + s.selfDestructed = true } func (s *stateObject) touch() { @@ -446,7 +446,7 @@ func (s *stateObject) deepCopy(db *StateDB) *stateObject { obj.dirtyStorage = s.dirtyStorage.Copy() obj.originStorage = s.originStorage.Copy() obj.pendingStorage = s.pendingStorage.Copy() - obj.suicided = s.suicided + obj.selfDestructed = s.selfDestructed obj.dirtyCode = s.dirtyCode obj.deleted = s.deleted return obj diff --git a/core/state/state_test.go b/core/state/state_test.go index b6c11728be..dc9df371d3 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -208,7 +208,7 @@ func TestSnapshot2(t *testing.T) { so0.SetBalance(big.NewInt(42)) so0.SetNonce(43) so0.SetCode(crypto.Keccak256Hash([]byte{'c', 'a', 'f', 'e'}), []byte{'c', 'a', 'f', 'e'}) - so0.suicided = false + so0.selfDestructed = false so0.deleted = false state.setStateObject(so0) @@ -220,7 +220,7 @@ func TestSnapshot2(t *testing.T) { so1.SetBalance(big.NewInt(52)) so1.SetNonce(53) so1.SetCode(crypto.Keccak256Hash([]byte{'c', 'a', 'f', 'e', '2'}), []byte{'c', 'a', 'f', 'e', '2'}) - so1.suicided = true + so1.selfDestructed = true so1.deleted = true state.setStateObject(so1) diff --git a/core/state/statedb.go b/core/state/statedb.go index 71a863add7..4b17e70255 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -271,7 +271,7 @@ func (s *StateDB) SubRefund(gas uint64) { } // Exist reports whether the given account address exists in the state. -// Notably this also returns true for suicided accounts. +// Notably this also returns true for self-destructed accounts. func (s *StateDB) Exist(addr common.Address) bool { return s.getStateObject(addr) != nil } @@ -397,10 +397,10 @@ func (s *StateDB) StorageTrie(addr common.Address) (Trie, error) { return cpy.getTrie(s.db) } -func (s *StateDB) HasSuicided(addr common.Address) bool { +func (s *StateDB) HasSelfDestructed(addr common.Address) bool { stateObject := s.getStateObject(addr) if stateObject != nil { - return stateObject.suicided + return stateObject.selfDestructed } return false } @@ -474,24 +474,23 @@ func (s *StateDB) SetStorage(addr common.Address, storage map[common.Hash]common } } -// Suicide marks the given account as suicided. +// SelfDestruct marks the given account as selfdestructed. // This clears the account balance. // // The account's state object is still available until the state is committed, -// getStateObject will return a non-nil account after Suicide. -func (s *StateDB) Suicide(addr common.Address) bool { +// getStateObject will return a non-nil account after SelfDestruct. +func (s *StateDB) SelfDestruct(addr common.Address) { stateObject := s.getStateObject(addr) if stateObject == nil { - return false + return } - s.journal.append(suicideChange{ + s.journal.append(selfDestructChange{ account: &addr, - prev: stateObject.suicided, + prev: stateObject.selfDestructed, prevbalance: new(big.Int).Set(stateObject.Balance()), }) - stateObject.markSuicided() + stateObject.markSelfdestructed() stateObject.data.Balance = new(big.Int) - return true } // SetTransientState sets transient storage for a given account. It @@ -892,7 +891,7 @@ func (s *StateDB) Finalise(deleteEmptyObjects bool) { // Thus, we can safely ignore it here continue } - if obj.suicided || (deleteEmptyObjects && obj.empty()) { + if obj.selfDestructed || (deleteEmptyObjects && obj.empty()) { obj.deleted = true // We need to maintain account deletions explicitly (will remain diff --git a/core/state/statedb_fuzz_test.go b/core/state/statedb_fuzz_test.go index d014b2b3cc..4dca96efd3 100644 --- a/core/state/statedb_fuzz_test.go +++ b/core/state/statedb_fuzz_test.go @@ -95,9 +95,9 @@ func newStateTestAction(addr common.Address, r *rand.Rand, index int) testAction }, }, { - name: "Suicide", + name: "Selfdestruct", fn: func(a testAction, s *StateDB) { - s.Suicide(addr) + s.SelfDestruct(addr) }, }, } diff --git a/core/state/statedb_test.go b/core/state/statedb_test.go index 8290a154ff..6848b01cb1 100644 --- a/core/state/statedb_test.go +++ b/core/state/statedb_test.go @@ -301,9 +301,9 @@ func newTestAction(addr common.Address, r *rand.Rand) testAction { }, }, { - name: "Suicide", + name: "SelfDestruct", fn: func(a testAction, s *StateDB) { - s.Suicide(addr) + s.SelfDestruct(addr) }, }, { @@ -453,7 +453,7 @@ func (test *snapshotTest) checkEqual(state, checkstate *StateDB) error { } // Check basic accessor methods. checkeq("Exist", state.Exist(addr), checkstate.Exist(addr)) - checkeq("HasSuicided", state.HasSuicided(addr), checkstate.HasSuicided(addr)) + checkeq("HasSelfdestructed", state.HasSelfDestructed(addr), checkstate.HasSelfDestructed(addr)) checkeq("GetBalance", state.GetBalance(addr), checkstate.GetBalance(addr)) checkeq("GetNonce", state.GetNonce(addr), checkstate.GetNonce(addr)) checkeq("GetCode", state.GetCode(addr), checkstate.GetCode(addr)) @@ -727,7 +727,7 @@ func TestDeleteCreateRevert(t *testing.T) { state, _ = New(root, state.db, state.snaps) // Simulate self-destructing in one transaction, then create-reverting in another - state.Suicide(addr) + state.SelfDestruct(addr) state.Finalise(true) id := state.Snapshot() diff --git a/core/vm/gas_table.go b/core/vm/gas_table.go index 79aba9d4ff..5153c8b7a3 100644 --- a/core/vm/gas_table.go +++ b/core/vm/gas_table.go @@ -472,7 +472,7 @@ func gasSelfdestruct(evm *EVM, contract *Contract, stack *Stack, mem *Memory, me } } - if !evm.StateDB.HasSuicided(contract.Address()) { + if !evm.StateDB.HasSelfDestructed(contract.Address()) { evm.StateDB.AddRefund(params.SelfdestructRefundGas) } return gas, nil diff --git a/core/vm/instructions.go b/core/vm/instructions.go index 505aef4127..4b6514ec8c 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -408,7 +408,7 @@ func opExtCodeCopy(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) // emptyCodeHash. If the precompile account is not transferred any amount on a private or // customized chain, the return value will be zero. // -// 5. Caller tries to get the code hash for an account which is marked as suicided +// 5. Caller tries to get the code hash for an account which is marked as self-destructed // in the current transaction, the code hash of this account should be returned. // // 6. Caller tries to get the code hash for an account which is marked as deleted, this @@ -821,7 +821,7 @@ func opSelfdestruct(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext beneficiary := scope.Stack.pop() balance := interpreter.evm.StateDB.GetBalance(scope.Contract.Address()) interpreter.evm.StateDB.AddBalance(beneficiary.Bytes20(), balance) - interpreter.evm.StateDB.Suicide(scope.Contract.Address()) + interpreter.evm.StateDB.SelfDestruct(scope.Contract.Address()) if tracer := interpreter.evm.Config.Tracer; tracer != nil { tracer.CaptureEnter(SELFDESTRUCT, scope.Contract.Address(), beneficiary.Bytes20(), []byte{}, 0, balance) tracer.CaptureExit([]byte{}, 0, nil) diff --git a/core/vm/interface.go b/core/vm/interface.go index b83f78307e..05fb480f96 100644 --- a/core/vm/interface.go +++ b/core/vm/interface.go @@ -51,11 +51,11 @@ type StateDB interface { GetTransientState(addr common.Address, key common.Hash) common.Hash SetTransientState(addr common.Address, key, value common.Hash) - Suicide(common.Address) bool - HasSuicided(common.Address) bool + SelfDestruct(common.Address) + HasSelfDestructed(common.Address) bool // Exist reports whether the given account exists in state. - // Notably this should also return true for suicided accounts. + // Notably this should also return true for self-destructed accounts. Exist(common.Address) bool // Empty returns whether the given account is empty. Empty // is defined according to EIP161 (balance = nonce = code = 0). diff --git a/core/vm/operations_acl.go b/core/vm/operations_acl.go index 551e1f5f11..04c6409ebd 100644 --- a/core/vm/operations_acl.go +++ b/core/vm/operations_acl.go @@ -235,7 +235,7 @@ func makeSelfdestructGasFn(refundsEnabled bool) gasFunc { if evm.StateDB.Empty(address) && evm.StateDB.GetBalance(contract.Address()).Sign() != 0 { gas += params.CreateBySelfdestructGas } - if refundsEnabled && !evm.StateDB.HasSuicided(contract.Address()) { + if refundsEnabled && !evm.StateDB.HasSelfDestructed(contract.Address()) { evm.StateDB.AddRefund(params.SelfdestructRefundGas) } return gas, nil diff --git a/eth/tracers/native/call_flat.go b/eth/tracers/native/call_flat.go index 8612c23387..266ab99001 100644 --- a/eth/tracers/native/call_flat.go +++ b/eth/tracers/native/call_flat.go @@ -248,7 +248,7 @@ func flatFromNested(input *callFrame, traceAddress []int, convertErrs bool, ctx case vm.CREATE, vm.CREATE2: frame = newFlatCreate(input) case vm.SELFDESTRUCT: - frame = newFlatSuicide(input) + frame = newFlatSelfdestruct(input) case vm.CALL, vm.STATICCALL, vm.CALLCODE, vm.DELEGATECALL: frame = newFlatCall(input) default: @@ -330,7 +330,7 @@ func newFlatCall(input *callFrame) *flatCallFrame { } } -func newFlatSuicide(input *callFrame) *flatCallFrame { +func newFlatSelfdestruct(input *callFrame) *flatCallFrame { return &flatCallFrame{ Type: "suicide", Action: flatCallAction{ diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 667e951b66..6956fe5b96 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -197,7 +197,7 @@ func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config, snapshotter bo } post := t.json.Post[subtest.Fork][subtest.Index] // N.B: We need to do this in a two-step process, because the first Commit takes care - // of suicides, and we need to touch the coinbase _after_ it has potentially suicided. + // of self-destructs, and we need to touch the coinbase _after_ it has potentially self-destructed. if root != common.Hash(post.Root) { return snaps, statedb, fmt.Errorf("post state root mismatch: got %x, want %x", root, post.Root) } @@ -275,7 +275,7 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh } // Add 0-value mining reward. This only makes a difference in the cases // where - // - the coinbase suicided, or + // - the coinbase self-destructed, or // - there are only 'bad' transactions, which aren't executed. In those cases, // the coinbase gets no txfee, so isn't created, and thus needs to be touched statedb.AddBalance(block.Coinbase(), new(big.Int)) From 99e000cb132e25c29c34b9ed55d99f1b132e73c2 Mon Sep 17 00:00:00 2001 From: ucwong Date: Sat, 15 Jul 2023 16:45:57 +0100 Subject: [PATCH 80/90] core/vm: use types.EmptyCodeHash (#27729) --- core/vm/evm.go | 7 ++----- core/vm/instructions_test.go | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/core/vm/evm.go b/core/vm/evm.go index 36336d8cbd..3f3c3751f8 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -21,15 +21,12 @@ import ( "sync/atomic" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" "github.com/holiman/uint256" ) -// emptyCodeHash is used by create to ensure deployment is disallowed to already -// deployed contract addresses (relevant after the account abstraction). -var emptyCodeHash = crypto.Keccak256Hash(nil) - type ( // CanTransferFunc is the signature of a transfer guard function CanTransferFunc func(StateDB, common.Address, *big.Int) bool @@ -438,7 +435,7 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, } // Ensure there's no existing contract already at the designated address contractHash := evm.StateDB.GetCodeHash(address) - if evm.StateDB.GetNonce(address) != 0 || (contractHash != (common.Hash{}) && contractHash != emptyCodeHash) { + if evm.StateDB.GetNonce(address) != 0 || (contractHash != (common.Hash{}) && contractHash != types.EmptyCodeHash) { return nil, common.Address{}, 0, ErrContractAddressCollision } // Create a new account on the state diff --git a/core/vm/instructions_test.go b/core/vm/instructions_test.go index 36886cb9ed..807073336d 100644 --- a/core/vm/instructions_test.go +++ b/core/vm/instructions_test.go @@ -725,7 +725,7 @@ func TestRandom(t *testing.T) { for _, tt := range []testcase{ {name: "empty hash", random: common.Hash{}}, {name: "1", random: common.Hash{0}}, - {name: "emptyCodeHash", random: emptyCodeHash}, + {name: "emptyCodeHash", random: types.EmptyCodeHash}, {name: "hash(0x010203)", random: crypto.Keccak256Hash([]byte{0x01, 0x02, 0x03})}, } { var ( From b058cf454b3bdc7e770e2b3cec83a0bcb48f55ee Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Sat, 15 Jul 2023 23:27:36 +0200 Subject: [PATCH 81/90] core, tests: EIP-4844 transaction processing logic (#27721) This updates the reference tests to the latest version and also adds logic to process EIP-4844 blob transactions into the state transition. We are now passing most Cancun fork tests. Co-authored-by: Marius van der Wijden Co-authored-by: Felix Lange --- cmd/evm/staterunner.go | 1 + core/block_validator.go | 14 +--- core/error.go | 4 + core/evm.go | 21 ++--- core/gen_genesis.go | 64 +++++++++------ core/genesis.go | 52 ++++++++---- core/state_processor.go | 2 +- core/state_processor_test.go | 154 ++++++++++++++++------------------- core/state_transition.go | 73 ++++++++++++++--- core/vm/evm.go | 15 ++-- tests/gen_sttransaction.go | 13 +++ tests/state_test.go | 13 +-- tests/state_test_util.go | 25 +++--- tests/testdata | 2 +- 14 files changed, 268 insertions(+), 185 deletions(-) diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index ed102e6054..231830f6d7 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -117,6 +117,7 @@ func runStateTest(fname string, cfg vm.Config, jsonOut, dump bool) error { // Test failed, mark as so and dump any state to aid debugging result.Pass, result.Error = false, err.Error() if dump && s != nil { + s, _ = state.New(*result.Root, s.Database(), nil) dump := s.RawDump(nil) result.State = &dump } diff --git a/core/block_validator.go b/core/block_validator.go index 1bca444451..4ea623cf47 100644 --- a/core/block_validator.go +++ b/core/block_validator.go @@ -86,18 +86,8 @@ func (v *BlockValidator) ValidateBody(block *types.Block) error { for _, tx := range block.Transactions() { // Count the number of blobs to validate against the header's dataGasUsed blobs += len(tx.BlobHashes()) - - // Validate the data blobs individually too - if tx.Type() == types.BlobTxType { - if len(tx.BlobHashes()) == 0 { - return errors.New("no-blob blob transaction present in block body") - } - for _, hash := range tx.BlobHashes() { - if hash[0] != params.BlobTxHashVersion { - return fmt.Errorf("blob hash version mismatch (have %d, supported %d)", hash[0], params.BlobTxHashVersion) - } - } - } + // The individual checks for blob validity (version-check + not empty) + // happens in the state_transition check. } if header.DataGasUsed != nil { if want := *header.DataGasUsed / params.BlobTxDataGasPerBlob; uint64(blobs) != want { // div because the header is surely good vs the body might be bloated diff --git a/core/error.go b/core/error.go index 872ba8d365..8967432a40 100644 --- a/core/error.go +++ b/core/error.go @@ -100,4 +100,8 @@ var ( // ErrSenderNoEOA is returned if the sender of a transaction is a contract. ErrSenderNoEOA = errors.New("sender not an eoa") + + // ErrBlobFeeCapTooLow is returned if the transaction fee cap is less than the + // data gas fee of the block. + ErrBlobFeeCapTooLow = errors.New("max fee per data gas less than block data gas fee") ) diff --git a/core/evm.go b/core/evm.go index b7ff779029..f4e6aa43f4 100644 --- a/core/evm.go +++ b/core/evm.go @@ -56,16 +56,17 @@ func NewEVMBlockContext(header *types.Header, chain ChainContext, author *common random = &header.MixDigest } return vm.BlockContext{ - CanTransfer: CanTransfer, - Transfer: Transfer, - GetHash: GetHashFn(header, chain), - Coinbase: beneficiary, - BlockNumber: new(big.Int).Set(header.Number), - Time: header.Time, - Difficulty: new(big.Int).Set(header.Difficulty), - BaseFee: baseFee, - GasLimit: header.GasLimit, - Random: random, + CanTransfer: CanTransfer, + Transfer: Transfer, + GetHash: GetHashFn(header, chain), + Coinbase: beneficiary, + BlockNumber: new(big.Int).Set(header.Number), + Time: header.Time, + Difficulty: new(big.Int).Set(header.Difficulty), + BaseFee: baseFee, + GasLimit: header.GasLimit, + Random: random, + ExcessDataGas: header.ExcessDataGas, } } diff --git a/core/gen_genesis.go b/core/gen_genesis.go index 4e0844e889..9a8b279d25 100644 --- a/core/gen_genesis.go +++ b/core/gen_genesis.go @@ -18,19 +18,21 @@ var _ = (*genesisSpecMarshaling)(nil) // MarshalJSON marshals as JSON. func (g Genesis) MarshalJSON() ([]byte, error) { type Genesis struct { - Config *params.ChainConfig `json:"config"` - Nonce math.HexOrDecimal64 `json:"nonce"` - Timestamp math.HexOrDecimal64 `json:"timestamp"` - ExtraData hexutil.Bytes `json:"extraData"` - GasLimit math.HexOrDecimal64 `json:"gasLimit" gencodec:"required"` - Difficulty *math.HexOrDecimal256 `json:"difficulty" gencodec:"required"` - Mixhash common.Hash `json:"mixHash"` - Coinbase common.Address `json:"coinbase"` - Alloc map[common.UnprefixedAddress]GenesisAccount `json:"alloc" gencodec:"required"` - Number math.HexOrDecimal64 `json:"number"` - GasUsed math.HexOrDecimal64 `json:"gasUsed"` - ParentHash common.Hash `json:"parentHash"` - BaseFee *math.HexOrDecimal256 `json:"baseFeePerGas"` + Config *params.ChainConfig `json:"config"` + Nonce math.HexOrDecimal64 `json:"nonce"` + Timestamp math.HexOrDecimal64 `json:"timestamp"` + ExtraData hexutil.Bytes `json:"extraData"` + GasLimit math.HexOrDecimal64 `json:"gasLimit" gencodec:"required"` + Difficulty *math.HexOrDecimal256 `json:"difficulty" gencodec:"required"` + Mixhash common.Hash `json:"mixHash"` + Coinbase common.Address `json:"coinbase"` + Alloc map[common.UnprefixedAddress]GenesisAccount `json:"alloc" gencodec:"required"` + Number math.HexOrDecimal64 `json:"number"` + GasUsed math.HexOrDecimal64 `json:"gasUsed"` + ParentHash common.Hash `json:"parentHash"` + BaseFee *math.HexOrDecimal256 `json:"baseFeePerGas"` + ExcessDataGas *math.HexOrDecimal64 `json:"excessDataGas"` + DataGasUsed *math.HexOrDecimal64 `json:"dataGasUsed"` } var enc Genesis enc.Config = g.Config @@ -51,25 +53,29 @@ func (g Genesis) MarshalJSON() ([]byte, error) { enc.GasUsed = math.HexOrDecimal64(g.GasUsed) enc.ParentHash = g.ParentHash enc.BaseFee = (*math.HexOrDecimal256)(g.BaseFee) + enc.ExcessDataGas = (*math.HexOrDecimal64)(g.ExcessDataGas) + enc.DataGasUsed = (*math.HexOrDecimal64)(g.DataGasUsed) return json.Marshal(&enc) } // UnmarshalJSON unmarshals from JSON. func (g *Genesis) UnmarshalJSON(input []byte) error { type Genesis struct { - Config *params.ChainConfig `json:"config"` - Nonce *math.HexOrDecimal64 `json:"nonce"` - Timestamp *math.HexOrDecimal64 `json:"timestamp"` - ExtraData *hexutil.Bytes `json:"extraData"` - GasLimit *math.HexOrDecimal64 `json:"gasLimit" gencodec:"required"` - Difficulty *math.HexOrDecimal256 `json:"difficulty" gencodec:"required"` - Mixhash *common.Hash `json:"mixHash"` - Coinbase *common.Address `json:"coinbase"` - Alloc map[common.UnprefixedAddress]GenesisAccount `json:"alloc" gencodec:"required"` - Number *math.HexOrDecimal64 `json:"number"` - GasUsed *math.HexOrDecimal64 `json:"gasUsed"` - ParentHash *common.Hash `json:"parentHash"` - BaseFee *math.HexOrDecimal256 `json:"baseFeePerGas"` + Config *params.ChainConfig `json:"config"` + Nonce *math.HexOrDecimal64 `json:"nonce"` + Timestamp *math.HexOrDecimal64 `json:"timestamp"` + ExtraData *hexutil.Bytes `json:"extraData"` + GasLimit *math.HexOrDecimal64 `json:"gasLimit" gencodec:"required"` + Difficulty *math.HexOrDecimal256 `json:"difficulty" gencodec:"required"` + Mixhash *common.Hash `json:"mixHash"` + Coinbase *common.Address `json:"coinbase"` + Alloc map[common.UnprefixedAddress]GenesisAccount `json:"alloc" gencodec:"required"` + Number *math.HexOrDecimal64 `json:"number"` + GasUsed *math.HexOrDecimal64 `json:"gasUsed"` + ParentHash *common.Hash `json:"parentHash"` + BaseFee *math.HexOrDecimal256 `json:"baseFeePerGas"` + ExcessDataGas *math.HexOrDecimal64 `json:"excessDataGas"` + DataGasUsed *math.HexOrDecimal64 `json:"dataGasUsed"` } var dec Genesis if err := json.Unmarshal(input, &dec); err != nil { @@ -120,5 +126,11 @@ func (g *Genesis) UnmarshalJSON(input []byte) error { if dec.BaseFee != nil { g.BaseFee = (*big.Int)(dec.BaseFee) } + if dec.ExcessDataGas != nil { + g.ExcessDataGas = (*uint64)(dec.ExcessDataGas) + } + if dec.DataGasUsed != nil { + g.DataGasUsed = (*uint64)(dec.DataGasUsed) + } return nil } diff --git a/core/genesis.go b/core/genesis.go index 013ef78601..0b255a8d37 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -59,10 +59,12 @@ type Genesis struct { // These fields are used for consensus tests. Please don't use them // in actual genesis blocks. - Number uint64 `json:"number"` - GasUsed uint64 `json:"gasUsed"` - ParentHash common.Hash `json:"parentHash"` - BaseFee *big.Int `json:"baseFeePerGas"` + Number uint64 `json:"number"` + GasUsed uint64 `json:"gasUsed"` + ParentHash common.Hash `json:"parentHash"` + BaseFee *big.Int `json:"baseFeePerGas"` // EIP-1559 + ExcessDataGas *uint64 `json:"excessDataGas"` // EIP-4844 + DataGasUsed *uint64 `json:"dataGasUsed"` // EIP-4844 } func ReadGenesis(db ethdb.Database) (*Genesis, error) { @@ -96,6 +98,9 @@ func ReadGenesis(db ethdb.Database) (*Genesis, error) { genesis.Difficulty = genesisHeader.Difficulty genesis.Mixhash = genesisHeader.MixDigest genesis.Coinbase = genesisHeader.Coinbase + genesis.BaseFee = genesisHeader.BaseFee + genesis.ExcessDataGas = genesisHeader.ExcessDataGas + genesis.DataGasUsed = genesisHeader.DataGasUsed return &genesis, nil } @@ -214,15 +219,17 @@ type GenesisAccount struct { // field type overrides for gencodec type genesisSpecMarshaling struct { - Nonce math.HexOrDecimal64 - Timestamp math.HexOrDecimal64 - ExtraData hexutil.Bytes - GasLimit math.HexOrDecimal64 - GasUsed math.HexOrDecimal64 - Number math.HexOrDecimal64 - Difficulty *math.HexOrDecimal256 - BaseFee *math.HexOrDecimal256 - Alloc map[common.UnprefixedAddress]GenesisAccount + Nonce math.HexOrDecimal64 + Timestamp math.HexOrDecimal64 + ExtraData hexutil.Bytes + GasLimit math.HexOrDecimal64 + GasUsed math.HexOrDecimal64 + Number math.HexOrDecimal64 + Difficulty *math.HexOrDecimal256 + Alloc map[common.UnprefixedAddress]GenesisAccount + BaseFee *math.HexOrDecimal256 + ExcessDataGas *math.HexOrDecimal64 + DataGasUsed *math.HexOrDecimal64 } type genesisAccountMarshaling struct { @@ -463,9 +470,22 @@ func (g *Genesis) ToBlock() *types.Block { } } var withdrawals []*types.Withdrawal - if g.Config != nil && g.Config.IsShanghai(big.NewInt(int64(g.Number)), g.Timestamp) { - head.WithdrawalsHash = &types.EmptyWithdrawalsHash - withdrawals = make([]*types.Withdrawal, 0) + if conf := g.Config; conf != nil { + num := big.NewInt(int64(g.Number)) + if conf.IsShanghai(num, g.Timestamp) { + head.WithdrawalsHash = &types.EmptyWithdrawalsHash + withdrawals = make([]*types.Withdrawal, 0) + } + if conf.IsCancun(num, g.Timestamp) { + head.ExcessDataGas = g.ExcessDataGas + head.DataGasUsed = g.DataGasUsed + if head.ExcessDataGas == nil { + head.ExcessDataGas = new(uint64) + } + if head.DataGasUsed == nil { + head.DataGasUsed = new(uint64) + } + } } return types.NewBlock(head, nil, nil, nil, trie.NewStackTrie(nil)).WithWithdrawals(withdrawals) } diff --git a/core/state_processor.go b/core/state_processor.go index 4837628e66..fcaf5a8ff3 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -157,6 +157,6 @@ func ApplyTransaction(config *params.ChainConfig, bc ChainContext, author *commo } // Create a new context to be used in the EVM environment blockContext := NewEVMBlockContext(header, bc, author) - vmenv := vm.NewEVM(blockContext, vm.TxContext{}, statedb, config, cfg) + vmenv := vm.NewEVM(blockContext, vm.TxContext{BlobHashes: tx.BlobHashes()}, statedb, config, cfg) return applyTransaction(msg, config, gp, statedb, header.Number, header.Hash(), tx, usedGas, vmenv) } diff --git a/core/state_processor_test.go b/core/state_processor_test.go index 5dbeed97a0..a8858ec7e3 100644 --- a/core/state_processor_test.go +++ b/core/state_processor_test.go @@ -33,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" + "github.com/holiman/uint256" "golang.org/x/crypto/sha3" ) @@ -45,19 +46,23 @@ func u64(val uint64) *uint64 { return &val } func TestStateProcessorErrors(t *testing.T) { var ( config = ¶ms.ChainConfig{ - ChainID: big.NewInt(1), - HomesteadBlock: big.NewInt(0), - EIP150Block: big.NewInt(0), - EIP155Block: big.NewInt(0), - EIP158Block: big.NewInt(0), - ByzantiumBlock: big.NewInt(0), - ConstantinopleBlock: big.NewInt(0), - PetersburgBlock: big.NewInt(0), - IstanbulBlock: big.NewInt(0), - MuirGlacierBlock: big.NewInt(0), - BerlinBlock: big.NewInt(0), - LondonBlock: big.NewInt(0), - Ethash: new(params.EthashConfig), + ChainID: big.NewInt(1), + HomesteadBlock: big.NewInt(0), + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), + ConstantinopleBlock: big.NewInt(0), + PetersburgBlock: big.NewInt(0), + IstanbulBlock: big.NewInt(0), + MuirGlacierBlock: big.NewInt(0), + BerlinBlock: big.NewInt(0), + LondonBlock: big.NewInt(0), + Ethash: new(params.EthashConfig), + TerminalTotalDifficulty: big.NewInt(0), + TerminalTotalDifficultyPassed: true, + ShanghaiTime: new(uint64), + CancunTime: new(uint64), } signer = types.LatestSigner(config) key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") @@ -89,6 +94,22 @@ func TestStateProcessorErrors(t *testing.T) { }), signer, key1) return tx } + var mkBlobTx = func(nonce uint64, to common.Address, gasLimit uint64, gasTipCap, gasFeeCap *big.Int, hashes []common.Hash) *types.Transaction { + tx, err := types.SignTx(types.NewTx(&types.BlobTx{ + Nonce: nonce, + GasTipCap: uint256.MustFromBig(gasTipCap), + GasFeeCap: uint256.MustFromBig(gasFeeCap), + Gas: gasLimit, + To: to, + BlobHashes: hashes, + Value: new(uint256.Int), + }), signer, key1) + if err != nil { + t.Fatal(err) + } + return tx + } + { // Tests against a 'recent' chain definition var ( db = rawdb.NewMemoryDatabase() @@ -105,8 +126,10 @@ func TestStateProcessorErrors(t *testing.T) { }, }, } - blockchain, _ = NewBlockChain(db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + blockchain, _ = NewBlockChain(db, nil, gspec, nil, beacon.New(ethash.NewFaker()), vm.Config{}, nil, nil) + tooBigInitCode = [params.MaxInitCodeSize + 1]byte{} ) + defer blockchain.Stop() bigNumber := new(big.Int).SetBytes(common.FromHex("0xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")) tooBigNumber := new(big.Int).Set(bigNumber) @@ -209,8 +232,26 @@ func TestStateProcessorErrors(t *testing.T) { }, want: "could not apply tx 0 [0xd82a0c2519acfeac9a948258c47e784acd20651d9d80f9a1c67b4137651c3a24]: insufficient funds for gas * price + value: address 0x71562b71999873DB5b286dF957af199Ec94617F7 have 1000000000000000000 want 2431633873983640103894990685182446064918669677978451844828609264166175722438635000", }, + { // ErrMaxInitCodeSizeExceeded + txs: []*types.Transaction{ + mkDynamicCreationTx(0, 500000, common.Big0, big.NewInt(params.InitialBaseFee), tooBigInitCode[:]), + }, + want: "could not apply tx 0 [0xd491405f06c92d118dd3208376fcee18a57c54bc52063ee4a26b1cf296857c25]: max initcode size exceeded: code size 49153 limit 49152", + }, + { // ErrIntrinsicGas: Not enough gas to cover init code + txs: []*types.Transaction{ + mkDynamicCreationTx(0, 54299, common.Big0, big.NewInt(params.InitialBaseFee), make([]byte, 320)), + }, + want: "could not apply tx 0 [0xfd49536a9b323769d8472fcb3ebb3689b707a349379baee3e2ee3fe7baae06a1]: intrinsic gas too low: have 54299, want 54300", + }, + { // ErrBlobFeeCapTooLow + txs: []*types.Transaction{ + mkBlobTx(0, common.Address{}, params.TxGas, big.NewInt(1), big.NewInt(1), []common.Hash{(common.Hash{1})}), + }, + want: "could not apply tx 0 [0x6c11015985ce82db691d7b2d017acda296db88b811c3c60dc71449c76256c716]: max fee per gas less than block base fee: address 0x71562b71999873DB5b286dF957af199Ec94617F7, maxFeePerGas: 1 baseFee: 875000000", + }, } { - block := GenerateBadBlock(gspec.ToBlock(), ethash.NewFaker(), tt.txs, gspec.Config) + block := GenerateBadBlock(gspec.ToBlock(), beacon.New(ethash.NewFaker()), tt.txs, gspec.Config) _, err := blockchain.InsertChain(types.Blocks{block}) if err == nil { t.Fatal("block imported without errors") @@ -284,7 +325,7 @@ func TestStateProcessorErrors(t *testing.T) { }, }, } - blockchain, _ = NewBlockChain(db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil, nil) + blockchain, _ = NewBlockChain(db, nil, gspec, nil, beacon.New(ethash.NewFaker()), vm.Config{}, nil, nil) ) defer blockchain.Stop() for i, tt := range []struct { @@ -298,73 +339,7 @@ func TestStateProcessorErrors(t *testing.T) { want: "could not apply tx 0 [0x88626ac0d53cb65308f2416103c62bb1f18b805573d4f96a3640bbbfff13c14f]: sender not an eoa: address 0x71562b71999873DB5b286dF957af199Ec94617F7, codehash: 0x9280914443471259d4570a8661015ae4a5b80186dbc619658fb494bebc3da3d1", }, } { - block := GenerateBadBlock(gspec.ToBlock(), ethash.NewFaker(), tt.txs, gspec.Config) - _, err := blockchain.InsertChain(types.Blocks{block}) - if err == nil { - t.Fatal("block imported without errors") - } - if have, want := err.Error(), tt.want; have != want { - t.Errorf("test %d:\nhave \"%v\"\nwant \"%v\"\n", i, have, want) - } - } - } - - // ErrMaxInitCodeSizeExceeded, for this we need extra Shanghai (EIP-3860) enabled. - { - var ( - db = rawdb.NewMemoryDatabase() - gspec = &Genesis{ - Config: ¶ms.ChainConfig{ - ChainID: big.NewInt(1), - HomesteadBlock: big.NewInt(0), - EIP150Block: big.NewInt(0), - EIP155Block: big.NewInt(0), - EIP158Block: big.NewInt(0), - ByzantiumBlock: big.NewInt(0), - ConstantinopleBlock: big.NewInt(0), - PetersburgBlock: big.NewInt(0), - IstanbulBlock: big.NewInt(0), - MuirGlacierBlock: big.NewInt(0), - BerlinBlock: big.NewInt(0), - LondonBlock: big.NewInt(0), - ArrowGlacierBlock: big.NewInt(0), - GrayGlacierBlock: big.NewInt(0), - MergeNetsplitBlock: big.NewInt(0), - TerminalTotalDifficulty: big.NewInt(0), - TerminalTotalDifficultyPassed: true, - ShanghaiTime: u64(0), - }, - Alloc: GenesisAlloc{ - common.HexToAddress("0x71562b71999873DB5b286dF957af199Ec94617F7"): GenesisAccount{ - Balance: big.NewInt(1000000000000000000), // 1 ether - Nonce: 0, - }, - }, - } - genesis = gspec.MustCommit(db) - blockchain, _ = NewBlockChain(db, nil, gspec, nil, beacon.New(ethash.NewFaker()), vm.Config{}, nil, nil) - tooBigInitCode = [params.MaxInitCodeSize + 1]byte{} - smallInitCode = [320]byte{} - ) - defer blockchain.Stop() - for i, tt := range []struct { - txs []*types.Transaction - want string - }{ - { // ErrMaxInitCodeSizeExceeded - txs: []*types.Transaction{ - mkDynamicCreationTx(0, 500000, common.Big0, misc.CalcBaseFee(config, genesis.Header()), tooBigInitCode[:]), - }, - want: "could not apply tx 0 [0x832b54a6c3359474a9f504b1003b2cc1b6fcaa18e4ef369eb45b5d40dad6378f]: max initcode size exceeded: code size 49153 limit 49152", - }, - { // ErrIntrinsicGas: Not enough gas to cover init code - txs: []*types.Transaction{ - mkDynamicCreationTx(0, 54299, common.Big0, misc.CalcBaseFee(config, genesis.Header()), smallInitCode[:]), - }, - want: "could not apply tx 0 [0x39b7436cb432d3662a25626474282c5c4c1a213326fd87e4e18a91477bae98b2]: intrinsic gas too low: have 54299, want 54300", - }, - } { - block := GenerateBadBlock(genesis, beacon.New(ethash.NewFaker()), tt.txs, gspec.Config) + block := GenerateBadBlock(gspec.ToBlock(), beacon.New(ethash.NewFaker()), tt.txs, gspec.Config) _, err := blockchain.InsertChain(types.Blocks{block}) if err == nil { t.Fatal("block imported without errors") @@ -412,6 +387,7 @@ func GenerateBadBlock(parent *types.Block, engine consensus.Engine, txs types.Tr hasher := sha3.NewLegacyKeccak256() hasher.Write(header.Number.Bytes()) var cumulativeGas uint64 + var nBlobs int for _, tx := range txs { txh := tx.Hash() hasher.Write(txh[:]) @@ -420,8 +396,20 @@ func GenerateBadBlock(parent *types.Block, engine consensus.Engine, txs types.Tr receipt.GasUsed = tx.Gas() receipts = append(receipts, receipt) cumulativeGas += tx.Gas() + nBlobs += len(tx.BlobHashes()) } header.Root = common.BytesToHash(hasher.Sum(nil)) + if config.IsCancun(header.Number, header.Time) { + var pExcess, pUsed = uint64(0), uint64(0) + if parent.ExcessDataGas() != nil { + pExcess = *parent.ExcessDataGas() + pUsed = *parent.DataGasUsed() + } + excess := misc.CalcExcessDataGas(pExcess, pUsed) + used := uint64(nBlobs * params.BlobTxDataGasPerBlob) + header.ExcessDataGas = &excess + header.DataGasUsed = &used + } // Assemble and return the final block for sealing if config.IsShanghai(header.Number, header.Time) { return types.NewBlockWithWithdrawals(header, txs, nil, receipts, []*types.Withdrawal{}, trie.NewStackTrie(nil)) diff --git a/core/state_transition.go b/core/state_transition.go index 022238c8d6..9fcdb74458 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -17,12 +17,14 @@ package core import ( + "errors" "fmt" "math" "math/big" "github.com/ethereum/go-ethereum/common" cmath "github.com/ethereum/go-ethereum/common/math" + "github.com/ethereum/go-ethereum/consensus/misc" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/params" @@ -125,17 +127,18 @@ func toWordSize(size uint64) uint64 { // A Message contains the data derived from a single transaction that is relevant to state // processing. type Message struct { - To *common.Address - From common.Address - Nonce uint64 - Value *big.Int - GasLimit uint64 - GasPrice *big.Int - GasFeeCap *big.Int - GasTipCap *big.Int - Data []byte - AccessList types.AccessList - BlobHashes []common.Hash + To *common.Address + From common.Address + Nonce uint64 + Value *big.Int + GasLimit uint64 + GasPrice *big.Int + GasFeeCap *big.Int + GasTipCap *big.Int + Data []byte + AccessList types.AccessList + BlobGasFeeCap *big.Int + BlobHashes []common.Hash // When SkipAccountChecks is true, the message nonce is not checked against the // account nonce in state. It also disables checking that the sender is an EOA. @@ -157,6 +160,7 @@ func TransactionToMessage(tx *types.Transaction, s types.Signer, baseFee *big.In AccessList: tx.AccessList(), SkipAccountChecks: false, BlobHashes: tx.BlobHashes(), + BlobGasFeeCap: tx.BlobGasFeeCap(), } // If baseFee provided, set gasPrice to effectiveGasPrice. if baseFee != nil { @@ -230,12 +234,28 @@ func (st *StateTransition) to() common.Address { func (st *StateTransition) buyGas() error { mgval := new(big.Int).SetUint64(st.msg.GasLimit) mgval = mgval.Mul(mgval, st.msg.GasPrice) - balanceCheck := mgval + balanceCheck := new(big.Int).Set(mgval) if st.msg.GasFeeCap != nil { - balanceCheck = new(big.Int).SetUint64(st.msg.GasLimit) + balanceCheck.SetUint64(st.msg.GasLimit) balanceCheck = balanceCheck.Mul(balanceCheck, st.msg.GasFeeCap) balanceCheck.Add(balanceCheck, st.msg.Value) } + if st.evm.ChainConfig().IsCancun(st.evm.Context.BlockNumber, st.evm.Context.Time) { + if dataGas := st.dataGasUsed(); dataGas > 0 { + if st.evm.Context.ExcessDataGas == nil { + // programming error + panic("missing field excess data gas") + } + // Check that the user has enough funds to cover dataGasUsed * tx.BlobGasFeeCap + blobBalanceCheck := new(big.Int).SetUint64(dataGas) + blobBalanceCheck.Mul(blobBalanceCheck, st.msg.BlobGasFeeCap) + balanceCheck.Add(balanceCheck, blobBalanceCheck) + // Pay for dataGasUsed * actual blob fee + blobFee := new(big.Int).SetUint64(dataGas) + blobFee.Mul(blobFee, misc.CalcBlobFee(*st.evm.Context.ExcessDataGas)) + mgval.Add(mgval, blobFee) + } + } if have, want := st.state.GetBalance(st.msg.From), balanceCheck; have.Cmp(want) < 0 { return fmt.Errorf("%w: address %v have %v want %v", ErrInsufficientFunds, st.msg.From.Hex(), have, want) } @@ -297,6 +317,28 @@ func (st *StateTransition) preCheck() error { } } } + // Check the blob version validity + if msg.BlobHashes != nil { + if len(msg.BlobHashes) == 0 { + return errors.New("blob transaction missing blob hashes") + } + for i, hash := range msg.BlobHashes { + if hash[0] != params.BlobTxHashVersion { + return fmt.Errorf("blob %d hash version mismatch (have %d, supported %d)", + i, hash[0], params.BlobTxHashVersion) + } + } + } + + if st.evm.ChainConfig().IsCancun(st.evm.Context.BlockNumber, st.evm.Context.Time) { + if st.dataGasUsed() > 0 { + // Check that the user is paying at least the current blob fee + if have, want := st.msg.BlobGasFeeCap, misc.CalcBlobFee(*st.evm.Context.ExcessDataGas); have.Cmp(want) < 0 { + return fmt.Errorf("%w: address %v have %v want %v", ErrBlobFeeCapTooLow, st.msg.From.Hex(), have, want) + } + } + } + return st.buyGas() } @@ -427,3 +469,8 @@ func (st *StateTransition) refundGas(refundQuotient uint64) { func (st *StateTransition) gasUsed() uint64 { return st.initialGas - st.gasRemaining } + +// dataGasUsed returns the amount of data gas used by the message. +func (st *StateTransition) dataGasUsed() uint64 { + return uint64(len(st.msg.BlobHashes) * params.BlobTxDataGasPerBlob) +} diff --git a/core/vm/evm.go b/core/vm/evm.go index 3f3c3751f8..54395136a4 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -67,13 +67,14 @@ type BlockContext struct { GetHash GetHashFunc // Block information - Coinbase common.Address // Provides information for COINBASE - GasLimit uint64 // Provides information for GASLIMIT - BlockNumber *big.Int // Provides information for NUMBER - Time uint64 // Provides information for TIME - Difficulty *big.Int // Provides information for DIFFICULTY - BaseFee *big.Int // Provides information for BASEFEE - Random *common.Hash // Provides information for PREVRANDAO + Coinbase common.Address // Provides information for COINBASE + GasLimit uint64 // Provides information for GASLIMIT + BlockNumber *big.Int // Provides information for NUMBER + Time uint64 // Provides information for TIME + Difficulty *big.Int // Provides information for DIFFICULTY + BaseFee *big.Int // Provides information for BASEFEE + Random *common.Hash // Provides information for PREVRANDAO + ExcessDataGas *uint64 // ExcessDataGas field in the header, needed to compute the data } // TxContext provides the EVM with information about a transaction. diff --git a/tests/gen_sttransaction.go b/tests/gen_sttransaction.go index 7693a207a5..1127883315 100644 --- a/tests/gen_sttransaction.go +++ b/tests/gen_sttransaction.go @@ -6,6 +6,7 @@ import ( "encoding/json" "math/big" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/math" "github.com/ethereum/go-ethereum/core/types" @@ -26,6 +27,8 @@ func (s stTransaction) MarshalJSON() ([]byte, error) { GasLimit []math.HexOrDecimal64 `json:"gasLimit"` Value []string `json:"value"` PrivateKey hexutil.Bytes `json:"secretKey"` + BlobVersionedHashes []common.Hash `json:"blobVersionedHashes,omitempty"` + BlobGasFeeCap *math.HexOrDecimal256 `json:"maxFeePerDataGas,omitempty"` } var enc stTransaction enc.GasPrice = (*math.HexOrDecimal256)(s.GasPrice) @@ -43,6 +46,8 @@ func (s stTransaction) MarshalJSON() ([]byte, error) { } enc.Value = s.Value enc.PrivateKey = s.PrivateKey + enc.BlobVersionedHashes = s.BlobVersionedHashes + enc.BlobGasFeeCap = (*math.HexOrDecimal256)(s.BlobGasFeeCap) return json.Marshal(&enc) } @@ -59,6 +64,8 @@ func (s *stTransaction) UnmarshalJSON(input []byte) error { GasLimit []math.HexOrDecimal64 `json:"gasLimit"` Value []string `json:"value"` PrivateKey *hexutil.Bytes `json:"secretKey"` + BlobVersionedHashes []common.Hash `json:"blobVersionedHashes,omitempty"` + BlobGasFeeCap *math.HexOrDecimal256 `json:"maxFeePerDataGas,omitempty"` } var dec stTransaction if err := json.Unmarshal(input, &dec); err != nil { @@ -97,5 +104,11 @@ func (s *stTransaction) UnmarshalJSON(input []byte) error { if dec.PrivateKey != nil { s.PrivateKey = *dec.PrivateKey } + if dec.BlobVersionedHashes != nil { + s.BlobVersionedHashes = dec.BlobVersionedHashes + } + if dec.BlobGasFeeCap != nil { + s.BlobGasFeeCap = (*big.Int)(dec.BlobGasFeeCap) + } return nil } diff --git a/tests/state_test.go b/tests/state_test.go index 9d3862e1dc..61ce8aab39 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -48,23 +48,24 @@ func TestState(t *testing.T) { st.slow(`^stStaticCall/static_Return50000`) st.slow(`^stSystemOperationsTest/CallRecursiveBomb`) st.slow(`^stTransactionTest/Opcodes_TransactionInit`) - // Very time consuming st.skipLoad(`^stTimeConsuming/`) st.skipLoad(`.*vmPerformance/loop.*`) - // Uses 1GB RAM per tested fork st.skipLoad(`^stStaticCall/static_Call1MB`) // Broken tests: - // - // The stEOF tests are generated with EOF as part of Shanghai, which - // is erroneous. Therefore, these tests are skipped. - st.skipLoad(`^EIPTests/stEOF/`) + // EOF is not part of cancun + st.skipLoad(`^stEOF/`) + // Expected failures: + // These EIP-4844 tests need to be regenerated. + st.fails(`stEIP4844-blobtransactions/opcodeBlobhashOutOfRange.json`, "test has incorrect state root") + st.fails(`stEIP4844-blobtransactions/opcodeBlobhBounds.json`, "test has incorrect state root") // For Istanbul, older tests were moved into LegacyTests for _, dir := range []string{ + filepath.Join(baseDir, "EIPTests", "StateTests"), stateTestDir, legacyStateTestDir, benchmarksDir, diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 6956fe5b96..0958e248b4 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -113,6 +113,8 @@ type stTransaction struct { GasLimit []uint64 `json:"gasLimit"` Value []string `json:"value"` PrivateKey []byte `json:"secretKey"` + BlobVersionedHashes []common.Hash `json:"blobVersionedHashes,omitempty"` + BlobGasFeeCap *big.Int `json:"maxFeePerDataGas,omitempty"` } type stTransactionMarshaling struct { @@ -122,6 +124,7 @@ type stTransactionMarshaling struct { Nonce math.HexOrDecimal64 GasLimit []math.HexOrDecimal64 PrivateKey hexutil.Bytes + BlobGasFeeCap *math.HexOrDecimal256 } // GetChainConfig takes a fork definition and returns a chain config. @@ -403,16 +406,18 @@ func (tx *stTransaction) toMessage(ps stPostState, baseFee *big.Int) (*core.Mess } msg := &core.Message{ - From: from, - To: to, - Nonce: tx.Nonce, - Value: value, - GasLimit: gasLimit, - GasPrice: gasPrice, - GasFeeCap: tx.MaxFeePerGas, - GasTipCap: tx.MaxPriorityFeePerGas, - Data: data, - AccessList: accessList, + From: from, + To: to, + Nonce: tx.Nonce, + Value: value, + GasLimit: gasLimit, + GasPrice: gasPrice, + GasFeeCap: tx.MaxFeePerGas, + GasTipCap: tx.MaxPriorityFeePerGas, + Data: data, + AccessList: accessList, + BlobHashes: tx.BlobVersionedHashes, + BlobGasFeeCap: tx.BlobGasFeeCap, } return msg, nil } diff --git a/tests/testdata b/tests/testdata index bac70c50a5..ee3fa4c86d 160000 --- a/tests/testdata +++ b/tests/testdata @@ -1 +1 @@ -Subproject commit bac70c50a579197af68af5fc6d8c7b6163b92c52 +Subproject commit ee3fa4c86d05f99f2717f83a6ad08008490ddf07 From 988d84aa7caf8e71ce441fa65f80d44216d9e00e Mon Sep 17 00:00:00 2001 From: jwasinger Date: Mon, 17 Jul 2023 19:02:18 +0200 Subject: [PATCH 82/90] core/state, core/vm: implement EIP 6780 (#27189) EIP-6780: SELFDESTRUCT only in same transaction > SELFDESTRUCT will recover all funds to the caller but not delete the account, except when called in the same transaction as creation --------- Co-authored-by: Martin Holst Swende --- core/state/state_object.go | 3 +++ core/state/statedb.go | 15 +++++++++++++++ core/vm/eips.go | 12 ++++++++++++ core/vm/instructions.go | 16 ++++++++++++++++ core/vm/interface.go | 2 ++ core/vm/jump_table.go | 2 ++ 6 files changed, 50 insertions(+) diff --git a/core/state/state_object.go b/core/state/state_object.go index ed5f2b2945..6fea47f05b 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -86,6 +86,9 @@ type stateObject struct { // or an account that is considered as empty will be marked as deleted at // the end of transaction and no longer accessible anymore. deleted bool + + // Flag whether the object was created in the current transaction + created bool } // empty returns whether the account is considered empty. diff --git a/core/state/statedb.go b/core/state/statedb.go index 4b17e70255..fdcc76310f 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -493,6 +493,17 @@ func (s *StateDB) SelfDestruct(addr common.Address) { stateObject.data.Balance = new(big.Int) } +func (s *StateDB) Selfdestruct6780(addr common.Address) { + stateObject := s.getStateObject(addr) + if stateObject == nil { + return + } + + if stateObject.created { + s.SelfDestruct(addr) + } +} + // SetTransientState sets transient storage for a given account. It // adds the change to the journal so that it can be rolled back // to its previous value if there is a revert. @@ -681,6 +692,9 @@ func (s *StateDB) createObject(addr common.Address) (newobj, prev *stateObject) delete(s.accountsOrigin, prev.addrHash) delete(s.storagesOrigin, prev.addrHash) } + + newobj.created = true + s.setStateObject(newobj) if prev != nil && !prev.deleted { return newobj, prev @@ -910,6 +924,7 @@ func (s *StateDB) Finalise(deleteEmptyObjects bool) { } else { obj.finalise(true) // Prefetch slots in the background } + obj.created = false s.stateObjectsPending[addr] = struct{}{} s.stateObjectsDirty[addr] = struct{}{} diff --git a/core/vm/eips.go b/core/vm/eips.go index 0daa122ca0..704c1ce127 100644 --- a/core/vm/eips.go +++ b/core/vm/eips.go @@ -27,6 +27,7 @@ import ( var activators = map[int]func(*JumpTable){ 5656: enable5656, + 6780: enable6780, 3855: enable3855, 3860: enable3860, 3529: enable3529, @@ -291,3 +292,14 @@ func enable4844(jt *JumpTable) { maxStack: maxStack(1, 1), } } + +// enable6780 applies EIP-6780 (deactivate SELFDESTRUCT) +func enable6780(jt *JumpTable) { + jt[SELFDESTRUCT] = &operation{ + execute: opSelfdestruct6780, + dynamicGas: gasSelfdestructEIP3529, + constantGas: params.SelfdestructGasEIP150, + minStack: minStack(1, 0), + maxStack: maxStack(1, 0), + } +} diff --git a/core/vm/instructions.go b/core/vm/instructions.go index 4b6514ec8c..2105201fce 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -829,6 +829,22 @@ func opSelfdestruct(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext return nil, errStopToken } +func opSelfdestruct6780(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byte, error) { + if interpreter.readOnly { + return nil, ErrWriteProtection + } + beneficiary := scope.Stack.pop() + balance := interpreter.evm.StateDB.GetBalance(scope.Contract.Address()) + interpreter.evm.StateDB.SubBalance(scope.Contract.Address(), balance) + interpreter.evm.StateDB.AddBalance(beneficiary.Bytes20(), balance) + interpreter.evm.StateDB.Selfdestruct6780(scope.Contract.Address()) + if tracer := interpreter.evm.Config.Tracer; tracer != nil { + tracer.CaptureEnter(SELFDESTRUCT, scope.Contract.Address(), beneficiary.Bytes20(), []byte{}, 0, balance) + tracer.CaptureExit([]byte{}, 0, nil) + } + return nil, errStopToken +} + // following functions are used by the instruction jump table // make log instruction function diff --git a/core/vm/interface.go b/core/vm/interface.go index 05fb480f96..26814d3d2f 100644 --- a/core/vm/interface.go +++ b/core/vm/interface.go @@ -54,6 +54,8 @@ type StateDB interface { SelfDestruct(common.Address) HasSelfDestructed(common.Address) bool + Selfdestruct6780(common.Address) + // Exist reports whether the given account exists in state. // Notably this should also return true for self-destructed accounts. Exist(common.Address) bool diff --git a/core/vm/jump_table.go b/core/vm/jump_table.go index a59fa87473..702b186615 100644 --- a/core/vm/jump_table.go +++ b/core/vm/jump_table.go @@ -85,6 +85,7 @@ func newCancunInstructionSet() JumpTable { enable4844(&instructionSet) // EIP-4844 (DATAHASH opcode) enable1153(&instructionSet) // EIP-1153 "Transient Storage" enable5656(&instructionSet) // EIP-5656 (MCOPY opcode) + enable6780(&instructionSet) // EIP-6780 SELFDESTRUCT only in same transaction return validate(instructionSet) } @@ -92,6 +93,7 @@ func newShanghaiInstructionSet() JumpTable { instructionSet := newMergeInstructionSet() enable3855(&instructionSet) // PUSH0 instruction enable3860(&instructionSet) // Limit and meter initcode + return validate(instructionSet) } From d4d88f9bce13ca9310bf28f5f26ea9f1915ba90d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Mon, 17 Jul 2023 20:48:11 +0200 Subject: [PATCH 83/90] les: remove obsolete code related to PoW header syncing (#27737) This change removes PoW header syncing related code from LES and also deletes duplicated packages les/catalyst, les/downloader and les/fetcher. These package copies were created because people wanted to make changes in their eth/ counterparts, but weren't able to adapt LES code to the API changes. --- cmd/geth/main.go | 3 - cmd/utils/flags.go | 34 - eth/ethconfig/config.go | 6 - eth/ethconfig/gen_config.go | 18 - les/api_backend.go | 3 +- les/api_test.go | 7 +- les/catalyst/api.go | 220 ---- les/catalyst/api_test.go | 251 ---- les/client.go | 27 +- les/client_handler.go | 186 +-- les/downloader/api.go | 166 --- les/downloader/downloader.go | 1979 ----------------------------- les/downloader/downloader_test.go | 1621 ----------------------- les/downloader/events.go | 25 - les/downloader/metrics.go | 45 - les/downloader/modes.go | 81 -- les/downloader/peer.go | 502 -------- les/downloader/queue.go | 913 ------------- les/downloader/queue_test.go | 438 ------- les/downloader/resultstore.go | 195 --- les/downloader/statesync.go | 638 ---------- les/downloader/testchain_test.go | 235 ---- les/downloader/types.go | 79 -- les/fetcher.go | 563 -------- les/fetcher/block_fetcher.go | 888 ------------- les/fetcher/block_fetcher_test.go | 901 ------------- les/fetcher_test.go | 189 --- les/handler_test.go | 2 +- les/odr_test.go | 5 + les/peer.go | 25 +- les/pruner.go | 99 -- les/pruner_test.go | 204 --- les/request_test.go | 5 + les/retrieve.go | 9 - les/sync.go | 56 - les/sync_test.go | 83 -- les/test_helper.go | 32 +- les/ulc.go | 54 - les/ulc_test.go | 162 --- 39 files changed, 54 insertions(+), 10895 deletions(-) delete mode 100644 les/catalyst/api.go delete mode 100644 les/catalyst/api_test.go delete mode 100644 les/downloader/api.go delete mode 100644 les/downloader/downloader.go delete mode 100644 les/downloader/downloader_test.go delete mode 100644 les/downloader/events.go delete mode 100644 les/downloader/metrics.go delete mode 100644 les/downloader/modes.go delete mode 100644 les/downloader/peer.go delete mode 100644 les/downloader/queue.go delete mode 100644 les/downloader/queue_test.go delete mode 100644 les/downloader/resultstore.go delete mode 100644 les/downloader/statesync.go delete mode 100644 les/downloader/testchain_test.go delete mode 100644 les/downloader/types.go delete mode 100644 les/fetcher.go delete mode 100644 les/fetcher/block_fetcher.go delete mode 100644 les/fetcher/block_fetcher_test.go delete mode 100644 les/fetcher_test.go delete mode 100644 les/pruner.go delete mode 100644 les/pruner_test.go delete mode 100644 les/sync.go delete mode 100644 les/sync_test.go delete mode 100644 les/ulc.go delete mode 100644 les/ulc_test.go diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 05e6b727d6..8ada24829c 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -93,9 +93,6 @@ var ( utils.LightMaxPeersFlag, utils.LightNoPruneFlag, utils.LightKDFFlag, - utils.UltraLightServersFlag, - utils.UltraLightFractionFlag, - utils.UltraLightOnlyAnnounceFlag, utils.LightNoSyncServeFlag, utils.EthRequiredBlocksFlag, utils.LegacyWhitelistFlag, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index feceb99d3e..8961e350ae 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -61,7 +61,6 @@ import ( "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/les" - lescatalyst "github.com/ethereum/go-ethereum/les/catalyst" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/metrics/exp" @@ -294,23 +293,6 @@ var ( Value: ethconfig.Defaults.LightPeers, Category: flags.LightCategory, } - UltraLightServersFlag = &cli.StringFlag{ - Name: "ulc.servers", - Usage: "List of trusted ultra-light servers", - Value: strings.Join(ethconfig.Defaults.UltraLightServers, ","), - Category: flags.LightCategory, - } - UltraLightFractionFlag = &cli.IntFlag{ - Name: "ulc.fraction", - Usage: "Minimum % of trusted ultra-light servers required to announce a new head", - Value: ethconfig.Defaults.UltraLightFraction, - Category: flags.LightCategory, - } - UltraLightOnlyAnnounceFlag = &cli.BoolFlag{ - Name: "ulc.onlyannounce", - Usage: "Ultra light server sends announcements only", - Category: flags.LightCategory, - } LightNoPruneFlag = &cli.BoolFlag{ Name: "light.nopruning", Usage: "Disable ancient light chain data pruning", @@ -1211,19 +1193,6 @@ func setLes(ctx *cli.Context, cfg *ethconfig.Config) { if ctx.IsSet(LightMaxPeersFlag.Name) { cfg.LightPeers = ctx.Int(LightMaxPeersFlag.Name) } - if ctx.IsSet(UltraLightServersFlag.Name) { - cfg.UltraLightServers = strings.Split(ctx.String(UltraLightServersFlag.Name), ",") - } - if ctx.IsSet(UltraLightFractionFlag.Name) { - cfg.UltraLightFraction = ctx.Int(UltraLightFractionFlag.Name) - } - if cfg.UltraLightFraction <= 0 && cfg.UltraLightFraction > 100 { - log.Error("Ultra light fraction is invalid", "had", cfg.UltraLightFraction, "updated", ethconfig.Defaults.UltraLightFraction) - cfg.UltraLightFraction = ethconfig.Defaults.UltraLightFraction - } - if ctx.IsSet(UltraLightOnlyAnnounceFlag.Name) { - cfg.UltraLightOnlyAnnounce = ctx.Bool(UltraLightOnlyAnnounceFlag.Name) - } if ctx.IsSet(LightNoPruneFlag.Name) { cfg.LightNoPrune = ctx.Bool(LightNoPruneFlag.Name) } @@ -1884,9 +1853,6 @@ func RegisterEthService(stack *node.Node, cfg *ethconfig.Config) (ethapi.Backend Fatalf("Failed to register the Ethereum service: %v", err) } stack.RegisterAPIs(tracers.APIs(backend.ApiBackend)) - if err := lescatalyst.Register(stack, backend); err != nil { - Fatalf("Failed to register the Engine API service: %v", err) - } return backend.ApiBackend, nil } backend, err := eth.New(stack, cfg) diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index dc798ec196..5de0055b51 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -61,7 +61,6 @@ var Defaults = Config{ NetworkId: 1, TxLookupLimit: 2350000, LightPeers: 100, - UltraLightFraction: 75, DatabaseCache: 512, TrieCleanCache: 154, TrieDirtyCache: 256, @@ -111,11 +110,6 @@ type Config struct { LightNoPrune bool `toml:",omitempty"` // Whether to disable light chain pruning LightNoSyncServe bool `toml:",omitempty"` // Whether to serve light clients before syncing - // Ultra Light client options - UltraLightServers []string `toml:",omitempty"` // List of trusted ultra light servers - UltraLightFraction int `toml:",omitempty"` // Percentage of trusted servers to accept an announcement - UltraLightOnlyAnnounce bool `toml:",omitempty"` // Whether to only announce headers, or also serve them - // Database options SkipBcVersionCheck bool `toml:"-"` DatabaseHandles int `toml:"-"` diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index 8f48467236..82b2d5b6b0 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -31,9 +31,6 @@ func (c Config) MarshalTOML() (interface{}, error) { LightPeers int `toml:",omitempty"` LightNoPrune bool `toml:",omitempty"` LightNoSyncServe bool `toml:",omitempty"` - UltraLightServers []string `toml:",omitempty"` - UltraLightFraction int `toml:",omitempty"` - UltraLightOnlyAnnounce bool `toml:",omitempty"` SkipBcVersionCheck bool `toml:"-"` DatabaseHandles int `toml:"-"` DatabaseCache int @@ -71,9 +68,6 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.LightPeers = c.LightPeers enc.LightNoPrune = c.LightNoPrune enc.LightNoSyncServe = c.LightNoSyncServe - enc.UltraLightServers = c.UltraLightServers - enc.UltraLightFraction = c.UltraLightFraction - enc.UltraLightOnlyAnnounce = c.UltraLightOnlyAnnounce enc.SkipBcVersionCheck = c.SkipBcVersionCheck enc.DatabaseHandles = c.DatabaseHandles enc.DatabaseCache = c.DatabaseCache @@ -115,9 +109,6 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { LightPeers *int `toml:",omitempty"` LightNoPrune *bool `toml:",omitempty"` LightNoSyncServe *bool `toml:",omitempty"` - UltraLightServers []string `toml:",omitempty"` - UltraLightFraction *int `toml:",omitempty"` - UltraLightOnlyAnnounce *bool `toml:",omitempty"` SkipBcVersionCheck *bool `toml:"-"` DatabaseHandles *int `toml:"-"` DatabaseCache *int @@ -188,15 +179,6 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.LightNoSyncServe != nil { c.LightNoSyncServe = *dec.LightNoSyncServe } - if dec.UltraLightServers != nil { - c.UltraLightServers = dec.UltraLightServers - } - if dec.UltraLightFraction != nil { - c.UltraLightFraction = *dec.UltraLightFraction - } - if dec.UltraLightOnlyAnnounce != nil { - c.UltraLightOnlyAnnounce = *dec.UltraLightOnlyAnnounce - } if dec.SkipBcVersionCheck != nil { c.SkipBcVersionCheck = *dec.SkipBcVersionCheck } diff --git a/les/api_backend.go b/les/api_backend.go index dee3e3bfd9..3e9dbadce8 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -57,7 +57,6 @@ func (b *LesApiBackend) CurrentBlock() *types.Header { } func (b *LesApiBackend) SetHead(number uint64) { - b.eth.handler.downloader.Cancel() b.eth.blockchain.SetHead(number) } @@ -264,7 +263,7 @@ func (b *LesApiBackend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEven } func (b *LesApiBackend) SyncProgress() ethereum.SyncProgress { - return b.eth.Downloader().Progress() + return ethereum.SyncProgress{} } func (b *LesApiBackend) ProtocolVersion() int { diff --git a/les/api_test.go b/les/api_test.go index ac895362a0..88a950afce 100644 --- a/les/api_test.go +++ b/les/api_test.go @@ -31,9 +31,8 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/eth" - ethdownloader "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/ethconfig" - "github.com/ethereum/go-ethereum/les/downloader" "github.com/ethereum/go-ethereum/les/flowcontrol" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" @@ -493,13 +492,13 @@ func testSim(t *testing.T, serverCount, clientCount int, serverDir, clientDir [] func newLesClientService(ctx *adapters.ServiceContext, stack *node.Node) (node.Lifecycle, error) { config := ethconfig.Defaults - config.SyncMode = (ethdownloader.SyncMode)(downloader.LightSync) + config.SyncMode = downloader.LightSync return New(stack, &config) } func newLesServerService(ctx *adapters.ServiceContext, stack *node.Node) (node.Lifecycle, error) { config := ethconfig.Defaults - config.SyncMode = (ethdownloader.SyncMode)(downloader.FullSync) + config.SyncMode = downloader.FullSync config.LightServ = testServerCapacity config.LightPeers = testMaxClients ethereum, err := eth.New(stack, &config) diff --git a/les/catalyst/api.go b/les/catalyst/api.go deleted file mode 100644 index 30b41101db..0000000000 --- a/les/catalyst/api.go +++ /dev/null @@ -1,220 +0,0 @@ -// Copyright 2022 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Package catalyst implements the temporary eth1/eth2 RPC integration. -package catalyst - -import ( - "errors" - "fmt" - - "github.com/ethereum/go-ethereum/beacon/engine" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" - "github.com/ethereum/go-ethereum/les" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/rpc" -) - -// Register adds catalyst APIs to the light client. -func Register(stack *node.Node, backend *les.LightEthereum) error { - log.Warn("Catalyst mode enabled", "protocol", "les") - stack.RegisterAPIs([]rpc.API{ - { - Namespace: "engine", - Service: NewConsensusAPI(backend), - Authenticated: true, - }, - }) - return nil -} - -type ConsensusAPI struct { - les *les.LightEthereum -} - -// NewConsensusAPI creates a new consensus api for the given backend. -// The underlying blockchain needs to have a valid terminal total difficulty set. -func NewConsensusAPI(les *les.LightEthereum) *ConsensusAPI { - if les.BlockChain().Config().TerminalTotalDifficulty == nil { - log.Warn("Catalyst started without valid total difficulty") - } - return &ConsensusAPI{les: les} -} - -// ForkchoiceUpdatedV1 has several responsibilities: -// -// We try to set our blockchain to the headBlock. -// -// If the method is called with an empty head block: we return success, which can be used -// to check if the catalyst mode is enabled. -// -// If the total difficulty was not reached: we return INVALID. -// -// If the finalizedBlockHash is set: we check if we have the finalizedBlockHash in our db, -// if not we start a sync. -// -// If there are payloadAttributes: we return an error since block creation is not -// supported in les mode. -func (api *ConsensusAPI) ForkchoiceUpdatedV1(heads engine.ForkchoiceStateV1, payloadAttributes *engine.PayloadAttributes) (engine.ForkChoiceResponse, error) { - if heads.HeadBlockHash == (common.Hash{}) { - log.Warn("Forkchoice requested update to zero hash") - return engine.STATUS_INVALID, nil // TODO(karalabe): Why does someone send us this? - } - if err := api.checkTerminalTotalDifficulty(heads.HeadBlockHash); err != nil { - if header := api.les.BlockChain().GetHeaderByHash(heads.HeadBlockHash); header == nil { - // TODO (MariusVanDerWijden) trigger sync - return engine.STATUS_SYNCING, nil - } - return engine.STATUS_INVALID, err - } - // If the finalized block is set, check if it is in our blockchain - if heads.FinalizedBlockHash != (common.Hash{}) { - if header := api.les.BlockChain().GetHeaderByHash(heads.FinalizedBlockHash); header == nil { - // TODO (MariusVanDerWijden) trigger sync - return engine.STATUS_SYNCING, nil - } - } - // SetHead - if err := api.setCanonical(heads.HeadBlockHash); err != nil { - return engine.STATUS_INVALID, err - } - if payloadAttributes != nil { - return engine.STATUS_INVALID, errors.New("not supported") - } - return api.validForkChoiceResponse(), nil -} - -// GetPayloadV1 returns a cached payload by id. It's not supported in les mode. -func (api *ConsensusAPI) GetPayloadV1(payloadID engine.PayloadID) (*engine.ExecutableData, error) { - return nil, engine.GenericServerError.With(errors.New("not supported in light client mode")) -} - -// ExecutePayloadV1 creates an Eth1 block, inserts it in the chain, and returns the status of the chain. -func (api *ConsensusAPI) ExecutePayloadV1(params engine.ExecutableData) (engine.PayloadStatusV1, error) { - block, err := engine.ExecutableDataToBlock(params) - if err != nil { - return api.invalid(), err - } - if !api.les.BlockChain().HasHeader(block.ParentHash(), block.NumberU64()-1) { - /* - TODO (MariusVanDerWijden) reenable once sync is merged - if err := api.eth.Downloader().BeaconSync(api.eth.SyncMode(), block.Header()); err != nil { - return SYNCING, err - } - */ - // TODO (MariusVanDerWijden) we should return nil here not empty hash - return engine.PayloadStatusV1{Status: engine.SYNCING, LatestValidHash: nil}, nil - } - parent := api.les.BlockChain().GetHeaderByHash(params.ParentHash) - if parent == nil { - return api.invalid(), fmt.Errorf("could not find parent %x", params.ParentHash) - } - td := api.les.BlockChain().GetTd(parent.Hash(), block.NumberU64()-1) - ttd := api.les.BlockChain().Config().TerminalTotalDifficulty - if td.Cmp(ttd) < 0 { - return api.invalid(), fmt.Errorf("can not execute payload on top of block with low td got: %v threshold %v", td, ttd) - } - if err = api.les.BlockChain().InsertHeader(block.Header()); err != nil { - return api.invalid(), err - } - if merger := api.les.Merger(); !merger.TDDReached() { - merger.ReachTTD() - } - hash := block.Hash() - return engine.PayloadStatusV1{Status: engine.VALID, LatestValidHash: &hash}, nil -} - -func (api *ConsensusAPI) validForkChoiceResponse() engine.ForkChoiceResponse { - currentHash := api.les.BlockChain().CurrentHeader().Hash() - return engine.ForkChoiceResponse{ - PayloadStatus: engine.PayloadStatusV1{Status: engine.VALID, LatestValidHash: ¤tHash}, - } -} - -// invalid returns a response "INVALID" with the latest valid hash set to the current head. -func (api *ConsensusAPI) invalid() engine.PayloadStatusV1 { - currentHash := api.les.BlockChain().CurrentHeader().Hash() - return engine.PayloadStatusV1{Status: engine.INVALID, LatestValidHash: ¤tHash} -} - -func (api *ConsensusAPI) checkTerminalTotalDifficulty(head common.Hash) error { - // shortcut if we entered PoS already - if api.les.Merger().PoSFinalized() { - return nil - } - // make sure the parent has enough terminal total difficulty - header := api.les.BlockChain().GetHeaderByHash(head) - if header == nil { - return errors.New("unknown header") - } - td := api.les.BlockChain().GetTd(header.Hash(), header.Number.Uint64()) - if td != nil && td.Cmp(api.les.BlockChain().Config().TerminalTotalDifficulty) < 0 { - return errors.New("invalid ttd") - } - return nil -} - -// setCanonical is called to perform a force choice. -func (api *ConsensusAPI) setCanonical(newHead common.Hash) error { - log.Info("Setting head", "head", newHead) - - headHeader := api.les.BlockChain().CurrentHeader() - if headHeader.Hash() == newHead { - return nil - } - newHeadHeader := api.les.BlockChain().GetHeaderByHash(newHead) - if newHeadHeader == nil { - return errors.New("unknown header") - } - if err := api.les.BlockChain().SetCanonical(newHeadHeader); err != nil { - return err - } - // Trigger the transition if it's the first `NewHead` event. - if merger := api.les.Merger(); !merger.PoSFinalized() { - merger.FinalizePoS() - } - return nil -} - -// ExchangeTransitionConfigurationV1 checks the given configuration against -// the configuration of the node. -func (api *ConsensusAPI) ExchangeTransitionConfigurationV1(config engine.TransitionConfigurationV1) (*engine.TransitionConfigurationV1, error) { - log.Trace("Engine API request received", "method", "ExchangeTransitionConfiguration", "ttd", config.TerminalTotalDifficulty) - if config.TerminalTotalDifficulty == nil { - return nil, errors.New("invalid terminal total difficulty") - } - - ttd := api.les.BlockChain().Config().TerminalTotalDifficulty - if ttd == nil || ttd.Cmp(config.TerminalTotalDifficulty.ToInt()) != 0 { - log.Warn("Invalid TTD configured", "geth", ttd, "beacon", config.TerminalTotalDifficulty) - return nil, fmt.Errorf("invalid ttd: execution %v consensus %v", ttd, config.TerminalTotalDifficulty) - } - - if config.TerminalBlockHash != (common.Hash{}) { - if hash := api.les.BlockChain().GetCanonicalHash(uint64(config.TerminalBlockNumber)); hash == config.TerminalBlockHash { - return &engine.TransitionConfigurationV1{ - TerminalTotalDifficulty: (*hexutil.Big)(ttd), - TerminalBlockHash: config.TerminalBlockHash, - TerminalBlockNumber: config.TerminalBlockNumber, - }, nil - } - return nil, errors.New("invalid terminal block hash") - } - - return &engine.TransitionConfigurationV1{TerminalTotalDifficulty: (*hexutil.Big)(ttd)}, nil -} diff --git a/les/catalyst/api_test.go b/les/catalyst/api_test.go deleted file mode 100644 index 2405499d32..0000000000 --- a/les/catalyst/api_test.go +++ /dev/null @@ -1,251 +0,0 @@ -// Copyright 2022 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package catalyst - -import ( - "math/big" - "testing" - - "github.com/ethereum/go-ethereum/beacon/engine" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus/ethash" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/eth/downloader" - "github.com/ethereum/go-ethereum/eth/ethconfig" - "github.com/ethereum/go-ethereum/les" - "github.com/ethereum/go-ethereum/node" - "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie" -) - -var ( - // testKey is a private key to use for funding a tester account. - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - - // testAddr is the Ethereum address of the tester account. - testAddr = crypto.PubkeyToAddress(testKey.PublicKey) - - testBalance = big.NewInt(2e18) -) - -func generatePreMergeChain(pre, post int) (*core.Genesis, []*types.Header, []*types.Block, []*types.Header, []*types.Block) { - config := *params.AllEthashProtocolChanges - genesis := &core.Genesis{ - Config: &config, - Alloc: core.GenesisAlloc{testAddr: {Balance: testBalance}}, - ExtraData: []byte("test genesis"), - Timestamp: 9000, - BaseFee: big.NewInt(params.InitialBaseFee), - } - // Pre-merge blocks - db, preBLocks, _ := core.GenerateChainWithGenesis(genesis, ethash.NewFaker(), pre, nil) - totalDifficulty := new(big.Int).Set(params.GenesisDifficulty) - - var preHeaders []*types.Header - for _, b := range preBLocks { - totalDifficulty.Add(totalDifficulty, b.Difficulty()) - preHeaders = append(preHeaders, b.Header()) - } - config.TerminalTotalDifficulty = totalDifficulty - // Post-merge blocks - postBlocks, _ := core.GenerateChain(genesis.Config, - preBLocks[len(preBLocks)-1], ethash.NewFaker(), db, post, - func(i int, b *core.BlockGen) { - b.SetPoS() - }) - - var postHeaders []*types.Header - for _, b := range postBlocks { - postHeaders = append(postHeaders, b.Header()) - } - - return genesis, preHeaders, preBLocks, postHeaders, postBlocks -} - -func TestSetHeadBeforeTotalDifficulty(t *testing.T) { - genesis, headers, blocks, _, _ := generatePreMergeChain(10, 0) - n, lesService := startLesService(t, genesis, headers) - defer n.Close() - - api := NewConsensusAPI(lesService) - fcState := engine.ForkchoiceStateV1{ - HeadBlockHash: blocks[5].Hash(), - SafeBlockHash: common.Hash{}, - FinalizedBlockHash: common.Hash{}, - } - if _, err := api.ForkchoiceUpdatedV1(fcState, nil); err == nil { - t.Errorf("fork choice updated before total terminal difficulty should fail") - } -} - -func TestExecutePayloadV1(t *testing.T) { - genesis, headers, _, _, postBlocks := generatePreMergeChain(10, 2) - n, lesService := startLesService(t, genesis, headers) - lesService.Merger().ReachTTD() - defer n.Close() - - api := NewConsensusAPI(lesService) - fcState := engine.ForkchoiceStateV1{ - HeadBlockHash: postBlocks[0].Hash(), - SafeBlockHash: common.Hash{}, - FinalizedBlockHash: common.Hash{}, - } - if _, err := api.ForkchoiceUpdatedV1(fcState, nil); err != nil { - t.Errorf("Failed to update head %v", err) - } - block := postBlocks[0] - - fakeBlock := types.NewBlock(&types.Header{ - ParentHash: block.ParentHash(), - UncleHash: crypto.Keccak256Hash(nil), - Coinbase: block.Coinbase(), - Root: block.Root(), - TxHash: crypto.Keccak256Hash(nil), - ReceiptHash: crypto.Keccak256Hash(nil), - Bloom: block.Bloom(), - Difficulty: big.NewInt(0), - Number: block.Number(), - GasLimit: block.GasLimit(), - GasUsed: block.GasUsed(), - Time: block.Time(), - Extra: block.Extra(), - MixDigest: block.MixDigest(), - Nonce: types.BlockNonce{}, - BaseFee: block.BaseFee(), - }, nil, nil, nil, trie.NewStackTrie(nil)) - - _, err := api.ExecutePayloadV1(engine.ExecutableData{ - ParentHash: fakeBlock.ParentHash(), - FeeRecipient: fakeBlock.Coinbase(), - StateRoot: fakeBlock.Root(), - ReceiptsRoot: fakeBlock.ReceiptHash(), - LogsBloom: fakeBlock.Bloom().Bytes(), - Random: fakeBlock.MixDigest(), - Number: fakeBlock.NumberU64(), - GasLimit: fakeBlock.GasLimit(), - GasUsed: fakeBlock.GasUsed(), - Timestamp: fakeBlock.Time(), - ExtraData: fakeBlock.Extra(), - BaseFeePerGas: fakeBlock.BaseFee(), - BlockHash: fakeBlock.Hash(), - Transactions: encodeTransactions(fakeBlock.Transactions()), - }) - if err != nil { - t.Errorf("Failed to execute payload %v", err) - } - headHeader := api.les.BlockChain().CurrentHeader() - if headHeader.Number.Uint64() != fakeBlock.NumberU64()-1 { - t.Fatal("Unexpected chain head update") - } - fcState = engine.ForkchoiceStateV1{ - HeadBlockHash: fakeBlock.Hash(), - SafeBlockHash: common.Hash{}, - FinalizedBlockHash: common.Hash{}, - } - if _, err := api.ForkchoiceUpdatedV1(fcState, nil); err != nil { - t.Fatal("Failed to update head") - } - headHeader = api.les.BlockChain().CurrentHeader() - if headHeader.Number.Uint64() != fakeBlock.NumberU64() { - t.Fatal("Failed to update chain head") - } -} - -func TestEth2DeepReorg(t *testing.T) { - // TODO (MariusVanDerWijden) TestEth2DeepReorg is currently broken, because it tries to reorg - // before the totalTerminalDifficulty threshold - /* - genesis, preMergeBlocks := generatePreMergeChain(core.TriesInMemory * 2) - n, ethservice := startEthService(t, genesis, preMergeBlocks) - defer n.Close() - - var ( - api = NewConsensusAPI(ethservice, nil) - parent = preMergeBlocks[len(preMergeBlocks)-core.TriesInMemory-1] - head = ethservice.BlockChain().CurrentBlock().NumberU64() - ) - if ethservice.BlockChain().HasBlockAndState(parent.Hash(), parent.NumberU64()) { - t.Errorf("Block %d not pruned", parent.NumberU64()) - } - for i := 0; i < 10; i++ { - execData, err := api.assembleBlock(AssembleBlockParams{ - ParentHash: parent.Hash(), - Timestamp: parent.Time() + 5, - }) - if err != nil { - t.Fatalf("Failed to create the executable data %v", err) - } - block, err := ExecutableDataToBlock(ethservice.BlockChain().Config(), parent.Header(), *execData) - if err != nil { - t.Fatalf("Failed to convert executable data to block %v", err) - } - newResp, err := api.ExecutePayload(*execData) - if err != nil || newResp.Status != "VALID" { - t.Fatalf("Failed to insert block: %v", err) - } - if ethservice.BlockChain().CurrentBlock().NumberU64() != head { - t.Fatalf("Chain head shouldn't be updated") - } - if err := api.setCanonical(block.Hash()); err != nil { - t.Fatalf("Failed to set head: %v", err) - } - if ethservice.BlockChain().CurrentBlock().NumberU64() != block.NumberU64() { - t.Fatalf("Chain head should be updated") - } - parent, head = block, block.NumberU64() - } - */ -} - -// startEthService creates a full node instance for testing. -func startLesService(t *testing.T, genesis *core.Genesis, headers []*types.Header) (*node.Node, *les.LightEthereum) { - t.Helper() - - n, err := node.New(&node.Config{}) - if err != nil { - t.Fatal("can't create node:", err) - } - ethcfg := ðconfig.Config{ - Genesis: genesis, - SyncMode: downloader.LightSync, - TrieDirtyCache: 256, - TrieCleanCache: 256, - LightPeers: 10, - } - lesService, err := les.New(n, ethcfg) - if err != nil { - t.Fatal("can't create eth service:", err) - } - if err := n.Start(); err != nil { - t.Fatal("can't start node:", err) - } - if _, err := lesService.BlockChain().InsertHeaderChain(headers); err != nil { - n.Close() - t.Fatal("can't import test headers:", err) - } - return n, lesService -} - -func encodeTransactions(txs []*types.Transaction) [][]byte { - var enc = make([][]byte, len(txs)) - for i, tx := range txs { - enc[i], _ = tx.MarshalBinary() - } - return enc -} diff --git a/les/client.go b/les/client.go index 78bd05c77b..132c857aa5 100644 --- a/les/client.go +++ b/les/client.go @@ -36,7 +36,6 @@ import ( "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/internal/shutdowncheck" - "github.com/ethereum/go-ethereum/les/downloader" "github.com/ethereum/go-ethereum/les/vflux" vfc "github.com/ethereum/go-ethereum/les/vflux/client" "github.com/ethereum/go-ethereum/light" @@ -64,7 +63,6 @@ type LightEthereum struct { blockchain *light.LightChain serverPool *vfc.ServerPool serverPoolIterator enode.Iterator - pruner *pruner merger *consensus.Merger bloomRequests chan chan *bloombits.Retrieval // Channel receiving bloom data retrieval requests @@ -146,7 +144,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { if leth.udpEnabled { prenegQuery = leth.prenegQuery } - leth.serverPool, leth.serverPoolIterator = vfc.NewServerPool(lesDb, []byte("serverpool:"), time.Second, prenegQuery, &mclock.System{}, config.UltraLightServers, requestList) + leth.serverPool, leth.serverPoolIterator = vfc.NewServerPool(lesDb, []byte("serverpool:"), time.Second, prenegQuery, &mclock.System{}, nil, requestList) leth.serverPool.AddMetrics(suggestedTimeoutGauge, totalValueGauge, serverSelectableGauge, serverConnectedGauge, sessionValueMeter, serverDialedMeter) leth.retriever = newRetrieveManager(peers, leth.reqDist, leth.serverPool.GetTimeout) @@ -170,9 +168,6 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { leth.chtIndexer.Start(leth.blockchain) leth.bloomIndexer.Start(leth.blockchain) - // Start a light chain pruner to delete useless historical data. - leth.pruner = newPruner(chainDb, leth.chtIndexer, leth.bloomTrieIndexer) - // Rewind the chain in case of an incompatible config upgrade. if compat, ok := genesisErr.(*params.ConfigCompatError); ok { log.Warn("Rewinding chain to upgrade configuration", "err", compat) @@ -191,7 +186,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { } leth.ApiBackend.gpo = gasprice.NewOracle(leth.ApiBackend, gpoParams) - leth.handler = newClientHandler(config.UltraLightServers, config.UltraLightFraction, leth) + leth.handler = newClientHandler(leth) leth.netRPCService = ethapi.NewNetAPI(leth.p2pServer, leth.config.NetworkId) // Register the backend on the node @@ -298,9 +293,6 @@ func (s *LightEthereum) APIs() []rpc.API { { Namespace: "eth", Service: &LightDummyAPI{}, - }, { - Namespace: "eth", - Service: downloader.NewDownloaderAPI(s.handler.downloader, s.eventMux), }, { Namespace: "net", Service: s.netRPCService, @@ -315,13 +307,12 @@ func (s *LightEthereum) ResetWithGenesisBlock(gb *types.Block) { s.blockchain.ResetWithGenesisBlock(gb) } -func (s *LightEthereum) BlockChain() *light.LightChain { return s.blockchain } -func (s *LightEthereum) TxPool() *light.TxPool { return s.txPool } -func (s *LightEthereum) Engine() consensus.Engine { return s.engine } -func (s *LightEthereum) LesVersion() int { return int(ClientProtocolVersions[0]) } -func (s *LightEthereum) Downloader() *downloader.Downloader { return s.handler.downloader } -func (s *LightEthereum) EventMux() *event.TypeMux { return s.eventMux } -func (s *LightEthereum) Merger() *consensus.Merger { return s.merger } +func (s *LightEthereum) BlockChain() *light.LightChain { return s.blockchain } +func (s *LightEthereum) TxPool() *light.TxPool { return s.txPool } +func (s *LightEthereum) Engine() consensus.Engine { return s.engine } +func (s *LightEthereum) LesVersion() int { return int(ClientProtocolVersions[0]) } +func (s *LightEthereum) EventMux() *event.TypeMux { return s.eventMux } +func (s *LightEthereum) Merger() *consensus.Merger { return s.merger } // Protocols returns all the currently configured network protocols to start. func (s *LightEthereum) Protocols() []p2p.Protocol { @@ -354,7 +345,6 @@ func (s *LightEthereum) Start() error { // Start bloom request workers. s.wg.Add(bloomServiceThreads) s.startBloomHandlers(params.BloomBitsBlocksClient) - s.handler.start() return nil } @@ -374,7 +364,6 @@ func (s *LightEthereum) Stop() error { s.handler.stop() s.txPool.Stop() s.engine.Close() - s.pruner.close() s.eventMux.Stop() // Clean shutdown marker as the last thing before closing db s.shutdownTracker.Stop() diff --git a/les/client_handler.go b/les/client_handler.go index 6e7d88fc5a..39965d4c96 100644 --- a/les/client_handler.go +++ b/les/client_handler.go @@ -17,9 +17,6 @@ package les import ( - "context" - "math/big" - "math/rand" "sync" "time" @@ -27,68 +24,37 @@ import ( "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/core/forkid" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/eth/protocols/eth" - "github.com/ethereum/go-ethereum/les/downloader" "github.com/ethereum/go-ethereum/light" - "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p" ) // clientHandler is responsible for receiving and processing all incoming server // responses. type clientHandler struct { - ulc *ulc forkFilter forkid.Filter - fetcher *lightFetcher - downloader *downloader.Downloader backend *LightEthereum closeCh chan struct{} wg sync.WaitGroup // WaitGroup used to track all connected peers. - - // Hooks used in the testing - syncStart func(header *types.Header) // Hook called when the syncing is started - syncEnd func(header *types.Header) // Hook called when the syncing is done } -func newClientHandler(ulcServers []string, ulcFraction int, backend *LightEthereum) *clientHandler { +func newClientHandler(backend *LightEthereum) *clientHandler { handler := &clientHandler{ forkFilter: forkid.NewFilter(backend.blockchain), backend: backend, closeCh: make(chan struct{}), } - if ulcServers != nil { - ulc, err := newULC(ulcServers, ulcFraction) - if err != nil { - log.Error("Failed to initialize ultra light client") - } - handler.ulc = ulc - log.Info("Enable ultra light client mode") - } - handler.fetcher = newLightFetcher(backend.blockchain, backend.engine, backend.peers, handler.ulc, backend.chainDb, backend.reqDist, handler.synchronise) - handler.downloader = downloader.New(0, backend.chainDb, backend.eventMux, nil, backend.blockchain, handler.removePeer) - handler.backend.peers.subscribe((*downloaderPeerNotify)(handler)) return handler } -func (h *clientHandler) start() { - h.fetcher.start() -} - func (h *clientHandler) stop() { close(h.closeCh) - h.downloader.Terminate() - h.fetcher.stop() h.wg.Wait() } // runPeer is the p2p protocol run function for the given version. func (h *clientHandler) runPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter) error { - trusted := false - if h.ulc != nil { - trusted = h.ulc.trusted(p.ID()) - } - peer := newServerPeer(int(version), h.backend.config.NetworkId, trusted, p, newMeteredMsgWriter(rw, int(version))) + peer := newServerPeer(int(version), h.backend.config.NetworkId, false, p, newMeteredMsgWriter(rw, int(version))) defer peer.close() h.wg.Add(1) defer h.wg.Done() @@ -136,12 +102,6 @@ func (h *clientHandler) handle(p *serverPeer, noInitAnnounce bool) error { serverConnectionGauge.Update(int64(h.backend.peers.len())) }() - // Discard all the announces after the transition - // Also discarding initial signal to prevent syncing during testing. - if !(noInitAnnounce || h.backend.merger.TDDReached()) { - h.fetcher.announce(p, &announceData{Hash: p.headInfo.Hash, Number: p.headInfo.Number, Td: p.headInfo.Td}) - } - // Mark the peer starts to be served. p.serving.Store(true) defer p.serving.Store(false) @@ -206,11 +166,6 @@ func (h *clientHandler) handleMsg(p *serverPeer) error { // Update peer head information first and then notify the announcement p.updateHead(req.Hash, req.Number, req.Td) - - // Discard all the announces after the transition - if !h.backend.merger.TDDReached() { - h.fetcher.announce(p, &req) - } } case msg.Code == BlockHeadersMsg: p.Log().Trace("Received block header response message") @@ -221,28 +176,13 @@ func (h *clientHandler) handleMsg(p *serverPeer) error { if err := msg.Decode(&resp); err != nil { return errResp(ErrDecode, "msg %v: %v", msg, err) } - headers := resp.Headers p.fcServer.ReceivedReply(resp.ReqID, resp.BV) p.answeredRequest(resp.ReqID) - // Filter out the explicitly requested header by the retriever - if h.backend.retriever.requested(resp.ReqID) { - deliverMsg = &Msg{ - MsgType: MsgBlockHeaders, - ReqID: resp.ReqID, - Obj: resp.Headers, - } - } else { - // Filter out any explicitly requested headers, deliver the rest to the downloader - filter := len(headers) == 1 - if filter { - headers = h.fetcher.deliverHeaders(p, resp.ReqID, resp.Headers) - } - if len(headers) != 0 || !filter { - if err := h.downloader.DeliverHeaders(p.id, headers); err != nil { - log.Debug("Failed to deliver headers", "err", err) - } - } + deliverMsg = &Msg{ + MsgType: MsgBlockHeaders, + ReqID: resp.ReqID, + Obj: resp.Headers, } case msg.Code == BlockBodiesMsg: p.Log().Trace("Received block bodies response") @@ -366,117 +306,3 @@ func (h *clientHandler) handleMsg(p *serverPeer) error { } return nil } - -func (h *clientHandler) removePeer(id string) { - h.backend.peers.unregister(id) -} - -type peerConnection struct { - handler *clientHandler - peer *serverPeer -} - -func (pc *peerConnection) Head() (common.Hash, *big.Int) { - return pc.peer.HeadAndTd() -} - -func (pc *peerConnection) RequestHeadersByHash(origin common.Hash, amount int, skip int, reverse bool) error { - rq := &distReq{ - getCost: func(dp distPeer) uint64 { - peer := dp.(*serverPeer) - return peer.getRequestCost(GetBlockHeadersMsg, amount) - }, - canSend: func(dp distPeer) bool { - return dp.(*serverPeer) == pc.peer - }, - request: func(dp distPeer) func() { - reqID := rand.Uint64() - peer := dp.(*serverPeer) - cost := peer.getRequestCost(GetBlockHeadersMsg, amount) - peer.fcServer.QueuedRequest(reqID, cost) - return func() { peer.requestHeadersByHash(reqID, origin, amount, skip, reverse) } - }, - } - _, ok := <-pc.handler.backend.reqDist.queue(rq) - if !ok { - return light.ErrNoPeers - } - return nil -} - -func (pc *peerConnection) RequestHeadersByNumber(origin uint64, amount int, skip int, reverse bool) error { - rq := &distReq{ - getCost: func(dp distPeer) uint64 { - peer := dp.(*serverPeer) - return peer.getRequestCost(GetBlockHeadersMsg, amount) - }, - canSend: func(dp distPeer) bool { - return dp.(*serverPeer) == pc.peer - }, - request: func(dp distPeer) func() { - reqID := rand.Uint64() - peer := dp.(*serverPeer) - cost := peer.getRequestCost(GetBlockHeadersMsg, amount) - peer.fcServer.QueuedRequest(reqID, cost) - return func() { peer.requestHeadersByNumber(reqID, origin, amount, skip, reverse) } - }, - } - _, ok := <-pc.handler.backend.reqDist.queue(rq) - if !ok { - return light.ErrNoPeers - } - return nil -} - -// RetrieveSingleHeaderByNumber requests a single header by the specified block -// number. This function will wait the response until it's timeout or delivered. -func (pc *peerConnection) RetrieveSingleHeaderByNumber(context context.Context, number uint64) (*types.Header, error) { - reqID := rand.Uint64() - rq := &distReq{ - getCost: func(dp distPeer) uint64 { - peer := dp.(*serverPeer) - return peer.getRequestCost(GetBlockHeadersMsg, 1) - }, - canSend: func(dp distPeer) bool { - return dp.(*serverPeer) == pc.peer - }, - request: func(dp distPeer) func() { - peer := dp.(*serverPeer) - cost := peer.getRequestCost(GetBlockHeadersMsg, 1) - peer.fcServer.QueuedRequest(reqID, cost) - return func() { peer.requestHeadersByNumber(reqID, number, 1, 0, false) } - }, - } - var header *types.Header - if err := pc.handler.backend.retriever.retrieve(context, reqID, rq, func(peer distPeer, msg *Msg) error { - if msg.MsgType != MsgBlockHeaders { - return errInvalidMessageType - } - headers := msg.Obj.([]*types.Header) - if len(headers) != 1 { - return errInvalidEntryCount - } - header = headers[0] - return nil - }, nil); err != nil { - return nil, err - } - return header, nil -} - -// downloaderPeerNotify implements peerSetNotify -type downloaderPeerNotify clientHandler - -func (d *downloaderPeerNotify) registerPeer(p *serverPeer) { - h := (*clientHandler)(d) - pc := &peerConnection{ - handler: h, - peer: p, - } - h.downloader.RegisterLightPeer(p.id, eth.ETH66, pc) -} - -func (d *downloaderPeerNotify) unregisterPeer(p *serverPeer) { - h := (*clientHandler)(d) - h.downloader.UnregisterPeer(p.id) -} diff --git a/les/downloader/api.go b/les/downloader/api.go deleted file mode 100644 index 21200b676c..0000000000 --- a/les/downloader/api.go +++ /dev/null @@ -1,166 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package downloader - -import ( - "context" - "sync" - - "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/rpc" -) - -// DownloaderAPI provides an API which gives information about the current synchronisation status. -// It offers only methods that operates on data that can be available to anyone without security risks. -type DownloaderAPI struct { - d *Downloader - mux *event.TypeMux - installSyncSubscription chan chan interface{} - uninstallSyncSubscription chan *uninstallSyncSubscriptionRequest -} - -// NewDownloaderAPI create a new PublicDownloaderAPI. The API has an internal event loop that -// listens for events from the downloader through the global event mux. In case it receives one of -// these events it broadcasts it to all syncing subscriptions that are installed through the -// installSyncSubscription channel. -func NewDownloaderAPI(d *Downloader, m *event.TypeMux) *DownloaderAPI { - api := &DownloaderAPI{ - d: d, - mux: m, - installSyncSubscription: make(chan chan interface{}), - uninstallSyncSubscription: make(chan *uninstallSyncSubscriptionRequest), - } - - go api.eventLoop() - - return api -} - -// eventLoop runs a loop until the event mux closes. It will install and uninstall new -// sync subscriptions and broadcasts sync status updates to the installed sync subscriptions. -func (api *DownloaderAPI) eventLoop() { - var ( - sub = api.mux.Subscribe(StartEvent{}, DoneEvent{}, FailedEvent{}) - syncSubscriptions = make(map[chan interface{}]struct{}) - ) - - for { - select { - case i := <-api.installSyncSubscription: - syncSubscriptions[i] = struct{}{} - case u := <-api.uninstallSyncSubscription: - delete(syncSubscriptions, u.c) - close(u.uninstalled) - case event := <-sub.Chan(): - if event == nil { - return - } - - var notification interface{} - switch event.Data.(type) { - case StartEvent: - notification = &SyncingResult{ - Syncing: true, - Status: api.d.Progress(), - } - case DoneEvent, FailedEvent: - notification = false - } - // broadcast - for c := range syncSubscriptions { - c <- notification - } - } - } -} - -// Syncing provides information when this nodes starts synchronising with the Ethereum network and when it's finished. -func (api *DownloaderAPI) Syncing(ctx context.Context) (*rpc.Subscription, error) { - notifier, supported := rpc.NotifierFromContext(ctx) - if !supported { - return &rpc.Subscription{}, rpc.ErrNotificationsUnsupported - } - - rpcSub := notifier.CreateSubscription() - - go func() { - statuses := make(chan interface{}) - sub := api.SubscribeSyncStatus(statuses) - - for { - select { - case status := <-statuses: - notifier.Notify(rpcSub.ID, status) - case <-rpcSub.Err(): - sub.Unsubscribe() - return - case <-notifier.Closed(): - sub.Unsubscribe() - return - } - } - }() - - return rpcSub, nil -} - -// SyncingResult provides information about the current synchronisation status for this node. -type SyncingResult struct { - Syncing bool `json:"syncing"` - Status ethereum.SyncProgress `json:"status"` -} - -// uninstallSyncSubscriptionRequest uninstalls a syncing subscription in the API event loop. -type uninstallSyncSubscriptionRequest struct { - c chan interface{} - uninstalled chan interface{} -} - -// SyncStatusSubscription represents a syncing subscription. -type SyncStatusSubscription struct { - api *DownloaderAPI // register subscription in event loop of this api instance - c chan interface{} // channel where events are broadcasted to - unsubOnce sync.Once // make sure unsubscribe logic is executed once -} - -// Unsubscribe uninstalls the subscription from the DownloadAPI event loop. -// The status channel that was passed to subscribeSyncStatus isn't used anymore -// after this method returns. -func (s *SyncStatusSubscription) Unsubscribe() { - s.unsubOnce.Do(func() { - req := uninstallSyncSubscriptionRequest{s.c, make(chan interface{})} - s.api.uninstallSyncSubscription <- &req - - for { - select { - case <-s.c: - // drop new status events until uninstall confirmation - continue - case <-req.uninstalled: - return - } - } - }) -} - -// SubscribeSyncStatus creates a subscription that will broadcast new synchronisation updates. -// The given channel must receive interface values, the result can either -func (api *DownloaderAPI) SubscribeSyncStatus(status chan interface{}) *SyncStatusSubscription { - api.installSyncSubscription <- status - return &SyncStatusSubscription{api: api, c: status} -} diff --git a/les/downloader/downloader.go b/les/downloader/downloader.go deleted file mode 100644 index 5ffcbfa423..0000000000 --- a/les/downloader/downloader.go +++ /dev/null @@ -1,1979 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Package downloader is a temporary package whilst working on the eth/66 blocking refactors. -// After that work is done, les needs to be refactored to use the new package, -// or alternatively use a stripped down version of it. Either way, we need to -// keep the changes scoped so duplicating temporarily seems the sanest. -package downloader - -import ( - "errors" - "fmt" - "math/big" - "sync" - "sync/atomic" - "time" - - "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/state/snapshot" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/eth/protocols/eth" - "github.com/ethereum/go-ethereum/eth/protocols/snap" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/metrics" - "github.com/ethereum/go-ethereum/params" -) - -var ( - MaxBlockFetch = 128 // Amount of blocks to be fetched per retrieval request - MaxHeaderFetch = 192 // Amount of block headers to be fetched per retrieval request - MaxSkeletonSize = 128 // Number of header fetches to need for a skeleton assembly - MaxReceiptFetch = 256 // Amount of transaction receipts to allow fetching per request - MaxStateFetch = 384 // Amount of node state values to allow fetching per request - - maxQueuedHeaders = 32 * 1024 // [eth/62] Maximum number of headers to queue for import (DOS protection) - maxHeadersProcess = 2048 // Number of header download results to import at once into the chain - maxResultsProcess = 2048 // Number of content download results to import at once into the chain - fullMaxForkAncestry uint64 = params.FullImmutabilityThreshold // Maximum chain reorganisation (locally redeclared so tests can reduce it) - lightMaxForkAncestry uint64 = params.LightImmutabilityThreshold // Maximum chain reorganisation (locally redeclared so tests can reduce it) - - reorgProtThreshold = 48 // Threshold number of recent blocks to disable mini reorg protection - reorgProtHeaderDelay = 2 // Number of headers to delay delivering to cover mini reorgs - - fsHeaderSafetyNet = 2048 // Number of headers to discard in case a chain violation is detected - fsHeaderContCheck = 3 * time.Second // Time interval to check for header continuations during state download - fsMinFullBlocks = 64 // Number of blocks to retrieve fully even in fast sync -) - -var ( - errBusy = errors.New("busy") - errUnknownPeer = errors.New("peer is unknown or unhealthy") - errBadPeer = errors.New("action from bad peer ignored") - errStallingPeer = errors.New("peer is stalling") - errUnsyncedPeer = errors.New("unsynced peer") - errNoPeers = errors.New("no peers to keep download active") - errTimeout = errors.New("timeout") - errEmptyHeaderSet = errors.New("empty header set by peer") - errPeersUnavailable = errors.New("no peers available or all tried for download") - errInvalidAncestor = errors.New("retrieved ancestor is invalid") - errInvalidChain = errors.New("retrieved hash chain is invalid") - errInvalidBody = errors.New("retrieved block body is invalid") - errInvalidReceipt = errors.New("retrieved receipt is invalid") - errCancelStateFetch = errors.New("state data download canceled (requested)") - errCancelContentProcessing = errors.New("content processing canceled (requested)") - errCanceled = errors.New("syncing canceled (requested)") - errNoSyncActive = errors.New("no sync active") - errTooOld = errors.New("peer's protocol version too old") - errNoAncestorFound = errors.New("no common ancestor found") -) - -type Downloader struct { - mode uint32 // Synchronisation mode defining the strategy used (per sync cycle), use d.getMode() to get the SyncMode - mux *event.TypeMux // Event multiplexer to announce sync operation events - - checkpoint uint64 // Checkpoint block number to enforce head against (e.g. fast sync) - genesis uint64 // Genesis block number to limit sync to (e.g. light client CHT) - queue *queue // Scheduler for selecting the hashes to download - peers *peerSet // Set of active peers from which download can proceed - - stateDB ethdb.Database // Database to state sync into (and deduplicate via) - - // Statistics - syncStatsChainOrigin uint64 // Origin block number where syncing started at - syncStatsChainHeight uint64 // Highest block number known when syncing started - syncStatsState stateSyncStats - syncStatsLock sync.RWMutex // Lock protecting the sync stats fields - - lightchain LightChain - blockchain BlockChain - - // Callbacks - dropPeer peerDropFn // Drops a peer for misbehaving - - // Status - synchroniseMock func(id string, hash common.Hash) error // Replacement for synchronise during testing - synchronising int32 - notified int32 - committed int32 - ancientLimit uint64 // The maximum block number which can be regarded as ancient data. - - // Channels - headerCh chan dataPack // Channel receiving inbound block headers - bodyCh chan dataPack // Channel receiving inbound block bodies - receiptCh chan dataPack // Channel receiving inbound receipts - bodyWakeCh chan bool // Channel to signal the block body fetcher of new tasks - receiptWakeCh chan bool // Channel to signal the receipt fetcher of new tasks - headerProcCh chan []*types.Header // Channel to feed the header processor new tasks - - // State sync - pivotHeader *types.Header // Pivot block header to dynamically push the syncing state root - pivotLock sync.RWMutex // Lock protecting pivot header reads from updates - - snapSync bool // Whether to run state sync over the snap protocol - SnapSyncer *snap.Syncer // TODO(karalabe): make private! hack for now - stateSyncStart chan *stateSync - trackStateReq chan *stateReq - stateCh chan dataPack // Channel receiving inbound node state data - - // Cancellation and termination - cancelPeer string // Identifier of the peer currently being used as the master (cancel on drop) - cancelCh chan struct{} // Channel to cancel mid-flight syncs - cancelLock sync.RWMutex // Lock to protect the cancel channel and peer in delivers - cancelWg sync.WaitGroup // Make sure all fetcher goroutines have exited. - - quitCh chan struct{} // Quit channel to signal termination - quitLock sync.Mutex // Lock to prevent double closes - - // Testing hooks - syncInitHook func(uint64, uint64) // Method to call upon initiating a new sync run - bodyFetchHook func([]*types.Header) // Method to call upon starting a block body fetch - receiptFetchHook func([]*types.Header) // Method to call upon starting a receipt fetch - chainInsertHook func([]*fetchResult) // Method to call upon inserting a chain of blocks (possibly in multiple invocations) -} - -// LightChain encapsulates functions required to synchronise a light chain. -type LightChain interface { - // HasHeader verifies a header's presence in the local chain. - HasHeader(common.Hash, uint64) bool - - // GetHeaderByHash retrieves a header from the local chain. - GetHeaderByHash(common.Hash) *types.Header - - // CurrentHeader retrieves the head header from the local chain. - CurrentHeader() *types.Header - - // GetTd returns the total difficulty of a local block. - GetTd(common.Hash, uint64) *big.Int - - // InsertHeaderChain inserts a batch of headers into the local chain. - InsertHeaderChain([]*types.Header) (int, error) - - // SetHead rewinds the local chain to a new head. - SetHead(uint64) error -} - -// BlockChain encapsulates functions required to sync a (full or fast) blockchain. -type BlockChain interface { - LightChain - - // HasBlock verifies a block's presence in the local chain. - HasBlock(common.Hash, uint64) bool - - // HasFastBlock verifies a fast block's presence in the local chain. - HasFastBlock(common.Hash, uint64) bool - - // GetBlockByHash retrieves a block from the local chain. - GetBlockByHash(common.Hash) *types.Block - - // CurrentBlock retrieves the head block from the local chain. - CurrentBlock() *types.Block - - // CurrentFastBlock retrieves the head fast block from the local chain. - CurrentFastBlock() *types.Block - - // FastSyncCommitHead directly commits the head block to a certain entity. - FastSyncCommitHead(common.Hash) error - - // InsertChain inserts a batch of blocks into the local chain. - InsertChain(types.Blocks) (int, error) - - // InsertReceiptChain inserts a batch of receipts into the local chain. - InsertReceiptChain(types.Blocks, []types.Receipts, uint64) (int, error) - - // Snapshots returns the blockchain snapshot tree to paused it during sync. - Snapshots() *snapshot.Tree -} - -// New creates a new downloader to fetch hashes and blocks from remote peers. -func New(checkpoint uint64, stateDb ethdb.Database, mux *event.TypeMux, chain BlockChain, lightchain LightChain, dropPeer peerDropFn) *Downloader { - if lightchain == nil { - lightchain = chain - } - dl := &Downloader{ - stateDB: stateDb, - mux: mux, - checkpoint: checkpoint, - queue: newQueue(blockCacheMaxItems, blockCacheInitialItems), - peers: newPeerSet(), - blockchain: chain, - lightchain: lightchain, - dropPeer: dropPeer, - headerCh: make(chan dataPack, 1), - bodyCh: make(chan dataPack, 1), - receiptCh: make(chan dataPack, 1), - bodyWakeCh: make(chan bool, 1), - receiptWakeCh: make(chan bool, 1), - headerProcCh: make(chan []*types.Header, 1), - quitCh: make(chan struct{}), - stateCh: make(chan dataPack), - SnapSyncer: snap.NewSyncer(stateDb, rawdb.HashScheme), - stateSyncStart: make(chan *stateSync), - //syncStatsState: stateSyncStats{ - // processed: rawdb.ReadFastTrieProgress(stateDb), - //}, - trackStateReq: make(chan *stateReq), - } - go dl.stateFetcher() - return dl -} - -// Progress retrieves the synchronisation boundaries, specifically the origin -// block where synchronisation started at (may have failed/suspended); the block -// or header sync is currently at; and the latest known block which the sync targets. -// -// In addition, during the state download phase of fast synchronisation the number -// of processed and the total number of known states are also returned. Otherwise -// these are zero. -func (d *Downloader) Progress() ethereum.SyncProgress { - // Lock the current stats and return the progress - d.syncStatsLock.RLock() - defer d.syncStatsLock.RUnlock() - - current := uint64(0) - mode := d.getMode() - switch { - case d.blockchain != nil && mode == FullSync: - current = d.blockchain.CurrentBlock().NumberU64() - case d.blockchain != nil && mode == FastSync: - current = d.blockchain.CurrentFastBlock().NumberU64() - case d.lightchain != nil: - current = d.lightchain.CurrentHeader().Number.Uint64() - default: - log.Error("Unknown downloader chain/mode combo", "light", d.lightchain != nil, "full", d.blockchain != nil, "mode", mode) - } - return ethereum.SyncProgress{ - StartingBlock: d.syncStatsChainOrigin, - CurrentBlock: current, - HighestBlock: d.syncStatsChainHeight, - //PulledStates: d.syncStatsState.processed, - //KnownStates: d.syncStatsState.processed + d.syncStatsState.pending, - } -} - -// Synchronising returns whether the downloader is currently retrieving blocks. -func (d *Downloader) Synchronising() bool { - return atomic.LoadInt32(&d.synchronising) > 0 -} - -// RegisterPeer injects a new download peer into the set of block source to be -// used for fetching hashes and blocks from. -func (d *Downloader) RegisterPeer(id string, version uint, peer Peer) error { - var logger log.Logger - if len(id) < 16 { - // Tests use short IDs, don't choke on them - logger = log.New("peer", id) - } else { - logger = log.New("peer", id[:8]) - } - logger.Trace("Registering sync peer") - if err := d.peers.Register(newPeerConnection(id, version, peer, logger)); err != nil { - logger.Error("Failed to register sync peer", "err", err) - return err - } - return nil -} - -// RegisterLightPeer injects a light client peer, wrapping it so it appears as a regular peer. -func (d *Downloader) RegisterLightPeer(id string, version uint, peer LightPeer) error { - return d.RegisterPeer(id, version, &lightPeerWrapper{peer}) -} - -// UnregisterPeer remove a peer from the known list, preventing any action from -// the specified peer. An effort is also made to return any pending fetches into -// the queue. -func (d *Downloader) UnregisterPeer(id string) error { - // Unregister the peer from the active peer set and revoke any fetch tasks - var logger log.Logger - if len(id) < 16 { - // Tests use short IDs, don't choke on them - logger = log.New("peer", id) - } else { - logger = log.New("peer", id[:8]) - } - logger.Trace("Unregistering sync peer") - if err := d.peers.Unregister(id); err != nil { - logger.Error("Failed to unregister sync peer", "err", err) - return err - } - d.queue.Revoke(id) - - return nil -} - -// Synchronise tries to sync up our local block chain with a remote peer, both -// adding various sanity checks as well as wrapping it with various log entries. -func (d *Downloader) Synchronise(id string, head common.Hash, td *big.Int, mode SyncMode) error { - err := d.synchronise(id, head, td, mode) - - switch err { - case nil, errBusy, errCanceled: - return err - } - if errors.Is(err, errInvalidChain) || errors.Is(err, errBadPeer) || errors.Is(err, errTimeout) || - errors.Is(err, errStallingPeer) || errors.Is(err, errUnsyncedPeer) || errors.Is(err, errEmptyHeaderSet) || - errors.Is(err, errPeersUnavailable) || errors.Is(err, errTooOld) || errors.Is(err, errInvalidAncestor) { - log.Warn("Synchronisation failed, dropping peer", "peer", id, "err", err) - if d.dropPeer == nil { - // The dropPeer method is nil when `--copydb` is used for a local copy. - // Timeouts can occur if e.g. compaction hits at the wrong time, and can be ignored - log.Warn("Downloader wants to drop peer, but peerdrop-function is not set", "peer", id) - } else { - d.dropPeer(id) - } - return err - } - log.Warn("Synchronisation failed, retrying", "err", err) - return err -} - -// synchronise will select the peer and use it for synchronising. If an empty string is given -// it will use the best peer possible and synchronize if its TD is higher than our own. If any of the -// checks fail an error will be returned. This method is synchronous -func (d *Downloader) synchronise(id string, hash common.Hash, td *big.Int, mode SyncMode) error { - // Mock out the synchronisation if testing - if d.synchroniseMock != nil { - return d.synchroniseMock(id, hash) - } - // Make sure only one goroutine is ever allowed past this point at once - if !atomic.CompareAndSwapInt32(&d.synchronising, 0, 1) { - return errBusy - } - defer atomic.StoreInt32(&d.synchronising, 0) - - // Post a user notification of the sync (only once per session) - if atomic.CompareAndSwapInt32(&d.notified, 0, 1) { - log.Info("Block synchronisation started") - } - // If snap sync was requested, create the snap scheduler and switch to fast - // sync mode. Long term we could drop fast sync or merge the two together, - // but until snap becomes prevalent, we should support both. TODO(karalabe). - if mode == SnapSync { - if !d.snapSync { - // Snap sync uses the snapshot namespace to store potentially flakey data until - // sync completely heals and finishes. Pause snapshot maintenance in the mean - // time to prevent access. - if snapshots := d.blockchain.Snapshots(); snapshots != nil { // Only nil in tests - snapshots.Disable() - } - log.Warn("Enabling snapshot sync prototype") - d.snapSync = true - } - mode = FastSync - } - // Reset the queue, peer set and wake channels to clean any internal leftover state - d.queue.Reset(blockCacheMaxItems, blockCacheInitialItems) - d.peers.Reset() - - for _, ch := range []chan bool{d.bodyWakeCh, d.receiptWakeCh} { - select { - case <-ch: - default: - } - } - for _, ch := range []chan dataPack{d.headerCh, d.bodyCh, d.receiptCh} { - for empty := false; !empty; { - select { - case <-ch: - default: - empty = true - } - } - } - for empty := false; !empty; { - select { - case <-d.headerProcCh: - default: - empty = true - } - } - // Create cancel channel for aborting mid-flight and mark the master peer - d.cancelLock.Lock() - d.cancelCh = make(chan struct{}) - d.cancelPeer = id - d.cancelLock.Unlock() - - defer d.Cancel() // No matter what, we can't leave the cancel channel open - - // Atomically set the requested sync mode - atomic.StoreUint32(&d.mode, uint32(mode)) - - // Retrieve the origin peer and initiate the downloading process - p := d.peers.Peer(id) - if p == nil { - return errUnknownPeer - } - return d.syncWithPeer(p, hash, td) -} - -func (d *Downloader) getMode() SyncMode { - return SyncMode(atomic.LoadUint32(&d.mode)) -} - -// syncWithPeer starts a block synchronization based on the hash chain from the -// specified peer and head hash. -func (d *Downloader) syncWithPeer(p *peerConnection, hash common.Hash, td *big.Int) (err error) { - d.mux.Post(StartEvent{}) - defer func() { - // reset on error - if err != nil { - d.mux.Post(FailedEvent{err}) - } else { - latest := d.lightchain.CurrentHeader() - d.mux.Post(DoneEvent{latest}) - } - }() - if p.version < eth.ETH66 { - return fmt.Errorf("%w: advertized %d < required %d", errTooOld, p.version, eth.ETH66) - } - mode := d.getMode() - - log.Debug("Synchronising with the network", "peer", p.id, "eth", p.version, "head", hash, "td", td, "mode", mode) - defer func(start time.Time) { - log.Debug("Synchronisation terminated", "elapsed", common.PrettyDuration(time.Since(start))) - }(time.Now()) - - // Look up the sync boundaries: the common ancestor and the target block - latest, pivot, err := d.fetchHead(p) - if err != nil { - return err - } - if mode == FastSync && pivot == nil { - // If no pivot block was returned, the head is below the min full block - // threshold (i.e. new chain). In that case we won't really fast sync - // anyway, but still need a valid pivot block to avoid some code hitting - // nil panics on an access. - pivot = d.blockchain.CurrentBlock().Header() - } - height := latest.Number.Uint64() - - origin, err := d.findAncestor(p, latest) - if err != nil { - return err - } - d.syncStatsLock.Lock() - if d.syncStatsChainHeight <= origin || d.syncStatsChainOrigin > origin { - d.syncStatsChainOrigin = origin - } - d.syncStatsChainHeight = height - d.syncStatsLock.Unlock() - - // Ensure our origin point is below any fast sync pivot point - if mode == FastSync { - if height <= uint64(fsMinFullBlocks) { - origin = 0 - } else { - pivotNumber := pivot.Number.Uint64() - if pivotNumber <= origin { - origin = pivotNumber - 1 - } - // Write out the pivot into the database so a rollback beyond it will - // reenable fast sync - rawdb.WriteLastPivotNumber(d.stateDB, pivotNumber) - } - } - d.committed = 1 - if mode == FastSync && pivot.Number.Uint64() != 0 { - d.committed = 0 - } - if mode == FastSync { - // Set the ancient data limitation. - // If we are running fast sync, all block data older than ancientLimit will be - // written to the ancient store. More recent data will be written to the active - // database and will wait for the freezer to migrate. - // - // If there is a checkpoint available, then calculate the ancientLimit through - // that. Otherwise calculate the ancient limit through the advertised height - // of the remote peer. - // - // The reason for picking checkpoint first is that a malicious peer can give us - // a fake (very high) height, forcing the ancient limit to also be very high. - // The peer would start to feed us valid blocks until head, resulting in all of - // the blocks might be written into the ancient store. A following mini-reorg - // could cause issues. - if d.checkpoint != 0 && d.checkpoint > fullMaxForkAncestry+1 { - d.ancientLimit = d.checkpoint - } else if height > fullMaxForkAncestry+1 { - d.ancientLimit = height - fullMaxForkAncestry - 1 - } else { - d.ancientLimit = 0 - } - frozen, _ := d.stateDB.Ancients() // Ignore the error here since light client can also hit here. - - // If a part of blockchain data has already been written into active store, - // disable the ancient style insertion explicitly. - if origin >= frozen && frozen != 0 { - d.ancientLimit = 0 - log.Info("Disabling direct-ancient mode", "origin", origin, "ancient", frozen-1) - } else if d.ancientLimit > 0 { - log.Debug("Enabling direct-ancient mode", "ancient", d.ancientLimit) - } - // Rewind the ancient store and blockchain if reorg happens. - if origin+1 < frozen { - if err := d.lightchain.SetHead(origin + 1); err != nil { - return err - } - } - } - // Initiate the sync using a concurrent header and content retrieval algorithm - d.queue.Prepare(origin+1, mode) - if d.syncInitHook != nil { - d.syncInitHook(origin, height) - } - fetchers := []func() error{ - func() error { return d.fetchHeaders(p, origin+1) }, // Headers are always retrieved - func() error { return d.fetchBodies(origin + 1) }, // Bodies are retrieved during normal and fast sync - func() error { return d.fetchReceipts(origin + 1) }, // Receipts are retrieved during fast sync - func() error { return d.processHeaders(origin+1, td) }, - } - if mode == FastSync { - d.pivotLock.Lock() - d.pivotHeader = pivot - d.pivotLock.Unlock() - - fetchers = append(fetchers, func() error { return d.processFastSyncContent() }) - } else if mode == FullSync { - fetchers = append(fetchers, d.processFullSyncContent) - } - return d.spawnSync(fetchers) -} - -// spawnSync runs d.process and all given fetcher functions to completion in -// separate goroutines, returning the first error that appears. -func (d *Downloader) spawnSync(fetchers []func() error) error { - errc := make(chan error, len(fetchers)) - d.cancelWg.Add(len(fetchers)) - for _, fn := range fetchers { - fn := fn - go func() { defer d.cancelWg.Done(); errc <- fn() }() - } - // Wait for the first error, then terminate the others. - var err error - for i := 0; i < len(fetchers); i++ { - if i == len(fetchers)-1 { - // Close the queue when all fetchers have exited. - // This will cause the block processor to end when - // it has processed the queue. - d.queue.Close() - } - if err = <-errc; err != nil && err != errCanceled { - break - } - } - d.queue.Close() - d.Cancel() - return err -} - -// cancel aborts all of the operations and resets the queue. However, cancel does -// not wait for the running download goroutines to finish. This method should be -// used when cancelling the downloads from inside the downloader. -func (d *Downloader) cancel() { - // Close the current cancel channel - d.cancelLock.Lock() - defer d.cancelLock.Unlock() - - if d.cancelCh != nil { - select { - case <-d.cancelCh: - // Channel was already closed - default: - close(d.cancelCh) - } - } -} - -// Cancel aborts all of the operations and waits for all download goroutines to -// finish before returning. -func (d *Downloader) Cancel() { - d.cancel() - d.cancelWg.Wait() -} - -// Terminate interrupts the downloader, canceling all pending operations. -// The downloader cannot be reused after calling Terminate. -func (d *Downloader) Terminate() { - // Close the termination channel (make sure double close is allowed) - d.quitLock.Lock() - select { - case <-d.quitCh: - default: - close(d.quitCh) - } - d.quitLock.Unlock() - - // Cancel any pending download requests - d.Cancel() -} - -// fetchHead retrieves the head header and prior pivot block (if available) from -// a remote peer. -func (d *Downloader) fetchHead(p *peerConnection) (head *types.Header, pivot *types.Header, err error) { - p.log.Debug("Retrieving remote chain head") - mode := d.getMode() - - // Request the advertised remote head block and wait for the response - latest, _ := p.peer.Head() - fetch := 1 - if mode == FastSync { - fetch = 2 // head + pivot headers - } - go p.peer.RequestHeadersByHash(latest, fetch, fsMinFullBlocks-1, true) - - ttl := d.peers.rates.TargetTimeout() - timeout := time.After(ttl) - for { - select { - case <-d.cancelCh: - return nil, nil, errCanceled - - case packet := <-d.headerCh: - // Discard anything not from the origin peer - if packet.PeerId() != p.id { - log.Debug("Received headers from incorrect peer", "peer", packet.PeerId()) - break - } - // Make sure the peer gave us at least one and at most the requested headers - headers := packet.(*headerPack).headers - if len(headers) == 0 || len(headers) > fetch { - return nil, nil, fmt.Errorf("%w: returned headers %d != requested %d", errBadPeer, len(headers), fetch) - } - // The first header needs to be the head, validate against the checkpoint - // and request. If only 1 header was returned, make sure there's no pivot - // or there was not one requested. - head := headers[0] - if (mode == FastSync || mode == LightSync) && head.Number.Uint64() < d.checkpoint { - return nil, nil, fmt.Errorf("%w: remote head %d below checkpoint %d", errUnsyncedPeer, head.Number, d.checkpoint) - } - if len(headers) == 1 { - if mode == FastSync && head.Number.Uint64() > uint64(fsMinFullBlocks) { - return nil, nil, fmt.Errorf("%w: no pivot included along head header", errBadPeer) - } - p.log.Debug("Remote head identified, no pivot", "number", head.Number, "hash", head.Hash()) - return head, nil, nil - } - // At this point we have 2 headers in total and the first is the - // validated head of the chain. Check the pivot number and return, - pivot := headers[1] - if pivot.Number.Uint64() != head.Number.Uint64()-uint64(fsMinFullBlocks) { - return nil, nil, fmt.Errorf("%w: remote pivot %d != requested %d", errInvalidChain, pivot.Number, head.Number.Uint64()-uint64(fsMinFullBlocks)) - } - return head, pivot, nil - - case <-timeout: - p.log.Debug("Waiting for head header timed out", "elapsed", ttl) - return nil, nil, errTimeout - - case <-d.bodyCh: - case <-d.receiptCh: - // Out of bounds delivery, ignore - } - } -} - -// calculateRequestSpan calculates what headers to request from a peer when trying to determine the -// common ancestor. -// It returns parameters to be used for peer.RequestHeadersByNumber: -// -// from - starting block number -// count - number of headers to request -// skip - number of headers to skip -// -// and also returns 'max', the last block which is expected to be returned by the remote peers, -// given the (from,count,skip) -func calculateRequestSpan(remoteHeight, localHeight uint64) (int64, int, int, uint64) { - var ( - from int - count int - MaxCount = MaxHeaderFetch / 16 - ) - // requestHead is the highest block that we will ask for. If requestHead is not offset, - // the highest block that we will get is 16 blocks back from head, which means we - // will fetch 14 or 15 blocks unnecessarily in the case the height difference - // between us and the peer is 1-2 blocks, which is most common - requestHead := int(remoteHeight) - 1 - if requestHead < 0 { - requestHead = 0 - } - // requestBottom is the lowest block we want included in the query - // Ideally, we want to include the one just below our own head - requestBottom := int(localHeight - 1) - if requestBottom < 0 { - requestBottom = 0 - } - totalSpan := requestHead - requestBottom - span := 1 + totalSpan/MaxCount - if span < 2 { - span = 2 - } - if span > 16 { - span = 16 - } - - count = 1 + totalSpan/span - if count > MaxCount { - count = MaxCount - } - if count < 2 { - count = 2 - } - from = requestHead - (count-1)*span - if from < 0 { - from = 0 - } - max := from + (count-1)*span - return int64(from), count, span - 1, uint64(max) -} - -// findAncestor tries to locate the common ancestor link of the local chain and -// a remote peers blockchain. In the general case when our node was in sync and -// on the correct chain, checking the top N links should already get us a match. -// In the rare scenario when we ended up on a long reorganisation (i.e. none of -// the head links match), we do a binary search to find the common ancestor. -func (d *Downloader) findAncestor(p *peerConnection, remoteHeader *types.Header) (uint64, error) { - // Figure out the valid ancestor range to prevent rewrite attacks - var ( - floor = int64(-1) - localHeight uint64 - remoteHeight = remoteHeader.Number.Uint64() - ) - mode := d.getMode() - switch mode { - case FullSync: - localHeight = d.blockchain.CurrentBlock().NumberU64() - case FastSync: - localHeight = d.blockchain.CurrentFastBlock().NumberU64() - default: - localHeight = d.lightchain.CurrentHeader().Number.Uint64() - } - p.log.Debug("Looking for common ancestor", "local", localHeight, "remote", remoteHeight) - - // Recap floor value for binary search - maxForkAncestry := fullMaxForkAncestry - if d.getMode() == LightSync { - maxForkAncestry = lightMaxForkAncestry - } - if localHeight >= maxForkAncestry { - // We're above the max reorg threshold, find the earliest fork point - floor = int64(localHeight - maxForkAncestry) - } - // If we're doing a light sync, ensure the floor doesn't go below the CHT, as - // all headers before that point will be missing. - if mode == LightSync { - // If we don't know the current CHT position, find it - if d.genesis == 0 { - header := d.lightchain.CurrentHeader() - for header != nil { - d.genesis = header.Number.Uint64() - if floor >= int64(d.genesis)-1 { - break - } - header = d.lightchain.GetHeaderByHash(header.ParentHash) - } - } - // We already know the "genesis" block number, cap floor to that - if floor < int64(d.genesis)-1 { - floor = int64(d.genesis) - 1 - } - } - - ancestor, err := d.findAncestorSpanSearch(p, mode, remoteHeight, localHeight, floor) - if err == nil { - return ancestor, nil - } - // The returned error was not nil. - // If the error returned does not reflect that a common ancestor was not found, return it. - // If the error reflects that a common ancestor was not found, continue to binary search, - // where the error value will be reassigned. - if !errors.Is(err, errNoAncestorFound) { - return 0, err - } - - ancestor, err = d.findAncestorBinarySearch(p, mode, remoteHeight, floor) - if err != nil { - return 0, err - } - return ancestor, nil -} - -func (d *Downloader) findAncestorSpanSearch(p *peerConnection, mode SyncMode, remoteHeight, localHeight uint64, floor int64) (commonAncestor uint64, err error) { - from, count, skip, max := calculateRequestSpan(remoteHeight, localHeight) - - p.log.Trace("Span searching for common ancestor", "count", count, "from", from, "skip", skip) - go p.peer.RequestHeadersByNumber(uint64(from), count, skip, false) - - // Wait for the remote response to the head fetch - number, hash := uint64(0), common.Hash{} - - ttl := d.peers.rates.TargetTimeout() - timeout := time.After(ttl) - - for finished := false; !finished; { - select { - case <-d.cancelCh: - return 0, errCanceled - - case packet := <-d.headerCh: - // Discard anything not from the origin peer - if packet.PeerId() != p.id { - log.Debug("Received headers from incorrect peer", "peer", packet.PeerId()) - break - } - // Make sure the peer actually gave something valid - headers := packet.(*headerPack).headers - if len(headers) == 0 { - p.log.Warn("Empty head header set") - return 0, errEmptyHeaderSet - } - // Make sure the peer's reply conforms to the request - for i, header := range headers { - expectNumber := from + int64(i)*int64(skip+1) - if number := header.Number.Int64(); number != expectNumber { - p.log.Warn("Head headers broke chain ordering", "index", i, "requested", expectNumber, "received", number) - return 0, fmt.Errorf("%w: %v", errInvalidChain, errors.New("head headers broke chain ordering")) - } - } - // Check if a common ancestor was found - finished = true - for i := len(headers) - 1; i >= 0; i-- { - // Skip any headers that underflow/overflow our requested set - if headers[i].Number.Int64() < from || headers[i].Number.Uint64() > max { - continue - } - // Otherwise check if we already know the header or not - h := headers[i].Hash() - n := headers[i].Number.Uint64() - - var known bool - switch mode { - case FullSync: - known = d.blockchain.HasBlock(h, n) - case FastSync: - known = d.blockchain.HasFastBlock(h, n) - default: - known = d.lightchain.HasHeader(h, n) - } - if known { - number, hash = n, h - break - } - } - - case <-timeout: - p.log.Debug("Waiting for head header timed out", "elapsed", ttl) - return 0, errTimeout - - case <-d.bodyCh: - case <-d.receiptCh: - // Out of bounds delivery, ignore - } - } - // If the head fetch already found an ancestor, return - if hash != (common.Hash{}) { - if int64(number) <= floor { - p.log.Warn("Ancestor below allowance", "number", number, "hash", hash, "allowance", floor) - return 0, errInvalidAncestor - } - p.log.Debug("Found common ancestor", "number", number, "hash", hash) - return number, nil - } - return 0, errNoAncestorFound -} - -func (d *Downloader) findAncestorBinarySearch(p *peerConnection, mode SyncMode, remoteHeight uint64, floor int64) (commonAncestor uint64, err error) { - hash := common.Hash{} - - // Ancestor not found, we need to binary search over our chain - start, end := uint64(0), remoteHeight - if floor > 0 { - start = uint64(floor) - } - p.log.Trace("Binary searching for common ancestor", "start", start, "end", end) - - for start+1 < end { - // Split our chain interval in two, and request the hash to cross check - check := (start + end) / 2 - - ttl := d.peers.rates.TargetTimeout() - timeout := time.After(ttl) - - go p.peer.RequestHeadersByNumber(check, 1, 0, false) - - // Wait until a reply arrives to this request - for arrived := false; !arrived; { - select { - case <-d.cancelCh: - return 0, errCanceled - - case packet := <-d.headerCh: - // Discard anything not from the origin peer - if packet.PeerId() != p.id { - log.Debug("Received headers from incorrect peer", "peer", packet.PeerId()) - break - } - // Make sure the peer actually gave something valid - headers := packet.(*headerPack).headers - if len(headers) != 1 { - p.log.Warn("Multiple headers for single request", "headers", len(headers)) - return 0, fmt.Errorf("%w: multiple headers (%d) for single request", errBadPeer, len(headers)) - } - arrived = true - - // Modify the search interval based on the response - h := headers[0].Hash() - n := headers[0].Number.Uint64() - - var known bool - switch mode { - case FullSync: - known = d.blockchain.HasBlock(h, n) - case FastSync: - known = d.blockchain.HasFastBlock(h, n) - default: - known = d.lightchain.HasHeader(h, n) - } - if !known { - end = check - break - } - header := d.lightchain.GetHeaderByHash(h) // Independent of sync mode, header surely exists - if header.Number.Uint64() != check { - p.log.Warn("Received non requested header", "number", header.Number, "hash", header.Hash(), "request", check) - return 0, fmt.Errorf("%w: non-requested header (%d)", errBadPeer, header.Number) - } - start = check - hash = h - - case <-timeout: - p.log.Debug("Waiting for search header timed out", "elapsed", ttl) - return 0, errTimeout - - case <-d.bodyCh: - case <-d.receiptCh: - // Out of bounds delivery, ignore - } - } - } - // Ensure valid ancestry and return - if int64(start) <= floor { - p.log.Warn("Ancestor below allowance", "number", start, "hash", hash, "allowance", floor) - return 0, errInvalidAncestor - } - p.log.Debug("Found common ancestor", "number", start, "hash", hash) - return start, nil -} - -// fetchHeaders keeps retrieving headers concurrently from the number -// requested, until no more are returned, potentially throttling on the way. To -// facilitate concurrency but still protect against malicious nodes sending bad -// headers, we construct a header chain skeleton using the "origin" peer we are -// syncing with, and fill in the missing headers using anyone else. Headers from -// other peers are only accepted if they map cleanly to the skeleton. If no one -// can fill in the skeleton - not even the origin peer - it's assumed invalid and -// the origin is dropped. -func (d *Downloader) fetchHeaders(p *peerConnection, from uint64) error { - p.log.Debug("Directing header downloads", "origin", from) - defer p.log.Debug("Header download terminated") - - // Create a timeout timer, and the associated header fetcher - skeleton := true // Skeleton assembly phase or finishing up - pivoting := false // Whether the next request is pivot verification - request := time.Now() // time of the last skeleton fetch request - timeout := time.NewTimer(0) // timer to dump a non-responsive active peer - <-timeout.C // timeout channel should be initially empty - defer timeout.Stop() - - var ttl time.Duration - getHeaders := func(from uint64) { - request = time.Now() - - ttl = d.peers.rates.TargetTimeout() - timeout.Reset(ttl) - - if skeleton { - p.log.Trace("Fetching skeleton headers", "count", MaxHeaderFetch, "from", from) - go p.peer.RequestHeadersByNumber(from+uint64(MaxHeaderFetch)-1, MaxSkeletonSize, MaxHeaderFetch-1, false) - } else { - p.log.Trace("Fetching full headers", "count", MaxHeaderFetch, "from", from) - go p.peer.RequestHeadersByNumber(from, MaxHeaderFetch, 0, false) - } - } - getNextPivot := func() { - pivoting = true - request = time.Now() - - ttl = d.peers.rates.TargetTimeout() - timeout.Reset(ttl) - - d.pivotLock.RLock() - pivot := d.pivotHeader.Number.Uint64() - d.pivotLock.RUnlock() - - p.log.Trace("Fetching next pivot header", "number", pivot+uint64(fsMinFullBlocks)) - go p.peer.RequestHeadersByNumber(pivot+uint64(fsMinFullBlocks), 2, fsMinFullBlocks-9, false) // move +64 when it's 2x64-8 deep - } - // Start pulling the header chain skeleton until all is done - ancestor := from - getHeaders(from) - - mode := d.getMode() - for { - select { - case <-d.cancelCh: - return errCanceled - - case packet := <-d.headerCh: - // Make sure the active peer is giving us the skeleton headers - if packet.PeerId() != p.id { - log.Debug("Received skeleton from incorrect peer", "peer", packet.PeerId()) - break - } - headerReqTimer.UpdateSince(request) - timeout.Stop() - - // If the pivot is being checked, move if it became stale and run the real retrieval - var pivot uint64 - - d.pivotLock.RLock() - if d.pivotHeader != nil { - pivot = d.pivotHeader.Number.Uint64() - } - d.pivotLock.RUnlock() - - if pivoting { - if packet.Items() == 2 { - // Retrieve the headers and do some sanity checks, just in case - headers := packet.(*headerPack).headers - - if have, want := headers[0].Number.Uint64(), pivot+uint64(fsMinFullBlocks); have != want { - log.Warn("Peer sent invalid next pivot", "have", have, "want", want) - return fmt.Errorf("%w: next pivot number %d != requested %d", errInvalidChain, have, want) - } - if have, want := headers[1].Number.Uint64(), pivot+2*uint64(fsMinFullBlocks)-8; have != want { - log.Warn("Peer sent invalid pivot confirmer", "have", have, "want", want) - return fmt.Errorf("%w: next pivot confirmer number %d != requested %d", errInvalidChain, have, want) - } - log.Warn("Pivot seemingly stale, moving", "old", pivot, "new", headers[0].Number) - pivot = headers[0].Number.Uint64() - - d.pivotLock.Lock() - d.pivotHeader = headers[0] - d.pivotLock.Unlock() - - // Write out the pivot into the database so a rollback beyond - // it will reenable fast sync and update the state root that - // the state syncer will be downloading. - rawdb.WriteLastPivotNumber(d.stateDB, pivot) - } - pivoting = false - getHeaders(from) - continue - } - // If the skeleton's finished, pull any remaining head headers directly from the origin - if skeleton && packet.Items() == 0 { - skeleton = false - getHeaders(from) - continue - } - // If no more headers are inbound, notify the content fetchers and return - if packet.Items() == 0 { - // Don't abort header fetches while the pivot is downloading - if atomic.LoadInt32(&d.committed) == 0 && pivot <= from { - p.log.Debug("No headers, waiting for pivot commit") - select { - case <-time.After(fsHeaderContCheck): - getHeaders(from) - continue - case <-d.cancelCh: - return errCanceled - } - } - // Pivot done (or not in fast sync) and no more headers, terminate the process - p.log.Debug("No more headers available") - select { - case d.headerProcCh <- nil: - return nil - case <-d.cancelCh: - return errCanceled - } - } - headers := packet.(*headerPack).headers - - // If we received a skeleton batch, resolve internals concurrently - if skeleton { - filled, proced, err := d.fillHeaderSkeleton(from, headers) - if err != nil { - p.log.Debug("Skeleton chain invalid", "err", err) - return fmt.Errorf("%w: %v", errInvalidChain, err) - } - headers = filled[proced:] - from += uint64(proced) - } else { - // If we're closing in on the chain head, but haven't yet reached it, delay - // the last few headers so mini reorgs on the head don't cause invalid hash - // chain errors. - if n := len(headers); n > 0 { - // Retrieve the current head we're at - var head uint64 - if mode == LightSync { - head = d.lightchain.CurrentHeader().Number.Uint64() - } else { - head = d.blockchain.CurrentFastBlock().NumberU64() - if full := d.blockchain.CurrentBlock().NumberU64(); head < full { - head = full - } - } - // If the head is below the common ancestor, we're actually deduplicating - // already existing chain segments, so use the ancestor as the fake head. - // Otherwise we might end up delaying header deliveries pointlessly. - if head < ancestor { - head = ancestor - } - // If the head is way older than this batch, delay the last few headers - if head+uint64(reorgProtThreshold) < headers[n-1].Number.Uint64() { - delay := reorgProtHeaderDelay - if delay > n { - delay = n - } - headers = headers[:n-delay] - } - } - } - // Insert all the new headers and fetch the next batch - if len(headers) > 0 { - p.log.Trace("Scheduling new headers", "count", len(headers), "from", from) - select { - case d.headerProcCh <- headers: - case <-d.cancelCh: - return errCanceled - } - from += uint64(len(headers)) - - // If we're still skeleton filling fast sync, check pivot staleness - // before continuing to the next skeleton filling - if skeleton && pivot > 0 { - getNextPivot() - } else { - getHeaders(from) - } - } else { - // No headers delivered, or all of them being delayed, sleep a bit and retry - p.log.Trace("All headers delayed, waiting") - select { - case <-time.After(fsHeaderContCheck): - getHeaders(from) - continue - case <-d.cancelCh: - return errCanceled - } - } - - case <-timeout.C: - if d.dropPeer == nil { - // The dropPeer method is nil when `--copydb` is used for a local copy. - // Timeouts can occur if e.g. compaction hits at the wrong time, and can be ignored - p.log.Warn("Downloader wants to drop peer, but peerdrop-function is not set", "peer", p.id) - break - } - // Header retrieval timed out, consider the peer bad and drop - p.log.Debug("Header request timed out", "elapsed", ttl) - headerTimeoutMeter.Mark(1) - d.dropPeer(p.id) - - // Finish the sync gracefully instead of dumping the gathered data though - for _, ch := range []chan bool{d.bodyWakeCh, d.receiptWakeCh} { - select { - case ch <- false: - case <-d.cancelCh: - } - } - select { - case d.headerProcCh <- nil: - case <-d.cancelCh: - } - return fmt.Errorf("%w: header request timed out", errBadPeer) - } - } -} - -// fillHeaderSkeleton concurrently retrieves headers from all our available peers -// and maps them to the provided skeleton header chain. -// -// Any partial results from the beginning of the skeleton is (if possible) forwarded -// immediately to the header processor to keep the rest of the pipeline full even -// in the case of header stalls. -// -// The method returns the entire filled skeleton and also the number of headers -// already forwarded for processing. -func (d *Downloader) fillHeaderSkeleton(from uint64, skeleton []*types.Header) ([]*types.Header, int, error) { - log.Debug("Filling up skeleton", "from", from) - d.queue.ScheduleSkeleton(from, skeleton) - - var ( - deliver = func(packet dataPack) (int, error) { - pack := packet.(*headerPack) - return d.queue.DeliverHeaders(pack.peerID, pack.headers, d.headerProcCh) - } - expire = func() map[string]int { return d.queue.ExpireHeaders(d.peers.rates.TargetTimeout()) } - reserve = func(p *peerConnection, count int) (*fetchRequest, bool, bool) { - return d.queue.ReserveHeaders(p, count), false, false - } - fetch = func(p *peerConnection, req *fetchRequest) error { return p.FetchHeaders(req.From, MaxHeaderFetch) } - capacity = func(p *peerConnection) int { return p.HeaderCapacity(d.peers.rates.TargetRoundTrip()) } - setIdle = func(p *peerConnection, accepted int, deliveryTime time.Time) { - p.SetHeadersIdle(accepted, deliveryTime) - } - ) - err := d.fetchParts(d.headerCh, deliver, d.queue.headerContCh, expire, - d.queue.PendingHeaders, d.queue.InFlightHeaders, reserve, - nil, fetch, d.queue.CancelHeaders, capacity, d.peers.HeaderIdlePeers, setIdle, "headers") - - log.Debug("Skeleton fill terminated", "err", err) - - filled, proced := d.queue.RetrieveHeaders() - return filled, proced, err -} - -// fetchBodies iteratively downloads the scheduled block bodies, taking any -// available peers, reserving a chunk of blocks for each, waiting for delivery -// and also periodically checking for timeouts. -func (d *Downloader) fetchBodies(from uint64) error { - log.Debug("Downloading block bodies", "origin", from) - - var ( - deliver = func(packet dataPack) (int, error) { - pack := packet.(*bodyPack) - return d.queue.DeliverBodies(pack.peerID, pack.transactions, pack.uncles) - } - expire = func() map[string]int { return d.queue.ExpireBodies(d.peers.rates.TargetTimeout()) } - fetch = func(p *peerConnection, req *fetchRequest) error { return p.FetchBodies(req) } - capacity = func(p *peerConnection) int { return p.BlockCapacity(d.peers.rates.TargetRoundTrip()) } - setIdle = func(p *peerConnection, accepted int, deliveryTime time.Time) { p.SetBodiesIdle(accepted, deliveryTime) } - ) - err := d.fetchParts(d.bodyCh, deliver, d.bodyWakeCh, expire, - d.queue.PendingBlocks, d.queue.InFlightBlocks, d.queue.ReserveBodies, - d.bodyFetchHook, fetch, d.queue.CancelBodies, capacity, d.peers.BodyIdlePeers, setIdle, "bodies") - - log.Debug("Block body download terminated", "err", err) - return err -} - -// fetchReceipts iteratively downloads the scheduled block receipts, taking any -// available peers, reserving a chunk of receipts for each, waiting for delivery -// and also periodically checking for timeouts. -func (d *Downloader) fetchReceipts(from uint64) error { - log.Debug("Downloading transaction receipts", "origin", from) - - var ( - deliver = func(packet dataPack) (int, error) { - pack := packet.(*receiptPack) - return d.queue.DeliverReceipts(pack.peerID, pack.receipts) - } - expire = func() map[string]int { return d.queue.ExpireReceipts(d.peers.rates.TargetTimeout()) } - fetch = func(p *peerConnection, req *fetchRequest) error { return p.FetchReceipts(req) } - capacity = func(p *peerConnection) int { return p.ReceiptCapacity(d.peers.rates.TargetRoundTrip()) } - setIdle = func(p *peerConnection, accepted int, deliveryTime time.Time) { - p.SetReceiptsIdle(accepted, deliveryTime) - } - ) - err := d.fetchParts(d.receiptCh, deliver, d.receiptWakeCh, expire, - d.queue.PendingReceipts, d.queue.InFlightReceipts, d.queue.ReserveReceipts, - d.receiptFetchHook, fetch, d.queue.CancelReceipts, capacity, d.peers.ReceiptIdlePeers, setIdle, "receipts") - - log.Debug("Transaction receipt download terminated", "err", err) - return err -} - -// fetchParts iteratively downloads scheduled block parts, taking any available -// peers, reserving a chunk of fetch requests for each, waiting for delivery and -// also periodically checking for timeouts. -// -// As the scheduling/timeout logic mostly is the same for all downloaded data -// types, this method is used by each for data gathering and is instrumented with -// various callbacks to handle the slight differences between processing them. -// -// The instrumentation parameters: -// - errCancel: error type to return if the fetch operation is cancelled (mostly makes logging nicer) -// - deliveryCh: channel from which to retrieve downloaded data packets (merged from all concurrent peers) -// - deliver: processing callback to deliver data packets into type specific download queues (usually within `queue`) -// - wakeCh: notification channel for waking the fetcher when new tasks are available (or sync completed) -// - expire: task callback method to abort requests that took too long and return the faulty peers (traffic shaping) -// - pending: task callback for the number of requests still needing download (detect completion/non-completability) -// - inFlight: task callback for the number of in-progress requests (wait for all active downloads to finish) -// - throttle: task callback to check if the processing queue is full and activate throttling (bound memory use) -// - reserve: task callback to reserve new download tasks to a particular peer (also signals partial completions) -// - fetchHook: tester callback to notify of new tasks being initiated (allows testing the scheduling logic) -// - fetch: network callback to actually send a particular download request to a physical remote peer -// - cancel: task callback to abort an in-flight download request and allow rescheduling it (in case of lost peer) -// - capacity: network callback to retrieve the estimated type-specific bandwidth capacity of a peer (traffic shaping) -// - idle: network callback to retrieve the currently (type specific) idle peers that can be assigned tasks -// - setIdle: network callback to set a peer back to idle and update its estimated capacity (traffic shaping) -// - kind: textual label of the type being downloaded to display in log messages -func (d *Downloader) fetchParts(deliveryCh chan dataPack, deliver func(dataPack) (int, error), wakeCh chan bool, - expire func() map[string]int, pending func() int, inFlight func() bool, reserve func(*peerConnection, int) (*fetchRequest, bool, bool), - fetchHook func([]*types.Header), fetch func(*peerConnection, *fetchRequest) error, cancel func(*fetchRequest), capacity func(*peerConnection) int, - idle func() ([]*peerConnection, int), setIdle func(*peerConnection, int, time.Time), kind string) error { - // Create a ticker to detect expired retrieval tasks - ticker := time.NewTicker(100 * time.Millisecond) - defer ticker.Stop() - - update := make(chan struct{}, 1) - - // Prepare the queue and fetch block parts until the block header fetcher's done - finished := false - for { - select { - case <-d.cancelCh: - return errCanceled - - case packet := <-deliveryCh: - deliveryTime := time.Now() - // If the peer was previously banned and failed to deliver its pack - // in a reasonable time frame, ignore its message. - if peer := d.peers.Peer(packet.PeerId()); peer != nil { - // Deliver the received chunk of data and check chain validity - accepted, err := deliver(packet) - if errors.Is(err, errInvalidChain) { - return err - } - // Unless a peer delivered something completely else than requested (usually - // caused by a timed out request which came through in the end), set it to - // idle. If the delivery's stale, the peer should have already been idled. - if !errors.Is(err, errStaleDelivery) { - setIdle(peer, accepted, deliveryTime) - } - // Issue a log to the user to see what's going on - switch { - case err == nil && packet.Items() == 0: - peer.log.Trace("Requested data not delivered", "type", kind) - case err == nil: - peer.log.Trace("Delivered new batch of data", "type", kind, "count", packet.Stats()) - default: - peer.log.Debug("Failed to deliver retrieved data", "type", kind, "err", err) - } - } - // Blocks assembled, try to update the progress - select { - case update <- struct{}{}: - default: - } - - case cont := <-wakeCh: - // The header fetcher sent a continuation flag, check if it's done - if !cont { - finished = true - } - // Headers arrive, try to update the progress - select { - case update <- struct{}{}: - default: - } - - case <-ticker.C: - // Sanity check update the progress - select { - case update <- struct{}{}: - default: - } - - case <-update: - // Short circuit if we lost all our peers - if d.peers.Len() == 0 { - return errNoPeers - } - // Check for fetch request timeouts and demote the responsible peers - for pid, fails := range expire() { - if peer := d.peers.Peer(pid); peer != nil { - // If a lot of retrieval elements expired, we might have overestimated the remote peer or perhaps - // ourselves. Only reset to minimal throughput but don't drop just yet. If even the minimal times - // out that sync wise we need to get rid of the peer. - // - // The reason the minimum threshold is 2 is because the downloader tries to estimate the bandwidth - // and latency of a peer separately, which requires pushing the measures capacity a bit and seeing - // how response times reacts, to it always requests one more than the minimum (i.e. min 2). - if fails > 2 { - peer.log.Trace("Data delivery timed out", "type", kind) - setIdle(peer, 0, time.Now()) - } else { - peer.log.Debug("Stalling delivery, dropping", "type", kind) - - if d.dropPeer == nil { - // The dropPeer method is nil when `--copydb` is used for a local copy. - // Timeouts can occur if e.g. compaction hits at the wrong time, and can be ignored - peer.log.Warn("Downloader wants to drop peer, but peerdrop-function is not set", "peer", pid) - } else { - d.dropPeer(pid) - - // If this peer was the master peer, abort sync immediately - d.cancelLock.RLock() - master := pid == d.cancelPeer - d.cancelLock.RUnlock() - - if master { - d.cancel() - return errTimeout - } - } - } - } - } - // If there's nothing more to fetch, wait or terminate - if pending() == 0 { - if !inFlight() && finished { - log.Debug("Data fetching completed", "type", kind) - return nil - } - break - } - // Send a download request to all idle peers, until throttled - progressed, throttled, running := false, false, inFlight() - idles, total := idle() - pendCount := pending() - for _, peer := range idles { - // Short circuit if throttling activated - if throttled { - break - } - // Short circuit if there is no more available task. - if pendCount = pending(); pendCount == 0 { - break - } - // Reserve a chunk of fetches for a peer. A nil can mean either that - // no more headers are available, or that the peer is known not to - // have them. - request, progress, throttle := reserve(peer, capacity(peer)) - if progress { - progressed = true - } - if throttle { - throttled = true - throttleCounter.Inc(1) - } - if request == nil { - continue - } - if request.From > 0 { - peer.log.Trace("Requesting new batch of data", "type", kind, "from", request.From) - } else { - peer.log.Trace("Requesting new batch of data", "type", kind, "count", len(request.Headers), "from", request.Headers[0].Number) - } - // Fetch the chunk and make sure any errors return the hashes to the queue - if fetchHook != nil { - fetchHook(request.Headers) - } - if err := fetch(peer, request); err != nil { - // Although we could try and make an attempt to fix this, this error really - // means that we've double allocated a fetch task to a peer. If that is the - // case, the internal state of the downloader and the queue is very wrong so - // better hard crash and note the error instead of silently accumulating into - // a much bigger issue. - panic(fmt.Sprintf("%v: %s fetch assignment failed", peer, kind)) - } - running = true - } - // Make sure that we have peers available for fetching. If all peers have been tried - // and all failed throw an error - if !progressed && !throttled && !running && len(idles) == total && pendCount > 0 { - return errPeersUnavailable - } - } - } -} - -// processHeaders takes batches of retrieved headers from an input channel and -// keeps processing and scheduling them into the header chain and downloader's -// queue until the stream ends or a failure occurs. -func (d *Downloader) processHeaders(origin uint64, td *big.Int) error { - // Keep a count of uncertain headers to roll back - var ( - rollback uint64 // Zero means no rollback (fine as you can't unroll the genesis) - rollbackErr error - mode = d.getMode() - ) - defer func() { - if rollback > 0 { - lastHeader, lastFastBlock, lastBlock := d.lightchain.CurrentHeader().Number, common.Big0, common.Big0 - if mode != LightSync { - lastFastBlock = d.blockchain.CurrentFastBlock().Number() - lastBlock = d.blockchain.CurrentBlock().Number() - } - if err := d.lightchain.SetHead(rollback - 1); err != nil { // -1 to target the parent of the first uncertain block - // We're already unwinding the stack, only print the error to make it more visible - log.Error("Failed to roll back chain segment", "head", rollback-1, "err", err) - } - curFastBlock, curBlock := common.Big0, common.Big0 - if mode != LightSync { - curFastBlock = d.blockchain.CurrentFastBlock().Number() - curBlock = d.blockchain.CurrentBlock().Number() - } - log.Warn("Rolled back chain segment", - "header", fmt.Sprintf("%d->%d", lastHeader, d.lightchain.CurrentHeader().Number), - "fast", fmt.Sprintf("%d->%d", lastFastBlock, curFastBlock), - "block", fmt.Sprintf("%d->%d", lastBlock, curBlock), "reason", rollbackErr) - } - }() - // Wait for batches of headers to process - gotHeaders := false - - for { - select { - case <-d.cancelCh: - rollbackErr = errCanceled - return errCanceled - - case headers := <-d.headerProcCh: - // Terminate header processing if we synced up - if len(headers) == 0 { - // Notify everyone that headers are fully processed - for _, ch := range []chan bool{d.bodyWakeCh, d.receiptWakeCh} { - select { - case ch <- false: - case <-d.cancelCh: - } - } - // If no headers were retrieved at all, the peer violated its TD promise that it had a - // better chain compared to ours. The only exception is if its promised blocks were - // already imported by other means (e.g. fetcher): - // - // R , L : Both at block 10 - // R: Mine block 11, and propagate it to L - // L: Queue block 11 for import - // L: Notice that R's head and TD increased compared to ours, start sync - // L: Import of block 11 finishes - // L: Sync begins, and finds common ancestor at 11 - // L: Request new headers up from 11 (R's TD was higher, it must have something) - // R: Nothing to give - if mode != LightSync { - head := d.blockchain.CurrentBlock() - if !gotHeaders && td.Cmp(d.blockchain.GetTd(head.Hash(), head.NumberU64())) > 0 { - return errStallingPeer - } - } - // If fast or light syncing, ensure promised headers are indeed delivered. This is - // needed to detect scenarios where an attacker feeds a bad pivot and then bails out - // of delivering the post-pivot blocks that would flag the invalid content. - // - // This check cannot be executed "as is" for full imports, since blocks may still be - // queued for processing when the header download completes. However, as long as the - // peer gave us something useful, we're already happy/progressed (above check). - if mode == FastSync || mode == LightSync { - head := d.lightchain.CurrentHeader() - if td.Cmp(d.lightchain.GetTd(head.Hash(), head.Number.Uint64())) > 0 { - return errStallingPeer - } - } - // Disable any rollback and return - rollback = 0 - return nil - } - // Otherwise split the chunk of headers into batches and process them - gotHeaders = true - for len(headers) > 0 { - // Terminate if something failed in between processing chunks - select { - case <-d.cancelCh: - rollbackErr = errCanceled - return errCanceled - default: - } - // Select the next chunk of headers to import - limit := maxHeadersProcess - if limit > len(headers) { - limit = len(headers) - } - chunk := headers[:limit] - - // In case of header only syncing, validate the chunk immediately - if mode == FastSync || mode == LightSync { - if n, err := d.lightchain.InsertHeaderChain(chunk); err != nil { - rollbackErr = err - - // If some headers were inserted, track them as uncertain - if mode == FastSync && n > 0 && rollback == 0 { - rollback = chunk[0].Number.Uint64() - } - log.Warn("Invalid header encountered", "number", chunk[n].Number, "hash", chunk[n].Hash(), "parent", chunk[n].ParentHash, "err", err) - return fmt.Errorf("%w: %v", errInvalidChain, err) - } - // All verifications passed, track all headers within the allotted limits - if mode == FastSync { - head := chunk[len(chunk)-1].Number.Uint64() - if head-rollback > uint64(fsHeaderSafetyNet) { - rollback = head - uint64(fsHeaderSafetyNet) - } else { - rollback = 1 - } - } - } - // Unless we're doing light chains, schedule the headers for associated content retrieval - if mode == FullSync || mode == FastSync { - // If we've reached the allowed number of pending headers, stall a bit - for d.queue.PendingBlocks() >= maxQueuedHeaders || d.queue.PendingReceipts() >= maxQueuedHeaders { - select { - case <-d.cancelCh: - rollbackErr = errCanceled - return errCanceled - case <-time.After(time.Second): - } - } - // Otherwise insert the headers for content retrieval - inserts := d.queue.Schedule(chunk, origin) - if len(inserts) != len(chunk) { - rollbackErr = fmt.Errorf("stale headers: len inserts %v len(chunk) %v", len(inserts), len(chunk)) - return fmt.Errorf("%w: stale headers", errBadPeer) - } - } - headers = headers[limit:] - origin += uint64(limit) - } - // Update the highest block number we know if a higher one is found. - d.syncStatsLock.Lock() - if d.syncStatsChainHeight < origin { - d.syncStatsChainHeight = origin - 1 - } - d.syncStatsLock.Unlock() - - // Signal the content downloaders of the availability of new tasks - for _, ch := range []chan bool{d.bodyWakeCh, d.receiptWakeCh} { - select { - case ch <- true: - default: - } - } - } - } -} - -// processFullSyncContent takes fetch results from the queue and imports them into the chain. -func (d *Downloader) processFullSyncContent() error { - for { - results := d.queue.Results(true) - if len(results) == 0 { - return nil - } - if d.chainInsertHook != nil { - d.chainInsertHook(results) - } - if err := d.importBlockResults(results); err != nil { - return err - } - } -} - -func (d *Downloader) importBlockResults(results []*fetchResult) error { - // Check for any early termination requests - if len(results) == 0 { - return nil - } - select { - case <-d.quitCh: - return errCancelContentProcessing - default: - } - // Retrieve the a batch of results to import - first, last := results[0].Header, results[len(results)-1].Header - log.Debug("Inserting downloaded chain", "items", len(results), - "firstnum", first.Number, "firsthash", first.Hash(), - "lastnum", last.Number, "lasthash", last.Hash(), - ) - blocks := make([]*types.Block, len(results)) - for i, result := range results { - blocks[i] = types.NewBlockWithHeader(result.Header).WithBody(result.Transactions, result.Uncles) - } - if index, err := d.blockchain.InsertChain(blocks); err != nil { - if index < len(results) { - log.Debug("Downloaded item processing failed", "number", results[index].Header.Number, "hash", results[index].Header.Hash(), "err", err) - } else { - // The InsertChain method in blockchain.go will sometimes return an out-of-bounds index, - // when it needs to preprocess blocks to import a sidechain. - // The importer will put together a new list of blocks to import, which is a superset - // of the blocks delivered from the downloader, and the indexing will be off. - log.Debug("Downloaded item processing failed on sidechain import", "index", index, "err", err) - } - return fmt.Errorf("%w: %v", errInvalidChain, err) - } - return nil -} - -// processFastSyncContent takes fetch results from the queue and writes them to the -// database. It also controls the synchronisation of state nodes of the pivot block. -func (d *Downloader) processFastSyncContent() error { - // Start syncing state of the reported head block. This should get us most of - // the state of the pivot block. - d.pivotLock.RLock() - sync := d.syncState(d.pivotHeader.Root) - d.pivotLock.RUnlock() - - defer func() { - // The `sync` object is replaced every time the pivot moves. We need to - // defer close the very last active one, hence the lazy evaluation vs. - // calling defer sync.Cancel() !!! - sync.Cancel() - }() - - closeOnErr := func(s *stateSync) { - if err := s.Wait(); err != nil && err != errCancelStateFetch && err != errCanceled && err != snap.ErrCancelled { - d.queue.Close() // wake up Results - } - } - go closeOnErr(sync) - - // To cater for moving pivot points, track the pivot block and subsequently - // accumulated download results separately. - var ( - oldPivot *fetchResult // Locked in pivot block, might change eventually - oldTail []*fetchResult // Downloaded content after the pivot - ) - for { - // Wait for the next batch of downloaded data to be available, and if the pivot - // block became stale, move the goalpost - results := d.queue.Results(oldPivot == nil) // Block if we're not monitoring pivot staleness - if len(results) == 0 { - // If pivot sync is done, stop - if oldPivot == nil { - return sync.Cancel() - } - // If sync failed, stop - select { - case <-d.cancelCh: - sync.Cancel() - return errCanceled - default: - } - } - if d.chainInsertHook != nil { - d.chainInsertHook(results) - } - // If we haven't downloaded the pivot block yet, check pivot staleness - // notifications from the header downloader - d.pivotLock.RLock() - pivot := d.pivotHeader - d.pivotLock.RUnlock() - - if oldPivot == nil { - if pivot.Root != sync.root { - sync.Cancel() - sync = d.syncState(pivot.Root) - - go closeOnErr(sync) - } - } else { - results = append(append([]*fetchResult{oldPivot}, oldTail...), results...) - } - // Split around the pivot block and process the two sides via fast/full sync - if atomic.LoadInt32(&d.committed) == 0 { - latest := results[len(results)-1].Header - // If the height is above the pivot block by 2 sets, it means the pivot - // become stale in the network and it was garbage collected, move to a - // new pivot. - // - // Note, we have `reorgProtHeaderDelay` number of blocks withheld, Those - // need to be taken into account, otherwise we're detecting the pivot move - // late and will drop peers due to unavailable state!!! - if height := latest.Number.Uint64(); height >= pivot.Number.Uint64()+2*uint64(fsMinFullBlocks)-uint64(reorgProtHeaderDelay) { - log.Warn("Pivot became stale, moving", "old", pivot.Number.Uint64(), "new", height-uint64(fsMinFullBlocks)+uint64(reorgProtHeaderDelay)) - pivot = results[len(results)-1-fsMinFullBlocks+reorgProtHeaderDelay].Header // must exist as lower old pivot is uncommitted - - d.pivotLock.Lock() - d.pivotHeader = pivot - d.pivotLock.Unlock() - - // Write out the pivot into the database so a rollback beyond it will - // reenable fast sync - rawdb.WriteLastPivotNumber(d.stateDB, pivot.Number.Uint64()) - } - } - P, beforeP, afterP := splitAroundPivot(pivot.Number.Uint64(), results) - if err := d.commitFastSyncData(beforeP, sync); err != nil { - return err - } - if P != nil { - // If new pivot block found, cancel old state retrieval and restart - if oldPivot != P { - sync.Cancel() - sync = d.syncState(P.Header.Root) - - go closeOnErr(sync) - oldPivot = P - } - // Wait for completion, occasionally checking for pivot staleness - select { - case <-sync.done: - if sync.err != nil { - return sync.err - } - if err := d.commitPivotBlock(P); err != nil { - return err - } - oldPivot = nil - - case <-time.After(time.Second): - oldTail = afterP - continue - } - } - // Fast sync done, pivot commit done, full import - if err := d.importBlockResults(afterP); err != nil { - return err - } - } -} - -func splitAroundPivot(pivot uint64, results []*fetchResult) (p *fetchResult, before, after []*fetchResult) { - if len(results) == 0 { - return nil, nil, nil - } - if lastNum := results[len(results)-1].Header.Number.Uint64(); lastNum < pivot { - // the pivot is somewhere in the future - return nil, results, nil - } - // This can also be optimized, but only happens very seldom - for _, result := range results { - num := result.Header.Number.Uint64() - switch { - case num < pivot: - before = append(before, result) - case num == pivot: - p = result - default: - after = append(after, result) - } - } - return p, before, after -} - -func (d *Downloader) commitFastSyncData(results []*fetchResult, stateSync *stateSync) error { - // Check for any early termination requests - if len(results) == 0 { - return nil - } - select { - case <-d.quitCh: - return errCancelContentProcessing - case <-stateSync.done: - if err := stateSync.Wait(); err != nil { - return err - } - default: - } - // Retrieve the a batch of results to import - first, last := results[0].Header, results[len(results)-1].Header - log.Debug("Inserting fast-sync blocks", "items", len(results), - "firstnum", first.Number, "firsthash", first.Hash(), - "lastnumn", last.Number, "lasthash", last.Hash(), - ) - blocks := make([]*types.Block, len(results)) - receipts := make([]types.Receipts, len(results)) - for i, result := range results { - blocks[i] = types.NewBlockWithHeader(result.Header).WithBody(result.Transactions, result.Uncles) - receipts[i] = result.Receipts - } - if index, err := d.blockchain.InsertReceiptChain(blocks, receipts, d.ancientLimit); err != nil { - log.Debug("Downloaded item processing failed", "number", results[index].Header.Number, "hash", results[index].Header.Hash(), "err", err) - return fmt.Errorf("%w: %v", errInvalidChain, err) - } - return nil -} - -func (d *Downloader) commitPivotBlock(result *fetchResult) error { - block := types.NewBlockWithHeader(result.Header).WithBody(result.Transactions, result.Uncles) - log.Debug("Committing fast sync pivot as new head", "number", block.Number(), "hash", block.Hash()) - - // Commit the pivot block as the new head, will require full sync from here on - if _, err := d.blockchain.InsertReceiptChain([]*types.Block{block}, []types.Receipts{result.Receipts}, d.ancientLimit); err != nil { - return err - } - if err := d.blockchain.FastSyncCommitHead(block.Hash()); err != nil { - return err - } - atomic.StoreInt32(&d.committed, 1) - return nil -} - -// DeliverHeaders injects a new batch of block headers received from a remote -// node into the download schedule. -func (d *Downloader) DeliverHeaders(id string, headers []*types.Header) error { - return d.deliver(d.headerCh, &headerPack{id, headers}, headerInMeter, headerDropMeter) -} - -// DeliverBodies injects a new batch of block bodies received from a remote node. -func (d *Downloader) DeliverBodies(id string, transactions [][]*types.Transaction, uncles [][]*types.Header) error { - return d.deliver(d.bodyCh, &bodyPack{id, transactions, uncles}, bodyInMeter, bodyDropMeter) -} - -// DeliverReceipts injects a new batch of receipts received from a remote node. -func (d *Downloader) DeliverReceipts(id string, receipts [][]*types.Receipt) error { - return d.deliver(d.receiptCh, &receiptPack{id, receipts}, receiptInMeter, receiptDropMeter) -} - -// DeliverNodeData injects a new batch of node state data received from a remote node. -func (d *Downloader) DeliverNodeData(id string, data [][]byte) error { - return d.deliver(d.stateCh, &statePack{id, data}, stateInMeter, stateDropMeter) -} - -// DeliverSnapPacket is invoked from a peer's message handler when it transmits a -// data packet for the local node to consume. -func (d *Downloader) DeliverSnapPacket(peer *snap.Peer, packet snap.Packet) error { - switch packet := packet.(type) { - case *snap.AccountRangePacket: - hashes, accounts, err := packet.Unpack() - if err != nil { - return err - } - return d.SnapSyncer.OnAccounts(peer, packet.ID, hashes, accounts, packet.Proof) - - case *snap.StorageRangesPacket: - hashset, slotset := packet.Unpack() - return d.SnapSyncer.OnStorage(peer, packet.ID, hashset, slotset, packet.Proof) - - case *snap.ByteCodesPacket: - return d.SnapSyncer.OnByteCodes(peer, packet.ID, packet.Codes) - - case *snap.TrieNodesPacket: - return d.SnapSyncer.OnTrieNodes(peer, packet.ID, packet.Nodes) - - default: - return fmt.Errorf("unexpected snap packet type: %T", packet) - } -} - -// deliver injects a new batch of data received from a remote node. -func (d *Downloader) deliver(destCh chan dataPack, packet dataPack, inMeter, dropMeter metrics.Meter) (err error) { - // Update the delivery metrics for both good and failed deliveries - inMeter.Mark(int64(packet.Items())) - defer func() { - if err != nil { - dropMeter.Mark(int64(packet.Items())) - } - }() - // Deliver or abort if the sync is canceled while queuing - d.cancelLock.RLock() - cancel := d.cancelCh - d.cancelLock.RUnlock() - if cancel == nil { - return errNoSyncActive - } - select { - case destCh <- packet: - return nil - case <-cancel: - return errNoSyncActive - } -} diff --git a/les/downloader/downloader_test.go b/les/downloader/downloader_test.go deleted file mode 100644 index ae0d5ccda3..0000000000 --- a/les/downloader/downloader_test.go +++ /dev/null @@ -1,1621 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package downloader - -import ( - "errors" - "fmt" - "math/big" - "strings" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/ethereum/go-ethereum" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/state/snapshot" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/eth/protocols/eth" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/trie" -) - -// Reduce some of the parameters to make the tester faster. -func init() { - fullMaxForkAncestry = 10000 - lightMaxForkAncestry = 10000 - blockCacheMaxItems = 1024 - fsHeaderContCheck = 500 * time.Millisecond -} - -// downloadTester is a test simulator for mocking out local block chain. -type downloadTester struct { - downloader *Downloader - - genesis *types.Block // Genesis blocks used by the tester and peers - stateDb ethdb.Database // Database used by the tester for syncing from peers - peerDb ethdb.Database // Database of the peers containing all data - peers map[string]*downloadTesterPeer - - ownHashes []common.Hash // Hash chain belonging to the tester - ownHeaders map[common.Hash]*types.Header // Headers belonging to the tester - ownBlocks map[common.Hash]*types.Block // Blocks belonging to the tester - ownReceipts map[common.Hash]types.Receipts // Receipts belonging to the tester - ownChainTd map[common.Hash]*big.Int // Total difficulties of the blocks in the local chain - - ancientHeaders map[common.Hash]*types.Header // Ancient headers belonging to the tester - ancientBlocks map[common.Hash]*types.Block // Ancient blocks belonging to the tester - ancientReceipts map[common.Hash]types.Receipts // Ancient receipts belonging to the tester - ancientChainTd map[common.Hash]*big.Int // Ancient total difficulties of the blocks in the local chain - - lock sync.RWMutex -} - -// newTester creates a new downloader test mocker. -func newTester() *downloadTester { - tester := &downloadTester{ - genesis: testGenesis, - peerDb: testDB, - peers: make(map[string]*downloadTesterPeer), - ownHashes: []common.Hash{testGenesis.Hash()}, - ownHeaders: map[common.Hash]*types.Header{testGenesis.Hash(): testGenesis.Header()}, - ownBlocks: map[common.Hash]*types.Block{testGenesis.Hash(): testGenesis}, - ownReceipts: map[common.Hash]types.Receipts{testGenesis.Hash(): nil}, - ownChainTd: map[common.Hash]*big.Int{testGenesis.Hash(): testGenesis.Difficulty()}, - - // Initialize ancient store with test genesis block - ancientHeaders: map[common.Hash]*types.Header{testGenesis.Hash(): testGenesis.Header()}, - ancientBlocks: map[common.Hash]*types.Block{testGenesis.Hash(): testGenesis}, - ancientReceipts: map[common.Hash]types.Receipts{testGenesis.Hash(): nil}, - ancientChainTd: map[common.Hash]*big.Int{testGenesis.Hash(): testGenesis.Difficulty()}, - } - tester.stateDb = rawdb.NewMemoryDatabase() - tester.stateDb.Put(testGenesis.Root().Bytes(), []byte{0x00}) - - tester.downloader = New(0, tester.stateDb, new(event.TypeMux), tester, nil, tester.dropPeer) - return tester -} - -// terminate aborts any operations on the embedded downloader and releases all -// held resources. -func (dl *downloadTester) terminate() { - dl.downloader.Terminate() -} - -// sync starts synchronizing with a remote peer, blocking until it completes. -func (dl *downloadTester) sync(id string, td *big.Int, mode SyncMode) error { - dl.lock.RLock() - hash := dl.peers[id].chain.headBlock().Hash() - // If no particular TD was requested, load from the peer's blockchain - if td == nil { - td = dl.peers[id].chain.td(hash) - } - dl.lock.RUnlock() - - // Synchronise with the chosen peer and ensure proper cleanup afterwards - err := dl.downloader.synchronise(id, hash, td, mode) - select { - case <-dl.downloader.cancelCh: - // Ok, downloader fully cancelled after sync cycle - default: - // Downloader is still accepting packets, can block a peer up - panic("downloader active post sync cycle") // panic will be caught by tester - } - return err -} - -// HasHeader checks if a header is present in the testers canonical chain. -func (dl *downloadTester) HasHeader(hash common.Hash, number uint64) bool { - return dl.GetHeaderByHash(hash) != nil -} - -// HasBlock checks if a block is present in the testers canonical chain. -func (dl *downloadTester) HasBlock(hash common.Hash, number uint64) bool { - return dl.GetBlockByHash(hash) != nil -} - -// HasFastBlock checks if a block is present in the testers canonical chain. -func (dl *downloadTester) HasFastBlock(hash common.Hash, number uint64) bool { - dl.lock.RLock() - defer dl.lock.RUnlock() - - if _, ok := dl.ancientReceipts[hash]; ok { - return true - } - _, ok := dl.ownReceipts[hash] - return ok -} - -// GetHeader retrieves a header from the testers canonical chain. -func (dl *downloadTester) GetHeaderByHash(hash common.Hash) *types.Header { - dl.lock.RLock() - defer dl.lock.RUnlock() - return dl.getHeaderByHash(hash) -} - -// getHeaderByHash returns the header if found either within ancients or own blocks) -// This method assumes that the caller holds at least the read-lock (dl.lock) -func (dl *downloadTester) getHeaderByHash(hash common.Hash) *types.Header { - header := dl.ancientHeaders[hash] - if header != nil { - return header - } - return dl.ownHeaders[hash] -} - -// GetBlock retrieves a block from the testers canonical chain. -func (dl *downloadTester) GetBlockByHash(hash common.Hash) *types.Block { - dl.lock.RLock() - defer dl.lock.RUnlock() - - block := dl.ancientBlocks[hash] - if block != nil { - return block - } - return dl.ownBlocks[hash] -} - -// CurrentHeader retrieves the current head header from the canonical chain. -func (dl *downloadTester) CurrentHeader() *types.Header { - dl.lock.RLock() - defer dl.lock.RUnlock() - - for i := len(dl.ownHashes) - 1; i >= 0; i-- { - if header := dl.ancientHeaders[dl.ownHashes[i]]; header != nil { - return header - } - if header := dl.ownHeaders[dl.ownHashes[i]]; header != nil { - return header - } - } - return dl.genesis.Header() -} - -// CurrentBlock retrieves the current head block from the canonical chain. -func (dl *downloadTester) CurrentBlock() *types.Block { - dl.lock.RLock() - defer dl.lock.RUnlock() - - for i := len(dl.ownHashes) - 1; i >= 0; i-- { - if block := dl.ancientBlocks[dl.ownHashes[i]]; block != nil { - if _, err := dl.stateDb.Get(block.Root().Bytes()); err == nil { - return block - } - return block - } - if block := dl.ownBlocks[dl.ownHashes[i]]; block != nil { - if _, err := dl.stateDb.Get(block.Root().Bytes()); err == nil { - return block - } - } - } - return dl.genesis -} - -// CurrentFastBlock retrieves the current head fast-sync block from the canonical chain. -func (dl *downloadTester) CurrentFastBlock() *types.Block { - dl.lock.RLock() - defer dl.lock.RUnlock() - - for i := len(dl.ownHashes) - 1; i >= 0; i-- { - if block := dl.ancientBlocks[dl.ownHashes[i]]; block != nil { - return block - } - if block := dl.ownBlocks[dl.ownHashes[i]]; block != nil { - return block - } - } - return dl.genesis -} - -// FastSyncCommitHead manually sets the head block to a given hash. -func (dl *downloadTester) FastSyncCommitHead(hash common.Hash) error { - // For now only check that the state trie is correct - if block := dl.GetBlockByHash(hash); block != nil { - _, err := trie.NewStateTrie(trie.StateTrieID(block.Root()), trie.NewDatabase(dl.stateDb)) - return err - } - return fmt.Errorf("non existent block: %x", hash[:4]) -} - -// GetTd retrieves the block's total difficulty from the canonical chain. -func (dl *downloadTester) GetTd(hash common.Hash, number uint64) *big.Int { - dl.lock.RLock() - defer dl.lock.RUnlock() - - return dl.getTd(hash) -} - -// getTd retrieves the block's total difficulty if found either within -// ancients or own blocks). -// This method assumes that the caller holds at least the read-lock (dl.lock) -func (dl *downloadTester) getTd(hash common.Hash) *big.Int { - if td := dl.ancientChainTd[hash]; td != nil { - return td - } - return dl.ownChainTd[hash] -} - -// InsertHeaderChain injects a new batch of headers into the simulated chain. -func (dl *downloadTester) InsertHeaderChain(headers []*types.Header) (i int, err error) { - dl.lock.Lock() - defer dl.lock.Unlock() - // Do a quick check, as the blockchain.InsertHeaderChain doesn't insert anything in case of errors - if dl.getHeaderByHash(headers[0].ParentHash) == nil { - return 0, fmt.Errorf("InsertHeaderChain: unknown parent at first position, parent of number %d", headers[0].Number) - } - var hashes []common.Hash - for i := 1; i < len(headers); i++ { - hash := headers[i-1].Hash() - if headers[i].ParentHash != headers[i-1].Hash() { - return i, fmt.Errorf("non-contiguous import at position %d", i) - } - hashes = append(hashes, hash) - } - hashes = append(hashes, headers[len(headers)-1].Hash()) - // Do a full insert if pre-checks passed - for i, header := range headers { - hash := hashes[i] - if dl.getHeaderByHash(hash) != nil { - continue - } - if dl.getHeaderByHash(header.ParentHash) == nil { - // This _should_ be impossible, due to precheck and induction - return i, fmt.Errorf("InsertHeaderChain: unknown parent at position %d", i) - } - dl.ownHashes = append(dl.ownHashes, hash) - dl.ownHeaders[hash] = header - - td := dl.getTd(header.ParentHash) - dl.ownChainTd[hash] = new(big.Int).Add(td, header.Difficulty) - } - return len(headers), nil -} - -// InsertChain injects a new batch of blocks into the simulated chain. -func (dl *downloadTester) InsertChain(blocks types.Blocks) (i int, err error) { - dl.lock.Lock() - defer dl.lock.Unlock() - for i, block := range blocks { - if parent, ok := dl.ownBlocks[block.ParentHash()]; !ok { - return i, fmt.Errorf("InsertChain: unknown parent at position %d / %d", i, len(blocks)) - } else if _, err := dl.stateDb.Get(parent.Root().Bytes()); err != nil { - return i, fmt.Errorf("InsertChain: unknown parent state %x: %v", parent.Root(), err) - } - if hdr := dl.getHeaderByHash(block.Hash()); hdr == nil { - dl.ownHashes = append(dl.ownHashes, block.Hash()) - dl.ownHeaders[block.Hash()] = block.Header() - } - dl.ownBlocks[block.Hash()] = block - dl.ownReceipts[block.Hash()] = make(types.Receipts, 0) - dl.stateDb.Put(block.Root().Bytes(), []byte{0x00}) - td := dl.getTd(block.ParentHash()) - dl.ownChainTd[block.Hash()] = new(big.Int).Add(td, block.Difficulty()) - } - return len(blocks), nil -} - -// InsertReceiptChain injects a new batch of receipts into the simulated chain. -func (dl *downloadTester) InsertReceiptChain(blocks types.Blocks, receipts []types.Receipts, ancientLimit uint64) (i int, err error) { - dl.lock.Lock() - defer dl.lock.Unlock() - - for i := 0; i < len(blocks) && i < len(receipts); i++ { - if _, ok := dl.ownHeaders[blocks[i].Hash()]; !ok { - return i, errors.New("unknown owner") - } - if _, ok := dl.ancientBlocks[blocks[i].ParentHash()]; !ok { - if _, ok := dl.ownBlocks[blocks[i].ParentHash()]; !ok { - return i, errors.New("InsertReceiptChain: unknown parent") - } - } - if blocks[i].NumberU64() <= ancientLimit { - dl.ancientBlocks[blocks[i].Hash()] = blocks[i] - dl.ancientReceipts[blocks[i].Hash()] = receipts[i] - - // Migrate from active db to ancient db - dl.ancientHeaders[blocks[i].Hash()] = blocks[i].Header() - dl.ancientChainTd[blocks[i].Hash()] = new(big.Int).Add(dl.ancientChainTd[blocks[i].ParentHash()], blocks[i].Difficulty()) - delete(dl.ownHeaders, blocks[i].Hash()) - delete(dl.ownChainTd, blocks[i].Hash()) - } else { - dl.ownBlocks[blocks[i].Hash()] = blocks[i] - dl.ownReceipts[blocks[i].Hash()] = receipts[i] - } - } - return len(blocks), nil -} - -// SetHead rewinds the local chain to a new head. -func (dl *downloadTester) SetHead(head uint64) error { - dl.lock.Lock() - defer dl.lock.Unlock() - - // Find the hash of the head to reset to - var hash common.Hash - for h, header := range dl.ownHeaders { - if header.Number.Uint64() == head { - hash = h - } - } - for h, header := range dl.ancientHeaders { - if header.Number.Uint64() == head { - hash = h - } - } - if hash == (common.Hash{}) { - return fmt.Errorf("unknown head to set: %d", head) - } - // Find the offset in the header chain - var offset int - for o, h := range dl.ownHashes { - if h == hash { - offset = o - break - } - } - // Remove all the hashes and associated data afterwards - for i := offset + 1; i < len(dl.ownHashes); i++ { - delete(dl.ownChainTd, dl.ownHashes[i]) - delete(dl.ownHeaders, dl.ownHashes[i]) - delete(dl.ownReceipts, dl.ownHashes[i]) - delete(dl.ownBlocks, dl.ownHashes[i]) - - delete(dl.ancientChainTd, dl.ownHashes[i]) - delete(dl.ancientHeaders, dl.ownHashes[i]) - delete(dl.ancientReceipts, dl.ownHashes[i]) - delete(dl.ancientBlocks, dl.ownHashes[i]) - } - dl.ownHashes = dl.ownHashes[:offset+1] - return nil -} - -// Rollback removes some recently added elements from the chain. -func (dl *downloadTester) Rollback(hashes []common.Hash) { -} - -// newPeer registers a new block download source into the downloader. -func (dl *downloadTester) newPeer(id string, version uint, chain *testChain) error { - dl.lock.Lock() - defer dl.lock.Unlock() - - peer := &downloadTesterPeer{dl: dl, id: id, chain: chain} - dl.peers[id] = peer - return dl.downloader.RegisterPeer(id, version, peer) -} - -// dropPeer simulates a hard peer removal from the connection pool. -func (dl *downloadTester) dropPeer(id string) { - dl.lock.Lock() - defer dl.lock.Unlock() - - delete(dl.peers, id) - dl.downloader.UnregisterPeer(id) -} - -// Snapshots implements the BlockChain interface for the downloader, but is a noop. -func (dl *downloadTester) Snapshots() *snapshot.Tree { - return nil -} - -type downloadTesterPeer struct { - dl *downloadTester - id string - chain *testChain - missingStates map[common.Hash]bool // State entries that fast sync should not return -} - -// Head constructs a function to retrieve a peer's current head hash -// and total difficulty. -func (dlp *downloadTesterPeer) Head() (common.Hash, *big.Int) { - b := dlp.chain.headBlock() - return b.Hash(), dlp.chain.td(b.Hash()) -} - -// RequestHeadersByHash constructs a GetBlockHeaders function based on a hashed -// origin; associated with a particular peer in the download tester. The returned -// function can be used to retrieve batches of headers from the particular peer. -func (dlp *downloadTesterPeer) RequestHeadersByHash(origin common.Hash, amount int, skip int, reverse bool) error { - result := dlp.chain.headersByHash(origin, amount, skip, reverse) - go dlp.dl.downloader.DeliverHeaders(dlp.id, result) - return nil -} - -// RequestHeadersByNumber constructs a GetBlockHeaders function based on a numbered -// origin; associated with a particular peer in the download tester. The returned -// function can be used to retrieve batches of headers from the particular peer. -func (dlp *downloadTesterPeer) RequestHeadersByNumber(origin uint64, amount int, skip int, reverse bool) error { - result := dlp.chain.headersByNumber(origin, amount, skip, reverse) - go dlp.dl.downloader.DeliverHeaders(dlp.id, result) - return nil -} - -// RequestBodies constructs a getBlockBodies method associated with a particular -// peer in the download tester. The returned function can be used to retrieve -// batches of block bodies from the particularly requested peer. -func (dlp *downloadTesterPeer) RequestBodies(hashes []common.Hash) error { - txs, uncles := dlp.chain.bodies(hashes) - go dlp.dl.downloader.DeliverBodies(dlp.id, txs, uncles) - return nil -} - -// RequestReceipts constructs a getReceipts method associated with a particular -// peer in the download tester. The returned function can be used to retrieve -// batches of block receipts from the particularly requested peer. -func (dlp *downloadTesterPeer) RequestReceipts(hashes []common.Hash) error { - receipts := dlp.chain.receipts(hashes) - go dlp.dl.downloader.DeliverReceipts(dlp.id, receipts) - return nil -} - -// RequestNodeData constructs a getNodeData method associated with a particular -// peer in the download tester. The returned function can be used to retrieve -// batches of node state data from the particularly requested peer. -func (dlp *downloadTesterPeer) RequestNodeData(hashes []common.Hash) error { - dlp.dl.lock.RLock() - defer dlp.dl.lock.RUnlock() - - results := make([][]byte, 0, len(hashes)) - for _, hash := range hashes { - if data, err := dlp.dl.peerDb.Get(hash.Bytes()); err == nil { - if !dlp.missingStates[hash] { - results = append(results, data) - } - } - } - go dlp.dl.downloader.DeliverNodeData(dlp.id, results) - return nil -} - -// assertOwnChain checks if the local chain contains the correct number of items -// of the various chain components. -func assertOwnChain(t *testing.T, tester *downloadTester, length int) { - // Mark this method as a helper to report errors at callsite, not in here - t.Helper() - - assertOwnForkedChain(t, tester, 1, []int{length}) -} - -// assertOwnForkedChain checks if the local forked chain contains the correct -// number of items of the various chain components. -func assertOwnForkedChain(t *testing.T, tester *downloadTester, common int, lengths []int) { - // Mark this method as a helper to report errors at callsite, not in here - t.Helper() - - // Initialize the counters for the first fork - headers, blocks, receipts := lengths[0], lengths[0], lengths[0] - - // Update the counters for each subsequent fork - for _, length := range lengths[1:] { - headers += length - common - blocks += length - common - receipts += length - common - } - if tester.downloader.getMode() == LightSync { - blocks, receipts = 1, 1 - } - if hs := len(tester.ownHeaders) + len(tester.ancientHeaders) - 1; hs != headers { - t.Fatalf("synchronised headers mismatch: have %v, want %v", hs, headers) - } - if bs := len(tester.ownBlocks) + len(tester.ancientBlocks) - 1; bs != blocks { - t.Fatalf("synchronised blocks mismatch: have %v, want %v", bs, blocks) - } - if rs := len(tester.ownReceipts) + len(tester.ancientReceipts) - 1; rs != receipts { - t.Fatalf("synchronised receipts mismatch: have %v, want %v", rs, receipts) - } -} - -func TestCanonicalSynchronisation66Full(t *testing.T) { testCanonSync(t, eth.ETH66, FullSync) } -func TestCanonicalSynchronisation66Fast(t *testing.T) { testCanonSync(t, eth.ETH66, FastSync) } -func TestCanonicalSynchronisation66Light(t *testing.T) { testCanonSync(t, eth.ETH66, LightSync) } - -func testCanonSync(t *testing.T, protocol uint, mode SyncMode) { - t.Parallel() - - tester := newTester() - defer tester.terminate() - - // Create a small enough block chain to download - chain := testChainBase.shorten(blockCacheMaxItems - 15) - tester.newPeer("peer", protocol, chain) - - // Synchronise with the peer and make sure all relevant data was retrieved - if err := tester.sync("peer", nil, mode); err != nil { - t.Fatalf("failed to synchronise blocks: %v", err) - } - assertOwnChain(t, tester, chain.len()) -} - -// Tests that if a large batch of blocks are being downloaded, it is throttled -// until the cached blocks are retrieved. -func TestThrottling66Full(t *testing.T) { testThrottling(t, eth.ETH66, FullSync) } -func TestThrottling66Fast(t *testing.T) { testThrottling(t, eth.ETH66, FastSync) } - -func testThrottling(t *testing.T, protocol uint, mode SyncMode) { - t.Parallel() - tester := newTester() - - // Create a long block chain to download and the tester - targetBlocks := testChainBase.len() - 1 - tester.newPeer("peer", protocol, testChainBase) - - // Wrap the importer to allow stepping - blocked, proceed := uint32(0), make(chan struct{}) - tester.downloader.chainInsertHook = func(results []*fetchResult) { - atomic.StoreUint32(&blocked, uint32(len(results))) - <-proceed - } - // Start a synchronisation concurrently - errc := make(chan error, 1) - go func() { - errc <- tester.sync("peer", nil, mode) - }() - // Iteratively take some blocks, always checking the retrieval count - for { - // Check the retrieval count synchronously (! reason for this ugly block) - tester.lock.RLock() - retrieved := len(tester.ownBlocks) - tester.lock.RUnlock() - if retrieved >= targetBlocks+1 { - break - } - // Wait a bit for sync to throttle itself - var cached, frozen int - for start := time.Now(); time.Since(start) < 3*time.Second; { - time.Sleep(25 * time.Millisecond) - - tester.lock.Lock() - tester.downloader.queue.lock.Lock() - tester.downloader.queue.resultCache.lock.Lock() - { - cached = tester.downloader.queue.resultCache.countCompleted() - frozen = int(atomic.LoadUint32(&blocked)) - retrieved = len(tester.ownBlocks) - } - tester.downloader.queue.resultCache.lock.Unlock() - tester.downloader.queue.lock.Unlock() - tester.lock.Unlock() - - if cached == blockCacheMaxItems || - cached == blockCacheMaxItems-reorgProtHeaderDelay || - retrieved+cached+frozen == targetBlocks+1 || - retrieved+cached+frozen == targetBlocks+1-reorgProtHeaderDelay { - break - } - } - // Make sure we filled up the cache, then exhaust it - time.Sleep(25 * time.Millisecond) // give it a chance to screw up - tester.lock.RLock() - retrieved = len(tester.ownBlocks) - tester.lock.RUnlock() - if cached != blockCacheMaxItems && cached != blockCacheMaxItems-reorgProtHeaderDelay && retrieved+cached+frozen != targetBlocks+1 && retrieved+cached+frozen != targetBlocks+1-reorgProtHeaderDelay { - t.Fatalf("block count mismatch: have %v, want %v (owned %v, blocked %v, target %v)", cached, blockCacheMaxItems, retrieved, frozen, targetBlocks+1) - } - - // Permit the blocked blocks to import - if atomic.LoadUint32(&blocked) > 0 { - atomic.StoreUint32(&blocked, uint32(0)) - proceed <- struct{}{} - } - } - // Check that we haven't pulled more blocks than available - assertOwnChain(t, tester, targetBlocks+1) - if err := <-errc; err != nil { - t.Fatalf("block synchronization failed: %v", err) - } - tester.terminate() -} - -// Tests that simple synchronization against a forked chain works correctly. In -// this test common ancestor lookup should *not* be short circuited, and a full -// binary search should be executed. -func TestForkedSync66Full(t *testing.T) { testForkedSync(t, eth.ETH66, FullSync) } -func TestForkedSync66Fast(t *testing.T) { testForkedSync(t, eth.ETH66, FastSync) } -func TestForkedSync66Light(t *testing.T) { testForkedSync(t, eth.ETH66, LightSync) } - -func testForkedSync(t *testing.T, protocol uint, mode SyncMode) { - t.Parallel() - - tester := newTester() - defer tester.terminate() - - chainA := testChainForkLightA.shorten(testChainBase.len() + 80) - chainB := testChainForkLightB.shorten(testChainBase.len() + 80) - tester.newPeer("fork A", protocol, chainA) - tester.newPeer("fork B", protocol, chainB) - // Synchronise with the peer and make sure all blocks were retrieved - if err := tester.sync("fork A", nil, mode); err != nil { - t.Fatalf("failed to synchronise blocks: %v", err) - } - assertOwnChain(t, tester, chainA.len()) - - // Synchronise with the second peer and make sure that fork is pulled too - if err := tester.sync("fork B", nil, mode); err != nil { - t.Fatalf("failed to synchronise blocks: %v", err) - } - assertOwnForkedChain(t, tester, testChainBase.len(), []int{chainA.len(), chainB.len()}) -} - -// Tests that synchronising against a much shorter but much heavier fork works -// correctly and is not dropped. -func TestHeavyForkedSync66Full(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, FullSync) } -func TestHeavyForkedSync66Fast(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, FastSync) } -func TestHeavyForkedSync66Light(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, LightSync) } - -func testHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { - t.Parallel() - - tester := newTester() - defer tester.terminate() - - chainA := testChainForkLightA.shorten(testChainBase.len() + 80) - chainB := testChainForkHeavy.shorten(testChainBase.len() + 80) - tester.newPeer("light", protocol, chainA) - tester.newPeer("heavy", protocol, chainB) - - // Synchronise with the peer and make sure all blocks were retrieved - if err := tester.sync("light", nil, mode); err != nil { - t.Fatalf("failed to synchronise blocks: %v", err) - } - assertOwnChain(t, tester, chainA.len()) - - // Synchronise with the second peer and make sure that fork is pulled too - if err := tester.sync("heavy", nil, mode); err != nil { - t.Fatalf("failed to synchronise blocks: %v", err) - } - assertOwnForkedChain(t, tester, testChainBase.len(), []int{chainA.len(), chainB.len()}) -} - -// Tests that chain forks are contained within a certain interval of the current -// chain head, ensuring that malicious peers cannot waste resources by feeding -// long dead chains. -func TestBoundedForkedSync66Full(t *testing.T) { testBoundedForkedSync(t, eth.ETH66, FullSync) } -func TestBoundedForkedSync66Fast(t *testing.T) { testBoundedForkedSync(t, eth.ETH66, FastSync) } -func TestBoundedForkedSync66Light(t *testing.T) { testBoundedForkedSync(t, eth.ETH66, LightSync) } - -func testBoundedForkedSync(t *testing.T, protocol uint, mode SyncMode) { - t.Parallel() - - tester := newTester() - defer tester.terminate() - - chainA := testChainForkLightA - chainB := testChainForkLightB - tester.newPeer("original", protocol, chainA) - tester.newPeer("rewriter", protocol, chainB) - - // Synchronise with the peer and make sure all blocks were retrieved - if err := tester.sync("original", nil, mode); err != nil { - t.Fatalf("failed to synchronise blocks: %v", err) - } - assertOwnChain(t, tester, chainA.len()) - - // Synchronise with the second peer and ensure that the fork is rejected to being too old - if err := tester.sync("rewriter", nil, mode); err != errInvalidAncestor { - t.Fatalf("sync failure mismatch: have %v, want %v", err, errInvalidAncestor) - } -} - -// Tests that chain forks are contained within a certain interval of the current -// chain head for short but heavy forks too. These are a bit special because they -// take different ancestor lookup paths. -func TestBoundedHeavyForkedSync66Full(t *testing.T) { - testBoundedHeavyForkedSync(t, eth.ETH66, FullSync) -} -func TestBoundedHeavyForkedSync66Fast(t *testing.T) { - testBoundedHeavyForkedSync(t, eth.ETH66, FastSync) -} -func TestBoundedHeavyForkedSync66Light(t *testing.T) { - testBoundedHeavyForkedSync(t, eth.ETH66, LightSync) -} - -func testBoundedHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { - t.Parallel() - tester := newTester() - - // Create a long enough forked chain - chainA := testChainForkLightA - chainB := testChainForkHeavy - tester.newPeer("original", protocol, chainA) - - // Synchronise with the peer and make sure all blocks were retrieved - if err := tester.sync("original", nil, mode); err != nil { - t.Fatalf("failed to synchronise blocks: %v", err) - } - assertOwnChain(t, tester, chainA.len()) - - tester.newPeer("heavy-rewriter", protocol, chainB) - // Synchronise with the second peer and ensure that the fork is rejected to being too old - if err := tester.sync("heavy-rewriter", nil, mode); err != errInvalidAncestor { - t.Fatalf("sync failure mismatch: have %v, want %v", err, errInvalidAncestor) - } - tester.terminate() -} - -// Tests that an inactive downloader will not accept incoming block headers, -// bodies and receipts. -func TestInactiveDownloader63(t *testing.T) { - t.Parallel() - - tester := newTester() - defer tester.terminate() - - // Check that neither block headers nor bodies are accepted - if err := tester.downloader.DeliverHeaders("bad peer", []*types.Header{}); err != errNoSyncActive { - t.Errorf("error mismatch: have %v, want %v", err, errNoSyncActive) - } - if err := tester.downloader.DeliverBodies("bad peer", [][]*types.Transaction{}, [][]*types.Header{}); err != errNoSyncActive { - t.Errorf("error mismatch: have %v, want %v", err, errNoSyncActive) - } - if err := tester.downloader.DeliverReceipts("bad peer", [][]*types.Receipt{}); err != errNoSyncActive { - t.Errorf("error mismatch: have %v, want %v", err, errNoSyncActive) - } -} - -// Tests that a canceled download wipes all previously accumulated state. -func TestCancel66Full(t *testing.T) { testCancel(t, eth.ETH66, FullSync) } -func TestCancel66Fast(t *testing.T) { testCancel(t, eth.ETH66, FastSync) } -func TestCancel66Light(t *testing.T) { testCancel(t, eth.ETH66, LightSync) } - -func testCancel(t *testing.T, protocol uint, mode SyncMode) { - t.Parallel() - - tester := newTester() - defer tester.terminate() - - chain := testChainBase.shorten(MaxHeaderFetch) - tester.newPeer("peer", protocol, chain) - - // Make sure canceling works with a pristine downloader - tester.downloader.Cancel() - if !tester.downloader.queue.Idle() { - t.Errorf("download queue not idle") - } - // Synchronise with the peer, but cancel afterwards - if err := tester.sync("peer", nil, mode); err != nil { - t.Fatalf("failed to synchronise blocks: %v", err) - } - tester.downloader.Cancel() - if !tester.downloader.queue.Idle() { - t.Errorf("download queue not idle") - } -} - -// Tests that synchronisation from multiple peers works as intended (multi thread sanity test). -func TestMultiSynchronisation66Full(t *testing.T) { testMultiSynchronisation(t, eth.ETH66, FullSync) } -func TestMultiSynchronisation66Fast(t *testing.T) { testMultiSynchronisation(t, eth.ETH66, FastSync) } -func TestMultiSynchronisation66Light(t *testing.T) { testMultiSynchronisation(t, eth.ETH66, LightSync) } - -func testMultiSynchronisation(t *testing.T, protocol uint, mode SyncMode) { - t.Parallel() - - tester := newTester() - defer tester.terminate() - - // Create various peers with various parts of the chain - targetPeers := 8 - chain := testChainBase.shorten(targetPeers * 100) - - for i := 0; i < targetPeers; i++ { - id := fmt.Sprintf("peer #%d", i) - tester.newPeer(id, protocol, chain.shorten(chain.len()/(i+1))) - } - if err := tester.sync("peer #0", nil, mode); err != nil { - t.Fatalf("failed to synchronise blocks: %v", err) - } - assertOwnChain(t, tester, chain.len()) -} - -// Tests that synchronisations behave well in multi-version protocol environments -// and not wreak havoc on other nodes in the network. -func TestMultiProtoSynchronisation66Full(t *testing.T) { testMultiProtoSync(t, eth.ETH66, FullSync) } -func TestMultiProtoSynchronisation66Fast(t *testing.T) { testMultiProtoSync(t, eth.ETH66, FastSync) } -func TestMultiProtoSynchronisation66Light(t *testing.T) { testMultiProtoSync(t, eth.ETH66, LightSync) } - -func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { - t.Parallel() - - tester := newTester() - defer tester.terminate() - - // Create a small enough block chain to download - chain := testChainBase.shorten(blockCacheMaxItems - 15) - - // Create peers of every type - tester.newPeer("peer 66", eth.ETH66, chain) - //tester.newPeer("peer 65", eth.ETH67, chain) - - // Synchronise with the requested peer and make sure all blocks were retrieved - if err := tester.sync(fmt.Sprintf("peer %d", protocol), nil, mode); err != nil { - t.Fatalf("failed to synchronise blocks: %v", err) - } - assertOwnChain(t, tester, chain.len()) - - // Check that no peers have been dropped off - for _, version := range []int{66} { - peer := fmt.Sprintf("peer %d", version) - if _, ok := tester.peers[peer]; !ok { - t.Errorf("%s dropped", peer) - } - } -} - -// Tests that if a block is empty (e.g. header only), no body request should be -// made, and instead the header should be assembled into a whole block in itself. -func TestEmptyShortCircuit66Full(t *testing.T) { testEmptyShortCircuit(t, eth.ETH66, FullSync) } -func TestEmptyShortCircuit66Fast(t *testing.T) { testEmptyShortCircuit(t, eth.ETH66, FastSync) } -func TestEmptyShortCircuit66Light(t *testing.T) { testEmptyShortCircuit(t, eth.ETH66, LightSync) } - -func testEmptyShortCircuit(t *testing.T, protocol uint, mode SyncMode) { - t.Parallel() - - tester := newTester() - defer tester.terminate() - - // Create a block chain to download - chain := testChainBase - tester.newPeer("peer", protocol, chain) - - // Instrument the downloader to signal body requests - bodiesHave, receiptsHave := int32(0), int32(0) - tester.downloader.bodyFetchHook = func(headers []*types.Header) { - atomic.AddInt32(&bodiesHave, int32(len(headers))) - } - tester.downloader.receiptFetchHook = func(headers []*types.Header) { - atomic.AddInt32(&receiptsHave, int32(len(headers))) - } - // Synchronise with the peer and make sure all blocks were retrieved - if err := tester.sync("peer", nil, mode); err != nil { - t.Fatalf("failed to synchronise blocks: %v", err) - } - assertOwnChain(t, tester, chain.len()) - - // Validate the number of block bodies that should have been requested - bodiesNeeded, receiptsNeeded := 0, 0 - for _, block := range chain.blockm { - if mode != LightSync && block != tester.genesis && (len(block.Transactions()) > 0 || len(block.Uncles()) > 0) { - bodiesNeeded++ - } - } - for _, receipt := range chain.receiptm { - if mode == FastSync && len(receipt) > 0 { - receiptsNeeded++ - } - } - if int(bodiesHave) != bodiesNeeded { - t.Errorf("body retrieval count mismatch: have %v, want %v", bodiesHave, bodiesNeeded) - } - if int(receiptsHave) != receiptsNeeded { - t.Errorf("receipt retrieval count mismatch: have %v, want %v", receiptsHave, receiptsNeeded) - } -} - -// Tests that headers are enqueued continuously, preventing malicious nodes from -// stalling the downloader by feeding gapped header chains. -func TestMissingHeaderAttack66Full(t *testing.T) { testMissingHeaderAttack(t, eth.ETH66, FullSync) } -func TestMissingHeaderAttack66Fast(t *testing.T) { testMissingHeaderAttack(t, eth.ETH66, FastSync) } -func TestMissingHeaderAttack66Light(t *testing.T) { testMissingHeaderAttack(t, eth.ETH66, LightSync) } - -func testMissingHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { - t.Parallel() - - tester := newTester() - defer tester.terminate() - - chain := testChainBase.shorten(blockCacheMaxItems - 15) - brokenChain := chain.shorten(chain.len()) - delete(brokenChain.headerm, brokenChain.chain[brokenChain.len()/2]) - tester.newPeer("attack", protocol, brokenChain) - - if err := tester.sync("attack", nil, mode); err == nil { - t.Fatalf("succeeded attacker synchronisation") - } - // Synchronise with the valid peer and make sure sync succeeds - tester.newPeer("valid", protocol, chain) - if err := tester.sync("valid", nil, mode); err != nil { - t.Fatalf("failed to synchronise blocks: %v", err) - } - assertOwnChain(t, tester, chain.len()) -} - -// Tests that if requested headers are shifted (i.e. first is missing), the queue -// detects the invalid numbering. -func TestShiftedHeaderAttack66Full(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH66, FullSync) } -func TestShiftedHeaderAttack66Fast(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH66, FastSync) } -func TestShiftedHeaderAttack66Light(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH66, LightSync) } - -func testShiftedHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { - t.Parallel() - - tester := newTester() - defer tester.terminate() - - chain := testChainBase.shorten(blockCacheMaxItems - 15) - - // Attempt a full sync with an attacker feeding shifted headers - brokenChain := chain.shorten(chain.len()) - delete(brokenChain.headerm, brokenChain.chain[1]) - delete(brokenChain.blockm, brokenChain.chain[1]) - delete(brokenChain.receiptm, brokenChain.chain[1]) - tester.newPeer("attack", protocol, brokenChain) - if err := tester.sync("attack", nil, mode); err == nil { - t.Fatalf("succeeded attacker synchronisation") - } - - // Synchronise with the valid peer and make sure sync succeeds - tester.newPeer("valid", protocol, chain) - if err := tester.sync("valid", nil, mode); err != nil { - t.Fatalf("failed to synchronise blocks: %v", err) - } - assertOwnChain(t, tester, chain.len()) -} - -// Tests that upon detecting an invalid header, the recent ones are rolled back -// for various failure scenarios. Afterwards a full sync is attempted to make -// sure no state was corrupted. -func TestInvalidHeaderRollback66Fast(t *testing.T) { testInvalidHeaderRollback(t, eth.ETH66, FastSync) } - -func testInvalidHeaderRollback(t *testing.T, protocol uint, mode SyncMode) { - t.Parallel() - - tester := newTester() - - // Create a small enough block chain to download - targetBlocks := 3*fsHeaderSafetyNet + 256 + fsMinFullBlocks - chain := testChainBase.shorten(targetBlocks) - - // Attempt to sync with an attacker that feeds junk during the fast sync phase. - // This should result in the last fsHeaderSafetyNet headers being rolled back. - missing := fsHeaderSafetyNet + MaxHeaderFetch + 1 - fastAttackChain := chain.shorten(chain.len()) - delete(fastAttackChain.headerm, fastAttackChain.chain[missing]) - tester.newPeer("fast-attack", protocol, fastAttackChain) - - if err := tester.sync("fast-attack", nil, mode); err == nil { - t.Fatalf("succeeded fast attacker synchronisation") - } - if head := tester.CurrentHeader().Number.Int64(); int(head) > MaxHeaderFetch { - t.Errorf("rollback head mismatch: have %v, want at most %v", head, MaxHeaderFetch) - } - - // Attempt to sync with an attacker that feeds junk during the block import phase. - // This should result in both the last fsHeaderSafetyNet number of headers being - // rolled back, and also the pivot point being reverted to a non-block status. - missing = 3*fsHeaderSafetyNet + MaxHeaderFetch + 1 - blockAttackChain := chain.shorten(chain.len()) - delete(fastAttackChain.headerm, fastAttackChain.chain[missing]) // Make sure the fast-attacker doesn't fill in - delete(blockAttackChain.headerm, blockAttackChain.chain[missing]) - tester.newPeer("block-attack", protocol, blockAttackChain) - - if err := tester.sync("block-attack", nil, mode); err == nil { - t.Fatalf("succeeded block attacker synchronisation") - } - if head := tester.CurrentHeader().Number.Int64(); int(head) > 2*fsHeaderSafetyNet+MaxHeaderFetch { - t.Errorf("rollback head mismatch: have %v, want at most %v", head, 2*fsHeaderSafetyNet+MaxHeaderFetch) - } - if mode == FastSync { - if head := tester.CurrentBlock().NumberU64(); head != 0 { - t.Errorf("fast sync pivot block #%d not rolled back", head) - } - } - - // Attempt to sync with an attacker that withholds promised blocks after the - // fast sync pivot point. This could be a trial to leave the node with a bad - // but already imported pivot block. - withholdAttackChain := chain.shorten(chain.len()) - tester.newPeer("withhold-attack", protocol, withholdAttackChain) - tester.downloader.syncInitHook = func(uint64, uint64) { - for i := missing; i < withholdAttackChain.len(); i++ { - delete(withholdAttackChain.headerm, withholdAttackChain.chain[i]) - } - tester.downloader.syncInitHook = nil - } - if err := tester.sync("withhold-attack", nil, mode); err == nil { - t.Fatalf("succeeded withholding attacker synchronisation") - } - if head := tester.CurrentHeader().Number.Int64(); int(head) > 2*fsHeaderSafetyNet+MaxHeaderFetch { - t.Errorf("rollback head mismatch: have %v, want at most %v", head, 2*fsHeaderSafetyNet+MaxHeaderFetch) - } - if mode == FastSync { - if head := tester.CurrentBlock().NumberU64(); head != 0 { - t.Errorf("fast sync pivot block #%d not rolled back", head) - } - } - - // synchronise with the valid peer and make sure sync succeeds. Since the last rollback - // should also disable fast syncing for this process, verify that we did a fresh full - // sync. Note, we can't assert anything about the receipts since we won't purge the - // database of them, hence we can't use assertOwnChain. - tester.newPeer("valid", protocol, chain) - if err := tester.sync("valid", nil, mode); err != nil { - t.Fatalf("failed to synchronise blocks: %v", err) - } - if hs := len(tester.ownHeaders); hs != chain.len() { - t.Fatalf("synchronised headers mismatch: have %v, want %v", hs, chain.len()) - } - if mode != LightSync { - if bs := len(tester.ownBlocks); bs != chain.len() { - t.Fatalf("synchronised blocks mismatch: have %v, want %v", bs, chain.len()) - } - } - tester.terminate() -} - -// Tests that a peer advertising a high TD doesn't get to stall the downloader -// afterwards by not sending any useful hashes. -func TestHighTDStarvationAttack66Full(t *testing.T) { - testHighTDStarvationAttack(t, eth.ETH66, FullSync) -} -func TestHighTDStarvationAttack66Fast(t *testing.T) { - testHighTDStarvationAttack(t, eth.ETH66, FastSync) -} -func TestHighTDStarvationAttack66Light(t *testing.T) { - testHighTDStarvationAttack(t, eth.ETH66, LightSync) -} - -func testHighTDStarvationAttack(t *testing.T, protocol uint, mode SyncMode) { - t.Parallel() - - tester := newTester() - - chain := testChainBase.shorten(1) - tester.newPeer("attack", protocol, chain) - if err := tester.sync("attack", big.NewInt(1000000), mode); err != errStallingPeer { - t.Fatalf("synchronisation error mismatch: have %v, want %v", err, errStallingPeer) - } - tester.terminate() -} - -// Tests that misbehaving peers are disconnected, whilst behaving ones are not. -func TestBlockHeaderAttackerDropping66(t *testing.T) { testBlockHeaderAttackerDropping(t, eth.ETH66) } - -func testBlockHeaderAttackerDropping(t *testing.T, protocol uint) { - t.Parallel() - - // Define the disconnection requirement for individual hash fetch errors - tests := []struct { - result error - drop bool - }{ - {nil, false}, // Sync succeeded, all is well - {errBusy, false}, // Sync is already in progress, no problem - {errUnknownPeer, false}, // Peer is unknown, was already dropped, don't double drop - {errBadPeer, true}, // Peer was deemed bad for some reason, drop it - {errStallingPeer, true}, // Peer was detected to be stalling, drop it - {errUnsyncedPeer, true}, // Peer was detected to be unsynced, drop it - {errNoPeers, false}, // No peers to download from, soft race, no issue - {errTimeout, true}, // No hashes received in due time, drop the peer - {errEmptyHeaderSet, true}, // No headers were returned as a response, drop as it's a dead end - {errPeersUnavailable, true}, // Nobody had the advertised blocks, drop the advertiser - {errInvalidAncestor, true}, // Agreed upon ancestor is not acceptable, drop the chain rewriter - {errInvalidChain, true}, // Hash chain was detected as invalid, definitely drop - {errInvalidBody, false}, // A bad peer was detected, but not the sync origin - {errInvalidReceipt, false}, // A bad peer was detected, but not the sync origin - {errCancelContentProcessing, false}, // Synchronisation was canceled, origin may be innocent, don't drop - } - // Run the tests and check disconnection status - tester := newTester() - defer tester.terminate() - chain := testChainBase.shorten(1) - - for i, tt := range tests { - // Register a new peer and ensure its presence - id := fmt.Sprintf("test %d", i) - if err := tester.newPeer(id, protocol, chain); err != nil { - t.Fatalf("test %d: failed to register new peer: %v", i, err) - } - if _, ok := tester.peers[id]; !ok { - t.Fatalf("test %d: registered peer not found", i) - } - // Simulate a synchronisation and check the required result - tester.downloader.synchroniseMock = func(string, common.Hash) error { return tt.result } - - tester.downloader.Synchronise(id, tester.genesis.Hash(), big.NewInt(1000), FullSync) - if _, ok := tester.peers[id]; !ok != tt.drop { - t.Errorf("test %d: peer drop mismatch for %v: have %v, want %v", i, tt.result, !ok, tt.drop) - } - } -} - -// Tests that synchronisation progress (origin block number, current block number -// and highest block number) is tracked and updated correctly. -func TestSyncProgress66Full(t *testing.T) { testSyncProgress(t, eth.ETH66, FullSync) } -func TestSyncProgress66Fast(t *testing.T) { testSyncProgress(t, eth.ETH66, FastSync) } -func TestSyncProgress66Light(t *testing.T) { testSyncProgress(t, eth.ETH66, LightSync) } - -func testSyncProgress(t *testing.T, protocol uint, mode SyncMode) { - t.Parallel() - - tester := newTester() - defer tester.terminate() - chain := testChainBase.shorten(blockCacheMaxItems - 15) - - // Set a sync init hook to catch progress changes - starting := make(chan struct{}) - progress := make(chan struct{}) - - tester.downloader.syncInitHook = func(origin, latest uint64) { - starting <- struct{}{} - <-progress - } - checkProgress(t, tester.downloader, "pristine", ethereum.SyncProgress{}) - - // Synchronise half the blocks and check initial progress - tester.newPeer("peer-half", protocol, chain.shorten(chain.len()/2)) - pending := new(sync.WaitGroup) - pending.Add(1) - - go func() { - defer pending.Done() - if err := tester.sync("peer-half", nil, mode); err != nil { - panic(fmt.Sprintf("failed to synchronise blocks: %v", err)) - } - }() - <-starting - checkProgress(t, tester.downloader, "initial", ethereum.SyncProgress{ - HighestBlock: uint64(chain.len()/2 - 1), - }) - progress <- struct{}{} - pending.Wait() - - // Synchronise all the blocks and check continuation progress - tester.newPeer("peer-full", protocol, chain) - pending.Add(1) - go func() { - defer pending.Done() - if err := tester.sync("peer-full", nil, mode); err != nil { - panic(fmt.Sprintf("failed to synchronise blocks: %v", err)) - } - }() - <-starting - checkProgress(t, tester.downloader, "completing", ethereum.SyncProgress{ - StartingBlock: uint64(chain.len()/2 - 1), - CurrentBlock: uint64(chain.len()/2 - 1), - HighestBlock: uint64(chain.len() - 1), - }) - - // Check final progress after successful sync - progress <- struct{}{} - pending.Wait() - checkProgress(t, tester.downloader, "final", ethereum.SyncProgress{ - StartingBlock: uint64(chain.len()/2 - 1), - CurrentBlock: uint64(chain.len() - 1), - HighestBlock: uint64(chain.len() - 1), - }) -} - -func checkProgress(t *testing.T, d *Downloader, stage string, want ethereum.SyncProgress) { - // Mark this method as a helper to report errors at callsite, not in here - t.Helper() - - p := d.Progress() - //p.KnownStates, p.PulledStates = 0, 0 - //want.KnownStates, want.PulledStates = 0, 0 - if p != want { - t.Fatalf("%s progress mismatch:\nhave %+v\nwant %+v", stage, p, want) - } -} - -// Tests that synchronisation progress (origin block number and highest block -// number) is tracked and updated correctly in case of a fork (or manual head -// revertal). -func TestForkedSyncProgress66Full(t *testing.T) { testForkedSyncProgress(t, eth.ETH66, FullSync) } -func TestForkedSyncProgress66Fast(t *testing.T) { testForkedSyncProgress(t, eth.ETH66, FastSync) } -func TestForkedSyncProgress66Light(t *testing.T) { testForkedSyncProgress(t, eth.ETH66, LightSync) } - -func testForkedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { - t.Parallel() - - tester := newTester() - defer tester.terminate() - chainA := testChainForkLightA.shorten(testChainBase.len() + MaxHeaderFetch) - chainB := testChainForkLightB.shorten(testChainBase.len() + MaxHeaderFetch) - - // Set a sync init hook to catch progress changes - starting := make(chan struct{}) - progress := make(chan struct{}) - - tester.downloader.syncInitHook = func(origin, latest uint64) { - starting <- struct{}{} - <-progress - } - checkProgress(t, tester.downloader, "pristine", ethereum.SyncProgress{}) - - // Synchronise with one of the forks and check progress - tester.newPeer("fork A", protocol, chainA) - pending := new(sync.WaitGroup) - pending.Add(1) - go func() { - defer pending.Done() - if err := tester.sync("fork A", nil, mode); err != nil { - panic(fmt.Sprintf("failed to synchronise blocks: %v", err)) - } - }() - <-starting - - checkProgress(t, tester.downloader, "initial", ethereum.SyncProgress{ - HighestBlock: uint64(chainA.len() - 1), - }) - progress <- struct{}{} - pending.Wait() - - // Simulate a successful sync above the fork - tester.downloader.syncStatsChainOrigin = tester.downloader.syncStatsChainHeight - - // Synchronise with the second fork and check progress resets - tester.newPeer("fork B", protocol, chainB) - pending.Add(1) - go func() { - defer pending.Done() - if err := tester.sync("fork B", nil, mode); err != nil { - panic(fmt.Sprintf("failed to synchronise blocks: %v", err)) - } - }() - <-starting - checkProgress(t, tester.downloader, "forking", ethereum.SyncProgress{ - StartingBlock: uint64(testChainBase.len()) - 1, - CurrentBlock: uint64(chainA.len() - 1), - HighestBlock: uint64(chainB.len() - 1), - }) - - // Check final progress after successful sync - progress <- struct{}{} - pending.Wait() - checkProgress(t, tester.downloader, "final", ethereum.SyncProgress{ - StartingBlock: uint64(testChainBase.len()) - 1, - CurrentBlock: uint64(chainB.len() - 1), - HighestBlock: uint64(chainB.len() - 1), - }) -} - -// Tests that if synchronisation is aborted due to some failure, then the progress -// origin is not updated in the next sync cycle, as it should be considered the -// continuation of the previous sync and not a new instance. -func TestFailedSyncProgress66Full(t *testing.T) { testFailedSyncProgress(t, eth.ETH66, FullSync) } -func TestFailedSyncProgress66Fast(t *testing.T) { testFailedSyncProgress(t, eth.ETH66, FastSync) } -func TestFailedSyncProgress66Light(t *testing.T) { testFailedSyncProgress(t, eth.ETH66, LightSync) } - -func testFailedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { - t.Parallel() - - tester := newTester() - defer tester.terminate() - chain := testChainBase.shorten(blockCacheMaxItems - 15) - - // Set a sync init hook to catch progress changes - starting := make(chan struct{}) - progress := make(chan struct{}) - - tester.downloader.syncInitHook = func(origin, latest uint64) { - starting <- struct{}{} - <-progress - } - checkProgress(t, tester.downloader, "pristine", ethereum.SyncProgress{}) - - // Attempt a full sync with a faulty peer - brokenChain := chain.shorten(chain.len()) - missing := brokenChain.len() / 2 - delete(brokenChain.headerm, brokenChain.chain[missing]) - delete(brokenChain.blockm, brokenChain.chain[missing]) - delete(brokenChain.receiptm, brokenChain.chain[missing]) - tester.newPeer("faulty", protocol, brokenChain) - - pending := new(sync.WaitGroup) - pending.Add(1) - go func() { - defer pending.Done() - if err := tester.sync("faulty", nil, mode); err == nil { - panic("succeeded faulty synchronisation") - } - }() - <-starting - checkProgress(t, tester.downloader, "initial", ethereum.SyncProgress{ - HighestBlock: uint64(brokenChain.len() - 1), - }) - progress <- struct{}{} - pending.Wait() - afterFailedSync := tester.downloader.Progress() - - // Synchronise with a good peer and check that the progress origin remind the same - // after a failure - tester.newPeer("valid", protocol, chain) - pending.Add(1) - go func() { - defer pending.Done() - if err := tester.sync("valid", nil, mode); err != nil { - panic(fmt.Sprintf("failed to synchronise blocks: %v", err)) - } - }() - <-starting - checkProgress(t, tester.downloader, "completing", afterFailedSync) - - // Check final progress after successful sync - progress <- struct{}{} - pending.Wait() - checkProgress(t, tester.downloader, "final", ethereum.SyncProgress{ - CurrentBlock: uint64(chain.len() - 1), - HighestBlock: uint64(chain.len() - 1), - }) -} - -// Tests that if an attacker fakes a chain height, after the attack is detected, -// the progress height is successfully reduced at the next sync invocation. -func TestFakedSyncProgress66Full(t *testing.T) { testFakedSyncProgress(t, eth.ETH66, FullSync) } -func TestFakedSyncProgress66Fast(t *testing.T) { testFakedSyncProgress(t, eth.ETH66, FastSync) } -func TestFakedSyncProgress66Light(t *testing.T) { testFakedSyncProgress(t, eth.ETH66, LightSync) } - -func testFakedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { - t.Parallel() - - tester := newTester() - defer tester.terminate() - chain := testChainBase.shorten(blockCacheMaxItems - 15) - - // Set a sync init hook to catch progress changes - starting := make(chan struct{}) - progress := make(chan struct{}) - tester.downloader.syncInitHook = func(origin, latest uint64) { - starting <- struct{}{} - <-progress - } - checkProgress(t, tester.downloader, "pristine", ethereum.SyncProgress{}) - - // Create and sync with an attacker that promises a higher chain than available. - brokenChain := chain.shorten(chain.len()) - numMissing := 5 - for i := brokenChain.len() - 2; i > brokenChain.len()-numMissing; i-- { - delete(brokenChain.headerm, brokenChain.chain[i]) - } - tester.newPeer("attack", protocol, brokenChain) - - pending := new(sync.WaitGroup) - pending.Add(1) - go func() { - defer pending.Done() - if err := tester.sync("attack", nil, mode); err == nil { - panic("succeeded attacker synchronisation") - } - }() - <-starting - checkProgress(t, tester.downloader, "initial", ethereum.SyncProgress{ - HighestBlock: uint64(brokenChain.len() - 1), - }) - progress <- struct{}{} - pending.Wait() - afterFailedSync := tester.downloader.Progress() - - // Synchronise with a good peer and check that the progress height has been reduced to - // the true value. - validChain := chain.shorten(chain.len() - numMissing) - tester.newPeer("valid", protocol, validChain) - pending.Add(1) - - go func() { - defer pending.Done() - if err := tester.sync("valid", nil, mode); err != nil { - panic(fmt.Sprintf("failed to synchronise blocks: %v", err)) - } - }() - <-starting - checkProgress(t, tester.downloader, "completing", ethereum.SyncProgress{ - CurrentBlock: afterFailedSync.CurrentBlock, - HighestBlock: uint64(validChain.len() - 1), - }) - - // Check final progress after successful sync. - progress <- struct{}{} - pending.Wait() - checkProgress(t, tester.downloader, "final", ethereum.SyncProgress{ - CurrentBlock: uint64(validChain.len() - 1), - HighestBlock: uint64(validChain.len() - 1), - }) -} - -// This test reproduces an issue where unexpected deliveries would -// block indefinitely if they arrived at the right time. -func TestDeliverHeadersHang66Full(t *testing.T) { testDeliverHeadersHang(t, eth.ETH66, FullSync) } -func TestDeliverHeadersHang66Fast(t *testing.T) { testDeliverHeadersHang(t, eth.ETH66, FastSync) } -func TestDeliverHeadersHang66Light(t *testing.T) { testDeliverHeadersHang(t, eth.ETH66, LightSync) } - -func testDeliverHeadersHang(t *testing.T, protocol uint, mode SyncMode) { - t.Parallel() - - master := newTester() - defer master.terminate() - chain := testChainBase.shorten(15) - - for i := 0; i < 200; i++ { - tester := newTester() - tester.peerDb = master.peerDb - tester.newPeer("peer", protocol, chain) - - // Whenever the downloader requests headers, flood it with - // a lot of unrequested header deliveries. - tester.downloader.peers.peers["peer"].peer = &floodingTestPeer{ - peer: tester.downloader.peers.peers["peer"].peer, - tester: tester, - } - if err := tester.sync("peer", nil, mode); err != nil { - t.Errorf("test %d: sync failed: %v", i, err) - } - tester.terminate() - } -} - -type floodingTestPeer struct { - peer Peer - tester *downloadTester -} - -func (ftp *floodingTestPeer) Head() (common.Hash, *big.Int) { return ftp.peer.Head() } -func (ftp *floodingTestPeer) RequestHeadersByHash(hash common.Hash, count int, skip int, reverse bool) error { - return ftp.peer.RequestHeadersByHash(hash, count, skip, reverse) -} -func (ftp *floodingTestPeer) RequestBodies(hashes []common.Hash) error { - return ftp.peer.RequestBodies(hashes) -} -func (ftp *floodingTestPeer) RequestReceipts(hashes []common.Hash) error { - return ftp.peer.RequestReceipts(hashes) -} -func (ftp *floodingTestPeer) RequestNodeData(hashes []common.Hash) error { - return ftp.peer.RequestNodeData(hashes) -} - -func (ftp *floodingTestPeer) RequestHeadersByNumber(from uint64, count, skip int, reverse bool) error { - deliveriesDone := make(chan struct{}, 500) - for i := 0; i < cap(deliveriesDone)-1; i++ { - peer := fmt.Sprintf("fake-peer%d", i) - go func() { - ftp.tester.downloader.DeliverHeaders(peer, []*types.Header{{}, {}, {}, {}}) - deliveriesDone <- struct{}{} - }() - } - - // None of the extra deliveries should block. - timeout := time.After(60 * time.Second) - launched := false - for i := 0; i < cap(deliveriesDone); i++ { - select { - case <-deliveriesDone: - if !launched { - // Start delivering the requested headers - // after one of the flooding responses has arrived. - go func() { - ftp.peer.RequestHeadersByNumber(from, count, skip, reverse) - deliveriesDone <- struct{}{} - }() - launched = true - } - case <-timeout: - panic("blocked") - } - } - return nil -} - -func TestRemoteHeaderRequestSpan(t *testing.T) { - testCases := []struct { - remoteHeight uint64 - localHeight uint64 - expected []int - }{ - // Remote is way higher. We should ask for the remote head and go backwards - {1500, 1000, - []int{1323, 1339, 1355, 1371, 1387, 1403, 1419, 1435, 1451, 1467, 1483, 1499}, - }, - {15000, 13006, - []int{14823, 14839, 14855, 14871, 14887, 14903, 14919, 14935, 14951, 14967, 14983, 14999}, - }, - // Remote is pretty close to us. We don't have to fetch as many - {1200, 1150, - []int{1149, 1154, 1159, 1164, 1169, 1174, 1179, 1184, 1189, 1194, 1199}, - }, - // Remote is equal to us (so on a fork with higher td) - // We should get the closest couple of ancestors - {1500, 1500, - []int{1497, 1499}, - }, - // We're higher than the remote! Odd - {1000, 1500, - []int{997, 999}, - }, - // Check some weird edgecases that it behaves somewhat rationally - {0, 1500, - []int{0, 2}, - }, - {6000000, 0, - []int{5999823, 5999839, 5999855, 5999871, 5999887, 5999903, 5999919, 5999935, 5999951, 5999967, 5999983, 5999999}, - }, - {0, 0, - []int{0, 2}, - }, - } - reqs := func(from, count, span int) []int { - var r []int - num := from - for len(r) < count { - r = append(r, num) - num += span + 1 - } - return r - } - for i, tt := range testCases { - from, count, span, max := calculateRequestSpan(tt.remoteHeight, tt.localHeight) - data := reqs(int(from), count, span) - - if max != uint64(data[len(data)-1]) { - t.Errorf("test %d: wrong last value %d != %d", i, data[len(data)-1], max) - } - failed := false - if len(data) != len(tt.expected) { - failed = true - t.Errorf("test %d: length wrong, expected %d got %d", i, len(tt.expected), len(data)) - } else { - for j, n := range data { - if n != tt.expected[j] { - failed = true - break - } - } - } - if failed { - res := strings.ReplaceAll(fmt.Sprint(data), " ", ",") - exp := strings.ReplaceAll(fmt.Sprint(tt.expected), " ", ",") - t.Logf("got: %v\n", res) - t.Logf("exp: %v\n", exp) - t.Errorf("test %d: wrong values", i) - } - } -} - -// Tests that peers below a pre-configured checkpoint block are prevented from -// being fast-synced from, avoiding potential cheap eclipse attacks. -func TestCheckpointEnforcement66Full(t *testing.T) { testCheckpointEnforcement(t, eth.ETH66, FullSync) } -func TestCheckpointEnforcement66Fast(t *testing.T) { testCheckpointEnforcement(t, eth.ETH66, FastSync) } -func TestCheckpointEnforcement66Light(t *testing.T) { - testCheckpointEnforcement(t, eth.ETH66, LightSync) -} - -func testCheckpointEnforcement(t *testing.T, protocol uint, mode SyncMode) { - t.Parallel() - - // Create a new tester with a particular hard coded checkpoint block - tester := newTester() - defer tester.terminate() - - tester.downloader.checkpoint = uint64(fsMinFullBlocks) + 256 - chain := testChainBase.shorten(int(tester.downloader.checkpoint) - 1) - - // Attempt to sync with the peer and validate the result - tester.newPeer("peer", protocol, chain) - - var expect error - if mode == FastSync || mode == LightSync { - expect = errUnsyncedPeer - } - if err := tester.sync("peer", nil, mode); !errors.Is(err, expect) { - t.Fatalf("block sync error mismatch: have %v, want %v", err, expect) - } - if mode == FastSync || mode == LightSync { - assertOwnChain(t, tester, 1) - } else { - assertOwnChain(t, tester, chain.len()) - } -} diff --git a/les/downloader/events.go b/les/downloader/events.go deleted file mode 100644 index 25255a3a72..0000000000 --- a/les/downloader/events.go +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package downloader - -import "github.com/ethereum/go-ethereum/core/types" - -type DoneEvent struct { - Latest *types.Header -} -type StartEvent struct{} -type FailedEvent struct{ Err error } diff --git a/les/downloader/metrics.go b/les/downloader/metrics.go deleted file mode 100644 index c38732043a..0000000000 --- a/les/downloader/metrics.go +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Contains the metrics collected by the downloader. - -package downloader - -import ( - "github.com/ethereum/go-ethereum/metrics" -) - -var ( - headerInMeter = metrics.NewRegisteredMeter("eth/downloader/headers/in", nil) - headerReqTimer = metrics.NewRegisteredTimer("eth/downloader/headers/req", nil) - headerDropMeter = metrics.NewRegisteredMeter("eth/downloader/headers/drop", nil) - headerTimeoutMeter = metrics.NewRegisteredMeter("eth/downloader/headers/timeout", nil) - - bodyInMeter = metrics.NewRegisteredMeter("eth/downloader/bodies/in", nil) - bodyReqTimer = metrics.NewRegisteredTimer("eth/downloader/bodies/req", nil) - bodyDropMeter = metrics.NewRegisteredMeter("eth/downloader/bodies/drop", nil) - bodyTimeoutMeter = metrics.NewRegisteredMeter("eth/downloader/bodies/timeout", nil) - - receiptInMeter = metrics.NewRegisteredMeter("eth/downloader/receipts/in", nil) - receiptReqTimer = metrics.NewRegisteredTimer("eth/downloader/receipts/req", nil) - receiptDropMeter = metrics.NewRegisteredMeter("eth/downloader/receipts/drop", nil) - receiptTimeoutMeter = metrics.NewRegisteredMeter("eth/downloader/receipts/timeout", nil) - - stateInMeter = metrics.NewRegisteredMeter("eth/downloader/states/in", nil) - stateDropMeter = metrics.NewRegisteredMeter("eth/downloader/states/drop", nil) - - throttleCounter = metrics.NewRegisteredCounter("eth/downloader/throttle", nil) -) diff --git a/les/downloader/modes.go b/les/downloader/modes.go deleted file mode 100644 index 3ea14d22d7..0000000000 --- a/les/downloader/modes.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package downloader - -import "fmt" - -// SyncMode represents the synchronisation mode of the downloader. -// It is a uint32 as it is used with atomic operations. -type SyncMode uint32 - -const ( - FullSync SyncMode = iota // Synchronise the entire blockchain history from full blocks - FastSync // Quickly download the headers, full sync only at the chain - SnapSync // Download the chain and the state via compact snapshots - LightSync // Download only the headers and terminate afterwards -) - -func (mode SyncMode) IsValid() bool { - return mode >= FullSync && mode <= LightSync -} - -// String implements the stringer interface. -func (mode SyncMode) String() string { - switch mode { - case FullSync: - return "full" - case FastSync: - return "fast" - case SnapSync: - return "snap" - case LightSync: - return "light" - default: - return "unknown" - } -} - -func (mode SyncMode) MarshalText() ([]byte, error) { - switch mode { - case FullSync: - return []byte("full"), nil - case FastSync: - return []byte("fast"), nil - case SnapSync: - return []byte("snap"), nil - case LightSync: - return []byte("light"), nil - default: - return nil, fmt.Errorf("unknown sync mode %d", mode) - } -} - -func (mode *SyncMode) UnmarshalText(text []byte) error { - switch string(text) { - case "full": - *mode = FullSync - case "fast": - *mode = FastSync - case "snap": - *mode = SnapSync - case "light": - *mode = LightSync - default: - return fmt.Errorf(`unknown sync mode %q, want "full", "fast" or "light"`, text) - } - return nil -} diff --git a/les/downloader/peer.go b/les/downloader/peer.go deleted file mode 100644 index c2161e2dae..0000000000 --- a/les/downloader/peer.go +++ /dev/null @@ -1,502 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Contains the active peer-set of the downloader, maintaining both failures -// as well as reputation metrics to prioritize the block retrievals. - -package downloader - -import ( - "errors" - "math/big" - "sort" - "sync" - "sync/atomic" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/eth/protocols/eth" - "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/p2p/msgrate" -) - -const ( - maxLackingHashes = 4096 // Maximum number of entries allowed on the list or lacking items -) - -var ( - errAlreadyFetching = errors.New("already fetching blocks from peer") - errAlreadyRegistered = errors.New("peer is already registered") - errNotRegistered = errors.New("peer is not registered") -) - -// peerConnection represents an active peer from which hashes and blocks are retrieved. -type peerConnection struct { - id string // Unique identifier of the peer - - headerIdle int32 // Current header activity state of the peer (idle = 0, active = 1) - blockIdle int32 // Current block activity state of the peer (idle = 0, active = 1) - receiptIdle int32 // Current receipt activity state of the peer (idle = 0, active = 1) - stateIdle int32 // Current node data activity state of the peer (idle = 0, active = 1) - - headerStarted time.Time // Time instance when the last header fetch was started - blockStarted time.Time // Time instance when the last block (body) fetch was started - receiptStarted time.Time // Time instance when the last receipt fetch was started - stateStarted time.Time // Time instance when the last node data fetch was started - - rates *msgrate.Tracker // Tracker to hone in on the number of items retrievable per second - lacking map[common.Hash]struct{} // Set of hashes not to request (didn't have previously) - - peer Peer - - version uint // Eth protocol version number to switch strategies - log log.Logger // Contextual logger to add extra infos to peer logs - lock sync.RWMutex -} - -// LightPeer encapsulates the methods required to synchronise with a remote light peer. -type LightPeer interface { - Head() (common.Hash, *big.Int) - RequestHeadersByHash(common.Hash, int, int, bool) error - RequestHeadersByNumber(uint64, int, int, bool) error -} - -// Peer encapsulates the methods required to synchronise with a remote full peer. -type Peer interface { - LightPeer - RequestBodies([]common.Hash) error - RequestReceipts([]common.Hash) error - RequestNodeData([]common.Hash) error -} - -// lightPeerWrapper wraps a LightPeer struct, stubbing out the Peer-only methods. -type lightPeerWrapper struct { - peer LightPeer -} - -func (w *lightPeerWrapper) Head() (common.Hash, *big.Int) { return w.peer.Head() } -func (w *lightPeerWrapper) RequestHeadersByHash(h common.Hash, amount int, skip int, reverse bool) error { - return w.peer.RequestHeadersByHash(h, amount, skip, reverse) -} -func (w *lightPeerWrapper) RequestHeadersByNumber(i uint64, amount int, skip int, reverse bool) error { - return w.peer.RequestHeadersByNumber(i, amount, skip, reverse) -} -func (w *lightPeerWrapper) RequestBodies([]common.Hash) error { - panic("RequestBodies not supported in light client mode sync") -} -func (w *lightPeerWrapper) RequestReceipts([]common.Hash) error { - panic("RequestReceipts not supported in light client mode sync") -} -func (w *lightPeerWrapper) RequestNodeData([]common.Hash) error { - panic("RequestNodeData not supported in light client mode sync") -} - -// newPeerConnection creates a new downloader peer. -func newPeerConnection(id string, version uint, peer Peer, logger log.Logger) *peerConnection { - return &peerConnection{ - id: id, - lacking: make(map[common.Hash]struct{}), - peer: peer, - version: version, - log: logger, - } -} - -// Reset clears the internal state of a peer entity. -func (p *peerConnection) Reset() { - p.lock.Lock() - defer p.lock.Unlock() - - atomic.StoreInt32(&p.headerIdle, 0) - atomic.StoreInt32(&p.blockIdle, 0) - atomic.StoreInt32(&p.receiptIdle, 0) - atomic.StoreInt32(&p.stateIdle, 0) - - p.lacking = make(map[common.Hash]struct{}) -} - -// FetchHeaders sends a header retrieval request to the remote peer. -func (p *peerConnection) FetchHeaders(from uint64, count int) error { - // Short circuit if the peer is already fetching - if !atomic.CompareAndSwapInt32(&p.headerIdle, 0, 1) { - return errAlreadyFetching - } - p.headerStarted = time.Now() - - // Issue the header retrieval request (absolute upwards without gaps) - go p.peer.RequestHeadersByNumber(from, count, 0, false) - - return nil -} - -// FetchBodies sends a block body retrieval request to the remote peer. -func (p *peerConnection) FetchBodies(request *fetchRequest) error { - // Short circuit if the peer is already fetching - if !atomic.CompareAndSwapInt32(&p.blockIdle, 0, 1) { - return errAlreadyFetching - } - p.blockStarted = time.Now() - - go func() { - // Convert the header set to a retrievable slice - hashes := make([]common.Hash, 0, len(request.Headers)) - for _, header := range request.Headers { - hashes = append(hashes, header.Hash()) - } - p.peer.RequestBodies(hashes) - }() - - return nil -} - -// FetchReceipts sends a receipt retrieval request to the remote peer. -func (p *peerConnection) FetchReceipts(request *fetchRequest) error { - // Short circuit if the peer is already fetching - if !atomic.CompareAndSwapInt32(&p.receiptIdle, 0, 1) { - return errAlreadyFetching - } - p.receiptStarted = time.Now() - - go func() { - // Convert the header set to a retrievable slice - hashes := make([]common.Hash, 0, len(request.Headers)) - for _, header := range request.Headers { - hashes = append(hashes, header.Hash()) - } - p.peer.RequestReceipts(hashes) - }() - - return nil -} - -// FetchNodeData sends a node state data retrieval request to the remote peer. -func (p *peerConnection) FetchNodeData(hashes []common.Hash) error { - // Short circuit if the peer is already fetching - if !atomic.CompareAndSwapInt32(&p.stateIdle, 0, 1) { - return errAlreadyFetching - } - p.stateStarted = time.Now() - - go p.peer.RequestNodeData(hashes) - - return nil -} - -// SetHeadersIdle sets the peer to idle, allowing it to execute new header retrieval -// requests. Its estimated header retrieval throughput is updated with that measured -// just now. -func (p *peerConnection) SetHeadersIdle(delivered int, deliveryTime time.Time) { - p.rates.Update(eth.BlockHeadersMsg, deliveryTime.Sub(p.headerStarted), delivered) - atomic.StoreInt32(&p.headerIdle, 0) -} - -// SetBodiesIdle sets the peer to idle, allowing it to execute block body retrieval -// requests. Its estimated body retrieval throughput is updated with that measured -// just now. -func (p *peerConnection) SetBodiesIdle(delivered int, deliveryTime time.Time) { - p.rates.Update(eth.BlockBodiesMsg, deliveryTime.Sub(p.blockStarted), delivered) - atomic.StoreInt32(&p.blockIdle, 0) -} - -// SetReceiptsIdle sets the peer to idle, allowing it to execute new receipt -// retrieval requests. Its estimated receipt retrieval throughput is updated -// with that measured just now. -func (p *peerConnection) SetReceiptsIdle(delivered int, deliveryTime time.Time) { - p.rates.Update(eth.ReceiptsMsg, deliveryTime.Sub(p.receiptStarted), delivered) - atomic.StoreInt32(&p.receiptIdle, 0) -} - -// SetNodeDataIdle sets the peer to idle, allowing it to execute new state trie -// data retrieval requests. Its estimated state retrieval throughput is updated -// with that measured just now. -func (p *peerConnection) SetNodeDataIdle(delivered int, deliveryTime time.Time) { - p.rates.Update(eth.NodeDataMsg, deliveryTime.Sub(p.stateStarted), delivered) - atomic.StoreInt32(&p.stateIdle, 0) -} - -// HeaderCapacity retrieves the peers header download allowance based on its -// previously discovered throughput. -func (p *peerConnection) HeaderCapacity(targetRTT time.Duration) int { - cap := p.rates.Capacity(eth.BlockHeadersMsg, targetRTT) - if cap > MaxHeaderFetch { - cap = MaxHeaderFetch - } - return cap -} - -// BlockCapacity retrieves the peers block download allowance based on its -// previously discovered throughput. -func (p *peerConnection) BlockCapacity(targetRTT time.Duration) int { - cap := p.rates.Capacity(eth.BlockBodiesMsg, targetRTT) - if cap > MaxBlockFetch { - cap = MaxBlockFetch - } - return cap -} - -// ReceiptCapacity retrieves the peers receipt download allowance based on its -// previously discovered throughput. -func (p *peerConnection) ReceiptCapacity(targetRTT time.Duration) int { - cap := p.rates.Capacity(eth.ReceiptsMsg, targetRTT) - if cap > MaxReceiptFetch { - cap = MaxReceiptFetch - } - return cap -} - -// NodeDataCapacity retrieves the peers state download allowance based on its -// previously discovered throughput. -func (p *peerConnection) NodeDataCapacity(targetRTT time.Duration) int { - cap := p.rates.Capacity(eth.NodeDataMsg, targetRTT) - if cap > MaxStateFetch { - cap = MaxStateFetch - } - return cap -} - -// MarkLacking appends a new entity to the set of items (blocks, receipts, states) -// that a peer is known not to have (i.e. have been requested before). If the -// set reaches its maximum allowed capacity, items are randomly dropped off. -func (p *peerConnection) MarkLacking(hash common.Hash) { - p.lock.Lock() - defer p.lock.Unlock() - - for len(p.lacking) >= maxLackingHashes { - for drop := range p.lacking { - delete(p.lacking, drop) - break - } - } - p.lacking[hash] = struct{}{} -} - -// Lacks retrieves whether the hash of a blockchain item is on the peers lacking -// list (i.e. whether we know that the peer does not have it). -func (p *peerConnection) Lacks(hash common.Hash) bool { - p.lock.RLock() - defer p.lock.RUnlock() - - _, ok := p.lacking[hash] - return ok -} - -// peerSet represents the collection of active peer participating in the chain -// download procedure. -type peerSet struct { - peers map[string]*peerConnection - rates *msgrate.Trackers // Set of rate trackers to give the sync a common beat - - newPeerFeed event.Feed - peerDropFeed event.Feed - - lock sync.RWMutex -} - -// newPeerSet creates a new peer set top track the active download sources. -func newPeerSet() *peerSet { - return &peerSet{ - peers: make(map[string]*peerConnection), - rates: msgrate.NewTrackers(log.New("proto", "eth")), - } -} - -// SubscribeNewPeers subscribes to peer arrival events. -func (ps *peerSet) SubscribeNewPeers(ch chan<- *peerConnection) event.Subscription { - return ps.newPeerFeed.Subscribe(ch) -} - -// SubscribePeerDrops subscribes to peer departure events. -func (ps *peerSet) SubscribePeerDrops(ch chan<- *peerConnection) event.Subscription { - return ps.peerDropFeed.Subscribe(ch) -} - -// Reset iterates over the current peer set, and resets each of the known peers -// to prepare for a next batch of block retrieval. -func (ps *peerSet) Reset() { - ps.lock.RLock() - defer ps.lock.RUnlock() - - for _, peer := range ps.peers { - peer.Reset() - } -} - -// Register injects a new peer into the working set, or returns an error if the -// peer is already known. -// -// The method also sets the starting throughput values of the new peer to the -// average of all existing peers, to give it a realistic chance of being used -// for data retrievals. -func (ps *peerSet) Register(p *peerConnection) error { - // Register the new peer with some meaningful defaults - ps.lock.Lock() - if _, ok := ps.peers[p.id]; ok { - ps.lock.Unlock() - return errAlreadyRegistered - } - p.rates = msgrate.NewTracker(ps.rates.MeanCapacities(), ps.rates.MedianRoundTrip()) - if err := ps.rates.Track(p.id, p.rates); err != nil { - ps.lock.Unlock() - return err - } - ps.peers[p.id] = p - ps.lock.Unlock() - - ps.newPeerFeed.Send(p) - return nil -} - -// Unregister removes a remote peer from the active set, disabling any further -// actions to/from that particular entity. -func (ps *peerSet) Unregister(id string) error { - ps.lock.Lock() - p, ok := ps.peers[id] - if !ok { - ps.lock.Unlock() - return errNotRegistered - } - delete(ps.peers, id) - ps.rates.Untrack(id) - ps.lock.Unlock() - - ps.peerDropFeed.Send(p) - return nil -} - -// Peer retrieves the registered peer with the given id. -func (ps *peerSet) Peer(id string) *peerConnection { - ps.lock.RLock() - defer ps.lock.RUnlock() - - return ps.peers[id] -} - -// Len returns if the current number of peers in the set. -func (ps *peerSet) Len() int { - ps.lock.RLock() - defer ps.lock.RUnlock() - - return len(ps.peers) -} - -// AllPeers retrieves a flat list of all the peers within the set. -func (ps *peerSet) AllPeers() []*peerConnection { - ps.lock.RLock() - defer ps.lock.RUnlock() - - list := make([]*peerConnection, 0, len(ps.peers)) - for _, p := range ps.peers { - list = append(list, p) - } - return list -} - -// HeaderIdlePeers retrieves a flat list of all the currently header-idle peers -// within the active peer set, ordered by their reputation. -func (ps *peerSet) HeaderIdlePeers() ([]*peerConnection, int) { - idle := func(p *peerConnection) bool { - return atomic.LoadInt32(&p.headerIdle) == 0 - } - throughput := func(p *peerConnection) int { - return p.rates.Capacity(eth.BlockHeadersMsg, time.Second) - } - return ps.idlePeers(eth.ETH66, eth.ETH67, idle, throughput) -} - -// BodyIdlePeers retrieves a flat list of all the currently body-idle peers within -// the active peer set, ordered by their reputation. -func (ps *peerSet) BodyIdlePeers() ([]*peerConnection, int) { - idle := func(p *peerConnection) bool { - return atomic.LoadInt32(&p.blockIdle) == 0 - } - throughput := func(p *peerConnection) int { - return p.rates.Capacity(eth.BlockBodiesMsg, time.Second) - } - return ps.idlePeers(eth.ETH66, eth.ETH67, idle, throughput) -} - -// ReceiptIdlePeers retrieves a flat list of all the currently receipt-idle peers -// within the active peer set, ordered by their reputation. -func (ps *peerSet) ReceiptIdlePeers() ([]*peerConnection, int) { - idle := func(p *peerConnection) bool { - return atomic.LoadInt32(&p.receiptIdle) == 0 - } - throughput := func(p *peerConnection) int { - return p.rates.Capacity(eth.ReceiptsMsg, time.Second) - } - return ps.idlePeers(eth.ETH66, eth.ETH67, idle, throughput) -} - -// NodeDataIdlePeers retrieves a flat list of all the currently node-data-idle -// peers within the active peer set, ordered by their reputation. -func (ps *peerSet) NodeDataIdlePeers() ([]*peerConnection, int) { - idle := func(p *peerConnection) bool { - return atomic.LoadInt32(&p.stateIdle) == 0 - } - throughput := func(p *peerConnection) int { - return p.rates.Capacity(eth.NodeDataMsg, time.Second) - } - return ps.idlePeers(eth.ETH66, eth.ETH67, idle, throughput) -} - -// idlePeers retrieves a flat list of all currently idle peers satisfying the -// protocol version constraints, using the provided function to check idleness. -// The resulting set of peers are sorted by their capacity. -func (ps *peerSet) idlePeers(minProtocol, maxProtocol uint, idleCheck func(*peerConnection) bool, capacity func(*peerConnection) int) ([]*peerConnection, int) { - ps.lock.RLock() - defer ps.lock.RUnlock() - - var ( - total = 0 - idle = make([]*peerConnection, 0, len(ps.peers)) - tps = make([]int, 0, len(ps.peers)) - ) - for _, p := range ps.peers { - if p.version >= minProtocol && p.version <= maxProtocol { - if idleCheck(p) { - idle = append(idle, p) - tps = append(tps, capacity(p)) - } - total++ - } - } - - // And sort them - sortPeers := &peerCapacitySort{idle, tps} - sort.Sort(sortPeers) - return sortPeers.p, total -} - -// peerCapacitySort implements sort.Interface. -// It sorts peer connections by capacity (descending). -type peerCapacitySort struct { - p []*peerConnection - tp []int -} - -func (ps *peerCapacitySort) Len() int { - return len(ps.p) -} - -func (ps *peerCapacitySort) Less(i, j int) bool { - return ps.tp[i] > ps.tp[j] -} - -func (ps *peerCapacitySort) Swap(i, j int) { - ps.p[i], ps.p[j] = ps.p[j], ps.p[i] - ps.tp[i], ps.tp[j] = ps.tp[j], ps.tp[i] -} diff --git a/les/downloader/queue.go b/les/downloader/queue.go deleted file mode 100644 index 6896b09b38..0000000000 --- a/les/downloader/queue.go +++ /dev/null @@ -1,913 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Contains the block download scheduler to collect download tasks and schedule -// them in an ordered, and throttled way. - -package downloader - -import ( - "errors" - "fmt" - "sync" - "sync/atomic" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/prque" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/metrics" - "github.com/ethereum/go-ethereum/trie" -) - -const ( - bodyType = uint(0) - receiptType = uint(1) -) - -var ( - blockCacheMaxItems = 8192 // Maximum number of blocks to cache before throttling the download - blockCacheInitialItems = 2048 // Initial number of blocks to start fetching, before we know the sizes of the blocks - blockCacheMemory = 256 * 1024 * 1024 // Maximum amount of memory to use for block caching - blockCacheSizeWeight = 0.1 // Multiplier to approximate the average block size based on past ones -) - -var ( - errNoFetchesPending = errors.New("no fetches pending") - errStaleDelivery = errors.New("stale delivery") -) - -// fetchRequest is a currently running data retrieval operation. -type fetchRequest struct { - Peer *peerConnection // Peer to which the request was sent - From uint64 // [eth/62] Requested chain element index (used for skeleton fills only) - Headers []*types.Header // [eth/62] Requested headers, sorted by request order - Time time.Time // Time when the request was made -} - -// fetchResult is a struct collecting partial results from data fetchers until -// all outstanding pieces complete and the result as a whole can be processed. -type fetchResult struct { - pending int32 // Flag telling what deliveries are outstanding - - Header *types.Header - Uncles []*types.Header - Transactions types.Transactions - Receipts types.Receipts -} - -func newFetchResult(header *types.Header, fastSync bool) *fetchResult { - item := &fetchResult{ - Header: header, - } - if !header.EmptyBody() { - item.pending |= (1 << bodyType) - } - if fastSync && !header.EmptyReceipts() { - item.pending |= (1 << receiptType) - } - return item -} - -// SetBodyDone flags the body as finished. -func (f *fetchResult) SetBodyDone() { - if v := atomic.LoadInt32(&f.pending); (v & (1 << bodyType)) != 0 { - atomic.AddInt32(&f.pending, -1) - } -} - -// AllDone checks if item is done. -func (f *fetchResult) AllDone() bool { - return atomic.LoadInt32(&f.pending) == 0 -} - -// SetReceiptsDone flags the receipts as finished. -func (f *fetchResult) SetReceiptsDone() { - if v := atomic.LoadInt32(&f.pending); (v & (1 << receiptType)) != 0 { - atomic.AddInt32(&f.pending, -2) - } -} - -// Done checks if the given type is done already -func (f *fetchResult) Done(kind uint) bool { - v := atomic.LoadInt32(&f.pending) - return v&(1< 0 -} - -// InFlightBlocks retrieves whether there are block fetch requests currently in -// flight. -func (q *queue) InFlightBlocks() bool { - q.lock.Lock() - defer q.lock.Unlock() - - return len(q.blockPendPool) > 0 -} - -// InFlightReceipts retrieves whether there are receipt fetch requests currently -// in flight. -func (q *queue) InFlightReceipts() bool { - q.lock.Lock() - defer q.lock.Unlock() - - return len(q.receiptPendPool) > 0 -} - -// Idle returns if the queue is fully idle or has some data still inside. -func (q *queue) Idle() bool { - q.lock.Lock() - defer q.lock.Unlock() - - queued := q.blockTaskQueue.Size() + q.receiptTaskQueue.Size() - pending := len(q.blockPendPool) + len(q.receiptPendPool) - - return (queued + pending) == 0 -} - -// ScheduleSkeleton adds a batch of header retrieval tasks to the queue to fill -// up an already retrieved header skeleton. -func (q *queue) ScheduleSkeleton(from uint64, skeleton []*types.Header) { - q.lock.Lock() - defer q.lock.Unlock() - - // No skeleton retrieval can be in progress, fail hard if so (huge implementation bug) - if q.headerResults != nil { - panic("skeleton assembly already in progress") - } - // Schedule all the header retrieval tasks for the skeleton assembly - q.headerTaskPool = make(map[uint64]*types.Header) - q.headerTaskQueue = prque.New[int64, uint64](nil) - q.headerPeerMiss = make(map[string]map[uint64]struct{}) // Reset availability to correct invalid chains - q.headerResults = make([]*types.Header, len(skeleton)*MaxHeaderFetch) - q.headerProced = 0 - q.headerOffset = from - q.headerContCh = make(chan bool, 1) - - for i, header := range skeleton { - index := from + uint64(i*MaxHeaderFetch) - - q.headerTaskPool[index] = header - q.headerTaskQueue.Push(index, -int64(index)) - } -} - -// RetrieveHeaders retrieves the header chain assemble based on the scheduled -// skeleton. -func (q *queue) RetrieveHeaders() ([]*types.Header, int) { - q.lock.Lock() - defer q.lock.Unlock() - - headers, proced := q.headerResults, q.headerProced - q.headerResults, q.headerProced = nil, 0 - - return headers, proced -} - -// Schedule adds a set of headers for the download queue for scheduling, returning -// the new headers encountered. -func (q *queue) Schedule(headers []*types.Header, from uint64) []*types.Header { - q.lock.Lock() - defer q.lock.Unlock() - - // Insert all the headers prioritised by the contained block number - inserts := make([]*types.Header, 0, len(headers)) - for _, header := range headers { - // Make sure chain order is honoured and preserved throughout - hash := header.Hash() - if header.Number == nil || header.Number.Uint64() != from { - log.Warn("Header broke chain ordering", "number", header.Number, "hash", hash, "expected", from) - break - } - if q.headerHead != (common.Hash{}) && q.headerHead != header.ParentHash { - log.Warn("Header broke chain ancestry", "number", header.Number, "hash", hash) - break - } - // Make sure no duplicate requests are executed - // We cannot skip this, even if the block is empty, since this is - // what triggers the fetchResult creation. - if _, ok := q.blockTaskPool[hash]; ok { - log.Warn("Header already scheduled for block fetch", "number", header.Number, "hash", hash) - } else { - q.blockTaskPool[hash] = header - q.blockTaskQueue.Push(header, -int64(header.Number.Uint64())) - } - // Queue for receipt retrieval - if q.mode == FastSync && !header.EmptyReceipts() { - if _, ok := q.receiptTaskPool[hash]; ok { - log.Warn("Header already scheduled for receipt fetch", "number", header.Number, "hash", hash) - } else { - q.receiptTaskPool[hash] = header - q.receiptTaskQueue.Push(header, -int64(header.Number.Uint64())) - } - } - inserts = append(inserts, header) - q.headerHead = hash - from++ - } - return inserts -} - -// Results retrieves and permanently removes a batch of fetch results from -// the cache. the result slice will be empty if the queue has been closed. -// Results can be called concurrently with Deliver and Schedule, -// but assumes that there are not two simultaneous callers to Results -func (q *queue) Results(block bool) []*fetchResult { - // Abort early if there are no items and non-blocking requested - if !block && !q.resultCache.HasCompletedItems() { - return nil - } - closed := false - for !closed && !q.resultCache.HasCompletedItems() { - // In order to wait on 'active', we need to obtain the lock. - // That may take a while, if someone is delivering at the same - // time, so after obtaining the lock, we check again if there - // are any results to fetch. - // Also, in-between we ask for the lock and the lock is obtained, - // someone can have closed the queue. In that case, we should - // return the available results and stop blocking - q.lock.Lock() - if q.resultCache.HasCompletedItems() || q.closed { - q.lock.Unlock() - break - } - // No items available, and not closed - q.active.Wait() - closed = q.closed - q.lock.Unlock() - } - // Regardless if closed or not, we can still deliver whatever we have - results := q.resultCache.GetCompleted(maxResultsProcess) - for _, result := range results { - // Recalculate the result item weights to prevent memory exhaustion - size := result.Header.Size() - for _, uncle := range result.Uncles { - size += uncle.Size() - } - for _, receipt := range result.Receipts { - size += receipt.Size() - } - for _, tx := range result.Transactions { - size += common.StorageSize(tx.Size()) - } - q.resultSize = common.StorageSize(blockCacheSizeWeight)*size + - (1-common.StorageSize(blockCacheSizeWeight))*q.resultSize - } - // Using the newly calibrated resultsize, figure out the new throttle limit - // on the result cache - throttleThreshold := uint64((common.StorageSize(blockCacheMemory) + q.resultSize - 1) / q.resultSize) - throttleThreshold = q.resultCache.SetThrottleThreshold(throttleThreshold) - - // Log some info at certain times - if time.Since(q.lastStatLog) > 60*time.Second { - q.lastStatLog = time.Now() - info := q.Stats() - info = append(info, "throttle", throttleThreshold) - log.Info("Downloader queue stats", info...) - } - return results -} - -func (q *queue) Stats() []interface{} { - q.lock.RLock() - defer q.lock.RUnlock() - - return q.stats() -} - -func (q *queue) stats() []interface{} { - return []interface{}{ - "receiptTasks", q.receiptTaskQueue.Size(), - "blockTasks", q.blockTaskQueue.Size(), - "itemSize", q.resultSize, - } -} - -// ReserveHeaders reserves a set of headers for the given peer, skipping any -// previously failed batches. -func (q *queue) ReserveHeaders(p *peerConnection, count int) *fetchRequest { - q.lock.Lock() - defer q.lock.Unlock() - - // Short circuit if the peer's already downloading something (sanity check to - // not corrupt state) - if _, ok := q.headerPendPool[p.id]; ok { - return nil - } - // Retrieve a batch of hashes, skipping previously failed ones - send, skip := uint64(0), []uint64{} - for send == 0 && !q.headerTaskQueue.Empty() { - from, _ := q.headerTaskQueue.Pop() - if q.headerPeerMiss[p.id] != nil { - if _, ok := q.headerPeerMiss[p.id][from]; ok { - skip = append(skip, from) - continue - } - } - send = from - } - // Merge all the skipped batches back - for _, from := range skip { - q.headerTaskQueue.Push(from, -int64(from)) - } - // Assemble and return the block download request - if send == 0 { - return nil - } - request := &fetchRequest{ - Peer: p, - From: send, - Time: time.Now(), - } - q.headerPendPool[p.id] = request - return request -} - -// ReserveBodies reserves a set of body fetches for the given peer, skipping any -// previously failed downloads. Beside the next batch of needed fetches, it also -// returns a flag whether empty blocks were queued requiring processing. -func (q *queue) ReserveBodies(p *peerConnection, count int) (*fetchRequest, bool, bool) { - q.lock.Lock() - defer q.lock.Unlock() - - return q.reserveHeaders(p, count, q.blockTaskPool, q.blockTaskQueue, q.blockPendPool, bodyType) -} - -// ReserveReceipts reserves a set of receipt fetches for the given peer, skipping -// any previously failed downloads. Beside the next batch of needed fetches, it -// also returns a flag whether empty receipts were queued requiring importing. -func (q *queue) ReserveReceipts(p *peerConnection, count int) (*fetchRequest, bool, bool) { - q.lock.Lock() - defer q.lock.Unlock() - - return q.reserveHeaders(p, count, q.receiptTaskPool, q.receiptTaskQueue, q.receiptPendPool, receiptType) -} - -// reserveHeaders reserves a set of data download operations for a given peer, -// skipping any previously failed ones. This method is a generic version used -// by the individual special reservation functions. -// -// Note, this method expects the queue lock to be already held for writing. The -// reason the lock is not obtained in here is because the parameters already need -// to access the queue, so they already need a lock anyway. -// -// Returns: -// -// item - the fetchRequest -// progress - whether any progress was made -// throttle - if the caller should throttle for a while -func (q *queue) reserveHeaders(p *peerConnection, count int, taskPool map[common.Hash]*types.Header, taskQueue *prque.Prque[int64, *types.Header], - pendPool map[string]*fetchRequest, kind uint) (*fetchRequest, bool, bool) { - // Short circuit if the pool has been depleted, or if the peer's already - // downloading something (sanity check not to corrupt state) - if taskQueue.Empty() { - return nil, false, true - } - if _, ok := pendPool[p.id]; ok { - return nil, false, false - } - // Retrieve a batch of tasks, skipping previously failed ones - send := make([]*types.Header, 0, count) - skip := make([]*types.Header, 0) - progress := false - throttled := false - for proc := 0; len(send) < count && !taskQueue.Empty(); proc++ { - // the task queue will pop items in order, so the highest prio block - // is also the lowest block number. - header, _ := taskQueue.Peek() - - // we can ask the resultcache if this header is within the - // "prioritized" segment of blocks. If it is not, we need to throttle - - stale, throttle, item, err := q.resultCache.AddFetch(header, q.mode == FastSync) - if stale { - // Don't put back in the task queue, this item has already been - // delivered upstream - taskQueue.PopItem() - progress = true - delete(taskPool, header.Hash()) - proc = proc - 1 - log.Error("Fetch reservation already delivered", "number", header.Number.Uint64()) - continue - } - if throttle { - // There are no resultslots available. Leave it in the task queue - // However, if there are any left as 'skipped', we should not tell - // the caller to throttle, since we still want some other - // peer to fetch those for us - throttled = len(skip) == 0 - break - } - if err != nil { - // this most definitely should _not_ happen - log.Warn("Failed to reserve headers", "err", err) - // There are no resultslots available. Leave it in the task queue - break - } - if item.Done(kind) { - // If it's a noop, we can skip this task - delete(taskPool, header.Hash()) - taskQueue.PopItem() - proc = proc - 1 - progress = true - continue - } - // Remove it from the task queue - taskQueue.PopItem() - // Otherwise unless the peer is known not to have the data, add to the retrieve list - if p.Lacks(header.Hash()) { - skip = append(skip, header) - } else { - send = append(send, header) - } - } - // Merge all the skipped headers back - for _, header := range skip { - taskQueue.Push(header, -int64(header.Number.Uint64())) - } - if q.resultCache.HasCompletedItems() { - // Wake Results, resultCache was modified - q.active.Signal() - } - // Assemble and return the block download request - if len(send) == 0 { - return nil, progress, throttled - } - request := &fetchRequest{ - Peer: p, - Headers: send, - Time: time.Now(), - } - pendPool[p.id] = request - return request, progress, throttled -} - -// CancelHeaders aborts a fetch request, returning all pending skeleton indexes to the queue. -func (q *queue) CancelHeaders(request *fetchRequest) { - q.lock.Lock() - defer q.lock.Unlock() - q.cancel(request, q.headerTaskQueue, q.headerPendPool) -} - -// CancelBodies aborts a body fetch request, returning all pending headers to the -// task queue. -func (q *queue) CancelBodies(request *fetchRequest) { - q.lock.Lock() - defer q.lock.Unlock() - q.cancel(request, q.blockTaskQueue, q.blockPendPool) -} - -// CancelReceipts aborts a body fetch request, returning all pending headers to -// the task queue. -func (q *queue) CancelReceipts(request *fetchRequest) { - q.lock.Lock() - defer q.lock.Unlock() - q.cancel(request, q.receiptTaskQueue, q.receiptPendPool) -} - -// Cancel aborts a fetch request, returning all pending hashes to the task queue. -func (q *queue) cancel(request *fetchRequest, taskQueue interface{}, pendPool map[string]*fetchRequest) { - if request.From > 0 { - taskQueue.(*prque.Prque[int64, uint64]).Push(request.From, -int64(request.From)) - } - for _, header := range request.Headers { - taskQueue.(*prque.Prque[int64, *types.Header]).Push(header, -int64(header.Number.Uint64())) - } - delete(pendPool, request.Peer.id) -} - -// Revoke cancels all pending requests belonging to a given peer. This method is -// meant to be called during a peer drop to quickly reassign owned data fetches -// to remaining nodes. -func (q *queue) Revoke(peerID string) { - q.lock.Lock() - defer q.lock.Unlock() - - if request, ok := q.blockPendPool[peerID]; ok { - for _, header := range request.Headers { - q.blockTaskQueue.Push(header, -int64(header.Number.Uint64())) - } - delete(q.blockPendPool, peerID) - } - if request, ok := q.receiptPendPool[peerID]; ok { - for _, header := range request.Headers { - q.receiptTaskQueue.Push(header, -int64(header.Number.Uint64())) - } - delete(q.receiptPendPool, peerID) - } -} - -// ExpireHeaders checks for in flight requests that exceeded a timeout allowance, -// canceling them and returning the responsible peers for penalisation. -func (q *queue) ExpireHeaders(timeout time.Duration) map[string]int { - q.lock.Lock() - defer q.lock.Unlock() - - return q.expire(timeout, q.headerPendPool, q.headerTaskQueue, headerTimeoutMeter) -} - -// ExpireBodies checks for in flight block body requests that exceeded a timeout -// allowance, canceling them and returning the responsible peers for penalisation. -func (q *queue) ExpireBodies(timeout time.Duration) map[string]int { - q.lock.Lock() - defer q.lock.Unlock() - - return q.expire(timeout, q.blockPendPool, q.blockTaskQueue, bodyTimeoutMeter) -} - -// ExpireReceipts checks for in flight receipt requests that exceeded a timeout -// allowance, canceling them and returning the responsible peers for penalisation. -func (q *queue) ExpireReceipts(timeout time.Duration) map[string]int { - q.lock.Lock() - defer q.lock.Unlock() - - return q.expire(timeout, q.receiptPendPool, q.receiptTaskQueue, receiptTimeoutMeter) -} - -// expire is the generic check that move expired tasks from a pending pool back -// into a task pool, returning all entities caught with expired tasks. -// -// Note, this method expects the queue lock to be already held. The -// reason the lock is not obtained in here is because the parameters already need -// to access the queue, so they already need a lock anyway. -func (q *queue) expire(timeout time.Duration, pendPool map[string]*fetchRequest, taskQueue interface{}, timeoutMeter metrics.Meter) map[string]int { - // Iterate over the expired requests and return each to the queue - expiries := make(map[string]int) - for id, request := range pendPool { - if time.Since(request.Time) > timeout { - // Update the metrics with the timeout - timeoutMeter.Mark(1) - - // Return any non satisfied requests to the pool - if request.From > 0 { - taskQueue.(*prque.Prque[int64, uint64]).Push(request.From, -int64(request.From)) - } - for _, header := range request.Headers { - taskQueue.(*prque.Prque[int64, *types.Header]).Push(header, -int64(header.Number.Uint64())) - } - // Add the peer to the expiry report along the number of failed requests - expiries[id] = len(request.Headers) - - // Remove the expired requests from the pending pool directly - delete(pendPool, id) - } - } - return expiries -} - -// DeliverHeaders injects a header retrieval response into the header results -// cache. This method either accepts all headers it received, or none of them -// if they do not map correctly to the skeleton. -// -// If the headers are accepted, the method makes an attempt to deliver the set -// of ready headers to the processor to keep the pipeline full. However it will -// not block to prevent stalling other pending deliveries. -func (q *queue) DeliverHeaders(id string, headers []*types.Header, headerProcCh chan []*types.Header) (int, error) { - q.lock.Lock() - defer q.lock.Unlock() - - var logger log.Logger - if len(id) < 16 { - // Tests use short IDs, don't choke on them - logger = log.New("peer", id) - } else { - logger = log.New("peer", id[:16]) - } - // Short circuit if the data was never requested - request := q.headerPendPool[id] - if request == nil { - return 0, errNoFetchesPending - } - headerReqTimer.UpdateSince(request.Time) - delete(q.headerPendPool, id) - - // Ensure headers can be mapped onto the skeleton chain - target := q.headerTaskPool[request.From].Hash() - - accepted := len(headers) == MaxHeaderFetch - if accepted { - if headers[0].Number.Uint64() != request.From { - logger.Trace("First header broke chain ordering", "number", headers[0].Number, "hash", headers[0].Hash(), "expected", request.From) - accepted = false - } else if headers[len(headers)-1].Hash() != target { - logger.Trace("Last header broke skeleton structure ", "number", headers[len(headers)-1].Number, "hash", headers[len(headers)-1].Hash(), "expected", target) - accepted = false - } - } - if accepted { - parentHash := headers[0].Hash() - for i, header := range headers[1:] { - hash := header.Hash() - if want := request.From + 1 + uint64(i); header.Number.Uint64() != want { - logger.Warn("Header broke chain ordering", "number", header.Number, "hash", hash, "expected", want) - accepted = false - break - } - if parentHash != header.ParentHash { - logger.Warn("Header broke chain ancestry", "number", header.Number, "hash", hash) - accepted = false - break - } - // Set-up parent hash for next round - parentHash = hash - } - } - // If the batch of headers wasn't accepted, mark as unavailable - if !accepted { - logger.Trace("Skeleton filling not accepted", "from", request.From) - - miss := q.headerPeerMiss[id] - if miss == nil { - q.headerPeerMiss[id] = make(map[uint64]struct{}) - miss = q.headerPeerMiss[id] - } - miss[request.From] = struct{}{} - - q.headerTaskQueue.Push(request.From, -int64(request.From)) - return 0, errors.New("delivery not accepted") - } - // Clean up a successful fetch and try to deliver any sub-results - copy(q.headerResults[request.From-q.headerOffset:], headers) - delete(q.headerTaskPool, request.From) - - ready := 0 - for q.headerProced+ready < len(q.headerResults) && q.headerResults[q.headerProced+ready] != nil { - ready += MaxHeaderFetch - } - if ready > 0 { - // Headers are ready for delivery, gather them and push forward (non blocking) - process := make([]*types.Header, ready) - copy(process, q.headerResults[q.headerProced:q.headerProced+ready]) - - select { - case headerProcCh <- process: - logger.Trace("Pre-scheduled new headers", "count", len(process), "from", process[0].Number) - q.headerProced += len(process) - default: - } - } - // Check for termination and return - if len(q.headerTaskPool) == 0 { - q.headerContCh <- false - } - return len(headers), nil -} - -// DeliverBodies injects a block body retrieval response into the results queue. -// The method returns the number of blocks bodies accepted from the delivery and -// also wakes any threads waiting for data delivery. -func (q *queue) DeliverBodies(id string, txLists [][]*types.Transaction, uncleLists [][]*types.Header) (int, error) { - q.lock.Lock() - defer q.lock.Unlock() - trieHasher := trie.NewStackTrie(nil) - validate := func(index int, header *types.Header) error { - if types.DeriveSha(types.Transactions(txLists[index]), trieHasher) != header.TxHash { - return errInvalidBody - } - if types.CalcUncleHash(uncleLists[index]) != header.UncleHash { - return errInvalidBody - } - return nil - } - - reconstruct := func(index int, result *fetchResult) { - result.Transactions = txLists[index] - result.Uncles = uncleLists[index] - result.SetBodyDone() - } - return q.deliver(id, q.blockTaskPool, q.blockTaskQueue, q.blockPendPool, - bodyReqTimer, len(txLists), validate, reconstruct) -} - -// DeliverReceipts injects a receipt retrieval response into the results queue. -// The method returns the number of transaction receipts accepted from the delivery -// and also wakes any threads waiting for data delivery. -func (q *queue) DeliverReceipts(id string, receiptList [][]*types.Receipt) (int, error) { - q.lock.Lock() - defer q.lock.Unlock() - trieHasher := trie.NewStackTrie(nil) - validate := func(index int, header *types.Header) error { - if types.DeriveSha(types.Receipts(receiptList[index]), trieHasher) != header.ReceiptHash { - return errInvalidReceipt - } - return nil - } - reconstruct := func(index int, result *fetchResult) { - result.Receipts = receiptList[index] - result.SetReceiptsDone() - } - return q.deliver(id, q.receiptTaskPool, q.receiptTaskQueue, q.receiptPendPool, - receiptReqTimer, len(receiptList), validate, reconstruct) -} - -// deliver injects a data retrieval response into the results queue. -// -// Note, this method expects the queue lock to be already held for writing. The -// reason this lock is not obtained in here is because the parameters already need -// to access the queue, so they already need a lock anyway. -func (q *queue) deliver(id string, taskPool map[common.Hash]*types.Header, - taskQueue *prque.Prque[int64, *types.Header], pendPool map[string]*fetchRequest, reqTimer metrics.Timer, - results int, validate func(index int, header *types.Header) error, - reconstruct func(index int, result *fetchResult)) (int, error) { - // Short circuit if the data was never requested - request := pendPool[id] - if request == nil { - return 0, errNoFetchesPending - } - reqTimer.UpdateSince(request.Time) - delete(pendPool, id) - - // If no data items were retrieved, mark them as unavailable for the origin peer - if results == 0 { - for _, header := range request.Headers { - request.Peer.MarkLacking(header.Hash()) - } - } - // Assemble each of the results with their headers and retrieved data parts - var ( - accepted int - failure error - i int - hashes []common.Hash - ) - for _, header := range request.Headers { - // Short circuit assembly if no more fetch results are found - if i >= results { - break - } - // Validate the fields - if err := validate(i, header); err != nil { - failure = err - break - } - hashes = append(hashes, header.Hash()) - i++ - } - - for _, header := range request.Headers[:i] { - if res, stale, err := q.resultCache.GetDeliverySlot(header.Number.Uint64()); err == nil && !stale { - reconstruct(accepted, res) - } else { - // else: between here and above, some other peer filled this result, - // or it was indeed a no-op. This should not happen, but if it does it's - // not something to panic about - log.Error("Delivery stale", "stale", stale, "number", header.Number.Uint64(), "err", err) - failure = errStaleDelivery - } - // Clean up a successful fetch - delete(taskPool, hashes[accepted]) - accepted++ - } - // Return all failed or missing fetches to the queue - for _, header := range request.Headers[accepted:] { - taskQueue.Push(header, -int64(header.Number.Uint64())) - } - // Wake up Results - if accepted > 0 { - q.active.Signal() - } - if failure == nil { - return accepted, nil - } - // If none of the data was good, it's a stale delivery - if accepted > 0 { - return accepted, fmt.Errorf("partial failure: %v", failure) - } - return accepted, fmt.Errorf("%w: %v", failure, errStaleDelivery) -} - -// Prepare configures the result cache to allow accepting and caching inbound -// fetch results. -func (q *queue) Prepare(offset uint64, mode SyncMode) { - q.lock.Lock() - defer q.lock.Unlock() - - // Prepare the queue for sync results - q.resultCache.Prepare(offset) - q.mode = mode -} diff --git a/les/downloader/queue_test.go b/les/downloader/queue_test.go deleted file mode 100644 index 6bffa90219..0000000000 --- a/les/downloader/queue_test.go +++ /dev/null @@ -1,438 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package downloader - -import ( - "fmt" - "math/big" - "math/rand" - "sync" - "testing" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus/ethash" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/params" -) - -// makeChain creates a chain of n blocks starting at and including parent. -// the returned hash chain is ordered head->parent. In addition, every 3rd block -// contains a transaction and every 5th an uncle to allow testing correct block -// reassembly. -func makeChain(n int, seed byte, parent *types.Block, empty bool) ([]*types.Block, []types.Receipts) { - blocks, receipts := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testDB, n, func(i int, block *core.BlockGen) { - block.SetCoinbase(common.Address{seed}) - // Add one tx to every secondblock - if !empty && i%2 == 0 { - signer := types.MakeSigner(params.TestChainConfig, block.Number(), block.Timestamp()) - tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddress), common.Address{seed}, big.NewInt(1000), params.TxGas, block.BaseFee(), nil), signer, testKey) - if err != nil { - panic(err) - } - block.AddTx(tx) - } - }) - return blocks, receipts -} - -type chainData struct { - blocks []*types.Block - offset int -} - -var chain *chainData -var emptyChain *chainData - -func init() { - // Create a chain of blocks to import - targetBlocks := 128 - blocks, _ := makeChain(targetBlocks, 0, testGenesis, false) - chain = &chainData{blocks, 0} - - blocks, _ = makeChain(targetBlocks, 0, testGenesis, true) - emptyChain = &chainData{blocks, 0} -} - -func (chain *chainData) headers() []*types.Header { - hdrs := make([]*types.Header, len(chain.blocks)) - for i, b := range chain.blocks { - hdrs[i] = b.Header() - } - return hdrs -} - -func (chain *chainData) Len() int { - return len(chain.blocks) -} - -func dummyPeer(id string) *peerConnection { - p := &peerConnection{ - id: id, - lacking: make(map[common.Hash]struct{}), - } - return p -} - -func TestBasics(t *testing.T) { - numOfBlocks := len(emptyChain.blocks) - numOfReceipts := len(emptyChain.blocks) / 2 - - q := newQueue(10, 10) - if !q.Idle() { - t.Errorf("new queue should be idle") - } - q.Prepare(1, FastSync) - if res := q.Results(false); len(res) != 0 { - t.Fatal("new queue should have 0 results") - } - - // Schedule a batch of headers - q.Schedule(chain.headers(), 1) - if q.Idle() { - t.Errorf("queue should not be idle") - } - if got, exp := q.PendingBlocks(), chain.Len(); got != exp { - t.Errorf("wrong pending block count, got %d, exp %d", got, exp) - } - // Only non-empty receipts get added to task-queue - if got, exp := q.PendingReceipts(), 64; got != exp { - t.Errorf("wrong pending receipt count, got %d, exp %d", got, exp) - } - // Items are now queued for downloading, next step is that we tell the - // queue that a certain peer will deliver them for us - { - peer := dummyPeer("peer-1") - fetchReq, _, throttle := q.ReserveBodies(peer, 50) - if !throttle { - // queue size is only 10, so throttling should occur - t.Fatal("should throttle") - } - // But we should still get the first things to fetch - if got, exp := len(fetchReq.Headers), 5; got != exp { - t.Fatalf("expected %d requests, got %d", exp, got) - } - if got, exp := fetchReq.Headers[0].Number.Uint64(), uint64(1); got != exp { - t.Fatalf("expected header %d, got %d", exp, got) - } - } - if exp, got := q.blockTaskQueue.Size(), numOfBlocks-10; exp != got { - t.Errorf("expected block task queue to be %d, got %d", exp, got) - } - if exp, got := q.receiptTaskQueue.Size(), numOfReceipts; exp != got { - t.Errorf("expected receipt task queue to be %d, got %d", exp, got) - } - { - peer := dummyPeer("peer-2") - fetchReq, _, throttle := q.ReserveBodies(peer, 50) - - // The second peer should hit throttling - if !throttle { - t.Fatalf("should throttle") - } - // And not get any fetches at all, since it was throttled to begin with - if fetchReq != nil { - t.Fatalf("should have no fetches, got %d", len(fetchReq.Headers)) - } - } - if exp, got := q.blockTaskQueue.Size(), numOfBlocks-10; exp != got { - t.Errorf("expected block task queue to be %d, got %d", exp, got) - } - if exp, got := q.receiptTaskQueue.Size(), numOfReceipts; exp != got { - t.Errorf("expected receipt task queue to be %d, got %d", exp, got) - } - { - // The receipt delivering peer should not be affected - // by the throttling of body deliveries - peer := dummyPeer("peer-3") - fetchReq, _, throttle := q.ReserveReceipts(peer, 50) - if !throttle { - // queue size is only 10, so throttling should occur - t.Fatal("should throttle") - } - // But we should still get the first things to fetch - if got, exp := len(fetchReq.Headers), 5; got != exp { - t.Fatalf("expected %d requests, got %d", exp, got) - } - if got, exp := fetchReq.Headers[0].Number.Uint64(), uint64(1); got != exp { - t.Fatalf("expected header %d, got %d", exp, got) - } - } - if exp, got := q.blockTaskQueue.Size(), numOfBlocks-10; exp != got { - t.Errorf("expected block task queue to be %d, got %d", exp, got) - } - if exp, got := q.receiptTaskQueue.Size(), numOfReceipts-5; exp != got { - t.Errorf("expected receipt task queue to be %d, got %d", exp, got) - } - if got, exp := q.resultCache.countCompleted(), 0; got != exp { - t.Errorf("wrong processable count, got %d, exp %d", got, exp) - } -} - -func TestEmptyBlocks(t *testing.T) { - numOfBlocks := len(emptyChain.blocks) - - q := newQueue(10, 10) - - q.Prepare(1, FastSync) - // Schedule a batch of headers - q.Schedule(emptyChain.headers(), 1) - if q.Idle() { - t.Errorf("queue should not be idle") - } - if got, exp := q.PendingBlocks(), len(emptyChain.blocks); got != exp { - t.Errorf("wrong pending block count, got %d, exp %d", got, exp) - } - if got, exp := q.PendingReceipts(), 0; got != exp { - t.Errorf("wrong pending receipt count, got %d, exp %d", got, exp) - } - // They won't be processable, because the fetchresults haven't been - // created yet - if got, exp := q.resultCache.countCompleted(), 0; got != exp { - t.Errorf("wrong processable count, got %d, exp %d", got, exp) - } - - // Items are now queued for downloading, next step is that we tell the - // queue that a certain peer will deliver them for us - // That should trigger all of them to suddenly become 'done' - { - // Reserve blocks - peer := dummyPeer("peer-1") - fetchReq, _, _ := q.ReserveBodies(peer, 50) - - // there should be nothing to fetch, blocks are empty - if fetchReq != nil { - t.Fatal("there should be no body fetch tasks remaining") - } - } - if q.blockTaskQueue.Size() != numOfBlocks-10 { - t.Errorf("expected block task queue to be %d, got %d", numOfBlocks-10, q.blockTaskQueue.Size()) - } - if q.receiptTaskQueue.Size() != 0 { - t.Errorf("expected receipt task queue to be %d, got %d", 0, q.receiptTaskQueue.Size()) - } - { - peer := dummyPeer("peer-3") - fetchReq, _, _ := q.ReserveReceipts(peer, 50) - - // there should be nothing to fetch, blocks are empty - if fetchReq != nil { - t.Fatal("there should be no receipt fetch tasks remaining") - } - } - if q.blockTaskQueue.Size() != numOfBlocks-10 { - t.Errorf("expected block task queue to be %d, got %d", numOfBlocks-10, q.blockTaskQueue.Size()) - } - if q.receiptTaskQueue.Size() != 0 { - t.Errorf("expected receipt task queue to be %d, got %d", 0, q.receiptTaskQueue.Size()) - } - if got, exp := q.resultCache.countCompleted(), 10; got != exp { - t.Errorf("wrong processable count, got %d, exp %d", got, exp) - } -} - -// XTestDelivery does some more extensive testing of events that happen, -// blocks that become known and peers that make reservations and deliveries. -// disabled since it's not really a unit-test, but can be executed to test -// some more advanced scenarios -func XTestDelivery(t *testing.T) { - // the outside network, holding blocks - blo, rec := makeChain(128, 0, testGenesis, false) - world := newNetwork() - world.receipts = rec - world.chain = blo - world.progress(10) - if false { - log.Root().SetHandler(log.StdoutHandler) - } - q := newQueue(10, 10) - var wg sync.WaitGroup - q.Prepare(1, FastSync) - wg.Add(1) - go func() { - // deliver headers - defer wg.Done() - c := 1 - for { - //fmt.Printf("getting headers from %d\n", c) - hdrs := world.headers(c) - l := len(hdrs) - //fmt.Printf("scheduling %d headers, first %d last %d\n", - // l, hdrs[0].Number.Uint64(), hdrs[len(hdrs)-1].Number.Uint64()) - q.Schedule(hdrs, uint64(c)) - c += l - } - }() - wg.Add(1) - go func() { - // collect results - defer wg.Done() - tot := 0 - for { - res := q.Results(true) - tot += len(res) - fmt.Printf("got %d results, %d tot\n", len(res), tot) - // Now we can forget about these - world.forget(res[len(res)-1].Header.Number.Uint64()) - } - }() - wg.Add(1) - go func() { - defer wg.Done() - // reserve body fetch - i := 4 - for { - peer := dummyPeer(fmt.Sprintf("peer-%d", i)) - f, _, _ := q.ReserveBodies(peer, rand.Intn(30)) - if f != nil { - var emptyList []*types.Header - var txs [][]*types.Transaction - var uncles [][]*types.Header - numToSkip := rand.Intn(len(f.Headers)) - for _, hdr := range f.Headers[0 : len(f.Headers)-numToSkip] { - txs = append(txs, world.getTransactions(hdr.Number.Uint64())) - uncles = append(uncles, emptyList) - } - time.Sleep(100 * time.Millisecond) - _, err := q.DeliverBodies(peer.id, txs, uncles) - if err != nil { - fmt.Printf("delivered %d bodies %v\n", len(txs), err) - } - } else { - i++ - time.Sleep(200 * time.Millisecond) - } - } - }() - go func() { - defer wg.Done() - // reserve receiptfetch - peer := dummyPeer("peer-3") - for { - f, _, _ := q.ReserveReceipts(peer, rand.Intn(50)) - if f != nil { - var rcs [][]*types.Receipt - for _, hdr := range f.Headers { - rcs = append(rcs, world.getReceipts(hdr.Number.Uint64())) - } - _, err := q.DeliverReceipts(peer.id, rcs) - if err != nil { - fmt.Printf("delivered %d receipts %v\n", len(rcs), err) - } - time.Sleep(100 * time.Millisecond) - } else { - time.Sleep(200 * time.Millisecond) - } - } - }() - wg.Add(1) - go func() { - defer wg.Done() - for i := 0; i < 50; i++ { - time.Sleep(300 * time.Millisecond) - //world.tick() - //fmt.Printf("trying to progress\n") - world.progress(rand.Intn(100)) - } - for i := 0; i < 50; i++ { - time.Sleep(2990 * time.Millisecond) - } - }() - wg.Add(1) - go func() { - defer wg.Done() - for { - time.Sleep(990 * time.Millisecond) - fmt.Printf("world block tip is %d\n", - world.chain[len(world.chain)-1].Header().Number.Uint64()) - fmt.Println(q.Stats()) - } - }() - wg.Wait() -} - -func newNetwork() *network { - var l sync.RWMutex - return &network{ - cond: sync.NewCond(&l), - offset: 1, // block 1 is at blocks[0] - } -} - -// represents the network -type network struct { - offset int - chain []*types.Block - receipts []types.Receipts - lock sync.RWMutex - cond *sync.Cond -} - -func (n *network) getTransactions(blocknum uint64) types.Transactions { - index := blocknum - uint64(n.offset) - return n.chain[index].Transactions() -} -func (n *network) getReceipts(blocknum uint64) types.Receipts { - index := blocknum - uint64(n.offset) - if got := n.chain[index].Header().Number.Uint64(); got != blocknum { - fmt.Printf("Err, got %d exp %d\n", got, blocknum) - panic("sd") - } - return n.receipts[index] -} - -func (n *network) forget(blocknum uint64) { - index := blocknum - uint64(n.offset) - n.chain = n.chain[index:] - n.receipts = n.receipts[index:] - n.offset = int(blocknum) -} -func (n *network) progress(numBlocks int) { - n.lock.Lock() - defer n.lock.Unlock() - //fmt.Printf("progressing...\n") - newBlocks, newR := makeChain(numBlocks, 0, n.chain[len(n.chain)-1], false) - n.chain = append(n.chain, newBlocks...) - n.receipts = append(n.receipts, newR...) - n.cond.Broadcast() -} - -func (n *network) headers(from int) []*types.Header { - numHeaders := 128 - var hdrs []*types.Header - index := from - n.offset - - for index >= len(n.chain) { - // wait for progress - n.cond.L.Lock() - //fmt.Printf("header going into wait\n") - n.cond.Wait() - index = from - n.offset - n.cond.L.Unlock() - } - n.lock.RLock() - defer n.lock.RUnlock() - for i, b := range n.chain[index:] { - hdrs = append(hdrs, b.Header()) - if i >= numHeaders { - break - } - } - return hdrs -} diff --git a/les/downloader/resultstore.go b/les/downloader/resultstore.go deleted file mode 100644 index 7fcade2946..0000000000 --- a/les/downloader/resultstore.go +++ /dev/null @@ -1,195 +0,0 @@ -// Copyright 2020 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package downloader - -import ( - "fmt" - "sync" - "sync/atomic" - - "github.com/ethereum/go-ethereum/core/types" -) - -// resultStore implements a structure for maintaining fetchResults, tracking their -// download-progress and delivering (finished) results. -type resultStore struct { - items []*fetchResult // Downloaded but not yet delivered fetch results - resultOffset uint64 // Offset of the first cached fetch result in the block chain - - // Internal index of first non-completed entry, updated atomically when needed. - // If all items are complete, this will equal length(items), so - // *important* : is not safe to use for indexing without checking against length - indexIncomplete int32 // atomic access - - // throttleThreshold is the limit up to which we _want_ to fill the - // results. If blocks are large, we want to limit the results to less - // than the number of available slots, and maybe only fill 1024 out of - // 8192 possible places. The queue will, at certain times, recalibrate - // this index. - throttleThreshold uint64 - - lock sync.RWMutex -} - -func newResultStore(size int) *resultStore { - return &resultStore{ - resultOffset: 0, - items: make([]*fetchResult, size), - throttleThreshold: uint64(size), - } -} - -// SetThrottleThreshold updates the throttling threshold based on the requested -// limit and the total queue capacity. It returns the (possibly capped) threshold -func (r *resultStore) SetThrottleThreshold(threshold uint64) uint64 { - r.lock.Lock() - defer r.lock.Unlock() - - limit := uint64(len(r.items)) - if threshold >= limit { - threshold = limit - } - r.throttleThreshold = threshold - return r.throttleThreshold -} - -// AddFetch adds a header for body/receipt fetching. This is used when the queue -// wants to reserve headers for fetching. -// -// It returns the following: -// -// stale - if true, this item is already passed, and should not be requested again -// throttled - if true, the store is at capacity, this particular header is not prio now -// item - the result to store data into -// err - any error that occurred -func (r *resultStore) AddFetch(header *types.Header, fastSync bool) (stale, throttled bool, item *fetchResult, err error) { - r.lock.Lock() - defer r.lock.Unlock() - - var index int - item, index, stale, throttled, err = r.getFetchResult(header.Number.Uint64()) - if err != nil || stale || throttled { - return stale, throttled, item, err - } - if item == nil { - item = newFetchResult(header, fastSync) - r.items[index] = item - } - return stale, throttled, item, err -} - -// GetDeliverySlot returns the fetchResult for the given header. If the 'stale' flag -// is true, that means the header has already been delivered 'upstream'. This method -// does not bubble up the 'throttle' flag, since it's moot at the point in time when -// the item is downloaded and ready for delivery -func (r *resultStore) GetDeliverySlot(headerNumber uint64) (*fetchResult, bool, error) { - r.lock.RLock() - defer r.lock.RUnlock() - - res, _, stale, _, err := r.getFetchResult(headerNumber) - return res, stale, err -} - -// getFetchResult returns the fetchResult corresponding to the given item, and -// the index where the result is stored. -func (r *resultStore) getFetchResult(headerNumber uint64) (item *fetchResult, index int, stale, throttle bool, err error) { - index = int(int64(headerNumber) - int64(r.resultOffset)) - throttle = index >= int(r.throttleThreshold) - stale = index < 0 - - if index >= len(r.items) { - err = fmt.Errorf("%w: index allocation went beyond available resultStore space "+ - "(index [%d] = header [%d] - resultOffset [%d], len(resultStore) = %d", errInvalidChain, - index, headerNumber, r.resultOffset, len(r.items)) - return nil, index, stale, throttle, err - } - if stale { - return nil, index, stale, throttle, nil - } - item = r.items[index] - return item, index, stale, throttle, nil -} - -// HasCompletedItems returns true if there are processable items available -// this method is cheaper than countCompleted -func (r *resultStore) HasCompletedItems() bool { - r.lock.RLock() - defer r.lock.RUnlock() - - if len(r.items) == 0 { - return false - } - if item := r.items[0]; item != nil && item.AllDone() { - return true - } - return false -} - -// countCompleted returns the number of items ready for delivery, stopping at -// the first non-complete item. -// -// The method assumes (at least) rlock is held. -func (r *resultStore) countCompleted() int { - // We iterate from the already known complete point, and see - // if any more has completed since last count - index := atomic.LoadInt32(&r.indexIncomplete) - for ; ; index++ { - if index >= int32(len(r.items)) { - break - } - result := r.items[index] - if result == nil || !result.AllDone() { - break - } - } - atomic.StoreInt32(&r.indexIncomplete, index) - return int(index) -} - -// GetCompleted returns the next batch of completed fetchResults -func (r *resultStore) GetCompleted(limit int) []*fetchResult { - r.lock.Lock() - defer r.lock.Unlock() - - completed := r.countCompleted() - if limit > completed { - limit = completed - } - results := make([]*fetchResult, limit) - copy(results, r.items[:limit]) - - // Delete the results from the cache and clear the tail. - copy(r.items, r.items[limit:]) - for i := len(r.items) - limit; i < len(r.items); i++ { - r.items[i] = nil - } - // Advance the expected block number of the first cache entry - r.resultOffset += uint64(limit) - atomic.AddInt32(&r.indexIncomplete, int32(-limit)) - - return results -} - -// Prepare initialises the offset with the given block number -func (r *resultStore) Prepare(offset uint64) { - r.lock.Lock() - defer r.lock.Unlock() - - if r.resultOffset < offset { - r.resultOffset = offset - } -} diff --git a/les/downloader/statesync.go b/les/downloader/statesync.go deleted file mode 100644 index 4dacade3fa..0000000000 --- a/les/downloader/statesync.go +++ /dev/null @@ -1,638 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package downloader - -import ( - "fmt" - "sync" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/state" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/trie" - "golang.org/x/crypto/sha3" -) - -// stateReq represents a batch of state fetch requests grouped together into -// a single data retrieval network packet. -type stateReq struct { - nItems uint16 // Number of items requested for download (max is 384, so uint16 is sufficient) - trieTasks map[string]*trieTask // Trie node download tasks to track previous attempts - codeTasks map[common.Hash]*codeTask // Byte code download tasks to track previous attempts - timeout time.Duration // Maximum round trip time for this to complete - timer *time.Timer // Timer to fire when the RTT timeout expires - peer *peerConnection // Peer that we're requesting from - delivered time.Time // Time when the packet was delivered (independent when we process it) - response [][]byte // Response data of the peer (nil for timeouts) - dropped bool // Flag whether the peer dropped off early -} - -// timedOut returns if this request timed out. -func (req *stateReq) timedOut() bool { - return req.response == nil -} - -// stateSyncStats is a collection of progress stats to report during a state trie -// sync to RPC requests as well as to display in user logs. -type stateSyncStats struct { - processed uint64 // Number of state entries processed - duplicate uint64 // Number of state entries downloaded twice - unexpected uint64 // Number of non-requested state entries received - pending uint64 // Number of still pending state entries -} - -// syncState starts downloading state with the given root hash. -func (d *Downloader) syncState(root common.Hash) *stateSync { - // Create the state sync - s := newStateSync(d, root) - select { - case d.stateSyncStart <- s: - // If we tell the statesync to restart with a new root, we also need - // to wait for it to actually also start -- when old requests have timed - // out or been delivered - <-s.started - case <-d.quitCh: - s.err = errCancelStateFetch - close(s.done) - } - return s -} - -// stateFetcher manages the active state sync and accepts requests -// on its behalf. -func (d *Downloader) stateFetcher() { - for { - select { - case s := <-d.stateSyncStart: - for next := s; next != nil; { - next = d.runStateSync(next) - } - case <-d.stateCh: - // Ignore state responses while no sync is running. - case <-d.quitCh: - return - } - } -} - -// runStateSync runs a state synchronisation until it completes or another root -// hash is requested to be switched over to. -func (d *Downloader) runStateSync(s *stateSync) *stateSync { - var ( - active = make(map[string]*stateReq) // Currently in-flight requests - finished []*stateReq // Completed or failed requests - timeout = make(chan *stateReq) // Timed out active requests - ) - log.Trace("State sync starting", "root", s.root) - - defer func() { - // Cancel active request timers on exit. Also set peers to idle so they're - // available for the next sync. - for _, req := range active { - req.timer.Stop() - req.peer.SetNodeDataIdle(int(req.nItems), time.Now()) - } - }() - go s.run() - defer s.Cancel() - - // Listen for peer departure events to cancel assigned tasks - peerDrop := make(chan *peerConnection, 1024) - peerSub := s.d.peers.SubscribePeerDrops(peerDrop) - defer peerSub.Unsubscribe() - - for { - // Enable sending of the first buffered element if there is one. - var ( - deliverReq *stateReq - deliverReqCh chan *stateReq - ) - if len(finished) > 0 { - deliverReq = finished[0] - deliverReqCh = s.deliver - } - - select { - // The stateSync lifecycle: - case next := <-d.stateSyncStart: - d.spindownStateSync(active, finished, timeout, peerDrop) - return next - - case <-s.done: - d.spindownStateSync(active, finished, timeout, peerDrop) - return nil - - // Send the next finished request to the current sync: - case deliverReqCh <- deliverReq: - // Shift out the first request, but also set the emptied slot to nil for GC - copy(finished, finished[1:]) - finished[len(finished)-1] = nil - finished = finished[:len(finished)-1] - - // Handle incoming state packs: - case pack := <-d.stateCh: - // Discard any data not requested (or previously timed out) - req := active[pack.PeerId()] - if req == nil { - log.Debug("Unrequested node data", "peer", pack.PeerId(), "len", pack.Items()) - continue - } - // Finalize the request and queue up for processing - req.timer.Stop() - req.response = pack.(*statePack).states - req.delivered = time.Now() - - finished = append(finished, req) - delete(active, pack.PeerId()) - - // Handle dropped peer connections: - case p := <-peerDrop: - // Skip if no request is currently pending - req := active[p.id] - if req == nil { - continue - } - // Finalize the request and queue up for processing - req.timer.Stop() - req.dropped = true - req.delivered = time.Now() - - finished = append(finished, req) - delete(active, p.id) - - // Handle timed-out requests: - case req := <-timeout: - // If the peer is already requesting something else, ignore the stale timeout. - // This can happen when the timeout and the delivery happens simultaneously, - // causing both pathways to trigger. - if active[req.peer.id] != req { - continue - } - req.delivered = time.Now() - // Move the timed out data back into the download queue - finished = append(finished, req) - delete(active, req.peer.id) - - // Track outgoing state requests: - case req := <-d.trackStateReq: - // If an active request already exists for this peer, we have a problem. In - // theory the trie node schedule must never assign two requests to the same - // peer. In practice however, a peer might receive a request, disconnect and - // immediately reconnect before the previous times out. In this case the first - // request is never honored, alas we must not silently overwrite it, as that - // causes valid requests to go missing and sync to get stuck. - if old := active[req.peer.id]; old != nil { - log.Warn("Busy peer assigned new state fetch", "peer", old.peer.id) - // Move the previous request to the finished set - old.timer.Stop() - old.dropped = true - old.delivered = time.Now() - finished = append(finished, old) - } - // Start a timer to notify the sync loop if the peer stalled. - req.timer = time.AfterFunc(req.timeout, func() { - timeout <- req - }) - active[req.peer.id] = req - } - } -} - -// spindownStateSync 'drains' the outstanding requests; some will be delivered and other -// will time out. This is to ensure that when the next stateSync starts working, all peers -// are marked as idle and de facto _are_ idle. -func (d *Downloader) spindownStateSync(active map[string]*stateReq, finished []*stateReq, timeout chan *stateReq, peerDrop chan *peerConnection) { - log.Trace("State sync spinning down", "active", len(active), "finished", len(finished)) - for len(active) > 0 { - var ( - req *stateReq - reason string - ) - select { - // Handle (drop) incoming state packs: - case pack := <-d.stateCh: - req = active[pack.PeerId()] - reason = "delivered" - // Handle dropped peer connections: - case p := <-peerDrop: - req = active[p.id] - reason = "peerdrop" - // Handle timed-out requests: - case req = <-timeout: - reason = "timeout" - } - if req == nil { - continue - } - req.peer.log.Trace("State peer marked idle (spindown)", "req.items", int(req.nItems), "reason", reason) - req.timer.Stop() - delete(active, req.peer.id) - req.peer.SetNodeDataIdle(int(req.nItems), time.Now()) - } - // The 'finished' set contains deliveries that we were going to pass to processing. - // Those are now moot, but we still need to set those peers as idle, which would - // otherwise have been done after processing - for _, req := range finished { - req.peer.SetNodeDataIdle(int(req.nItems), time.Now()) - } -} - -// stateSync schedules requests for downloading a particular state trie defined -// by a given state root. -type stateSync struct { - d *Downloader // Downloader instance to access and manage current peerset - - root common.Hash // State root currently being synced - sched *trie.Sync // State trie sync scheduler defining the tasks - keccak crypto.KeccakState // Keccak256 hasher to verify deliveries with - - trieTasks map[string]*trieTask // Set of trie node tasks currently queued for retrieval, indexed by path - codeTasks map[common.Hash]*codeTask // Set of byte code tasks currently queued for retrieval, indexed by hash - - numUncommitted int - bytesUncommitted int - - started chan struct{} // Started is signalled once the sync loop starts - - deliver chan *stateReq // Delivery channel multiplexing peer responses - cancel chan struct{} // Channel to signal a termination request - cancelOnce sync.Once // Ensures cancel only ever gets called once - done chan struct{} // Channel to signal termination completion - err error // Any error hit during sync (set before completion) -} - -// trieTask represents a single trie node download task, containing a set of -// peers already attempted retrieval from to detect stalled syncs and abort. -type trieTask struct { - hash common.Hash - path [][]byte - attempts map[string]struct{} -} - -// codeTask represents a single byte code download task, containing a set of -// peers already attempted retrieval from to detect stalled syncs and abort. -type codeTask struct { - attempts map[string]struct{} -} - -// newStateSync creates a new state trie download scheduler. This method does not -// yet start the sync. The user needs to call run to initiate. -func newStateSync(d *Downloader, root common.Hash) *stateSync { - // Hack the node scheme here. It's a dead code is not used - // by light client at all. Just aim for passing tests. - return &stateSync{ - d: d, - root: root, - sched: state.NewStateSync(root, d.stateDB, nil, rawdb.HashScheme), - keccak: sha3.NewLegacyKeccak256().(crypto.KeccakState), - trieTasks: make(map[string]*trieTask), - codeTasks: make(map[common.Hash]*codeTask), - deliver: make(chan *stateReq), - cancel: make(chan struct{}), - done: make(chan struct{}), - started: make(chan struct{}), - } -} - -// run starts the task assignment and response processing loop, blocking until -// it finishes, and finally notifying any goroutines waiting for the loop to -// finish. -func (s *stateSync) run() { - close(s.started) - if s.d.snapSync { - s.err = s.d.SnapSyncer.Sync(s.root, s.cancel) - } else { - s.err = s.loop() - } - close(s.done) -} - -// Wait blocks until the sync is done or canceled. -func (s *stateSync) Wait() error { - <-s.done - return s.err -} - -// Cancel cancels the sync and waits until it has shut down. -func (s *stateSync) Cancel() error { - s.cancelOnce.Do(func() { - close(s.cancel) - }) - return s.Wait() -} - -// loop is the main event loop of a state trie sync. It it responsible for the -// assignment of new tasks to peers (including sending it to them) as well as -// for the processing of inbound data. Note, that the loop does not directly -// receive data from peers, rather those are buffered up in the downloader and -// pushed here async. The reason is to decouple processing from data receipt -// and timeouts. -func (s *stateSync) loop() (err error) { - // Listen for new peer events to assign tasks to them - newPeer := make(chan *peerConnection, 1024) - peerSub := s.d.peers.SubscribeNewPeers(newPeer) - defer peerSub.Unsubscribe() - defer func() { - cerr := s.commit(true) - if err == nil { - err = cerr - } - }() - - // Keep assigning new tasks until the sync completes or aborts - for s.sched.Pending() > 0 { - if err = s.commit(false); err != nil { - return err - } - s.assignTasks() - // Tasks assigned, wait for something to happen - select { - case <-newPeer: - // New peer arrived, try to assign it download tasks - - case <-s.cancel: - return errCancelStateFetch - - case <-s.d.cancelCh: - return errCanceled - - case req := <-s.deliver: - // Response, disconnect or timeout triggered, drop the peer if stalling - log.Trace("Received node data response", "peer", req.peer.id, "count", len(req.response), "dropped", req.dropped, "timeout", !req.dropped && req.timedOut()) - if req.nItems <= 2 && !req.dropped && req.timedOut() { - // 2 items are the minimum requested, if even that times out, we've no use of - // this peer at the moment. - log.Warn("Stalling state sync, dropping peer", "peer", req.peer.id) - if s.d.dropPeer == nil { - // The dropPeer method is nil when `--copydb` is used for a local copy. - // Timeouts can occur if e.g. compaction hits at the wrong time, and can be ignored - req.peer.log.Warn("Downloader wants to drop peer, but peerdrop-function is not set", "peer", req.peer.id) - } else { - s.d.dropPeer(req.peer.id) - - // If this peer was the master peer, abort sync immediately - s.d.cancelLock.RLock() - master := req.peer.id == s.d.cancelPeer - s.d.cancelLock.RUnlock() - - if master { - s.d.cancel() - return errTimeout - } - } - } - // Process all the received blobs and check for stale delivery - delivered, err := s.process(req) - req.peer.SetNodeDataIdle(delivered, req.delivered) - if err != nil { - log.Warn("Node data write error", "err", err) - return err - } - } - } - return nil -} - -func (s *stateSync) commit(force bool) error { - if !force && s.bytesUncommitted < ethdb.IdealBatchSize { - return nil - } - start := time.Now() - b := s.d.stateDB.NewBatch() - if err := s.sched.Commit(b); err != nil { - return err - } - if err := b.Write(); err != nil { - return fmt.Errorf("DB write error: %v", err) - } - s.updateStats(s.numUncommitted, 0, 0, time.Since(start)) - s.numUncommitted = 0 - s.bytesUncommitted = 0 - return nil -} - -// assignTasks attempts to assign new tasks to all idle peers, either from the -// batch currently being retried, or fetching new data from the trie sync itself. -func (s *stateSync) assignTasks() { - // Iterate over all idle peers and try to assign them state fetches - peers, _ := s.d.peers.NodeDataIdlePeers() - for _, p := range peers { - // Assign a batch of fetches proportional to the estimated latency/bandwidth - cap := p.NodeDataCapacity(s.d.peers.rates.TargetRoundTrip()) - req := &stateReq{peer: p, timeout: s.d.peers.rates.TargetTimeout()} - - nodes, _, codes := s.fillTasks(cap, req) - - // If the peer was assigned tasks to fetch, send the network request - if len(nodes)+len(codes) > 0 { - req.peer.log.Trace("Requesting batch of state data", "nodes", len(nodes), "codes", len(codes), "root", s.root) - select { - case s.d.trackStateReq <- req: - req.peer.FetchNodeData(append(nodes, codes...)) // Unified retrieval under eth/6x - case <-s.cancel: - case <-s.d.cancelCh: - } - } - } -} - -// fillTasks fills the given request object with a maximum of n state download -// tasks to send to the remote peer. -func (s *stateSync) fillTasks(n int, req *stateReq) (nodes []common.Hash, paths []trie.SyncPath, codes []common.Hash) { - // Refill available tasks from the scheduler. - if fill := n - (len(s.trieTasks) + len(s.codeTasks)); fill > 0 { - paths, hashes, codes := s.sched.Missing(fill) - for i, path := range paths { - s.trieTasks[path] = &trieTask{ - hash: hashes[i], - path: trie.NewSyncPath([]byte(path)), - attempts: make(map[string]struct{}), - } - } - for _, hash := range codes { - s.codeTasks[hash] = &codeTask{ - attempts: make(map[string]struct{}), - } - } - } - // Find tasks that haven't been tried with the request's peer. Prefer code - // over trie nodes as those can be written to disk and forgotten about. - nodes = make([]common.Hash, 0, n) - paths = make([]trie.SyncPath, 0, n) - codes = make([]common.Hash, 0, n) - - req.trieTasks = make(map[string]*trieTask, n) - req.codeTasks = make(map[common.Hash]*codeTask, n) - - for hash, t := range s.codeTasks { - // Stop when we've gathered enough requests - if len(nodes)+len(codes) == n { - break - } - // Skip any requests we've already tried from this peer - if _, ok := t.attempts[req.peer.id]; ok { - continue - } - // Assign the request to this peer - t.attempts[req.peer.id] = struct{}{} - codes = append(codes, hash) - req.codeTasks[hash] = t - delete(s.codeTasks, hash) - } - for path, t := range s.trieTasks { - // Stop when we've gathered enough requests - if len(nodes)+len(codes) == n { - break - } - // Skip any requests we've already tried from this peer - if _, ok := t.attempts[req.peer.id]; ok { - continue - } - // Assign the request to this peer - t.attempts[req.peer.id] = struct{}{} - - nodes = append(nodes, t.hash) - paths = append(paths, t.path) - - req.trieTasks[path] = t - delete(s.trieTasks, path) - } - req.nItems = uint16(len(nodes) + len(codes)) - return nodes, paths, codes -} - -// process iterates over a batch of delivered state data, injecting each item -// into a running state sync, re-queuing any items that were requested but not -// delivered. Returns whether the peer actually managed to deliver anything of -// value, and any error that occurred. -func (s *stateSync) process(req *stateReq) (int, error) { - // Collect processing stats and update progress if valid data was received - duplicate, unexpected, successful := 0, 0, 0 - - defer func(start time.Time) { - if duplicate > 0 || unexpected > 0 { - s.updateStats(0, duplicate, unexpected, time.Since(start)) - } - }(time.Now()) - - // Iterate over all the delivered data and inject one-by-one into the trie - for _, blob := range req.response { - hash, err := s.processNodeData(req.trieTasks, req.codeTasks, blob) - switch err { - case nil: - s.numUncommitted++ - s.bytesUncommitted += len(blob) - successful++ - case trie.ErrNotRequested: - unexpected++ - case trie.ErrAlreadyProcessed: - duplicate++ - default: - return successful, fmt.Errorf("invalid state node %s: %v", hash.TerminalString(), err) - } - } - // Put unfulfilled tasks back into the retry queue - npeers := s.d.peers.Len() - for path, task := range req.trieTasks { - // If the node did deliver something, missing items may be due to a protocol - // limit or a previous timeout + delayed delivery. Both cases should permit - // the node to retry the missing items (to avoid single-peer stalls). - if len(req.response) > 0 || req.timedOut() { - delete(task.attempts, req.peer.id) - } - // If we've requested the node too many times already, it may be a malicious - // sync where nobody has the right data. Abort. - if len(task.attempts) >= npeers { - return successful, fmt.Errorf("trie node %s failed with all peers (%d tries, %d peers)", task.hash.TerminalString(), len(task.attempts), npeers) - } - // Missing item, place into the retry queue. - s.trieTasks[path] = task - } - for hash, task := range req.codeTasks { - // If the node did deliver something, missing items may be due to a protocol - // limit or a previous timeout + delayed delivery. Both cases should permit - // the node to retry the missing items (to avoid single-peer stalls). - if len(req.response) > 0 || req.timedOut() { - delete(task.attempts, req.peer.id) - } - // If we've requested the node too many times already, it may be a malicious - // sync where nobody has the right data. Abort. - if len(task.attempts) >= npeers { - return successful, fmt.Errorf("byte code %s failed with all peers (%d tries, %d peers)", hash.TerminalString(), len(task.attempts), npeers) - } - // Missing item, place into the retry queue. - s.codeTasks[hash] = task - } - return successful, nil -} - -// processNodeData tries to inject a trie node data blob delivered from a remote -// peer into the state trie, returning whether anything useful was written or any -// error occurred. -// -// If multiple requests correspond to the same hash, this method will inject the -// blob as a result for the first one only, leaving the remaining duplicates to -// be fetched again. -func (s *stateSync) processNodeData(nodeTasks map[string]*trieTask, codeTasks map[common.Hash]*codeTask, blob []byte) (common.Hash, error) { - var hash common.Hash - s.keccak.Reset() - s.keccak.Write(blob) - s.keccak.Read(hash[:]) - - if _, present := codeTasks[hash]; present { - err := s.sched.ProcessCode(trie.CodeSyncResult{ - Hash: hash, - Data: blob, - }) - delete(codeTasks, hash) - return hash, err - } - for path, task := range nodeTasks { - if task.hash == hash { - err := s.sched.ProcessNode(trie.NodeSyncResult{ - Path: path, - Data: blob, - }) - delete(nodeTasks, path) - return hash, err - } - } - return common.Hash{}, trie.ErrNotRequested -} - -// updateStats bumps the various state sync progress counters and displays a log -// message for the user to see. -func (s *stateSync) updateStats(written, duplicate, unexpected int, duration time.Duration) { - s.d.syncStatsLock.Lock() - defer s.d.syncStatsLock.Unlock() - - s.d.syncStatsState.pending = uint64(s.sched.Pending()) - s.d.syncStatsState.processed += uint64(written) - s.d.syncStatsState.duplicate += uint64(duplicate) - s.d.syncStatsState.unexpected += uint64(unexpected) - - if written > 0 || duplicate > 0 || unexpected > 0 { - log.Info("Imported new state entries", "count", written, "elapsed", common.PrettyDuration(duration), "processed", s.d.syncStatsState.processed, "pending", s.d.syncStatsState.pending, "trieretry", len(s.trieTasks), "coderetry", len(s.codeTasks), "duplicate", s.d.syncStatsState.duplicate, "unexpected", s.d.syncStatsState.unexpected) - } - //if written > 0 { - //rawdb.WriteFastTrieProgress(s.d.stateDB, s.d.syncStatsState.processed) - //} -} diff --git a/les/downloader/testchain_test.go b/les/downloader/testchain_test.go deleted file mode 100644 index dff0844cb8..0000000000 --- a/les/downloader/testchain_test.go +++ /dev/null @@ -1,235 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package downloader - -import ( - "fmt" - "math/big" - "sync" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus/ethash" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/params" -) - -// Test chain parameters. -var ( - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - testAddress = crypto.PubkeyToAddress(testKey.PublicKey) - testDB = rawdb.NewMemoryDatabase() - - gspec = core.Genesis{ - Alloc: core.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, - BaseFee: big.NewInt(params.InitialBaseFee), - } - testGenesis = gspec.MustCommit(testDB) -) - -// The common prefix of all test chains: -var testChainBase = newTestChain(blockCacheMaxItems+200, testGenesis) - -// Different forks on top of the base chain: -var testChainForkLightA, testChainForkLightB, testChainForkHeavy *testChain - -func init() { - var forkLen = int(fullMaxForkAncestry + 50) - var wg sync.WaitGroup - wg.Add(3) - go func() { testChainForkLightA = testChainBase.makeFork(forkLen, false, 1); wg.Done() }() - go func() { testChainForkLightB = testChainBase.makeFork(forkLen, false, 2); wg.Done() }() - go func() { testChainForkHeavy = testChainBase.makeFork(forkLen, true, 3); wg.Done() }() - wg.Wait() -} - -type testChain struct { - genesis *types.Block - chain []common.Hash - headerm map[common.Hash]*types.Header - blockm map[common.Hash]*types.Block - receiptm map[common.Hash][]*types.Receipt - tdm map[common.Hash]*big.Int -} - -// newTestChain creates a blockchain of the given length. -func newTestChain(length int, genesis *types.Block) *testChain { - tc := new(testChain).copy(length) - tc.genesis = genesis - tc.chain = append(tc.chain, genesis.Hash()) - tc.headerm[tc.genesis.Hash()] = tc.genesis.Header() - tc.tdm[tc.genesis.Hash()] = tc.genesis.Difficulty() - tc.blockm[tc.genesis.Hash()] = tc.genesis - tc.generate(length-1, 0, genesis, false) - return tc -} - -// makeFork creates a fork on top of the test chain. -func (tc *testChain) makeFork(length int, heavy bool, seed byte) *testChain { - fork := tc.copy(tc.len() + length) - fork.generate(length, seed, tc.headBlock(), heavy) - return fork -} - -// shorten creates a copy of the chain with the given length. It panics if the -// length is longer than the number of available blocks. -func (tc *testChain) shorten(length int) *testChain { - if length > tc.len() { - panic(fmt.Errorf("can't shorten test chain to %d blocks, it's only %d blocks long", length, tc.len())) - } - return tc.copy(length) -} - -func (tc *testChain) copy(newlen int) *testChain { - cpy := &testChain{ - genesis: tc.genesis, - headerm: make(map[common.Hash]*types.Header, newlen), - blockm: make(map[common.Hash]*types.Block, newlen), - receiptm: make(map[common.Hash][]*types.Receipt, newlen), - tdm: make(map[common.Hash]*big.Int, newlen), - } - for i := 0; i < len(tc.chain) && i < newlen; i++ { - hash := tc.chain[i] - cpy.chain = append(cpy.chain, tc.chain[i]) - cpy.tdm[hash] = tc.tdm[hash] - cpy.blockm[hash] = tc.blockm[hash] - cpy.headerm[hash] = tc.headerm[hash] - cpy.receiptm[hash] = tc.receiptm[hash] - } - return cpy -} - -// generate creates a chain of n blocks starting at and including parent. -// the returned hash chain is ordered head->parent. In addition, every 22th block -// contains a transaction and every 5th an uncle to allow testing correct block -// reassembly. -func (tc *testChain) generate(n int, seed byte, parent *types.Block, heavy bool) { - // start := time.Now() - // defer func() { fmt.Printf("test chain generated in %v\n", time.Since(start)) }() - - blocks, receipts := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testDB, n, func(i int, block *core.BlockGen) { - block.SetCoinbase(common.Address{seed}) - // If a heavy chain is requested, delay blocks to raise difficulty - if heavy { - block.OffsetTime(-1) - } - // Include transactions to the miner to make blocks more interesting. - if parent == tc.genesis && i%22 == 0 { - signer := types.MakeSigner(params.TestChainConfig, block.Number(), block.Timestamp()) - tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddress), common.Address{seed}, big.NewInt(1000), params.TxGas, block.BaseFee(), nil), signer, testKey) - if err != nil { - panic(err) - } - block.AddTx(tx) - } - // if the block number is a multiple of 5, add a bonus uncle to the block - if i > 0 && i%5 == 0 { - block.AddUncle(&types.Header{ - ParentHash: block.PrevBlock(i - 1).Hash(), - Number: big.NewInt(block.Number().Int64() - 1), - }) - } - }) - - // Convert the block-chain into a hash-chain and header/block maps - td := new(big.Int).Set(tc.td(parent.Hash())) - for i, b := range blocks { - td := td.Add(td, b.Difficulty()) - hash := b.Hash() - tc.chain = append(tc.chain, hash) - tc.blockm[hash] = b - tc.headerm[hash] = b.Header() - tc.receiptm[hash] = receipts[i] - tc.tdm[hash] = new(big.Int).Set(td) - } -} - -// len returns the total number of blocks in the chain. -func (tc *testChain) len() int { - return len(tc.chain) -} - -// headBlock returns the head of the chain. -func (tc *testChain) headBlock() *types.Block { - return tc.blockm[tc.chain[len(tc.chain)-1]] -} - -// td returns the total difficulty of the given block. -func (tc *testChain) td(hash common.Hash) *big.Int { - return tc.tdm[hash] -} - -// headersByHash returns headers in order from the given hash. -func (tc *testChain) headersByHash(origin common.Hash, amount int, skip int, reverse bool) []*types.Header { - num, _ := tc.hashToNumber(origin) - return tc.headersByNumber(num, amount, skip, reverse) -} - -// headersByNumber returns headers from the given number. -func (tc *testChain) headersByNumber(origin uint64, amount int, skip int, reverse bool) []*types.Header { - result := make([]*types.Header, 0, amount) - - if !reverse { - for num := origin; num < uint64(len(tc.chain)) && len(result) < amount; num += uint64(skip) + 1 { - if header, ok := tc.headerm[tc.chain[int(num)]]; ok { - result = append(result, header) - } - } - } else { - for num := int64(origin); num >= 0 && len(result) < amount; num -= int64(skip) + 1 { - if header, ok := tc.headerm[tc.chain[int(num)]]; ok { - result = append(result, header) - } - } - } - return result -} - -// receipts returns the receipts of the given block hashes. -func (tc *testChain) receipts(hashes []common.Hash) [][]*types.Receipt { - results := make([][]*types.Receipt, 0, len(hashes)) - for _, hash := range hashes { - if receipt, ok := tc.receiptm[hash]; ok { - results = append(results, receipt) - } - } - return results -} - -// bodies returns the block bodies of the given block hashes. -func (tc *testChain) bodies(hashes []common.Hash) ([][]*types.Transaction, [][]*types.Header) { - transactions := make([][]*types.Transaction, 0, len(hashes)) - uncles := make([][]*types.Header, 0, len(hashes)) - for _, hash := range hashes { - if block, ok := tc.blockm[hash]; ok { - transactions = append(transactions, block.Transactions()) - uncles = append(uncles, block.Uncles()) - } - } - return transactions, uncles -} - -func (tc *testChain) hashToNumber(target common.Hash) (uint64, bool) { - for num, hash := range tc.chain { - if hash == target { - return uint64(num), true - } - } - return 0, false -} diff --git a/les/downloader/types.go b/les/downloader/types.go deleted file mode 100644 index ff70bfa0e3..0000000000 --- a/les/downloader/types.go +++ /dev/null @@ -1,79 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package downloader - -import ( - "fmt" - - "github.com/ethereum/go-ethereum/core/types" -) - -// peerDropFn is a callback type for dropping a peer detected as malicious. -type peerDropFn func(id string) - -// dataPack is a data message returned by a peer for some query. -type dataPack interface { - PeerId() string - Items() int - Stats() string -} - -// headerPack is a batch of block headers returned by a peer. -type headerPack struct { - peerID string - headers []*types.Header -} - -func (p *headerPack) PeerId() string { return p.peerID } -func (p *headerPack) Items() int { return len(p.headers) } -func (p *headerPack) Stats() string { return fmt.Sprintf("%d", len(p.headers)) } - -// bodyPack is a batch of block bodies returned by a peer. -type bodyPack struct { - peerID string - transactions [][]*types.Transaction - uncles [][]*types.Header -} - -func (p *bodyPack) PeerId() string { return p.peerID } -func (p *bodyPack) Items() int { - if len(p.transactions) <= len(p.uncles) { - return len(p.transactions) - } - return len(p.uncles) -} -func (p *bodyPack) Stats() string { return fmt.Sprintf("%d:%d", len(p.transactions), len(p.uncles)) } - -// receiptPack is a batch of receipts returned by a peer. -type receiptPack struct { - peerID string - receipts [][]*types.Receipt -} - -func (p *receiptPack) PeerId() string { return p.peerID } -func (p *receiptPack) Items() int { return len(p.receipts) } -func (p *receiptPack) Stats() string { return fmt.Sprintf("%d", len(p.receipts)) } - -// statePack is a batch of states returned by a peer. -type statePack struct { - peerID string - states [][]byte -} - -func (p *statePack) PeerId() string { return p.peerID } -func (p *statePack) Items() int { return len(p.states) } -func (p *statePack) Stats() string { return fmt.Sprintf("%d", len(p.states)) } diff --git a/les/fetcher.go b/les/fetcher.go deleted file mode 100644 index df81ca4d9b..0000000000 --- a/les/fetcher.go +++ /dev/null @@ -1,563 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package les - -import ( - "math/big" - "math/rand" - "sync" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/les/fetcher" - "github.com/ethereum/go-ethereum/light" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/p2p/enode" -) - -const ( - blockDelayTimeout = 10 * time.Second // Timeout for retrieving the headers from the peer - gatherSlack = 100 * time.Millisecond // Interval used to collate almost-expired requests - cachedAnnosThreshold = 64 // The maximum queued announcements -) - -// announce represents an new block announcement from the les server. -type announce struct { - data *announceData - trust bool - peerid enode.ID -} - -// request represents a record when the header request is sent. -type request struct { - reqid uint64 - peerid enode.ID - sendAt time.Time - hash common.Hash -} - -// response represents a response packet from network as well as a channel -// to return all un-requested data. -type response struct { - reqid uint64 - headers []*types.Header - peerid enode.ID - remain chan []*types.Header -} - -// fetcherPeer holds the fetcher-specific information for each active peer -type fetcherPeer struct { - latest *announceData // The latest announcement sent from the peer - - // These following two fields can track the latest announces - // from the peer with limited size for caching. We hold the - // assumption that all enqueued announces are td-monotonic. - announces map[common.Hash]*announce // Announcement map - fifo []common.Hash // FIFO announces list -} - -// addAnno enqueues an new trusted announcement. If the queued announces overflow, -// evict from the oldest. -func (fp *fetcherPeer) addAnno(anno *announce) { - // Short circuit if the anno already exists. In normal case it should - // never happen since only monotonic anno is accepted. But the adversary - // may feed us fake announces with higher td but same hash. In this case, - // ignore the anno anyway. - hash := anno.data.Hash - if _, exist := fp.announces[hash]; exist { - return - } - fp.announces[hash] = anno - fp.fifo = append(fp.fifo, hash) - - // Evict oldest if the announces are oversized. - if len(fp.fifo)-cachedAnnosThreshold > 0 { - for i := 0; i < len(fp.fifo)-cachedAnnosThreshold; i++ { - delete(fp.announces, fp.fifo[i]) - } - copy(fp.fifo, fp.fifo[len(fp.fifo)-cachedAnnosThreshold:]) - fp.fifo = fp.fifo[:cachedAnnosThreshold] - } -} - -// forwardAnno removes all announces from the map with a number lower than -// the provided threshold. -func (fp *fetcherPeer) forwardAnno(td *big.Int) []*announce { - var ( - cutset int - evicted []*announce - ) - for ; cutset < len(fp.fifo); cutset++ { - anno := fp.announces[fp.fifo[cutset]] - if anno == nil { - continue // In theory it should never ever happen - } - if anno.data.Td.Cmp(td) > 0 { - break - } - evicted = append(evicted, anno) - delete(fp.announces, anno.data.Hash) - } - if cutset > 0 { - copy(fp.fifo, fp.fifo[cutset:]) - fp.fifo = fp.fifo[:len(fp.fifo)-cutset] - } - return evicted -} - -// lightFetcher implements retrieval of newly announced headers. It reuses -// the eth.BlockFetcher as the underlying fetcher but adding more additional -// rules: e.g. evict "timeout" peers. -type lightFetcher struct { - // Various handlers - ulc *ulc - chaindb ethdb.Database - reqDist *requestDistributor - peerset *serverPeerSet // The global peerset of light client which shared by all components - chain *light.LightChain // The local light chain which maintains the canonical header chain. - fetcher *fetcher.BlockFetcher // The underlying fetcher which takes care block header retrieval. - - // Peerset maintained by fetcher - plock sync.RWMutex - peers map[enode.ID]*fetcherPeer - - // Various channels - announceCh chan *announce - requestCh chan *request - deliverCh chan *response - syncDone chan *types.Header - - closeCh chan struct{} - wg sync.WaitGroup - - // Callback - synchronise func(peer *serverPeer) - - // Test fields or hooks - newHeadHook func(*types.Header) -} - -// newLightFetcher creates a light fetcher instance. -func newLightFetcher(chain *light.LightChain, engine consensus.Engine, peers *serverPeerSet, ulc *ulc, chaindb ethdb.Database, reqDist *requestDistributor, syncFn func(p *serverPeer)) *lightFetcher { - // Construct the fetcher by offering all necessary APIs - validator := func(header *types.Header) error { - // Disable seal verification explicitly if we are running in ulc mode. - return engine.VerifyHeader(chain, header) - } - heighter := func() uint64 { return chain.CurrentHeader().Number.Uint64() } - dropper := func(id string) { peers.unregister(id) } - inserter := func(headers []*types.Header) (int, error) { return chain.InsertHeaderChain(headers) } - f := &lightFetcher{ - ulc: ulc, - peerset: peers, - chaindb: chaindb, - chain: chain, - reqDist: reqDist, - fetcher: fetcher.NewBlockFetcher(true, chain.GetHeaderByHash, nil, validator, nil, heighter, inserter, nil, dropper), - peers: make(map[enode.ID]*fetcherPeer), - synchronise: syncFn, - announceCh: make(chan *announce), - requestCh: make(chan *request), - deliverCh: make(chan *response), - syncDone: make(chan *types.Header), - closeCh: make(chan struct{}), - } - peers.subscribe(f) - return f -} - -func (f *lightFetcher) start() { - f.wg.Add(1) - f.fetcher.Start() - go f.mainloop() -} - -func (f *lightFetcher) stop() { - close(f.closeCh) - f.fetcher.Stop() - f.wg.Wait() -} - -// registerPeer adds an new peer to the fetcher's peer set -func (f *lightFetcher) registerPeer(p *serverPeer) { - f.plock.Lock() - defer f.plock.Unlock() - - f.peers[p.ID()] = &fetcherPeer{announces: make(map[common.Hash]*announce)} -} - -// unregisterPeer removes the specified peer from the fetcher's peer set -func (f *lightFetcher) unregisterPeer(p *serverPeer) { - f.plock.Lock() - defer f.plock.Unlock() - - delete(f.peers, p.ID()) -} - -// peer returns the peer from the fetcher peerset. -func (f *lightFetcher) peer(id enode.ID) *fetcherPeer { - f.plock.RLock() - defer f.plock.RUnlock() - - return f.peers[id] -} - -// forEachPeer iterates the fetcher peerset, abort the iteration if the -// callback returns false. -func (f *lightFetcher) forEachPeer(check func(id enode.ID, p *fetcherPeer) bool) { - f.plock.RLock() - defer f.plock.RUnlock() - - for id, peer := range f.peers { - if !check(id, peer) { - return - } - } -} - -// mainloop is the main event loop of the light fetcher, which is responsible for -// -// - announcement maintenance(ulc) -// -// If we are running in ultra light client mode, then all announcements from -// the trusted servers are maintained. If the same announcements from trusted -// servers reach the threshold, then the relevant header is requested for retrieval. -// -// - block header retrieval -// Whenever we receive announce with higher td compared with local chain, the -// request will be made for header retrieval. -// -// - re-sync trigger -// If the local chain lags too much, then the fetcher will enter "synchronise" -// mode to retrieve missing headers in batch. -func (f *lightFetcher) mainloop() { - defer f.wg.Done() - - var ( - syncInterval = uint64(1) // Interval used to trigger a light resync. - syncing bool // Indicator whether the client is syncing - - ulc = f.ulc != nil - headCh = make(chan core.ChainHeadEvent, 100) - fetching = make(map[uint64]*request) - requestTimer = time.NewTimer(0) - - // Local status - localHead = f.chain.CurrentHeader() - localTd = f.chain.GetTd(localHead.Hash(), localHead.Number.Uint64()) - ) - defer requestTimer.Stop() - sub := f.chain.SubscribeChainHeadEvent(headCh) - defer sub.Unsubscribe() - - // reset updates the local status with given header. - reset := func(header *types.Header) { - localHead = header - localTd = f.chain.GetTd(header.Hash(), header.Number.Uint64()) - } - // trustedHeader returns an indicator whether the header is regarded as - // trusted. If we are running in the ulc mode, only when we receive enough - // same announcement from trusted server, the header will be trusted. - trustedHeader := func(hash common.Hash, number uint64) (bool, []enode.ID) { - var ( - agreed []enode.ID - trusted bool - ) - f.forEachPeer(func(id enode.ID, p *fetcherPeer) bool { - if anno := p.announces[hash]; anno != nil && anno.trust && anno.data.Number == number { - agreed = append(agreed, id) - if 100*len(agreed)/len(f.ulc.keys) >= f.ulc.fraction { - trusted = true - return false // abort iteration - } - } - return true - }) - return trusted, agreed - } - for { - select { - case anno := <-f.announceCh: - peerid, data := anno.peerid, anno.data - log.Debug("Received new announce", "peer", peerid, "number", data.Number, "hash", data.Hash, "reorg", data.ReorgDepth) - - peer := f.peer(peerid) - if peer == nil { - log.Debug("Receive announce from unknown peer", "peer", peerid) - continue - } - // Announced tds should be strictly monotonic, drop the peer if - // the announce is out-of-order. - if peer.latest != nil && data.Td.Cmp(peer.latest.Td) <= 0 { - f.peerset.unregister(peerid.String()) - log.Debug("Non-monotonic td", "peer", peerid, "current", data.Td, "previous", peer.latest.Td) - continue - } - peer.latest = data - - // Filter out any stale announce, the local chain is ahead of announce - if localTd != nil && data.Td.Cmp(localTd) <= 0 { - continue - } - peer.addAnno(anno) - - // If we are not syncing, try to trigger a single retrieval or re-sync - if !ulc && !syncing { - // Two scenarios lead to re-sync: - // - reorg happens - // - local chain lags - // We can't retrieve the parent of the announce by single retrieval - // in both cases, so resync is necessary. - if data.Number > localHead.Number.Uint64()+syncInterval || data.ReorgDepth > 0 { - syncing = true - go f.startSync(peerid) - log.Debug("Trigger light sync", "peer", peerid, "local", localHead.Number, "localhash", localHead.Hash(), "remote", data.Number, "remotehash", data.Hash) - continue - } - f.fetcher.Notify(peerid.String(), data.Hash, data.Number, time.Now(), f.requestHeaderByHash(peerid), nil) - log.Debug("Trigger header retrieval", "peer", peerid, "number", data.Number, "hash", data.Hash) - } - // Keep collecting announces from trusted server even we are syncing. - if ulc && anno.trust { - // Notify underlying fetcher to retrieve header or trigger a resync if - // we have receive enough announcements from trusted server. - trusted, agreed := trustedHeader(data.Hash, data.Number) - if trusted && !syncing { - if data.Number > localHead.Number.Uint64()+syncInterval || data.ReorgDepth > 0 { - syncing = true - go f.startSync(peerid) - log.Debug("Trigger trusted light sync", "local", localHead.Number, "localhash", localHead.Hash(), "remote", data.Number, "remotehash", data.Hash) - continue - } - p := agreed[rand.Intn(len(agreed))] - f.fetcher.Notify(p.String(), data.Hash, data.Number, time.Now(), f.requestHeaderByHash(p), nil) - log.Debug("Trigger trusted header retrieval", "number", data.Number, "hash", data.Hash) - } - } - - case req := <-f.requestCh: - fetching[req.reqid] = req // Tracking all in-flight requests for response latency statistic. - if len(fetching) == 1 { - f.rescheduleTimer(fetching, requestTimer) - } - - case <-requestTimer.C: - for reqid, request := range fetching { - if time.Since(request.sendAt) > blockDelayTimeout-gatherSlack { - delete(fetching, reqid) - f.peerset.unregister(request.peerid.String()) - log.Debug("Request timeout", "peer", request.peerid, "reqid", reqid) - } - } - f.rescheduleTimer(fetching, requestTimer) - - case resp := <-f.deliverCh: - if req := fetching[resp.reqid]; req != nil { - delete(fetching, resp.reqid) - f.rescheduleTimer(fetching, requestTimer) - - // The underlying fetcher does not check the consistency of request and response. - // The adversary can send the fake announces with invalid hash and number but always - // delivery some mismatched header. So it can't be punished by the underlying fetcher. - // We have to add two more rules here to detect. - if len(resp.headers) != 1 { - f.peerset.unregister(req.peerid.String()) - log.Debug("Deliver more than requested", "peer", req.peerid, "reqid", req.reqid) - continue - } - if resp.headers[0].Hash() != req.hash { - f.peerset.unregister(req.peerid.String()) - log.Debug("Deliver invalid header", "peer", req.peerid, "reqid", req.reqid) - continue - } - resp.remain <- f.fetcher.FilterHeaders(resp.peerid.String(), resp.headers, time.Now()) - } else { - // Discard the entire packet no matter it's a timeout response or unexpected one. - resp.remain <- resp.headers - } - - case ev := <-headCh: - // Short circuit if we are still syncing. - if syncing { - continue - } - reset(ev.Block.Header()) - - // Clean stale announcements from les-servers. - var droplist []enode.ID - f.forEachPeer(func(id enode.ID, p *fetcherPeer) bool { - removed := p.forwardAnno(localTd) - for _, anno := range removed { - if header := f.chain.GetHeaderByHash(anno.data.Hash); header != nil { - if header.Number.Uint64() != anno.data.Number { - droplist = append(droplist, id) - break - } - // In theory td should exists. - td := f.chain.GetTd(anno.data.Hash, anno.data.Number) - if td != nil && td.Cmp(anno.data.Td) != 0 { - droplist = append(droplist, id) - break - } - } - } - return true - }) - for _, id := range droplist { - f.peerset.unregister(id.String()) - log.Debug("Kicked out peer for invalid announcement") - } - if f.newHeadHook != nil { - f.newHeadHook(localHead) - } - - case origin := <-f.syncDone: - syncing = false // Reset the status - - // Rewind all untrusted headers for ulc mode. - if ulc { - head := f.chain.CurrentHeader() - ancestor := rawdb.FindCommonAncestor(f.chaindb, origin, head) - - // Recap the ancestor with genesis header in case the ancestor - // is not found. It can happen the original head is before the - // checkpoint while the synced headers are after it. In this - // case there is no ancestor between them. - if ancestor == nil { - ancestor = f.chain.Genesis().Header() - } - var untrusted []common.Hash - for head.Number.Cmp(ancestor.Number) > 0 { - hash, number := head.Hash(), head.Number.Uint64() - if trusted, _ := trustedHeader(hash, number); trusted { - break - } - untrusted = append(untrusted, hash) - head = f.chain.GetHeader(head.ParentHash, number-1) - if head == nil { - break // all the synced headers will be dropped - } - } - if len(untrusted) > 0 { - for i, j := 0, len(untrusted)-1; i < j; i, j = i+1, j-1 { - untrusted[i], untrusted[j] = untrusted[j], untrusted[i] - } - f.chain.Rollback(untrusted) - } - } - // Reset local status. - reset(f.chain.CurrentHeader()) - if f.newHeadHook != nil { - f.newHeadHook(localHead) - } - log.Debug("light sync finished", "number", localHead.Number, "hash", localHead.Hash()) - - case <-f.closeCh: - return - } - } -} - -// announce processes a new announcement message received from a peer. -func (f *lightFetcher) announce(p *serverPeer, head *announceData) { - select { - case f.announceCh <- &announce{peerid: p.ID(), trust: p.trusted, data: head}: - case <-f.closeCh: - return - } -} - -// trackRequest sends a reqID to main loop for in-flight request tracking. -func (f *lightFetcher) trackRequest(peerid enode.ID, reqid uint64, hash common.Hash) { - select { - case f.requestCh <- &request{reqid: reqid, peerid: peerid, sendAt: time.Now(), hash: hash}: - case <-f.closeCh: - } -} - -// requestHeaderByHash constructs a header retrieval request and sends it to -// local request distributor. -// -// Note, we rely on the underlying eth/fetcher to retrieve and validate the -// response, so that we have to obey the rule of eth/fetcher which only accepts -// the response from given peer. -func (f *lightFetcher) requestHeaderByHash(peerid enode.ID) func(common.Hash) error { - return func(hash common.Hash) error { - req := &distReq{ - getCost: func(dp distPeer) uint64 { return dp.(*serverPeer).getRequestCost(GetBlockHeadersMsg, 1) }, - canSend: func(dp distPeer) bool { return dp.(*serverPeer).ID() == peerid }, - request: func(dp distPeer) func() { - peer, id := dp.(*serverPeer), rand.Uint64() - cost := peer.getRequestCost(GetBlockHeadersMsg, 1) - peer.fcServer.QueuedRequest(id, cost) - - return func() { - f.trackRequest(peer.ID(), id, hash) - peer.requestHeadersByHash(id, hash, 1, 0, false) - } - }, - } - f.reqDist.queue(req) - return nil - } -} - -// startSync invokes synchronisation callback to start syncing. -func (f *lightFetcher) startSync(id enode.ID) { - defer func(header *types.Header) { - f.syncDone <- header - }(f.chain.CurrentHeader()) - - peer := f.peerset.peer(id.String()) - if peer == nil || peer.onlyAnnounce { - return - } - f.synchronise(peer) -} - -// deliverHeaders delivers header download request responses for processing -func (f *lightFetcher) deliverHeaders(peer *serverPeer, reqid uint64, headers []*types.Header) []*types.Header { - remain := make(chan []*types.Header, 1) - select { - case f.deliverCh <- &response{reqid: reqid, headers: headers, peerid: peer.ID(), remain: remain}: - case <-f.closeCh: - return nil - } - return <-remain -} - -// rescheduleTimer resets the specified timeout timer to the next request timeout. -func (f *lightFetcher) rescheduleTimer(requests map[uint64]*request, timer *time.Timer) { - // Short circuit if no inflight requests - if len(requests) == 0 { - timer.Stop() - return - } - // Otherwise find the earliest expiring request - earliest := time.Now() - for _, req := range requests { - if earliest.After(req.sendAt) { - earliest = req.sendAt - } - } - timer.Reset(blockDelayTimeout - time.Since(earliest)) -} diff --git a/les/fetcher/block_fetcher.go b/les/fetcher/block_fetcher.go deleted file mode 100644 index 085ecb2d66..0000000000 --- a/les/fetcher/block_fetcher.go +++ /dev/null @@ -1,888 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Package fetcher is a temporary package whilst working on the eth/66 blocking refactors. -// After that work is done, les needs to be refactored to use the new package, -// or alternatively use a stripped down version of it. Either way, we need to -// keep the changes scoped so duplicating temporarily seems the sanest. -package fetcher - -import ( - "errors" - "math/rand" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/prque" - "github.com/ethereum/go-ethereum/consensus" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/metrics" - "github.com/ethereum/go-ethereum/trie" -) - -const ( - lightTimeout = time.Millisecond // Time allowance before an announced header is explicitly requested - arriveTimeout = 500 * time.Millisecond // Time allowance before an announced block/transaction is explicitly requested - gatherSlack = 100 * time.Millisecond // Interval used to collate almost-expired announces with fetches - fetchTimeout = 5 * time.Second // Maximum allotted time to return an explicitly requested block/transaction -) - -const ( - maxUncleDist = 7 // Maximum allowed backward distance from the chain head - maxQueueDist = 32 // Maximum allowed distance from the chain head to queue - hashLimit = 256 // Maximum number of unique blocks or headers a peer may have announced - blockLimit = 64 // Maximum number of unique blocks a peer may have delivered -) - -var ( - blockAnnounceInMeter = metrics.NewRegisteredMeter("eth/fetcher/block/announces/in", nil) - blockAnnounceOutTimer = metrics.NewRegisteredTimer("eth/fetcher/block/announces/out", nil) - blockAnnounceDropMeter = metrics.NewRegisteredMeter("eth/fetcher/block/announces/drop", nil) - blockAnnounceDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/block/announces/dos", nil) - - blockBroadcastInMeter = metrics.NewRegisteredMeter("eth/fetcher/block/broadcasts/in", nil) - blockBroadcastOutTimer = metrics.NewRegisteredTimer("eth/fetcher/block/broadcasts/out", nil) - blockBroadcastDropMeter = metrics.NewRegisteredMeter("eth/fetcher/block/broadcasts/drop", nil) - blockBroadcastDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/block/broadcasts/dos", nil) - - headerFetchMeter = metrics.NewRegisteredMeter("eth/fetcher/block/headers", nil) - bodyFetchMeter = metrics.NewRegisteredMeter("eth/fetcher/block/bodies", nil) - - headerFilterInMeter = metrics.NewRegisteredMeter("eth/fetcher/block/filter/headers/in", nil) - headerFilterOutMeter = metrics.NewRegisteredMeter("eth/fetcher/block/filter/headers/out", nil) - bodyFilterInMeter = metrics.NewRegisteredMeter("eth/fetcher/block/filter/bodies/in", nil) - bodyFilterOutMeter = metrics.NewRegisteredMeter("eth/fetcher/block/filter/bodies/out", nil) -) - -var errTerminated = errors.New("terminated") - -// HeaderRetrievalFn is a callback type for retrieving a header from the local chain. -type HeaderRetrievalFn func(common.Hash) *types.Header - -// blockRetrievalFn is a callback type for retrieving a block from the local chain. -type blockRetrievalFn func(common.Hash) *types.Block - -// headerRequesterFn is a callback type for sending a header retrieval request. -type headerRequesterFn func(common.Hash) error - -// bodyRequesterFn is a callback type for sending a body retrieval request. -type bodyRequesterFn func([]common.Hash) error - -// headerVerifierFn is a callback type to verify a block's header for fast propagation. -type headerVerifierFn func(header *types.Header) error - -// blockBroadcasterFn is a callback type for broadcasting a block to connected peers. -type blockBroadcasterFn func(block *types.Block, propagate bool) - -// chainHeightFn is a callback type to retrieve the current chain height. -type chainHeightFn func() uint64 - -// headersInsertFn is a callback type to insert a batch of headers into the local chain. -type headersInsertFn func(headers []*types.Header) (int, error) - -// chainInsertFn is a callback type to insert a batch of blocks into the local chain. -type chainInsertFn func(types.Blocks) (int, error) - -// peerDropFn is a callback type for dropping a peer detected as malicious. -type peerDropFn func(id string) - -// blockAnnounce is the hash notification of the availability of a new block in the -// network. -type blockAnnounce struct { - hash common.Hash // Hash of the block being announced - number uint64 // Number of the block being announced (0 = unknown | old protocol) - header *types.Header // Header of the block partially reassembled (new protocol) - time time.Time // Timestamp of the announcement - - origin string // Identifier of the peer originating the notification - - fetchHeader headerRequesterFn // Fetcher function to retrieve the header of an announced block - fetchBodies bodyRequesterFn // Fetcher function to retrieve the body of an announced block -} - -// headerFilterTask represents a batch of headers needing fetcher filtering. -type headerFilterTask struct { - peer string // The source peer of block headers - headers []*types.Header // Collection of headers to filter - time time.Time // Arrival time of the headers -} - -// bodyFilterTask represents a batch of block bodies (transactions and uncles) -// needing fetcher filtering. -type bodyFilterTask struct { - peer string // The source peer of block bodies - transactions [][]*types.Transaction // Collection of transactions per block bodies - uncles [][]*types.Header // Collection of uncles per block bodies - time time.Time // Arrival time of the blocks' contents -} - -// blockOrHeaderInject represents a schedules import operation. -type blockOrHeaderInject struct { - origin string - - header *types.Header // Used for light mode fetcher which only cares about header. - block *types.Block // Used for normal mode fetcher which imports full block. -} - -// number returns the block number of the injected object. -func (inject *blockOrHeaderInject) number() uint64 { - if inject.header != nil { - return inject.header.Number.Uint64() - } - return inject.block.NumberU64() -} - -// number returns the block hash of the injected object. -func (inject *blockOrHeaderInject) hash() common.Hash { - if inject.header != nil { - return inject.header.Hash() - } - return inject.block.Hash() -} - -// BlockFetcher is responsible for accumulating block announcements from various peers -// and scheduling them for retrieval. -type BlockFetcher struct { - light bool // The indicator whether it's a light fetcher or normal one. - - // Various event channels - notify chan *blockAnnounce - inject chan *blockOrHeaderInject - - headerFilter chan chan *headerFilterTask - bodyFilter chan chan *bodyFilterTask - - done chan common.Hash - quit chan struct{} - - // Announce states - announces map[string]int // Per peer blockAnnounce counts to prevent memory exhaustion - announced map[common.Hash][]*blockAnnounce // Announced blocks, scheduled for fetching - fetching map[common.Hash]*blockAnnounce // Announced blocks, currently fetching - fetched map[common.Hash][]*blockAnnounce // Blocks with headers fetched, scheduled for body retrieval - completing map[common.Hash]*blockAnnounce // Blocks with headers, currently body-completing - - // Block cache - queue *prque.Prque[int64, *blockOrHeaderInject] // Queue containing the import operations (block number sorted) - queues map[string]int // Per peer block counts to prevent memory exhaustion - queued map[common.Hash]*blockOrHeaderInject // Set of already queued blocks (to dedup imports) - - // Callbacks - getHeader HeaderRetrievalFn // Retrieves a header from the local chain - getBlock blockRetrievalFn // Retrieves a block from the local chain - verifyHeader headerVerifierFn // Checks if a block's headers have a valid proof of work - broadcastBlock blockBroadcasterFn // Broadcasts a block to connected peers - chainHeight chainHeightFn // Retrieves the current chain's height - insertHeaders headersInsertFn // Injects a batch of headers into the chain - insertChain chainInsertFn // Injects a batch of blocks into the chain - dropPeer peerDropFn // Drops a peer for misbehaving - - // Testing hooks - announceChangeHook func(common.Hash, bool) // Method to call upon adding or deleting a hash from the blockAnnounce list - queueChangeHook func(common.Hash, bool) // Method to call upon adding or deleting a block from the import queue - fetchingHook func([]common.Hash) // Method to call upon starting a block (eth/61) or header (eth/62) fetch - completingHook func([]common.Hash) // Method to call upon starting a block body fetch (eth/62) - importedHook func(*types.Header, *types.Block) // Method to call upon successful header or block import (both eth/61 and eth/62) -} - -// NewBlockFetcher creates a block fetcher to retrieve blocks based on hash announcements. -func NewBlockFetcher(light bool, getHeader HeaderRetrievalFn, getBlock blockRetrievalFn, verifyHeader headerVerifierFn, broadcastBlock blockBroadcasterFn, chainHeight chainHeightFn, insertHeaders headersInsertFn, insertChain chainInsertFn, dropPeer peerDropFn) *BlockFetcher { - return &BlockFetcher{ - light: light, - notify: make(chan *blockAnnounce), - inject: make(chan *blockOrHeaderInject), - headerFilter: make(chan chan *headerFilterTask), - bodyFilter: make(chan chan *bodyFilterTask), - done: make(chan common.Hash), - quit: make(chan struct{}), - announces: make(map[string]int), - announced: make(map[common.Hash][]*blockAnnounce), - fetching: make(map[common.Hash]*blockAnnounce), - fetched: make(map[common.Hash][]*blockAnnounce), - completing: make(map[common.Hash]*blockAnnounce), - queue: prque.New[int64, *blockOrHeaderInject](nil), - queues: make(map[string]int), - queued: make(map[common.Hash]*blockOrHeaderInject), - getHeader: getHeader, - getBlock: getBlock, - verifyHeader: verifyHeader, - broadcastBlock: broadcastBlock, - chainHeight: chainHeight, - insertHeaders: insertHeaders, - insertChain: insertChain, - dropPeer: dropPeer, - } -} - -// Start boots up the announcement based synchroniser, accepting and processing -// hash notifications and block fetches until termination requested. -func (f *BlockFetcher) Start() { - go f.loop() -} - -// Stop terminates the announcement based synchroniser, canceling all pending -// operations. -func (f *BlockFetcher) Stop() { - close(f.quit) -} - -// Notify announces the fetcher of the potential availability of a new block in -// the network. -func (f *BlockFetcher) Notify(peer string, hash common.Hash, number uint64, time time.Time, - headerFetcher headerRequesterFn, bodyFetcher bodyRequesterFn) error { - block := &blockAnnounce{ - hash: hash, - number: number, - time: time, - origin: peer, - fetchHeader: headerFetcher, - fetchBodies: bodyFetcher, - } - select { - case f.notify <- block: - return nil - case <-f.quit: - return errTerminated - } -} - -// Enqueue tries to fill gaps the fetcher's future import queue. -func (f *BlockFetcher) Enqueue(peer string, block *types.Block) error { - op := &blockOrHeaderInject{ - origin: peer, - block: block, - } - select { - case f.inject <- op: - return nil - case <-f.quit: - return errTerminated - } -} - -// FilterHeaders extracts all the headers that were explicitly requested by the fetcher, -// returning those that should be handled differently. -func (f *BlockFetcher) FilterHeaders(peer string, headers []*types.Header, time time.Time) []*types.Header { - log.Trace("Filtering headers", "peer", peer, "headers", len(headers)) - - // Send the filter channel to the fetcher - filter := make(chan *headerFilterTask) - - select { - case f.headerFilter <- filter: - case <-f.quit: - return nil - } - // Request the filtering of the header list - select { - case filter <- &headerFilterTask{peer: peer, headers: headers, time: time}: - case <-f.quit: - return nil - } - // Retrieve the headers remaining after filtering - select { - case task := <-filter: - return task.headers - case <-f.quit: - return nil - } -} - -// FilterBodies extracts all the block bodies that were explicitly requested by -// the fetcher, returning those that should be handled differently. -func (f *BlockFetcher) FilterBodies(peer string, transactions [][]*types.Transaction, uncles [][]*types.Header, time time.Time) ([][]*types.Transaction, [][]*types.Header) { - log.Trace("Filtering bodies", "peer", peer, "txs", len(transactions), "uncles", len(uncles)) - - // Send the filter channel to the fetcher - filter := make(chan *bodyFilterTask) - - select { - case f.bodyFilter <- filter: - case <-f.quit: - return nil, nil - } - // Request the filtering of the body list - select { - case filter <- &bodyFilterTask{peer: peer, transactions: transactions, uncles: uncles, time: time}: - case <-f.quit: - return nil, nil - } - // Retrieve the bodies remaining after filtering - select { - case task := <-filter: - return task.transactions, task.uncles - case <-f.quit: - return nil, nil - } -} - -// Loop is the main fetcher loop, checking and processing various notification -// events. -func (f *BlockFetcher) loop() { - // Iterate the block fetching until a quit is requested - var ( - fetchTimer = time.NewTimer(0) - completeTimer = time.NewTimer(0) - ) - <-fetchTimer.C // clear out the channel - <-completeTimer.C - defer fetchTimer.Stop() - defer completeTimer.Stop() - - for { - // Clean up any expired block fetches - for hash, announce := range f.fetching { - if time.Since(announce.time) > fetchTimeout { - f.forgetHash(hash) - } - } - // Import any queued blocks that could potentially fit - height := f.chainHeight() - for !f.queue.Empty() { - op := f.queue.PopItem() - hash := op.hash() - if f.queueChangeHook != nil { - f.queueChangeHook(hash, false) - } - // If too high up the chain or phase, continue later - number := op.number() - if number > height+1 { - f.queue.Push(op, -int64(number)) - if f.queueChangeHook != nil { - f.queueChangeHook(hash, true) - } - break - } - // Otherwise if fresh and still unknown, try and import - if (number+maxUncleDist < height) || (f.light && f.getHeader(hash) != nil) || (!f.light && f.getBlock(hash) != nil) { - f.forgetBlock(hash) - continue - } - if f.light { - f.importHeaders(op.origin, op.header) - } else { - f.importBlocks(op.origin, op.block) - } - } - // Wait for an outside event to occur - select { - case <-f.quit: - // BlockFetcher terminating, abort all operations - return - - case notification := <-f.notify: - // A block was announced, make sure the peer isn't DOSing us - blockAnnounceInMeter.Mark(1) - - count := f.announces[notification.origin] + 1 - if count > hashLimit { - log.Debug("Peer exceeded outstanding announces", "peer", notification.origin, "limit", hashLimit) - blockAnnounceDOSMeter.Mark(1) - break - } - // If we have a valid block number, check that it's potentially useful - if notification.number > 0 { - if dist := int64(notification.number) - int64(f.chainHeight()); dist < -maxUncleDist || dist > maxQueueDist { - log.Debug("Peer discarded announcement", "peer", notification.origin, "number", notification.number, "hash", notification.hash, "distance", dist) - blockAnnounceDropMeter.Mark(1) - break - } - } - // All is well, schedule the announce if block's not yet downloading - if _, ok := f.fetching[notification.hash]; ok { - break - } - if _, ok := f.completing[notification.hash]; ok { - break - } - f.announces[notification.origin] = count - f.announced[notification.hash] = append(f.announced[notification.hash], notification) - if f.announceChangeHook != nil && len(f.announced[notification.hash]) == 1 { - f.announceChangeHook(notification.hash, true) - } - if len(f.announced) == 1 { - f.rescheduleFetch(fetchTimer) - } - - case op := <-f.inject: - // A direct block insertion was requested, try and fill any pending gaps - blockBroadcastInMeter.Mark(1) - - // Now only direct block injection is allowed, drop the header injection - // here silently if we receive. - if f.light { - continue - } - f.enqueue(op.origin, nil, op.block) - - case hash := <-f.done: - // A pending import finished, remove all traces of the notification - f.forgetHash(hash) - f.forgetBlock(hash) - - case <-fetchTimer.C: - // At least one block's timer ran out, check for needing retrieval - request := make(map[string][]common.Hash) - - for hash, announces := range f.announced { - // In current LES protocol(les2/les3), only header announce is - // available, no need to wait too much time for header broadcast. - timeout := arriveTimeout - gatherSlack - if f.light { - timeout = 0 - } - if time.Since(announces[0].time) > timeout { - // Pick a random peer to retrieve from, reset all others - announce := announces[rand.Intn(len(announces))] - f.forgetHash(hash) - - // If the block still didn't arrive, queue for fetching - if (f.light && f.getHeader(hash) == nil) || (!f.light && f.getBlock(hash) == nil) { - request[announce.origin] = append(request[announce.origin], hash) - f.fetching[hash] = announce - } - } - } - // Send out all block header requests - for peer, hashes := range request { - log.Trace("Fetching scheduled headers", "peer", peer, "list", hashes) - - // Create a closure of the fetch and schedule in on a new thread - fetchHeader, hashes := f.fetching[hashes[0]].fetchHeader, hashes - go func() { - if f.fetchingHook != nil { - f.fetchingHook(hashes) - } - for _, hash := range hashes { - headerFetchMeter.Mark(1) - fetchHeader(hash) // Suboptimal, but protocol doesn't allow batch header retrievals - } - }() - } - // Schedule the next fetch if blocks are still pending - f.rescheduleFetch(fetchTimer) - - case <-completeTimer.C: - // At least one header's timer ran out, retrieve everything - request := make(map[string][]common.Hash) - - for hash, announces := range f.fetched { - // Pick a random peer to retrieve from, reset all others - announce := announces[rand.Intn(len(announces))] - f.forgetHash(hash) - - // If the block still didn't arrive, queue for completion - if f.getBlock(hash) == nil { - request[announce.origin] = append(request[announce.origin], hash) - f.completing[hash] = announce - } - } - // Send out all block body requests - for peer, hashes := range request { - log.Trace("Fetching scheduled bodies", "peer", peer, "list", hashes) - - // Create a closure of the fetch and schedule in on a new thread - if f.completingHook != nil { - f.completingHook(hashes) - } - bodyFetchMeter.Mark(int64(len(hashes))) - go f.completing[hashes[0]].fetchBodies(hashes) - } - // Schedule the next fetch if blocks are still pending - f.rescheduleComplete(completeTimer) - - case filter := <-f.headerFilter: - // Headers arrived from a remote peer. Extract those that were explicitly - // requested by the fetcher, and return everything else so it's delivered - // to other parts of the system. - var task *headerFilterTask - select { - case task = <-filter: - case <-f.quit: - return - } - headerFilterInMeter.Mark(int64(len(task.headers))) - - // Split the batch of headers into unknown ones (to return to the caller), - // known incomplete ones (requiring body retrievals) and completed blocks. - unknown, incomplete, complete, lightHeaders := []*types.Header{}, []*blockAnnounce{}, []*types.Block{}, []*blockAnnounce{} - for _, header := range task.headers { - hash := header.Hash() - - // Filter fetcher-requested headers from other synchronisation algorithms - if announce := f.fetching[hash]; announce != nil && announce.origin == task.peer && f.fetched[hash] == nil && f.completing[hash] == nil && f.queued[hash] == nil { - // If the delivered header does not match the promised number, drop the announcer - if header.Number.Uint64() != announce.number { - log.Trace("Invalid block number fetched", "peer", announce.origin, "hash", header.Hash(), "announced", announce.number, "provided", header.Number) - f.dropPeer(announce.origin) - f.forgetHash(hash) - continue - } - // Collect all headers only if we are running in light - // mode and the headers are not imported by other means. - if f.light { - if f.getHeader(hash) == nil { - announce.header = header - lightHeaders = append(lightHeaders, announce) - } - f.forgetHash(hash) - continue - } - // Only keep if not imported by other means - if f.getBlock(hash) == nil { - announce.header = header - announce.time = task.time - - // If the block is empty (header only), short circuit into the final import queue - if header.TxHash == types.EmptyTxsHash && header.UncleHash == types.EmptyUncleHash { - log.Trace("Block empty, skipping body retrieval", "peer", announce.origin, "number", header.Number, "hash", header.Hash()) - - block := types.NewBlockWithHeader(header) - block.ReceivedAt = task.time - - complete = append(complete, block) - f.completing[hash] = announce - continue - } - // Otherwise add to the list of blocks needing completion - incomplete = append(incomplete, announce) - } else { - log.Trace("Block already imported, discarding header", "peer", announce.origin, "number", header.Number, "hash", header.Hash()) - f.forgetHash(hash) - } - } else { - // BlockFetcher doesn't know about it, add to the return list - unknown = append(unknown, header) - } - } - headerFilterOutMeter.Mark(int64(len(unknown))) - select { - case filter <- &headerFilterTask{headers: unknown, time: task.time}: - case <-f.quit: - return - } - // Schedule the retrieved headers for body completion - for _, announce := range incomplete { - hash := announce.header.Hash() - if _, ok := f.completing[hash]; ok { - continue - } - f.fetched[hash] = append(f.fetched[hash], announce) - if len(f.fetched) == 1 { - f.rescheduleComplete(completeTimer) - } - } - // Schedule the header for light fetcher import - for _, announce := range lightHeaders { - f.enqueue(announce.origin, announce.header, nil) - } - // Schedule the header-only blocks for import - for _, block := range complete { - if announce := f.completing[block.Hash()]; announce != nil { - f.enqueue(announce.origin, nil, block) - } - } - - case filter := <-f.bodyFilter: - // Block bodies arrived, extract any explicitly requested blocks, return the rest - var task *bodyFilterTask - select { - case task = <-filter: - case <-f.quit: - return - } - bodyFilterInMeter.Mark(int64(len(task.transactions))) - blocks := []*types.Block{} - // abort early if there's nothing explicitly requested - if len(f.completing) > 0 { - for i := 0; i < len(task.transactions) && i < len(task.uncles); i++ { - // Match up a body to any possible completion request - var ( - matched = false - uncleHash common.Hash // calculated lazily and reused - txnHash common.Hash // calculated lazily and reused - ) - for hash, announce := range f.completing { - if f.queued[hash] != nil || announce.origin != task.peer { - continue - } - if uncleHash == (common.Hash{}) { - uncleHash = types.CalcUncleHash(task.uncles[i]) - } - if uncleHash != announce.header.UncleHash { - continue - } - if txnHash == (common.Hash{}) { - txnHash = types.DeriveSha(types.Transactions(task.transactions[i]), trie.NewStackTrie(nil)) - } - if txnHash != announce.header.TxHash { - continue - } - // Mark the body matched, reassemble if still unknown - matched = true - if f.getBlock(hash) == nil { - block := types.NewBlockWithHeader(announce.header).WithBody(task.transactions[i], task.uncles[i]) - block.ReceivedAt = task.time - blocks = append(blocks, block) - } else { - f.forgetHash(hash) - } - } - if matched { - task.transactions = append(task.transactions[:i], task.transactions[i+1:]...) - task.uncles = append(task.uncles[:i], task.uncles[i+1:]...) - i-- - continue - } - } - } - bodyFilterOutMeter.Mark(int64(len(task.transactions))) - select { - case filter <- task: - case <-f.quit: - return - } - // Schedule the retrieved blocks for ordered import - for _, block := range blocks { - if announce := f.completing[block.Hash()]; announce != nil { - f.enqueue(announce.origin, nil, block) - } - } - } - } -} - -// rescheduleFetch resets the specified fetch timer to the next blockAnnounce timeout. -func (f *BlockFetcher) rescheduleFetch(fetch *time.Timer) { - // Short circuit if no blocks are announced - if len(f.announced) == 0 { - return - } - // Schedule announcement retrieval quickly for light mode - // since server won't send any headers to client. - if f.light { - fetch.Reset(lightTimeout) - return - } - // Otherwise find the earliest expiring announcement - earliest := time.Now() - for _, announces := range f.announced { - if earliest.After(announces[0].time) { - earliest = announces[0].time - } - } - fetch.Reset(arriveTimeout - time.Since(earliest)) -} - -// rescheduleComplete resets the specified completion timer to the next fetch timeout. -func (f *BlockFetcher) rescheduleComplete(complete *time.Timer) { - // Short circuit if no headers are fetched - if len(f.fetched) == 0 { - return - } - // Otherwise find the earliest expiring announcement - earliest := time.Now() - for _, announces := range f.fetched { - if earliest.After(announces[0].time) { - earliest = announces[0].time - } - } - complete.Reset(gatherSlack - time.Since(earliest)) -} - -// enqueue schedules a new header or block import operation, if the component -// to be imported has not yet been seen. -func (f *BlockFetcher) enqueue(peer string, header *types.Header, block *types.Block) { - var ( - hash common.Hash - number uint64 - ) - if header != nil { - hash, number = header.Hash(), header.Number.Uint64() - } else { - hash, number = block.Hash(), block.NumberU64() - } - // Ensure the peer isn't DOSing us - count := f.queues[peer] + 1 - if count > blockLimit { - log.Debug("Discarded delivered header or block, exceeded allowance", "peer", peer, "number", number, "hash", hash, "limit", blockLimit) - blockBroadcastDOSMeter.Mark(1) - f.forgetHash(hash) - return - } - // Discard any past or too distant blocks - if dist := int64(number) - int64(f.chainHeight()); dist < -maxUncleDist || dist > maxQueueDist { - log.Debug("Discarded delivered header or block, too far away", "peer", peer, "number", number, "hash", hash, "distance", dist) - blockBroadcastDropMeter.Mark(1) - f.forgetHash(hash) - return - } - // Schedule the block for future importing - if _, ok := f.queued[hash]; !ok { - op := &blockOrHeaderInject{origin: peer} - if header != nil { - op.header = header - } else { - op.block = block - } - f.queues[peer] = count - f.queued[hash] = op - f.queue.Push(op, -int64(number)) - if f.queueChangeHook != nil { - f.queueChangeHook(hash, true) - } - log.Debug("Queued delivered header or block", "peer", peer, "number", number, "hash", hash, "queued", f.queue.Size()) - } -} - -// importHeaders spawns a new goroutine to run a header insertion into the chain. -// If the header's number is at the same height as the current import phase, it -// updates the phase states accordingly. -func (f *BlockFetcher) importHeaders(peer string, header *types.Header) { - hash := header.Hash() - log.Debug("Importing propagated header", "peer", peer, "number", header.Number, "hash", hash) - - go func() { - defer func() { f.done <- hash }() - // If the parent's unknown, abort insertion - parent := f.getHeader(header.ParentHash) - if parent == nil { - log.Debug("Unknown parent of propagated header", "peer", peer, "number", header.Number, "hash", hash, "parent", header.ParentHash) - return - } - // Validate the header and if something went wrong, drop the peer - if err := f.verifyHeader(header); err != nil && err != consensus.ErrFutureBlock { - log.Debug("Propagated header verification failed", "peer", peer, "number", header.Number, "hash", hash, "err", err) - f.dropPeer(peer) - return - } - // Run the actual import and log any issues - if _, err := f.insertHeaders([]*types.Header{header}); err != nil { - log.Debug("Propagated header import failed", "peer", peer, "number", header.Number, "hash", hash, "err", err) - return - } - // Invoke the testing hook if needed - if f.importedHook != nil { - f.importedHook(header, nil) - } - }() -} - -// importBlocks spawns a new goroutine to run a block insertion into the chain. If the -// block's number is at the same height as the current import phase, it updates -// the phase states accordingly. -func (f *BlockFetcher) importBlocks(peer string, block *types.Block) { - hash := block.Hash() - - // Run the import on a new thread - log.Debug("Importing propagated block", "peer", peer, "number", block.Number(), "hash", hash) - go func() { - defer func() { f.done <- hash }() - - // If the parent's unknown, abort insertion - parent := f.getBlock(block.ParentHash()) - if parent == nil { - log.Debug("Unknown parent of propagated block", "peer", peer, "number", block.Number(), "hash", hash, "parent", block.ParentHash()) - return - } - // Quickly validate the header and propagate the block if it passes - switch err := f.verifyHeader(block.Header()); err { - case nil: - // All ok, quickly propagate to our peers - blockBroadcastOutTimer.UpdateSince(block.ReceivedAt) - go f.broadcastBlock(block, true) - - case consensus.ErrFutureBlock: - // Weird future block, don't fail, but neither propagate - - default: - // Something went very wrong, drop the peer - log.Debug("Propagated block verification failed", "peer", peer, "number", block.Number(), "hash", hash, "err", err) - f.dropPeer(peer) - return - } - // Run the actual import and log any issues - if _, err := f.insertChain(types.Blocks{block}); err != nil { - log.Debug("Propagated block import failed", "peer", peer, "number", block.Number(), "hash", hash, "err", err) - return - } - // If import succeeded, broadcast the block - blockAnnounceOutTimer.UpdateSince(block.ReceivedAt) - go f.broadcastBlock(block, false) - - // Invoke the testing hook if needed - if f.importedHook != nil { - f.importedHook(nil, block) - } - }() -} - -// forgetHash removes all traces of a block announcement from the fetcher's -// internal state. -func (f *BlockFetcher) forgetHash(hash common.Hash) { - // Remove all pending announces and decrement DOS counters - if announceMap, ok := f.announced[hash]; ok { - for _, announce := range announceMap { - f.announces[announce.origin]-- - if f.announces[announce.origin] <= 0 { - delete(f.announces, announce.origin) - } - } - delete(f.announced, hash) - if f.announceChangeHook != nil { - f.announceChangeHook(hash, false) - } - } - // Remove any pending fetches and decrement the DOS counters - if announce := f.fetching[hash]; announce != nil { - f.announces[announce.origin]-- - if f.announces[announce.origin] <= 0 { - delete(f.announces, announce.origin) - } - delete(f.fetching, hash) - } - - // Remove any pending completion requests and decrement the DOS counters - for _, announce := range f.fetched[hash] { - f.announces[announce.origin]-- - if f.announces[announce.origin] <= 0 { - delete(f.announces, announce.origin) - } - } - delete(f.fetched, hash) - - // Remove any pending completions and decrement the DOS counters - if announce := f.completing[hash]; announce != nil { - f.announces[announce.origin]-- - if f.announces[announce.origin] <= 0 { - delete(f.announces, announce.origin) - } - delete(f.completing, hash) - } -} - -// forgetBlock removes all traces of a queued block from the fetcher's internal -// state. -func (f *BlockFetcher) forgetBlock(hash common.Hash) { - if insert := f.queued[hash]; insert != nil { - f.queues[insert.origin]-- - if f.queues[insert.origin] == 0 { - delete(f.queues, insert.origin) - } - delete(f.queued, hash) - } -} diff --git a/les/fetcher/block_fetcher_test.go b/les/fetcher/block_fetcher_test.go deleted file mode 100644 index f29cf58805..0000000000 --- a/les/fetcher/block_fetcher_test.go +++ /dev/null @@ -1,901 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package fetcher - -import ( - "errors" - "math/big" - "sync" - "sync/atomic" - "testing" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/consensus/ethash" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/params" - "github.com/ethereum/go-ethereum/trie" -) - -var ( - testdb = rawdb.NewMemoryDatabase() - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - testAddress = crypto.PubkeyToAddress(testKey.PublicKey) - - gspec = core.Genesis{ - Alloc: core.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, - BaseFee: big.NewInt(params.InitialBaseFee), - } - genesis = gspec.MustCommit(testdb) - unknownBlock = types.NewBlock(&types.Header{Root: types.EmptyRootHash, GasLimit: params.GenesisGasLimit, BaseFee: big.NewInt(params.InitialBaseFee)}, nil, nil, nil, trie.NewStackTrie(nil)) -) - -// makeChain creates a chain of n blocks starting at and including parent. -// the returned hash chain is ordered head->parent. In addition, every 3rd block -// contains a transaction and every 5th an uncle to allow testing correct block -// reassembly. -func makeChain(n int, seed byte, parent *types.Block) ([]common.Hash, map[common.Hash]*types.Block) { - blocks, _ := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testdb, n, func(i int, block *core.BlockGen) { - block.SetCoinbase(common.Address{seed}) - - // If the block number is multiple of 3, send a bonus transaction to the miner - if parent == genesis && i%3 == 0 { - signer := types.MakeSigner(params.TestChainConfig, block.Number(), block.Timestamp()) - tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddress), common.Address{seed}, big.NewInt(1000), params.TxGas, block.BaseFee(), nil), signer, testKey) - if err != nil { - panic(err) - } - block.AddTx(tx) - } - // If the block number is a multiple of 5, add a bonus uncle to the block - if i > 0 && i%5 == 0 { - block.AddUncle(&types.Header{ParentHash: block.PrevBlock(i - 2).Hash(), Number: big.NewInt(int64(i - 1))}) - } - }) - hashes := make([]common.Hash, n+1) - hashes[len(hashes)-1] = parent.Hash() - blockm := make(map[common.Hash]*types.Block, n+1) - blockm[parent.Hash()] = parent - for i, b := range blocks { - hashes[len(hashes)-i-2] = b.Hash() - blockm[b.Hash()] = b - } - return hashes, blockm -} - -// fetcherTester is a test simulator for mocking out local block chain. -type fetcherTester struct { - fetcher *BlockFetcher - - hashes []common.Hash // Hash chain belonging to the tester - headers map[common.Hash]*types.Header // Headers belonging to the tester - blocks map[common.Hash]*types.Block // Blocks belonging to the tester - drops map[string]bool // Map of peers dropped by the fetcher - - lock sync.RWMutex -} - -// newTester creates a new fetcher test mocker. -func newTester(light bool) *fetcherTester { - tester := &fetcherTester{ - hashes: []common.Hash{genesis.Hash()}, - headers: map[common.Hash]*types.Header{genesis.Hash(): genesis.Header()}, - blocks: map[common.Hash]*types.Block{genesis.Hash(): genesis}, - drops: make(map[string]bool), - } - tester.fetcher = NewBlockFetcher(light, tester.getHeader, tester.getBlock, tester.verifyHeader, tester.broadcastBlock, tester.chainHeight, tester.insertHeaders, tester.insertChain, tester.dropPeer) - tester.fetcher.Start() - - return tester -} - -// getHeader retrieves a header from the tester's block chain. -func (f *fetcherTester) getHeader(hash common.Hash) *types.Header { - f.lock.RLock() - defer f.lock.RUnlock() - - return f.headers[hash] -} - -// getBlock retrieves a block from the tester's block chain. -func (f *fetcherTester) getBlock(hash common.Hash) *types.Block { - f.lock.RLock() - defer f.lock.RUnlock() - - return f.blocks[hash] -} - -// verifyHeader is a nop placeholder for the block header verification. -func (f *fetcherTester) verifyHeader(header *types.Header) error { - return nil -} - -// broadcastBlock is a nop placeholder for the block broadcasting. -func (f *fetcherTester) broadcastBlock(block *types.Block, propagate bool) { -} - -// chainHeight retrieves the current height (block number) of the chain. -func (f *fetcherTester) chainHeight() uint64 { - f.lock.RLock() - defer f.lock.RUnlock() - - if f.fetcher.light { - return f.headers[f.hashes[len(f.hashes)-1]].Number.Uint64() - } - return f.blocks[f.hashes[len(f.hashes)-1]].NumberU64() -} - -// insertChain injects a new headers into the simulated chain. -func (f *fetcherTester) insertHeaders(headers []*types.Header) (int, error) { - f.lock.Lock() - defer f.lock.Unlock() - - for i, header := range headers { - // Make sure the parent in known - if _, ok := f.headers[header.ParentHash]; !ok { - return i, errors.New("unknown parent") - } - // Discard any new blocks if the same height already exists - if header.Number.Uint64() <= f.headers[f.hashes[len(f.hashes)-1]].Number.Uint64() { - return i, nil - } - // Otherwise build our current chain - f.hashes = append(f.hashes, header.Hash()) - f.headers[header.Hash()] = header - } - return 0, nil -} - -// insertChain injects a new blocks into the simulated chain. -func (f *fetcherTester) insertChain(blocks types.Blocks) (int, error) { - f.lock.Lock() - defer f.lock.Unlock() - - for i, block := range blocks { - // Make sure the parent in known - if _, ok := f.blocks[block.ParentHash()]; !ok { - return i, errors.New("unknown parent") - } - // Discard any new blocks if the same height already exists - if block.NumberU64() <= f.blocks[f.hashes[len(f.hashes)-1]].NumberU64() { - return i, nil - } - // Otherwise build our current chain - f.hashes = append(f.hashes, block.Hash()) - f.blocks[block.Hash()] = block - } - return 0, nil -} - -// dropPeer is an emulator for the peer removal, simply accumulating the various -// peers dropped by the fetcher. -func (f *fetcherTester) dropPeer(peer string) { - f.lock.Lock() - defer f.lock.Unlock() - - f.drops[peer] = true -} - -// makeHeaderFetcher retrieves a block header fetcher associated with a simulated peer. -func (f *fetcherTester) makeHeaderFetcher(peer string, blocks map[common.Hash]*types.Block, drift time.Duration) headerRequesterFn { - closure := make(map[common.Hash]*types.Block) - for hash, block := range blocks { - closure[hash] = block - } - // Create a function that return a header from the closure - return func(hash common.Hash) error { - // Gather the blocks to return - headers := make([]*types.Header, 0, 1) - if block, ok := closure[hash]; ok { - headers = append(headers, block.Header()) - } - // Return on a new thread - go f.fetcher.FilterHeaders(peer, headers, time.Now().Add(drift)) - - return nil - } -} - -// makeBodyFetcher retrieves a block body fetcher associated with a simulated peer. -func (f *fetcherTester) makeBodyFetcher(peer string, blocks map[common.Hash]*types.Block, drift time.Duration) bodyRequesterFn { - closure := make(map[common.Hash]*types.Block) - for hash, block := range blocks { - closure[hash] = block - } - // Create a function that returns blocks from the closure - return func(hashes []common.Hash) error { - // Gather the block bodies to return - transactions := make([][]*types.Transaction, 0, len(hashes)) - uncles := make([][]*types.Header, 0, len(hashes)) - - for _, hash := range hashes { - if block, ok := closure[hash]; ok { - transactions = append(transactions, block.Transactions()) - uncles = append(uncles, block.Uncles()) - } - } - // Return on a new thread - go f.fetcher.FilterBodies(peer, transactions, uncles, time.Now().Add(drift)) - - return nil - } -} - -// verifyFetchingEvent verifies that one single event arrive on a fetching channel. -func verifyFetchingEvent(t *testing.T, fetching chan []common.Hash, arrive bool) { - if arrive { - select { - case <-fetching: - case <-time.After(time.Second): - t.Fatalf("fetching timeout") - } - } else { - select { - case <-fetching: - t.Fatalf("fetching invoked") - case <-time.After(10 * time.Millisecond): - } - } -} - -// verifyCompletingEvent verifies that one single event arrive on an completing channel. -func verifyCompletingEvent(t *testing.T, completing chan []common.Hash, arrive bool) { - if arrive { - select { - case <-completing: - case <-time.After(time.Second): - t.Fatalf("completing timeout") - } - } else { - select { - case <-completing: - t.Fatalf("completing invoked") - case <-time.After(10 * time.Millisecond): - } - } -} - -// verifyImportEvent verifies that one single event arrive on an import channel. -func verifyImportEvent(t *testing.T, imported chan interface{}, arrive bool) { - if arrive { - select { - case <-imported: - case <-time.After(time.Second): - t.Fatalf("import timeout") - } - } else { - select { - case <-imported: - t.Fatalf("import invoked") - case <-time.After(20 * time.Millisecond): - } - } -} - -// verifyImportCount verifies that exactly count number of events arrive on an -// import hook channel. -func verifyImportCount(t *testing.T, imported chan interface{}, count int) { - for i := 0; i < count; i++ { - select { - case <-imported: - case <-time.After(time.Second): - t.Fatalf("block %d: import timeout", i+1) - } - } - verifyImportDone(t, imported) -} - -// verifyImportDone verifies that no more events are arriving on an import channel. -func verifyImportDone(t *testing.T, imported chan interface{}) { - select { - case <-imported: - t.Fatalf("extra block imported") - case <-time.After(50 * time.Millisecond): - } -} - -// verifyChainHeight verifies the chain height is as expected. -func verifyChainHeight(t *testing.T, fetcher *fetcherTester, height uint64) { - if fetcher.chainHeight() != height { - t.Fatalf("chain height mismatch, got %d, want %d", fetcher.chainHeight(), height) - } -} - -// Tests that a fetcher accepts block/header announcements and initiates retrievals -// for them, successfully importing into the local chain. -func TestFullSequentialAnnouncements(t *testing.T) { testSequentialAnnouncements(t, false) } -func TestLightSequentialAnnouncements(t *testing.T) { testSequentialAnnouncements(t, true) } - -func testSequentialAnnouncements(t *testing.T, light bool) { - // Create a chain of blocks to import - targetBlocks := 4 * hashLimit - hashes, blocks := makeChain(targetBlocks, 0, genesis) - - tester := newTester(light) - headerFetcher := tester.makeHeaderFetcher("valid", blocks, -gatherSlack) - bodyFetcher := tester.makeBodyFetcher("valid", blocks, 0) - - // Iteratively announce blocks until all are imported - imported := make(chan interface{}) - tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { - if light { - if header == nil { - t.Fatalf("Fetcher try to import empty header") - } - imported <- header - } else { - if block == nil { - t.Fatalf("Fetcher try to import empty block") - } - imported <- block - } - } - for i := len(hashes) - 2; i >= 0; i-- { - tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) - verifyImportEvent(t, imported, true) - } - verifyImportDone(t, imported) - verifyChainHeight(t, tester, uint64(len(hashes)-1)) -} - -// Tests that if blocks are announced by multiple peers (or even the same buggy -// peer), they will only get downloaded at most once. -func TestFullConcurrentAnnouncements(t *testing.T) { testConcurrentAnnouncements(t, false) } -func TestLightConcurrentAnnouncements(t *testing.T) { testConcurrentAnnouncements(t, true) } - -func testConcurrentAnnouncements(t *testing.T, light bool) { - // Create a chain of blocks to import - targetBlocks := 4 * hashLimit - hashes, blocks := makeChain(targetBlocks, 0, genesis) - - // Assemble a tester with a built in counter for the requests - tester := newTester(light) - firstHeaderFetcher := tester.makeHeaderFetcher("first", blocks, -gatherSlack) - firstBodyFetcher := tester.makeBodyFetcher("first", blocks, 0) - secondHeaderFetcher := tester.makeHeaderFetcher("second", blocks, -gatherSlack) - secondBodyFetcher := tester.makeBodyFetcher("second", blocks, 0) - - counter := uint32(0) - firstHeaderWrapper := func(hash common.Hash) error { - atomic.AddUint32(&counter, 1) - return firstHeaderFetcher(hash) - } - secondHeaderWrapper := func(hash common.Hash) error { - atomic.AddUint32(&counter, 1) - return secondHeaderFetcher(hash) - } - // Iteratively announce blocks until all are imported - imported := make(chan interface{}) - tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { - if light { - if header == nil { - t.Fatalf("Fetcher try to import empty header") - } - imported <- header - } else { - if block == nil { - t.Fatalf("Fetcher try to import empty block") - } - imported <- block - } - } - for i := len(hashes) - 2; i >= 0; i-- { - tester.fetcher.Notify("first", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), firstHeaderWrapper, firstBodyFetcher) - tester.fetcher.Notify("second", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout+time.Millisecond), secondHeaderWrapper, secondBodyFetcher) - tester.fetcher.Notify("second", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout-time.Millisecond), secondHeaderWrapper, secondBodyFetcher) - verifyImportEvent(t, imported, true) - } - verifyImportDone(t, imported) - - // Make sure no blocks were retrieved twice - if int(counter) != targetBlocks { - t.Fatalf("retrieval count mismatch: have %v, want %v", counter, targetBlocks) - } - verifyChainHeight(t, tester, uint64(len(hashes)-1)) -} - -// Tests that announcements arriving while a previous is being fetched still -// results in a valid import. -func TestFullOverlappingAnnouncements(t *testing.T) { testOverlappingAnnouncements(t, false) } -func TestLightOverlappingAnnouncements(t *testing.T) { testOverlappingAnnouncements(t, true) } - -func testOverlappingAnnouncements(t *testing.T, light bool) { - // Create a chain of blocks to import - targetBlocks := 4 * hashLimit - hashes, blocks := makeChain(targetBlocks, 0, genesis) - - tester := newTester(light) - headerFetcher := tester.makeHeaderFetcher("valid", blocks, -gatherSlack) - bodyFetcher := tester.makeBodyFetcher("valid", blocks, 0) - - // Iteratively announce blocks, but overlap them continuously - overlap := 16 - imported := make(chan interface{}, len(hashes)-1) - for i := 0; i < overlap; i++ { - imported <- nil - } - tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { - if light { - if header == nil { - t.Fatalf("Fetcher try to import empty header") - } - imported <- header - } else { - if block == nil { - t.Fatalf("Fetcher try to import empty block") - } - imported <- block - } - } - - for i := len(hashes) - 2; i >= 0; i-- { - tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) - select { - case <-imported: - case <-time.After(time.Second): - t.Fatalf("block %d: import timeout", len(hashes)-i) - } - } - // Wait for all the imports to complete and check count - verifyImportCount(t, imported, overlap) - verifyChainHeight(t, tester, uint64(len(hashes)-1)) -} - -// Tests that announces already being retrieved will not be duplicated. -func TestFullPendingDeduplication(t *testing.T) { testPendingDeduplication(t, false) } -func TestLightPendingDeduplication(t *testing.T) { testPendingDeduplication(t, true) } - -func testPendingDeduplication(t *testing.T, light bool) { - // Create a hash and corresponding block - hashes, blocks := makeChain(1, 0, genesis) - - // Assemble a tester with a built in counter and delayed fetcher - tester := newTester(light) - headerFetcher := tester.makeHeaderFetcher("repeater", blocks, -gatherSlack) - bodyFetcher := tester.makeBodyFetcher("repeater", blocks, 0) - - delay := 50 * time.Millisecond - counter := uint32(0) - headerWrapper := func(hash common.Hash) error { - atomic.AddUint32(&counter, 1) - - // Simulate a long running fetch - go func() { - time.Sleep(delay) - headerFetcher(hash) - }() - return nil - } - checkNonExist := func() bool { - return tester.getBlock(hashes[0]) == nil - } - if light { - checkNonExist = func() bool { - return tester.getHeader(hashes[0]) == nil - } - } - // Announce the same block many times until it's fetched (wait for any pending ops) - for checkNonExist() { - tester.fetcher.Notify("repeater", hashes[0], 1, time.Now().Add(-arriveTimeout), headerWrapper, bodyFetcher) - time.Sleep(time.Millisecond) - } - time.Sleep(delay) - - // Check that all blocks were imported and none fetched twice - if int(counter) != 1 { - t.Fatalf("retrieval count mismatch: have %v, want %v", counter, 1) - } - verifyChainHeight(t, tester, 1) -} - -// Tests that announcements retrieved in a random order are cached and eventually -// imported when all the gaps are filled in. -func TestFullRandomArrivalImport(t *testing.T) { testRandomArrivalImport(t, false) } -func TestLightRandomArrivalImport(t *testing.T) { testRandomArrivalImport(t, true) } - -func testRandomArrivalImport(t *testing.T, light bool) { - // Create a chain of blocks to import, and choose one to delay - targetBlocks := maxQueueDist - hashes, blocks := makeChain(targetBlocks, 0, genesis) - skip := targetBlocks / 2 - - tester := newTester(light) - headerFetcher := tester.makeHeaderFetcher("valid", blocks, -gatherSlack) - bodyFetcher := tester.makeBodyFetcher("valid", blocks, 0) - - // Iteratively announce blocks, skipping one entry - imported := make(chan interface{}, len(hashes)-1) - tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { - if light { - if header == nil { - t.Fatalf("Fetcher try to import empty header") - } - imported <- header - } else { - if block == nil { - t.Fatalf("Fetcher try to import empty block") - } - imported <- block - } - } - for i := len(hashes) - 1; i >= 0; i-- { - if i != skip { - tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) - time.Sleep(time.Millisecond) - } - } - // Finally announce the skipped entry and check full import - tester.fetcher.Notify("valid", hashes[skip], uint64(len(hashes)-skip-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) - verifyImportCount(t, imported, len(hashes)-1) - verifyChainHeight(t, tester, uint64(len(hashes)-1)) -} - -// Tests that direct block enqueues (due to block propagation vs. hash announce) -// are correctly schedule, filling and import queue gaps. -func TestQueueGapFill(t *testing.T) { - // Create a chain of blocks to import, and choose one to not announce at all - targetBlocks := maxQueueDist - hashes, blocks := makeChain(targetBlocks, 0, genesis) - skip := targetBlocks / 2 - - tester := newTester(false) - headerFetcher := tester.makeHeaderFetcher("valid", blocks, -gatherSlack) - bodyFetcher := tester.makeBodyFetcher("valid", blocks, 0) - - // Iteratively announce blocks, skipping one entry - imported := make(chan interface{}, len(hashes)-1) - tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { imported <- block } - - for i := len(hashes) - 1; i >= 0; i-- { - if i != skip { - tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) - time.Sleep(time.Millisecond) - } - } - // Fill the missing block directly as if propagated - tester.fetcher.Enqueue("valid", blocks[hashes[skip]]) - verifyImportCount(t, imported, len(hashes)-1) - verifyChainHeight(t, tester, uint64(len(hashes)-1)) -} - -// Tests that blocks arriving from various sources (multiple propagations, hash -// announces, etc) do not get scheduled for import multiple times. -func TestImportDeduplication(t *testing.T) { - // Create two blocks to import (one for duplication, the other for stalling) - hashes, blocks := makeChain(2, 0, genesis) - - // Create the tester and wrap the importer with a counter - tester := newTester(false) - headerFetcher := tester.makeHeaderFetcher("valid", blocks, -gatherSlack) - bodyFetcher := tester.makeBodyFetcher("valid", blocks, 0) - - counter := uint32(0) - tester.fetcher.insertChain = func(blocks types.Blocks) (int, error) { - atomic.AddUint32(&counter, uint32(len(blocks))) - return tester.insertChain(blocks) - } - // Instrument the fetching and imported events - fetching := make(chan []common.Hash) - imported := make(chan interface{}, len(hashes)-1) - tester.fetcher.fetchingHook = func(hashes []common.Hash) { fetching <- hashes } - tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { imported <- block } - - // Announce the duplicating block, wait for retrieval, and also propagate directly - tester.fetcher.Notify("valid", hashes[0], 1, time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) - <-fetching - - tester.fetcher.Enqueue("valid", blocks[hashes[0]]) - tester.fetcher.Enqueue("valid", blocks[hashes[0]]) - tester.fetcher.Enqueue("valid", blocks[hashes[0]]) - - // Fill the missing block directly as if propagated, and check import uniqueness - tester.fetcher.Enqueue("valid", blocks[hashes[1]]) - verifyImportCount(t, imported, 2) - - if counter != 2 { - t.Fatalf("import invocation count mismatch: have %v, want %v", counter, 2) - } -} - -// Tests that blocks with numbers much lower or higher than out current head get -// discarded to prevent wasting resources on useless blocks from faulty peers. -func TestDistantPropagationDiscarding(t *testing.T) { - // Create a long chain to import and define the discard boundaries - hashes, blocks := makeChain(3*maxQueueDist, 0, genesis) - head := hashes[len(hashes)/2] - - low, high := len(hashes)/2+maxUncleDist+1, len(hashes)/2-maxQueueDist-1 - - // Create a tester and simulate a head block being the middle of the above chain - tester := newTester(false) - - tester.lock.Lock() - tester.hashes = []common.Hash{head} - tester.blocks = map[common.Hash]*types.Block{head: blocks[head]} - tester.lock.Unlock() - - // Ensure that a block with a lower number than the threshold is discarded - tester.fetcher.Enqueue("lower", blocks[hashes[low]]) - time.Sleep(10 * time.Millisecond) - if !tester.fetcher.queue.Empty() { - t.Fatalf("fetcher queued stale block") - } - // Ensure that a block with a higher number than the threshold is discarded - tester.fetcher.Enqueue("higher", blocks[hashes[high]]) - time.Sleep(10 * time.Millisecond) - if !tester.fetcher.queue.Empty() { - t.Fatalf("fetcher queued future block") - } -} - -// Tests that announcements with numbers much lower or higher than out current -// head get discarded to prevent wasting resources on useless blocks from faulty -// peers. -func TestFullDistantAnnouncementDiscarding(t *testing.T) { testDistantAnnouncementDiscarding(t, false) } -func TestLightDistantAnnouncementDiscarding(t *testing.T) { testDistantAnnouncementDiscarding(t, true) } - -func testDistantAnnouncementDiscarding(t *testing.T, light bool) { - // Create a long chain to import and define the discard boundaries - hashes, blocks := makeChain(3*maxQueueDist, 0, genesis) - head := hashes[len(hashes)/2] - - low, high := len(hashes)/2+maxUncleDist+1, len(hashes)/2-maxQueueDist-1 - - // Create a tester and simulate a head block being the middle of the above chain - tester := newTester(light) - - tester.lock.Lock() - tester.hashes = []common.Hash{head} - tester.headers = map[common.Hash]*types.Header{head: blocks[head].Header()} - tester.blocks = map[common.Hash]*types.Block{head: blocks[head]} - tester.lock.Unlock() - - headerFetcher := tester.makeHeaderFetcher("lower", blocks, -gatherSlack) - bodyFetcher := tester.makeBodyFetcher("lower", blocks, 0) - - fetching := make(chan struct{}, 2) - tester.fetcher.fetchingHook = func(hashes []common.Hash) { fetching <- struct{}{} } - - // Ensure that a block with a lower number than the threshold is discarded - tester.fetcher.Notify("lower", hashes[low], blocks[hashes[low]].NumberU64(), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) - select { - case <-time.After(50 * time.Millisecond): - case <-fetching: - t.Fatalf("fetcher requested stale header") - } - // Ensure that a block with a higher number than the threshold is discarded - tester.fetcher.Notify("higher", hashes[high], blocks[hashes[high]].NumberU64(), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) - select { - case <-time.After(50 * time.Millisecond): - case <-fetching: - t.Fatalf("fetcher requested future header") - } -} - -// Tests that peers announcing blocks with invalid numbers (i.e. not matching -// the headers provided afterwards) get dropped as malicious. -func TestFullInvalidNumberAnnouncement(t *testing.T) { testInvalidNumberAnnouncement(t, false) } -func TestLightInvalidNumberAnnouncement(t *testing.T) { testInvalidNumberAnnouncement(t, true) } - -func testInvalidNumberAnnouncement(t *testing.T, light bool) { - // Create a single block to import and check numbers against - hashes, blocks := makeChain(1, 0, genesis) - - tester := newTester(light) - badHeaderFetcher := tester.makeHeaderFetcher("bad", blocks, -gatherSlack) - badBodyFetcher := tester.makeBodyFetcher("bad", blocks, 0) - - imported := make(chan interface{}) - announced := make(chan interface{}) - tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { - if light { - if header == nil { - t.Fatalf("Fetcher try to import empty header") - } - imported <- header - } else { - if block == nil { - t.Fatalf("Fetcher try to import empty block") - } - imported <- block - } - } - // Announce a block with a bad number, check for immediate drop - tester.fetcher.announceChangeHook = func(hash common.Hash, b bool) { - announced <- nil - } - tester.fetcher.Notify("bad", hashes[0], 2, time.Now().Add(-arriveTimeout), badHeaderFetcher, badBodyFetcher) - verifyAnnounce := func() { - for i := 0; i < 2; i++ { - select { - case <-announced: - continue - case <-time.After(1 * time.Second): - t.Fatal("announce timeout") - return - } - } - } - verifyAnnounce() - verifyImportEvent(t, imported, false) - tester.lock.RLock() - dropped := tester.drops["bad"] - tester.lock.RUnlock() - - if !dropped { - t.Fatalf("peer with invalid numbered announcement not dropped") - } - goodHeaderFetcher := tester.makeHeaderFetcher("good", blocks, -gatherSlack) - goodBodyFetcher := tester.makeBodyFetcher("good", blocks, 0) - // Make sure a good announcement passes without a drop - tester.fetcher.Notify("good", hashes[0], 1, time.Now().Add(-arriveTimeout), goodHeaderFetcher, goodBodyFetcher) - verifyAnnounce() - verifyImportEvent(t, imported, true) - - tester.lock.RLock() - dropped = tester.drops["good"] - tester.lock.RUnlock() - - if dropped { - t.Fatalf("peer with valid numbered announcement dropped") - } - verifyImportDone(t, imported) -} - -// Tests that if a block is empty (i.e. header only), no body request should be -// made, and instead the header should be assembled into a whole block in itself. -func TestEmptyBlockShortCircuit(t *testing.T) { - // Create a chain of blocks to import - hashes, blocks := makeChain(32, 0, genesis) - - tester := newTester(false) - headerFetcher := tester.makeHeaderFetcher("valid", blocks, -gatherSlack) - bodyFetcher := tester.makeBodyFetcher("valid", blocks, 0) - - // Add a monitoring hook for all internal events - fetching := make(chan []common.Hash) - tester.fetcher.fetchingHook = func(hashes []common.Hash) { fetching <- hashes } - - completing := make(chan []common.Hash) - tester.fetcher.completingHook = func(hashes []common.Hash) { completing <- hashes } - - imported := make(chan interface{}) - tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { - if block == nil { - t.Fatalf("Fetcher try to import empty block") - } - imported <- block - } - // Iteratively announce blocks until all are imported - for i := len(hashes) - 2; i >= 0; i-- { - tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) - - // All announces should fetch the header - verifyFetchingEvent(t, fetching, true) - - // Only blocks with data contents should request bodies - verifyCompletingEvent(t, completing, len(blocks[hashes[i]].Transactions()) > 0 || len(blocks[hashes[i]].Uncles()) > 0) - - // Irrelevant of the construct, import should succeed - verifyImportEvent(t, imported, true) - } - verifyImportDone(t, imported) -} - -// Tests that a peer is unable to use unbounded memory with sending infinite -// block announcements to a node, but that even in the face of such an attack, -// the fetcher remains operational. -func TestHashMemoryExhaustionAttack(t *testing.T) { - // Create a tester with instrumented import hooks - tester := newTester(false) - - imported, announces := make(chan interface{}), int32(0) - tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { imported <- block } - tester.fetcher.announceChangeHook = func(hash common.Hash, added bool) { - if added { - atomic.AddInt32(&announces, 1) - } else { - atomic.AddInt32(&announces, -1) - } - } - // Create a valid chain and an infinite junk chain - targetBlocks := hashLimit + 2*maxQueueDist - hashes, blocks := makeChain(targetBlocks, 0, genesis) - validHeaderFetcher := tester.makeHeaderFetcher("valid", blocks, -gatherSlack) - validBodyFetcher := tester.makeBodyFetcher("valid", blocks, 0) - - attack, _ := makeChain(targetBlocks, 0, unknownBlock) - attackerHeaderFetcher := tester.makeHeaderFetcher("attacker", nil, -gatherSlack) - attackerBodyFetcher := tester.makeBodyFetcher("attacker", nil, 0) - - // Feed the tester a huge hashset from the attacker, and a limited from the valid peer - for i := 0; i < len(attack); i++ { - if i < maxQueueDist { - tester.fetcher.Notify("valid", hashes[len(hashes)-2-i], uint64(i+1), time.Now(), validHeaderFetcher, validBodyFetcher) - } - tester.fetcher.Notify("attacker", attack[i], 1 /* don't distance drop */, time.Now(), attackerHeaderFetcher, attackerBodyFetcher) - } - if count := atomic.LoadInt32(&announces); count != hashLimit+maxQueueDist { - t.Fatalf("queued announce count mismatch: have %d, want %d", count, hashLimit+maxQueueDist) - } - // Wait for fetches to complete - verifyImportCount(t, imported, maxQueueDist) - - // Feed the remaining valid hashes to ensure DOS protection state remains clean - for i := len(hashes) - maxQueueDist - 2; i >= 0; i-- { - tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), validHeaderFetcher, validBodyFetcher) - verifyImportEvent(t, imported, true) - } - verifyImportDone(t, imported) -} - -// Tests that blocks sent to the fetcher (either through propagation or via hash -// announces and retrievals) don't pile up indefinitely, exhausting available -// system memory. -func TestBlockMemoryExhaustionAttack(t *testing.T) { - // Create a tester with instrumented import hooks - tester := newTester(false) - - imported, enqueued := make(chan interface{}), int32(0) - tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { imported <- block } - tester.fetcher.queueChangeHook = func(hash common.Hash, added bool) { - if added { - atomic.AddInt32(&enqueued, 1) - } else { - atomic.AddInt32(&enqueued, -1) - } - } - // Create a valid chain and a batch of dangling (but in range) blocks - targetBlocks := hashLimit + 2*maxQueueDist - hashes, blocks := makeChain(targetBlocks, 0, genesis) - attack := make(map[common.Hash]*types.Block) - for i := byte(0); len(attack) < blockLimit+2*maxQueueDist; i++ { - hashes, blocks := makeChain(maxQueueDist-1, i, unknownBlock) - for _, hash := range hashes[:maxQueueDist-2] { - attack[hash] = blocks[hash] - } - } - // Try to feed all the attacker blocks make sure only a limited batch is accepted - for _, block := range attack { - tester.fetcher.Enqueue("attacker", block) - } - time.Sleep(200 * time.Millisecond) - if queued := atomic.LoadInt32(&enqueued); queued != blockLimit { - t.Fatalf("queued block count mismatch: have %d, want %d", queued, blockLimit) - } - // Queue up a batch of valid blocks, and check that a new peer is allowed to do so - for i := 0; i < maxQueueDist-1; i++ { - tester.fetcher.Enqueue("valid", blocks[hashes[len(hashes)-3-i]]) - } - time.Sleep(100 * time.Millisecond) - if queued := atomic.LoadInt32(&enqueued); queued != blockLimit+maxQueueDist-1 { - t.Fatalf("queued block count mismatch: have %d, want %d", queued, blockLimit+maxQueueDist-1) - } - // Insert the missing piece (and sanity check the import) - tester.fetcher.Enqueue("valid", blocks[hashes[len(hashes)-2]]) - verifyImportCount(t, imported, maxQueueDist) - - // Insert the remaining blocks in chunks to ensure clean DOS protection - for i := maxQueueDist; i < len(hashes)-1; i++ { - tester.fetcher.Enqueue("valid", blocks[hashes[len(hashes)-2-i]]) - verifyImportEvent(t, imported, true) - } - verifyImportDone(t, imported) -} diff --git a/les/fetcher_test.go b/les/fetcher_test.go deleted file mode 100644 index 7c523f797e..0000000000 --- a/les/fetcher_test.go +++ /dev/null @@ -1,189 +0,0 @@ -// Copyright 2019 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package les - -import ( - "math/big" - "testing" - "time" - - "github.com/ethereum/go-ethereum/consensus/ethash" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/params" -) - -// verifyImportEvent verifies that one single event arrive on an import channel. -func verifyImportEvent(t *testing.T, imported chan interface{}, arrive bool) { - if arrive { - select { - case <-imported: - case <-time.After(time.Second): - t.Fatalf("import timeout") - } - } else { - select { - case <-imported: - t.Fatalf("import invoked") - case <-time.After(20 * time.Millisecond): - } - } -} - -// verifyImportDone verifies that no more events are arriving on an import channel. -func verifyImportDone(t *testing.T, imported chan interface{}) { - select { - case <-imported: - t.Fatalf("extra block imported") - case <-time.After(50 * time.Millisecond): - } -} - -// verifyChainHeight verifies the chain height is as expected. -func verifyChainHeight(t *testing.T, fetcher *lightFetcher, height uint64) { - local := fetcher.chain.CurrentHeader().Number.Uint64() - if local != height { - t.Fatalf("chain height mismatch, got %d, want %d", local, height) - } -} - -func TestSequentialAnnouncementsLes2(t *testing.T) { testSequentialAnnouncements(t, 2) } -func TestSequentialAnnouncementsLes3(t *testing.T) { testSequentialAnnouncements(t, 3) } - -func testSequentialAnnouncements(t *testing.T, protocol int) { - netconfig := testnetConfig{ - blocks: 4, - protocol: protocol, - nopruning: true, - } - s, c, teardown := newClientServerEnv(t, netconfig) - defer teardown() - - // Create connected peer pair, the initial signal from LES server - // is discarded to prevent syncing. - p1, _, err := newTestPeerPair("peer", protocol, s.handler, c.handler, true) - if err != nil { - t.Fatalf("Failed to create peer pair %v", err) - } - importCh := make(chan interface{}) - c.handler.fetcher.newHeadHook = func(header *types.Header) { - importCh <- header - } - for i := uint64(1); i <= s.backend.Blockchain().CurrentHeader().Number.Uint64(); i++ { - header := s.backend.Blockchain().GetHeaderByNumber(i) - hash, number := header.Hash(), header.Number.Uint64() - td := rawdb.ReadTd(s.db, hash, number) - - announce := announceData{hash, number, td, 0, nil} - if p1.cpeer.announceType == announceTypeSigned { - announce.sign(s.handler.server.privateKey) - } - p1.cpeer.sendAnnounce(announce) - verifyImportEvent(t, importCh, true) - } - verifyImportDone(t, importCh) - verifyChainHeight(t, c.handler.fetcher, 4) -} - -func TestGappedAnnouncementsLes2(t *testing.T) { testGappedAnnouncements(t, 2) } -func TestGappedAnnouncementsLes3(t *testing.T) { testGappedAnnouncements(t, 3) } - -func testGappedAnnouncements(t *testing.T, protocol int) { - netconfig := testnetConfig{ - blocks: 4, - protocol: protocol, - nopruning: true, - } - s, c, teardown := newClientServerEnv(t, netconfig) - defer teardown() - - // Create connected peer pair, the initial signal from LES server - // is discarded to prevent syncing. - peer, _, err := newTestPeerPair("peer", protocol, s.handler, c.handler, true) - if err != nil { - t.Fatalf("Failed to create peer pair %v", err) - } - done := make(chan *types.Header, 1) - c.handler.fetcher.newHeadHook = func(header *types.Header) { done <- header } - - // Prepare announcement by latest header. - latest := s.backend.Blockchain().CurrentHeader() - hash, number := latest.Hash(), latest.Number.Uint64() - td := rawdb.ReadTd(s.db, hash, number) - - // Sign the announcement if necessary. - announce := announceData{hash, number, td, 0, nil} - if peer.cpeer.announceType == announceTypeSigned { - announce.sign(s.handler.server.privateKey) - } - peer.cpeer.sendAnnounce(announce) - - <-done // Wait syncing - verifyChainHeight(t, c.handler.fetcher, 4) - - // Send a reorged announcement - blocks, _ := core.GenerateChain(rawdb.ReadChainConfig(s.db, s.backend.Blockchain().Genesis().Hash()), s.backend.Blockchain().GetBlockByNumber(3), - ethash.NewFaker(), s.db, 2, func(i int, gen *core.BlockGen) { - gen.OffsetTime(-9) // higher block difficulty - }) - s.backend.Blockchain().InsertChain(blocks) - - <-done // Wait syncing - verifyChainHeight(t, c.handler.fetcher, 5) -} - -func TestInvalidAnnouncesLES2(t *testing.T) { testInvalidAnnounces(t, lpv2) } -func TestInvalidAnnouncesLES3(t *testing.T) { testInvalidAnnounces(t, lpv3) } -func TestInvalidAnnouncesLES4(t *testing.T) { testInvalidAnnounces(t, lpv4) } - -func testInvalidAnnounces(t *testing.T, protocol int) { - netconfig := testnetConfig{ - blocks: 4, - protocol: protocol, - nopruning: true, - } - s, c, teardown := newClientServerEnv(t, netconfig) - defer teardown() - - // Create connected peer pair, the initial signal from LES server - // is discarded to prevent syncing. - peer, _, err := newTestPeerPair("peer", lpv3, s.handler, c.handler, true) - if err != nil { - t.Fatalf("Failed to create peer pair %v", err) - } - done := make(chan *types.Header, 1) - c.handler.fetcher.newHeadHook = func(header *types.Header) { done <- header } - - // Prepare announcement by latest header. - headerOne := s.backend.Blockchain().GetHeaderByNumber(1) - hash, number := headerOne.Hash(), headerOne.Number.Uint64() - td := big.NewInt(params.GenesisDifficulty.Int64() + 200) // bad td - - // Sign the announcement if necessary. - announce := announceData{hash, number, td, 0, nil} - if peer.cpeer.announceType == announceTypeSigned { - announce.sign(s.handler.server.privateKey) - } - peer.cpeer.sendAnnounce(announce) - <-done // Wait syncing - - // Ensure the bad peer is evicted - if c.handler.backend.peers.len() != 0 { - t.Fatalf("Failed to evict invalid peer") - } -} diff --git a/les/handler_test.go b/les/handler_test.go index 31681fe368..81a34a25e8 100644 --- a/les/handler_test.go +++ b/les/handler_test.go @@ -31,7 +31,7 @@ import ( "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/les/downloader" + "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/params" diff --git a/les/odr_test.go b/les/odr_test.go index 2db9acd433..853b4ab565 100644 --- a/les/odr_test.go +++ b/les/odr_test.go @@ -16,6 +16,10 @@ package les +// Note: these tests are disabled now because they cannot work with the old sync +// mechanism removed but will be useful again once the PoS ultralight mode is implemented + +/* import ( "bytes" "context" @@ -451,3 +455,4 @@ func randomHash() common.Hash { } return hash } +*/ diff --git a/les/peer.go b/les/peer.go index 909d912e94..48381689ef 100644 --- a/les/peer.go +++ b/les/peer.go @@ -998,9 +998,6 @@ func (p *clientPeer) Handshake(td *big.Int, head common.Hash, headNum uint64, ge recentTx -= blockSafetyMargin - txIndexRecentOffset } } - if server.config.UltraLightOnlyAnnounce { - recentTx = txIndexDisabled - } if recentTx != txIndexUnlimited && p.version < lpv4 { return errors.New("Cannot serve old clients without a complete tx index") } @@ -1009,20 +1006,18 @@ func (p *clientPeer) Handshake(td *big.Int, head common.Hash, headNum uint64, ge p.headInfo = blockInfo{Hash: head, Number: headNum, Td: td} return p.handshake(td, head, headNum, genesis, forkID, forkFilter, func(lists *keyValueList) { // Add some information which services server can offer. - if !server.config.UltraLightOnlyAnnounce { - *lists = (*lists).add("serveHeaders", nil) - *lists = (*lists).add("serveChainSince", uint64(0)) - *lists = (*lists).add("serveStateSince", uint64(0)) + *lists = (*lists).add("serveHeaders", nil) + *lists = (*lists).add("serveChainSince", uint64(0)) + *lists = (*lists).add("serveStateSince", uint64(0)) - // If local ethereum node is running in archive mode, advertise ourselves we have - // all version state data. Otherwise only recent state is available. - stateRecent := uint64(core.TriesInMemory - blockSafetyMargin) - if server.archiveMode { - stateRecent = 0 - } - *lists = (*lists).add("serveRecentState", stateRecent) - *lists = (*lists).add("txRelay", nil) + // If local ethereum node is running in archive mode, advertise ourselves we have + // all version state data. Otherwise only recent state is available. + stateRecent := uint64(core.TriesInMemory - blockSafetyMargin) + if server.archiveMode { + stateRecent = 0 } + *lists = (*lists).add("serveRecentState", stateRecent) + *lists = (*lists).add("txRelay", nil) if p.version >= lpv4 { *lists = (*lists).add("recentTxLookup", recentTx) } diff --git a/les/pruner.go b/les/pruner.go deleted file mode 100644 index d115a61a70..0000000000 --- a/les/pruner.go +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright 2020 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package les - -import ( - "sync" - "time" - - "github.com/ethereum/go-ethereum/common/math" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/log" -) - -// pruner is responsible for pruning historical light chain data. -type pruner struct { - db ethdb.Database - indexers []*core.ChainIndexer - closeCh chan struct{} - wg sync.WaitGroup -} - -// newPruner returns a light chain pruner instance. -func newPruner(db ethdb.Database, indexers ...*core.ChainIndexer) *pruner { - pruner := &pruner{ - db: db, - indexers: indexers, - closeCh: make(chan struct{}), - } - pruner.wg.Add(1) - go pruner.loop() - return pruner -} - -// close notifies all background goroutines belonging to pruner to exit. -func (p *pruner) close() { - close(p.closeCh) - p.wg.Wait() -} - -// loop periodically queries the status of chain indexers and prunes useless -// historical chain data. Notably, whenever Geth restarts, it will iterate -// all historical sections even they don't exist at all(below checkpoint) so -// that light client can prune cached chain data that was ODRed after pruning -// that section. -func (p *pruner) loop() { - defer p.wg.Done() - - // cleanTicker is the ticker used to trigger a history clean 2 times a day. - var cleanTicker = time.NewTicker(12 * time.Hour) - defer cleanTicker.Stop() - - // pruning finds the sections that have been processed by all indexers - // and deletes all historical chain data. - // Note, if some indexers don't support pruning(e.g. eth.BloomIndexer), - // pruning operations can be silently ignored. - pruning := func() { - min := uint64(math.MaxUint64) - for _, indexer := range p.indexers { - sections, _, _ := indexer.Sections() - if sections < min { - min = sections - } - } - // Always keep the latest section data in database. - if min < 2 || len(p.indexers) == 0 { - return - } - for _, indexer := range p.indexers { - if err := indexer.Prune(min - 2); err != nil { - log.Debug("Failed to prune historical data", "err", err) - return - } - } - p.db.Compact(nil, nil) // Compact entire database, ensure all removed data are deleted. - } - for { - pruning() - select { - case <-cleanTicker.C: - case <-p.closeCh: - return - } - } -} diff --git a/les/pruner_test.go b/les/pruner_test.go deleted file mode 100644 index 1672414937..0000000000 --- a/les/pruner_test.go +++ /dev/null @@ -1,204 +0,0 @@ -// Copyright 2020 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package les - -import ( - "bytes" - "context" - "encoding/binary" - "testing" - "time" - - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/light" -) - -func TestLightPruner(t *testing.T) { - var ( - waitIndexers = func(cIndexer, bIndexer, btIndexer *core.ChainIndexer) { - for { - cs, _, _ := cIndexer.Sections() - bts, _, _ := btIndexer.Sections() - if cs >= 3 && bts >= 3 { - break - } - time.Sleep(10 * time.Millisecond) - } - } - config = light.TestClientIndexerConfig - netconfig = testnetConfig{ - blocks: int(3*config.ChtSize + config.ChtConfirms), - protocol: 3, - indexFn: waitIndexers, - connect: true, - } - ) - server, client, tearDown := newClientServerEnv(t, netconfig) - defer tearDown() - - // checkDB iterates the chain with given prefix, resolves the block number - // with given callback and ensures this entry should exist or not. - checkDB := func(from, to uint64, prefix []byte, resolve func(key, value []byte) *uint64, exist bool) bool { - it := client.db.NewIterator(prefix, nil) - defer it.Release() - - var next = from - for it.Next() { - number := resolve(it.Key(), it.Value()) - if number == nil || *number < from { - continue - } else if *number > to { - return true - } - if exist { - if *number != next { - return false - } - next++ - } else { - return false - } - } - return true - } - // checkPruned checks and ensures the stale chain data has been pruned. - checkPruned := func(from, to uint64) { - // Iterate canonical hash - if !checkDB(from, to, []byte("h"), func(key, value []byte) *uint64 { - if len(key) == 1+8+1 && bytes.Equal(key[9:10], []byte("n")) { - n := binary.BigEndian.Uint64(key[1:9]) - return &n - } - return nil - }, false) { - t.Fatalf("canonical hash mappings are not properly pruned") - } - // Iterate header - if !checkDB(from, to, []byte("h"), func(key, value []byte) *uint64 { - if len(key) == 1+8+32 { - n := binary.BigEndian.Uint64(key[1:9]) - return &n - } - return nil - }, false) { - t.Fatalf("headers are not properly pruned") - } - // Iterate body - if !checkDB(from, to, []byte("b"), func(key, value []byte) *uint64 { - if len(key) == 1+8+32 { - n := binary.BigEndian.Uint64(key[1:9]) - return &n - } - return nil - }, false) { - t.Fatalf("block bodies are not properly pruned") - } - // Iterate receipts - if !checkDB(from, to, []byte("r"), func(key, value []byte) *uint64 { - if len(key) == 1+8+32 { - n := binary.BigEndian.Uint64(key[1:9]) - return &n - } - return nil - }, false) { - t.Fatalf("receipts are not properly pruned") - } - // Iterate td - if !checkDB(from, to, []byte("h"), func(key, value []byte) *uint64 { - if len(key) == 1+8+32+1 && bytes.Equal(key[41:42], []byte("t")) { - n := binary.BigEndian.Uint64(key[1:9]) - return &n - } - return nil - }, false) { - t.Fatalf("tds are not properly pruned") - } - } - // Start light pruner. - time.Sleep(1500 * time.Millisecond) // Ensure light client has finished the syncing and indexing - newPruner(client.db, client.chtIndexer, client.bloomTrieIndexer) - - time.Sleep(1500 * time.Millisecond) // Ensure pruner have enough time to prune data. - checkPruned(1, config.ChtSize-1) - - // Ensure all APIs still work after pruning. - var cases = []struct { - from, to uint64 - methodName string - method func(uint64) bool - }{ - { - 1, 10, "GetHeaderByNumber", - func(n uint64) bool { - _, err := light.GetHeaderByNumber(context.Background(), client.handler.backend.odr, n) - return err == nil - }, - }, - { - 11, 20, "GetCanonicalHash", - func(n uint64) bool { - _, err := light.GetCanonicalHash(context.Background(), client.handler.backend.odr, n) - return err == nil - }, - }, - { - 21, 30, "GetTd", - func(n uint64) bool { - _, err := light.GetTd(context.Background(), client.handler.backend.odr, server.handler.blockchain.GetHeaderByNumber(n).Hash(), n) - return err == nil - }, - }, - { - 31, 40, "GetBodyRLP", - func(n uint64) bool { - _, err := light.GetBodyRLP(context.Background(), client.handler.backend.odr, server.handler.blockchain.GetHeaderByNumber(n).Hash(), n) - return err == nil - }, - }, - { - 41, 50, "GetBlock", - func(n uint64) bool { - _, err := light.GetBlock(context.Background(), client.handler.backend.odr, server.handler.blockchain.GetHeaderByNumber(n).Hash(), n) - return err == nil - }, - }, - { - 51, 60, "GetBlockReceipts", - func(n uint64) bool { - _, err := light.GetBlockReceipts(context.Background(), client.handler.backend.odr, server.handler.blockchain.GetHeaderByNumber(n).Hash(), n) - return err == nil - }, - }, - } - for _, c := range cases { - for i := c.from; i <= c.to; i++ { - if !c.method(i) { - t.Fatalf("rpc method %s failed, number %d", c.methodName, i) - } - } - } - // Check GetBloombits - _, err := light.GetBloomBits(context.Background(), client.handler.backend.odr, 0, []uint64{0}) - if err != nil { - t.Fatalf("Failed to retrieve bloombits of pruned section: %v", err) - } - - // Ensure the ODR cached data can be cleaned by pruner. - newPruner(client.db, client.chtIndexer, client.bloomTrieIndexer) - time.Sleep(50 * time.Millisecond) // Ensure pruner have enough time to prune data. - checkPruned(1, config.ChtSize-1) // Ensure all cached data(by odr) is cleaned. -} diff --git a/les/request_test.go b/les/request_test.go index 30ae2c2bee..5e354b7efd 100644 --- a/les/request_test.go +++ b/les/request_test.go @@ -16,6 +16,10 @@ package les +// Note: these tests are disabled now because they cannot work with the old sync +// mechanism removed but will be useful again once the PoS ultralight mode is implemented + +/* import ( "context" "testing" @@ -122,3 +126,4 @@ func testAccess(t *testing.T, protocol int, fn accessTestFn) { } test(5) } +*/ diff --git a/les/retrieve.go b/les/retrieve.go index 2b9e239e9a..728f960a54 100644 --- a/les/retrieve.go +++ b/les/retrieve.go @@ -153,15 +153,6 @@ func (rm *retrieveManager) sendReq(reqID uint64, req *distReq, val validatorFunc return r } -// requested reports whether the request with given reqid is sent by the retriever. -func (rm *retrieveManager) requested(reqId uint64) bool { - rm.lock.RLock() - defer rm.lock.RUnlock() - - _, ok := rm.sentReqs[reqId] - return ok -} - // deliver is called by the LES protocol manager to deliver reply messages to waiting requests func (rm *retrieveManager) deliver(peer distPeer, msg *Msg) error { rm.lock.RLock() diff --git a/les/sync.go b/les/sync.go deleted file mode 100644 index bd06077aae..0000000000 --- a/les/sync.go +++ /dev/null @@ -1,56 +0,0 @@ -// Copyright 2016 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package les - -import ( - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/les/downloader" - "github.com/ethereum/go-ethereum/log" -) - -// synchronise tries to sync up our local chain with a remote peer. -func (h *clientHandler) synchronise(peer *serverPeer) { - // Short circuit if the peer is nil. - if peer == nil { - return - } - // Make sure the peer's TD is higher than our own. - latest := h.backend.blockchain.CurrentHeader() - currentTd := rawdb.ReadTd(h.backend.chainDb, latest.Hash(), latest.Number.Uint64()) - if currentTd != nil && peer.Td().Cmp(currentTd) < 0 { - return - } - // Notify testing framework if syncing has completed (for testing purpose). - defer func() { - if h.syncEnd != nil { - h.syncEnd(h.backend.blockchain.CurrentHeader()) - } - }() - start := time.Now() - if h.syncStart != nil { - h.syncStart(h.backend.blockchain.CurrentHeader()) - } - // Fetch the remaining block headers based on the current chain header. - if err := h.downloader.Synchronise(peer.id, peer.Head(), peer.Td(), downloader.LightSync); err != nil { - log.Debug("Synchronise failed", "reason", err) - return - } - log.Debug("Synchronise finished", "elapsed", common.PrettyDuration(time.Since(start))) -} diff --git a/les/sync_test.go b/les/sync_test.go deleted file mode 100644 index 356a88ffbe..0000000000 --- a/les/sync_test.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2019 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package les - -import ( - "fmt" - "testing" - "time" - - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/light" -) - -// Test light syncing which will download all headers from genesis. -func TestLightSyncingLes3(t *testing.T) { testSyncing(t, lpv3) } - -func testSyncing(t *testing.T, protocol int) { - config := light.TestServerIndexerConfig - - waitIndexers := func(cIndexer, bIndexer, btIndexer *core.ChainIndexer) { - for { - cs, _, _ := cIndexer.Sections() - bts, _, _ := btIndexer.Sections() - if cs >= 1 && bts >= 1 { - break - } - time.Sleep(10 * time.Millisecond) - } - } - // Generate 128+1 blocks (totally 1 CHT section) - netconfig := testnetConfig{ - blocks: int(config.ChtSize + config.ChtConfirms), - protocol: protocol, - indexFn: waitIndexers, - nopruning: true, - } - server, client, tearDown := newClientServerEnv(t, netconfig) - defer tearDown() - - expected := config.ChtSize + config.ChtConfirms - - done := make(chan error) - client.handler.syncEnd = func(header *types.Header) { - if header.Number.Uint64() == expected { - done <- nil - } else { - done <- fmt.Errorf("blockchain length mismatch, want %d, got %d", expected, header.Number) - } - } - - // Create connected peer pair. - peer1, peer2, err := newTestPeerPair("peer", protocol, server.handler, client.handler, false) - if err != nil { - t.Fatalf("Failed to connect testing peers %v", err) - } - defer peer1.close() - defer peer2.close() - - select { - case err := <-done: - if err != nil { - t.Error("sync failed", err) - } - return - case <-time.NewTimer(10 * time.Second).C: - t.Error("checkpoint syncing timeout") - } -} diff --git a/les/test_helper.go b/les/test_helper.go index 057903557a..b03bca14bf 100644 --- a/les/test_helper.go +++ b/les/test_helper.go @@ -177,7 +177,7 @@ func testIndexers(db ethdb.Database, odr light.OdrBackend, config *light.Indexer return indexers[:] } -func newTestClientHandler(backend *backends.SimulatedBackend, odr *LesOdr, indexers []*core.ChainIndexer, db ethdb.Database, peers *serverPeerSet, ulcServers []string, ulcFraction int) (*clientHandler, func()) { +func newTestClientHandler(backend *backends.SimulatedBackend, odr *LesOdr, indexers []*core.ChainIndexer, db ethdb.Database, peers *serverPeerSet) (*clientHandler, func()) { var ( evmux = new(event.TypeMux) engine = ethash.NewFaker() @@ -210,9 +210,8 @@ func newTestClientHandler(backend *backends.SimulatedBackend, odr *LesOdr, index eventMux: evmux, merger: consensus.NewMerger(rawdb.NewMemoryDatabase()), } - client.handler = newClientHandler(ulcServers, ulcFraction, client) + client.handler = newClientHandler(client) - client.handler.start() return client.handler, func() { client.handler.stop() } @@ -307,7 +306,8 @@ func (p *testPeer) handshakeWithServer(t *testing.T, td *big.Int, head common.Ha } // handshakeWithClient executes the handshake with the remote client peer. -func (p *testPeer) handshakeWithClient(t *testing.T, td *big.Int, head common.Hash, headNum uint64, genesis common.Hash, forkID forkid.ID, costList RequestCostList, recentTxLookup uint64) { +// (used by temporarily disabled tests) +/*func (p *testPeer) handshakeWithClient(t *testing.T, td *big.Int, head common.Hash, headNum uint64, genesis common.Hash, forkID forkid.ID, costList RequestCostList, recentTxLookup uint64) { // It only works for the simulated client peer if p.speer == nil { t.Fatal("handshake for server peer only") @@ -337,7 +337,7 @@ func (p *testPeer) handshakeWithClient(t *testing.T, td *big.Int, head common.Ha if err := p2p.Send(p.app, StatusMsg, &sendList); err != nil { t.Fatalf("status send: %v", err) } -} +}*/ // close terminates the local side of the peer, notifying the remote protocol // manager of termination. @@ -405,7 +405,8 @@ type testClient struct { } // newRawPeer creates a new server peer connects to the server and do the handshake. -func (client *testClient) newRawPeer(t *testing.T, name string, version int, recentTxLookup uint64) (*testPeer, func(), <-chan error) { +// (used by temporarily disabled tests) +/*func (client *testClient) newRawPeer(t *testing.T, name string, version int, recentTxLookup uint64) (*testPeer, func(), <-chan error) { // Create a message pipe to communicate through app, net := p2p.MsgPipe() @@ -453,7 +454,7 @@ func (client *testClient) newRawPeer(t *testing.T, name string, version int, rec tp.close() } return tp, closePeer, errCh -} +}*/ // testServer represents a server object for testing with necessary auxiliary fields. type testServer struct { @@ -521,14 +522,12 @@ func (server *testServer) newRawPeer(t *testing.T, name string, version int) (*t // testnetConfig wraps all the configurations for testing network. type testnetConfig struct { - blocks int - protocol int - indexFn indexerCallback - ulcServers []string - ulcFraction int - simClock bool - connect bool - nopruning bool + blocks int + protocol int + indexFn indexerCallback + simClock bool + connect bool + nopruning bool } func newClientServerEnv(t *testing.T, config testnetConfig) (*testServer, *testClient, func()) { @@ -553,7 +552,7 @@ func newClientServerEnv(t *testing.T, config testnetConfig) (*testServer, *testC odr.SetIndexers(ccIndexer, cbIndexer, cbtIndexer) server, b, serverClose := newTestServerHandler(config.blocks, sindexers, sdb, clock) - client, clientClose := newTestClientHandler(b, odr, cIndexers, cdb, speers, config.ulcServers, config.ulcFraction) + client, clientClose := newTestClientHandler(b, odr, cIndexers, cdb, speers) scIndexer.Start(server.blockchain) sbIndexer.Start(server.blockchain) @@ -569,7 +568,6 @@ func newClientServerEnv(t *testing.T, config testnetConfig) (*testServer, *testC ) if config.connect { done := make(chan struct{}) - client.syncEnd = func(_ *types.Header) { close(done) } cpeer, speer, err = newTestPeerPair("peer", config.protocol, server, client, false) if err != nil { t.Fatalf("Failed to connect testing peers %v", err) diff --git a/les/ulc.go b/les/ulc.go deleted file mode 100644 index b97217e796..0000000000 --- a/les/ulc.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2019 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package les - -import ( - "errors" - - "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/p2p/enode" -) - -type ulc struct { - keys map[string]bool - fraction int -} - -// newULC creates and returns an ultra light client instance. -func newULC(servers []string, fraction int) (*ulc, error) { - keys := make(map[string]bool) - for _, id := range servers { - node, err := enode.Parse(enode.ValidSchemes, id) - if err != nil { - log.Warn("Failed to parse trusted server", "id", id, "err", err) - continue - } - keys[node.ID().String()] = true - } - if len(keys) == 0 { - return nil, errors.New("no trusted servers") - } - return &ulc{ - keys: keys, - fraction: fraction, - }, nil -} - -// trusted return an indicator that whether the specified peer is trusted. -func (u *ulc) trusted(p enode.ID) bool { - return u.keys[p.String()] -} diff --git a/les/ulc_test.go b/les/ulc_test.go deleted file mode 100644 index 791bc28853..0000000000 --- a/les/ulc_test.go +++ /dev/null @@ -1,162 +0,0 @@ -// Copyright 2018 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package les - -import ( - "crypto/rand" - "fmt" - "net" - "testing" - "time" - - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/p2p" - "github.com/ethereum/go-ethereum/p2p/enode" -) - -func TestULCAnnounceThresholdLes2(t *testing.T) { testULCAnnounceThreshold(t, 2) } -func TestULCAnnounceThresholdLes3(t *testing.T) { testULCAnnounceThreshold(t, 3) } - -func testULCAnnounceThreshold(t *testing.T, protocol int) { - // todo figure out why it takes fetcher so longer to fetcher the announced header. - t.Skip("Sometimes it can failed") - - // newTestLightPeer creates node with light sync mode - newTestLightPeer := func(t *testing.T, protocol int, ulcServers []string, ulcFraction int) (*testClient, func()) { - netconfig := testnetConfig{ - protocol: protocol, - ulcServers: ulcServers, - ulcFraction: ulcFraction, - nopruning: true, - } - _, c, teardown := newClientServerEnv(t, netconfig) - return c, teardown - } - - var cases = []struct { - height []int - threshold int - expect uint64 - }{ - {[]int{1}, 100, 1}, - {[]int{0, 0, 0}, 100, 0}, - {[]int{1, 2, 3}, 30, 3}, - {[]int{1, 2, 3}, 60, 2}, - {[]int{3, 2, 1}, 67, 1}, - {[]int{3, 2, 1}, 100, 1}, - } - for _, testcase := range cases { - var ( - servers []*testServer - teardowns []func() - nodes []*enode.Node - ids []string - ) - for i := 0; i < len(testcase.height); i++ { - s, n, teardown := newTestServerPeer(t, 0, protocol, nil) - - servers = append(servers, s) - nodes = append(nodes, n) - teardowns = append(teardowns, teardown) - ids = append(ids, n.String()) - } - c, teardown := newTestLightPeer(t, protocol, ids, testcase.threshold) - - // Connect all servers. - for i := 0; i < len(servers); i++ { - connect(servers[i].handler, nodes[i].ID(), c.handler, protocol, false) - } - for i := 0; i < len(servers); i++ { - for j := 0; j < testcase.height[i]; j++ { - servers[i].backend.Commit() - } - } - time.Sleep(1500 * time.Millisecond) // Ensure the fetcher has done its work. - head := c.handler.backend.blockchain.CurrentHeader().Number.Uint64() - if head != testcase.expect { - t.Fatalf("chain height mismatch, want %d, got %d", testcase.expect, head) - } - - // Release all servers and client resources. - teardown() - for i := 0; i < len(teardowns); i++ { - teardowns[i]() - } - } -} - -func connect(server *serverHandler, serverId enode.ID, client *clientHandler, protocol int, noInitAnnounce bool) (*serverPeer, *clientPeer, error) { - // Create a message pipe to communicate through - app, net := p2p.MsgPipe() - - var id enode.ID - rand.Read(id[:]) - - peer1 := newServerPeer(protocol, NetworkId, true, p2p.NewPeer(serverId, "", nil), net) // Mark server as trusted - peer2 := newClientPeer(protocol, NetworkId, p2p.NewPeer(id, "", nil), app) - - // Start the peerLight on a new thread - errc1 := make(chan error, 1) - errc2 := make(chan error, 1) - go func() { - select { - case <-server.closeCh: - errc1 <- p2p.DiscQuitting - case errc1 <- server.handle(peer2): - } - }() - go func() { - select { - case <-client.closeCh: - errc1 <- p2p.DiscQuitting - case errc1 <- client.handle(peer1, noInitAnnounce): - } - }() - // Ensure the connection is established or exits when any error occurs - for { - select { - case err := <-errc1: - return nil, nil, fmt.Errorf("failed to establish protocol connection %v", err) - case err := <-errc2: - return nil, nil, fmt.Errorf("failed to establish protocol connection %v", err) - default: - } - if peer1.serving.Load() && peer2.serving.Load() { - break - } - time.Sleep(50 * time.Millisecond) - } - return peer1, peer2, nil -} - -// newTestServerPeer creates server peer. -func newTestServerPeer(t *testing.T, blocks int, protocol int, indexFn indexerCallback) (*testServer, *enode.Node, func()) { - netconfig := testnetConfig{ - blocks: blocks, - protocol: protocol, - indexFn: indexFn, - nopruning: true, - } - s, _, teardown := newClientServerEnv(t, netconfig) - key, err := crypto.GenerateKey() - if err != nil { - t.Fatal("generate key err:", err) - } - s.handler.server.privateKey = key - n := enode.NewV4(&key.PublicKey, net.ParseIP("127.0.0.1"), 35000, 35000) - return s, n, teardown -} From e86ad526404681f52f67753c843d7a65274721b6 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 18 Jul 2023 09:44:16 +0200 Subject: [PATCH 84/90] beacon/engine, eth/catalyst: EIP-4844 updates for the engine API (#27736) This is a spin-out from the EIP-4844 devnet branch, containing just the Engine API modifications and nothing else. The newPayloadV3 endpoint won't really work in this version, but we need the data structures for testing so I'd like to get this in early. Co-authored-by: Marius van der Wijden --- beacon/engine/gen_ed.go | 12 ++++++ beacon/engine/gen_epe.go | 6 +++ beacon/engine/types.go | 51 +++++++++++++++++++++++--- eth/catalyst/api.go | 63 ++++++++++++++++++++------------ eth/catalyst/api_test.go | 42 +++++++++++++++++++-- eth/catalyst/simulated_beacon.go | 2 +- miner/payload_building.go | 8 ++-- 7 files changed, 146 insertions(+), 38 deletions(-) diff --git a/beacon/engine/gen_ed.go b/beacon/engine/gen_ed.go index 336dfc6cc7..884b7160e8 100644 --- a/beacon/engine/gen_ed.go +++ b/beacon/engine/gen_ed.go @@ -32,6 +32,8 @@ func (e ExecutableData) MarshalJSON() ([]byte, error) { BlockHash common.Hash `json:"blockHash" gencodec:"required"` Transactions []hexutil.Bytes `json:"transactions" gencodec:"required"` Withdrawals []*types.Withdrawal `json:"withdrawals"` + DataGasUsed *hexutil.Uint64 `json:"dataGasUsed"` + ExcessDataGas *hexutil.Uint64 `json:"excessDataGas"` } var enc ExecutableData enc.ParentHash = e.ParentHash @@ -54,6 +56,8 @@ func (e ExecutableData) MarshalJSON() ([]byte, error) { } } enc.Withdrawals = e.Withdrawals + enc.DataGasUsed = (*hexutil.Uint64)(e.DataGasUsed) + enc.ExcessDataGas = (*hexutil.Uint64)(e.ExcessDataGas) return json.Marshal(&enc) } @@ -75,6 +79,8 @@ func (e *ExecutableData) UnmarshalJSON(input []byte) error { BlockHash *common.Hash `json:"blockHash" gencodec:"required"` Transactions []hexutil.Bytes `json:"transactions" gencodec:"required"` Withdrawals []*types.Withdrawal `json:"withdrawals"` + DataGasUsed *hexutil.Uint64 `json:"dataGasUsed"` + ExcessDataGas *hexutil.Uint64 `json:"excessDataGas"` } var dec ExecutableData if err := json.Unmarshal(input, &dec); err != nil { @@ -142,5 +148,11 @@ func (e *ExecutableData) UnmarshalJSON(input []byte) error { if dec.Withdrawals != nil { e.Withdrawals = dec.Withdrawals } + if dec.DataGasUsed != nil { + e.DataGasUsed = (*uint64)(dec.DataGasUsed) + } + if dec.ExcessDataGas != nil { + e.ExcessDataGas = (*uint64)(dec.ExcessDataGas) + } return nil } diff --git a/beacon/engine/gen_epe.go b/beacon/engine/gen_epe.go index cc66cef6cd..664a50fc5b 100644 --- a/beacon/engine/gen_epe.go +++ b/beacon/engine/gen_epe.go @@ -17,10 +17,12 @@ func (e ExecutionPayloadEnvelope) MarshalJSON() ([]byte, error) { type ExecutionPayloadEnvelope struct { ExecutionPayload *ExecutableData `json:"executionPayload" gencodec:"required"` BlockValue *hexutil.Big `json:"blockValue" gencodec:"required"` + BlobsBundle *BlobsBundleV1 `json:"blobsBundle"` } var enc ExecutionPayloadEnvelope enc.ExecutionPayload = e.ExecutionPayload enc.BlockValue = (*hexutil.Big)(e.BlockValue) + enc.BlobsBundle = e.BlobsBundle return json.Marshal(&enc) } @@ -29,6 +31,7 @@ func (e *ExecutionPayloadEnvelope) UnmarshalJSON(input []byte) error { type ExecutionPayloadEnvelope struct { ExecutionPayload *ExecutableData `json:"executionPayload" gencodec:"required"` BlockValue *hexutil.Big `json:"blockValue" gencodec:"required"` + BlobsBundle *BlobsBundleV1 `json:"blobsBundle"` } var dec ExecutionPayloadEnvelope if err := json.Unmarshal(input, &dec); err != nil { @@ -42,5 +45,8 @@ func (e *ExecutionPayloadEnvelope) UnmarshalJSON(input []byte) error { return errors.New("missing required field 'blockValue' for ExecutionPayloadEnvelope") } e.BlockValue = (*big.Int)(dec.BlockValue) + if dec.BlobsBundle != nil { + e.BlobsBundle = dec.BlobsBundle + } return nil } diff --git a/beacon/engine/types.go b/beacon/engine/types.go index 07ebe544b4..d96bece856 100644 --- a/beacon/engine/types.go +++ b/beacon/engine/types.go @@ -23,6 +23,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/ethereum/go-ethereum/trie" ) @@ -61,6 +62,8 @@ type ExecutableData struct { BlockHash common.Hash `json:"blockHash" gencodec:"required"` Transactions [][]byte `json:"transactions" gencodec:"required"` Withdrawals []*types.Withdrawal `json:"withdrawals"` + DataGasUsed *uint64 `json:"dataGasUsed"` + ExcessDataGas *uint64 `json:"excessDataGas"` } // JSON type overrides for executableData. @@ -73,6 +76,8 @@ type executableDataMarshaling struct { ExtraData hexutil.Bytes LogsBloom hexutil.Bytes Transactions []hexutil.Bytes + DataGasUsed *hexutil.Uint64 + ExcessDataGas *hexutil.Uint64 } //go:generate go run github.com/fjl/gencodec -type ExecutionPayloadEnvelope -field-override executionPayloadEnvelopeMarshaling -out gen_epe.go @@ -80,6 +85,13 @@ type executableDataMarshaling struct { type ExecutionPayloadEnvelope struct { ExecutionPayload *ExecutableData `json:"executionPayload" gencodec:"required"` BlockValue *big.Int `json:"blockValue" gencodec:"required"` + BlobsBundle *BlobsBundleV1 `json:"blobsBundle"` +} + +type BlobsBundleV1 struct { + Commitments []hexutil.Bytes `json:"commitments"` + Proofs []hexutil.Bytes `json:"proofs"` + Blobs []hexutil.Bytes `json:"blobs"` } // JSON type overrides for ExecutionPayloadEnvelope. @@ -152,14 +164,15 @@ func decodeTransactions(enc [][]byte) ([]*types.Transaction, error) { // ExecutableDataToBlock constructs a block from executable data. // It verifies that the following fields: // -// len(extraData) <= 32 -// uncleHash = emptyUncleHash -// difficulty = 0 +// len(extraData) <= 32 +// uncleHash = emptyUncleHash +// difficulty = 0 +// if versionedHashes != nil, versionedHashes match to blob transactions // // and that the blockhash of the constructed block matches the parameters. Nil // Withdrawals value will propagate through the returned block. Empty // Withdrawals value must be passed via non-nil, length 0 value in params. -func ExecutableDataToBlock(params ExecutableData) (*types.Block, error) { +func ExecutableDataToBlock(params ExecutableData, versionedHashes []common.Hash) (*types.Block, error) { txs, err := decodeTransactions(params.Transactions) if err != nil { return nil, err @@ -174,6 +187,18 @@ func ExecutableDataToBlock(params ExecutableData) (*types.Block, error) { if params.BaseFeePerGas != nil && (params.BaseFeePerGas.Sign() == -1 || params.BaseFeePerGas.BitLen() > 256) { return nil, fmt.Errorf("invalid baseFeePerGas: %v", params.BaseFeePerGas) } + var blobHashes []common.Hash + for _, tx := range txs { + blobHashes = append(blobHashes, tx.BlobHashes()...) + } + if len(blobHashes) != len(versionedHashes) { + return nil, fmt.Errorf("invalid number of versionedHashes: %v blobHashes: %v", versionedHashes, blobHashes) + } + for i := 0; i < len(blobHashes); i++ { + if blobHashes[i] != versionedHashes[i] { + return nil, fmt.Errorf("invalid versionedHash at %v: %v blobHashes: %v", i, versionedHashes, blobHashes) + } + } // Only set withdrawalsRoot if it is non-nil. This allows CLs to use // ExecutableData before withdrawals are enabled by marshaling // Withdrawals as the json null value. @@ -199,6 +224,8 @@ func ExecutableDataToBlock(params ExecutableData) (*types.Block, error) { Extra: params.ExtraData, MixDigest: params.Random, WithdrawalsHash: withdrawalsRoot, + ExcessDataGas: params.ExcessDataGas, + DataGasUsed: params.DataGasUsed, } block := types.NewBlockWithHeader(header).WithBody(txs, nil /* uncles */).WithWithdrawals(params.Withdrawals) if block.Hash() != params.BlockHash { @@ -209,7 +236,7 @@ func ExecutableDataToBlock(params ExecutableData) (*types.Block, error) { // BlockToExecutableData constructs the ExecutableData structure by filling the // fields from the given block. It assumes the given block is post-merge block. -func BlockToExecutableData(block *types.Block, fees *big.Int) *ExecutionPayloadEnvelope { +func BlockToExecutableData(block *types.Block, fees *big.Int, blobs []kzg4844.Blob, commitments []kzg4844.Commitment, proofs []kzg4844.Proof) *ExecutionPayloadEnvelope { data := &ExecutableData{ BlockHash: block.Hash(), ParentHash: block.ParentHash(), @@ -226,8 +253,20 @@ func BlockToExecutableData(block *types.Block, fees *big.Int) *ExecutionPayloadE Random: block.MixDigest(), ExtraData: block.Extra(), Withdrawals: block.Withdrawals(), + DataGasUsed: block.DataGasUsed(), + ExcessDataGas: block.ExcessDataGas(), } - return &ExecutionPayloadEnvelope{ExecutionPayload: data, BlockValue: fees} + blobsBundle := BlobsBundleV1{ + Commitments: make([]hexutil.Bytes, 0), + Blobs: make([]hexutil.Bytes, 0), + Proofs: make([]hexutil.Bytes, 0), + } + for i := range blobs { + blobsBundle.Blobs = append(blobsBundle.Blobs, hexutil.Bytes(blobs[i][:])) + blobsBundle.Commitments = append(blobsBundle.Commitments, hexutil.Bytes(commitments[i][:])) + blobsBundle.Proofs = append(blobsBundle.Proofs, hexutil.Bytes(proofs[i][:])) + } + return &ExecutionPayloadEnvelope{ExecutionPayload: data, BlockValue: fees, BlobsBundle: &blobsBundle} } // ExecutionPayloadBodyV1 is used in the response to GetPayloadBodiesByHashV1 and GetPayloadBodiesByRangeV1 diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index e4b5533b75..2e3a236e00 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -81,8 +81,10 @@ var caps = []string{ "engine_exchangeTransitionConfigurationV1", "engine_getPayloadV1", "engine_getPayloadV2", + "engine_getPayloadV3", "engine_newPayloadV1", "engine_newPayloadV2", + "engine_newPayloadV3", "engine_getPayloadBodiesByHashV1", "engine_getPayloadBodiesByRangeV1", } @@ -405,23 +407,13 @@ func (api *ConsensusAPI) GetPayloadV2(payloadID engine.PayloadID) (*engine.Execu return api.getPayload(payloadID) } -func (api *ConsensusAPI) getPayload(payloadID engine.PayloadID) (*engine.ExecutionPayloadEnvelope, error) { - log.Trace("Engine API request received", "method", "GetPayload", "id", payloadID) - data := api.localBlocks.get(payloadID, false) - if data == nil { - return nil, engine.UnknownPayload - } - return data, nil +// GetPayloadV3 returns a cached payload by id. +func (api *ConsensusAPI) GetPayloadV3(payloadID engine.PayloadID) (*engine.ExecutionPayloadEnvelope, error) { + return api.getPayload(payloadID) } -// getFullPayload returns a cached payload by it. The difference is that this -// function always expects a non-empty payload, but can also return empty one -// if no transaction is executable. -// -// Note, this function is not a part of standard engine API, meant to be used -// by consensus client mock in dev mode. -func (api *ConsensusAPI) getFullPayload(payloadID engine.PayloadID) (*engine.ExecutionPayloadEnvelope, error) { - log.Trace("Engine API request received", "method", "GetFullPayload", "id", payloadID) +func (api *ConsensusAPI) getPayload(payloadID engine.PayloadID) (*engine.ExecutionPayloadEnvelope, error) { + log.Trace("Engine API request received", "method", "GetPayload", "id", payloadID) data := api.localBlocks.get(payloadID, true) if data == nil { return nil, engine.UnknownPayload @@ -434,7 +426,7 @@ func (api *ConsensusAPI) NewPayloadV1(params engine.ExecutableData) (engine.Payl if params.Withdrawals != nil { return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("withdrawals not supported in V1")) } - return api.newPayload(params) + return api.newPayload(params, nil) } // NewPayloadV2 creates an Eth1 block, inserts it in the chain, and returns the status of the chain. @@ -446,10 +438,29 @@ func (api *ConsensusAPI) NewPayloadV2(params engine.ExecutableData) (engine.Payl } else if params.Withdrawals != nil { return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("non-nil withdrawals pre-shanghai")) } - return api.newPayload(params) + if api.eth.BlockChain().Config().IsCancun(new(big.Int).SetUint64(params.Number), params.Timestamp) { + return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("newPayloadV2 called post-cancun")) + } + return api.newPayload(params, nil) } -func (api *ConsensusAPI) newPayload(params engine.ExecutableData) (engine.PayloadStatusV1, error) { +// NewPayloadV3 creates an Eth1 block, inserts it in the chain, and returns the status of the chain. +func (api *ConsensusAPI) NewPayloadV3(params engine.ExecutableData, versionedHashes *[]common.Hash) (engine.PayloadStatusV1, error) { + if !api.eth.BlockChain().Config().IsCancun(new(big.Int).SetUint64(params.Number), params.Timestamp) { + return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(errors.New("newPayloadV3 called pre-cancun")) + } + + if params.ExcessDataGas == nil { + return engine.PayloadStatusV1{Status: engine.INVALID}, engine.InvalidParams.With(fmt.Errorf("nil excessDataGas post-cancun")) + } + var hashes []common.Hash + if versionedHashes != nil { + hashes = *versionedHashes + } + return api.newPayload(params, hashes) +} + +func (api *ConsensusAPI) newPayload(params engine.ExecutableData, versionedHashes []common.Hash) (engine.PayloadStatusV1, error) { // The locking here is, strictly, not required. Without these locks, this can happen: // // 1. NewPayload( execdata-N ) is invoked from the CL. It goes all the way down to @@ -467,9 +478,9 @@ func (api *ConsensusAPI) newPayload(params engine.ExecutableData) (engine.Payloa defer api.newPayloadLock.Unlock() log.Trace("Engine API request received", "method", "NewPayload", "number", params.Number, "hash", params.BlockHash) - block, err := engine.ExecutableDataToBlock(params) + block, err := engine.ExecutableDataToBlock(params, versionedHashes) if err != nil { - log.Debug("Invalid NewPayload params", "params", params, "error", err) + log.Warn("Invalid NewPayload params", "params", params, "error", err) return engine.PayloadStatusV1{Status: engine.INVALID}, nil } // Stash away the last update to warn the user if the beacon client goes offline @@ -730,8 +741,8 @@ func (api *ConsensusAPI) ExchangeCapabilities([]string) []string { return caps } -// GetPayloadBodiesByHashV1 implements engine_getPayloadBodiesByHashV1 which -// allows for retrieval of a list of block bodies by the engine api. +// GetPayloadBodiesByHashV1 implements engine_getPayloadBodiesByHashV1 which allows for retrieval of a list +// of block bodies by the engine api. func (api *ConsensusAPI) GetPayloadBodiesByHashV1(hashes []common.Hash) []*engine.ExecutionPayloadBodyV1 { var bodies = make([]*engine.ExecutionPayloadBodyV1, len(hashes)) for i, hash := range hashes { @@ -741,8 +752,8 @@ func (api *ConsensusAPI) GetPayloadBodiesByHashV1(hashes []common.Hash) []*engin return bodies } -// GetPayloadBodiesByRangeV1 implements engine_getPayloadBodiesByRangeV1 which -// allows for retrieval of a range of block bodies by the engine api. +// GetPayloadBodiesByRangeV1 implements engine_getPayloadBodiesByRangeV1 which allows for retrieval of a range +// of block bodies by the engine api. func (api *ConsensusAPI) GetPayloadBodiesByRangeV1(start, count hexutil.Uint64) ([]*engine.ExecutionPayloadBodyV1, error) { if start == 0 || count == 0 { return nil, engine.InvalidParams.With(fmt.Errorf("invalid start or count, start: %v count: %v", start, count)) @@ -768,19 +779,23 @@ func getBody(block *types.Block) *engine.ExecutionPayloadBodyV1 { if block == nil { return nil } + var ( body = block.Body() txs = make([]hexutil.Bytes, len(body.Transactions)) withdrawals = body.Withdrawals ) + for j, tx := range body.Transactions { data, _ := tx.MarshalBinary() txs[j] = hexutil.Bytes(data) } + // Post-shanghai withdrawals MUST be set to empty slice instead of nil if withdrawals == nil && block.Header().WithdrawalsHash != nil { withdrawals = make([]*types.Withdrawal, 0) } + return &engine.ExecutionPayloadBodyV1{ TransactionData: txs, Withdrawals: withdrawals, diff --git a/eth/catalyst/api_test.go b/eth/catalyst/api_test.go index 254a510ab4..05ad3def48 100644 --- a/eth/catalyst/api_test.go +++ b/eth/catalyst/api_test.go @@ -38,6 +38,7 @@ import ( "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/ethconfig" @@ -322,7 +323,7 @@ func TestEth2NewBlock(t *testing.T) { if err != nil { t.Fatalf("Failed to create the executable data %v", err) } - block, err := engine.ExecutableDataToBlock(*execData) + block, err := engine.ExecutableDataToBlock(*execData, nil) if err != nil { t.Fatalf("Failed to convert executable data to block %v", err) } @@ -364,7 +365,7 @@ func TestEth2NewBlock(t *testing.T) { if err != nil { t.Fatalf("Failed to create the executable data %v", err) } - block, err := engine.ExecutableDataToBlock(*execData) + block, err := engine.ExecutableDataToBlock(*execData, nil) if err != nil { t.Fatalf("Failed to convert executable data to block %v", err) } @@ -996,7 +997,7 @@ func TestSimultaneousNewBlock(t *testing.T) { t.Fatal(testErr) } } - block, err := engine.ExecutableDataToBlock(*execData) + block, err := engine.ExecutableDataToBlock(*execData, nil) if err != nil { t.Fatalf("Failed to convert executable data to block %v", err) } @@ -1519,3 +1520,38 @@ func equalBody(a *types.Body, b *engine.ExecutionPayloadBodyV1) bool { } return reflect.DeepEqual(a.Withdrawals, b.Withdrawals) } + +func TestBlockToPayloadWithBlobs(t *testing.T) { + header := types.Header{} + var txs []*types.Transaction + + inner := types.BlobTx{ + BlobHashes: make([]common.Hash, 1), + } + + txs = append(txs, types.NewTx(&inner)) + + blobs := make([]kzg4844.Blob, 1) + commitments := make([]kzg4844.Commitment, 1) + proofs := make([]kzg4844.Proof, 1) + + block := types.NewBlock(&header, txs, nil, nil, trie.NewStackTrie(nil)) + envelope := engine.BlockToExecutableData(block, nil, blobs, commitments, proofs) + var want int + for _, tx := range txs { + want += len(tx.BlobHashes()) + } + if got := len(envelope.BlobsBundle.Commitments); got != want { + t.Fatalf("invalid number of commitments: got %v, want %v", got, want) + } + if got := len(envelope.BlobsBundle.Proofs); got != want { + t.Fatalf("invalid number of proofs: got %v, want %v", got, want) + } + if got := len(envelope.BlobsBundle.Blobs); got != want { + t.Fatalf("invalid number of blobs: got %v, want %v", got, want) + } + _, err := engine.ExecutableDataToBlock(*envelope.ExecutionPayload, make([]common.Hash, 1)) + if err != nil { + t.Error(err) + } +} diff --git a/eth/catalyst/simulated_beacon.go b/eth/catalyst/simulated_beacon.go index f0956d262c..d4b08b7ba8 100644 --- a/eth/catalyst/simulated_beacon.go +++ b/eth/catalyst/simulated_beacon.go @@ -151,7 +151,7 @@ func (c *SimulatedBeacon) sealBlock(withdrawals []*types.Withdrawal) error { return fmt.Errorf("error calling forkchoice update: %v", err) } - envelope, err := c.engineAPI.getFullPayload(*fcResponse.PayloadID) + envelope, err := c.engineAPI.getPayload(*fcResponse.PayloadID) if err != nil { return fmt.Errorf("error retrieving payload: %v", err) } diff --git a/miner/payload_building.go b/miner/payload_building.go index 9b13a4a695..299196a3cd 100644 --- a/miner/payload_building.go +++ b/miner/payload_building.go @@ -120,9 +120,9 @@ func (payload *Payload) Resolve() *engine.ExecutionPayloadEnvelope { close(payload.stop) } if payload.full != nil { - return engine.BlockToExecutableData(payload.full, payload.fullFees) + return engine.BlockToExecutableData(payload.full, payload.fullFees, nil, nil, nil) } - return engine.BlockToExecutableData(payload.empty, big.NewInt(0)) + return engine.BlockToExecutableData(payload.empty, big.NewInt(0), nil, nil, nil) } // ResolveEmpty is basically identical to Resolve, but it expects empty block only. @@ -131,7 +131,7 @@ func (payload *Payload) ResolveEmpty() *engine.ExecutionPayloadEnvelope { payload.lock.Lock() defer payload.lock.Unlock() - return engine.BlockToExecutableData(payload.empty, big.NewInt(0)) + return engine.BlockToExecutableData(payload.empty, big.NewInt(0), nil, nil, nil) } // ResolveFull is basically identical to Resolve, but it expects full block only. @@ -157,7 +157,7 @@ func (payload *Payload) ResolveFull() *engine.ExecutionPayloadEnvelope { default: close(payload.stop) } - return engine.BlockToExecutableData(payload.full, payload.fullFees) + return engine.BlockToExecutableData(payload.full, payload.fullFees, nil, nil, nil) } // buildPayload builds the payload according to the provided parameters. From 7f756dc1185d7f1eeeacb1d12341606b7135f9ea Mon Sep 17 00:00:00 2001 From: Pierre Grimaud Date: Tue, 18 Jul 2023 14:07:25 +0200 Subject: [PATCH 85/90] cmd/evm/testdata: fix typos in docs (#27742) --- cmd/evm/testdata/3/readme.md | 2 +- cmd/evm/testdata/4/readme.md | 2 +- cmd/evm/testdata/8/readme.md | 2 +- cmd/evm/testdata/9/readme.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/evm/testdata/3/readme.md b/cmd/evm/testdata/3/readme.md index aa1f041df2..246c58ef3b 100644 --- a/cmd/evm/testdata/3/readme.md +++ b/cmd/evm/testdata/3/readme.md @@ -1,2 +1,2 @@ -These files exemplify a transition where a transaction (excuted on block 5) requests +These files exemplify a transition where a transaction (executed on block 5) requests the blockhash for block `1`. diff --git a/cmd/evm/testdata/4/readme.md b/cmd/evm/testdata/4/readme.md index 9918edfb20..eede41a9fd 100644 --- a/cmd/evm/testdata/4/readme.md +++ b/cmd/evm/testdata/4/readme.md @@ -1,3 +1,3 @@ -These files exemplify a transition where a transaction (excuted on block 5) requests +These files exemplify a transition where a transaction (executed on block 5) requests the blockhash for block `4`, but where the hash for that block is missing. It's expected that executing these should cause `exit` with errorcode `4`. diff --git a/cmd/evm/testdata/8/readme.md b/cmd/evm/testdata/8/readme.md index c991fb2177..4dffdab91e 100644 --- a/cmd/evm/testdata/8/readme.md +++ b/cmd/evm/testdata/8/readme.md @@ -7,7 +7,7 @@ This test contains testcases for EIP-2930, which uses transactions with access l The alloc portion contains one contract (`0x000000000000000000000000000000000000aaaa`), containing the following code: `0x5854505854`: `PC ;SLOAD; POP; PC; SLOAD`. -Essentialy, this contract does `SLOAD(0)` and `SLOAD(3)`. +Essentially, this contract does `SLOAD(0)` and `SLOAD(3)`. The alloc also contains some funds on `0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b`. diff --git a/cmd/evm/testdata/9/readme.md b/cmd/evm/testdata/9/readme.md index b144644655..5394780288 100644 --- a/cmd/evm/testdata/9/readme.md +++ b/cmd/evm/testdata/9/readme.md @@ -7,7 +7,7 @@ This test contains testcases for EIP-1559, which uses an new transaction type an The alloc portion contains one contract (`0x000000000000000000000000000000000000aaaa`), containing the following code: `0x58585454`: `PC; PC; SLOAD; SLOAD`. -Essentialy, this contract does `SLOAD(0)` and `SLOAD(1)`. +Essentially, this contract does `SLOAD(0)` and `SLOAD(1)`. The alloc also contains some funds on `0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b`. From a46f4173cdeb61f34030e214876dd10b0e346862 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Mon, 24 Jul 2023 12:21:26 +0200 Subject: [PATCH 86/90] ethclient/gethclient: gofmt -s (#27762) --- ethclient/gethclient/gethclient_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ethclient/gethclient/gethclient_test.go b/ethclient/gethclient/gethclient_test.go index c2c11d0770..5a0f4d2534 100644 --- a/ethclient/gethclient/gethclient_test.go +++ b/ethclient/gethclient/gethclient_test.go @@ -399,17 +399,17 @@ func testCallContract(t *testing.T, client *rpc.Client) { func TestOverrideAccountMarshal(t *testing.T) { om := map[common.Address]OverrideAccount{ - common.Address{0x11}: OverrideAccount{ + {0x11}: { // Zero-valued nonce is not overriddden, but simply dropped by the encoder. Nonce: 0, }, - common.Address{0xaa}: OverrideAccount{ + {0xaa}: { Nonce: 5, }, - common.Address{0xbb}: OverrideAccount{ + {0xbb}: { Code: []byte{1}, }, - common.Address{0xcc}: OverrideAccount{ + {0xcc}: { // 'code', 'balance', 'state' should be set when input is // a non-nil but empty value. Code: []byte{}, From 88f3d61468a34fcb4d02f80edd0b061f446466aa Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Mon, 24 Jul 2023 18:22:09 +0800 Subject: [PATCH 87/90] all: expose block number information to statedb (#27753) * core/state: clean up * all: add block number infomration to statedb * core, trie: rename blockNumber to block --- cmd/evm/internal/t8ntool/execution.go | 4 +-- cmd/evm/runner.go | 3 +- core/blockchain.go | 2 +- core/blockchain_test.go | 2 +- core/chain_makers.go | 2 +- core/genesis.go | 4 +-- core/state/dump.go | 4 +-- core/state/snapshot/generate.go | 2 +- core/state/snapshot/generate_test.go | 2 +- core/state/state_object.go | 38 +++++++++++++------------- core/state/state_test.go | 12 ++++---- core/state/statedb.go | 29 +++++++++++--------- core/state/statedb_fuzz_test.go | 2 +- core/state/statedb_test.go | 22 +++++++-------- core/state/sync_test.go | 4 +-- eth/api_debug_test.go | 4 +-- eth/protocols/snap/sync_test.go | 8 +++--- eth/state_accessor.go | 2 +- light/postprocess.go | 4 +-- tests/fuzzers/stacktrie/trie_fuzzer.go | 2 +- tests/fuzzers/trie/trie-fuzzer.go | 2 +- tests/state_test_util.go | 4 +-- trie/database.go | 2 +- trie/iterator_test.go | 18 ++++++------ trie/secure_trie_test.go | 2 +- trie/sync_test.go | 6 ++-- trie/tracer_test.go | 18 ++++++------ trie/trie_test.go | 20 +++++++------- 28 files changed, 113 insertions(+), 111 deletions(-) diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index 338427bd46..37f9febd9c 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -266,7 +266,7 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, statedb.AddBalance(w.Address, amount) } // Commit block - root, err := statedb.Commit(chainConfig.IsEIP158(vmContext.BlockNumber)) + root, err := statedb.Commit(vmContext.BlockNumber.Uint64(), chainConfig.IsEIP158(vmContext.BlockNumber)) if err != nil { return nil, nil, NewError(ErrorEVM, fmt.Errorf("could not commit state: %v", err)) } @@ -307,7 +307,7 @@ func MakePreState(db ethdb.Database, accounts core.GenesisAlloc) *state.StateDB } } // Commit and re-open to start with a clean state. - root, _ := statedb.Commit(false) + root, _ := statedb.Commit(0, false) statedb, _ = state.New(root, sdb, nil) return statedb } diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index e61661a7af..ed7989e872 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -280,8 +280,7 @@ func runCmd(ctx *cli.Context) error { output, leftOverGas, stats, err := timedExec(bench, execFunc) if ctx.Bool(DumpFlag.Name) { - statedb.Commit(true) - statedb.IntermediateRoot(true) + statedb.Commit(genesisConfig.Number, true) fmt.Println(string(statedb.Dump(nil))) } diff --git a/core/blockchain.go b/core/blockchain.go index b760a301fb..a8c9d44724 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1336,7 +1336,7 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. log.Crit("Failed to write block into disk", "err", err) } // Commit all cached state changes into underlying memory database. - root, err := state.Commit(bc.chainConfig.IsEIP158(block.Number())) + root, err := state.Commit(block.NumberU64(), bc.chainConfig.IsEIP158(block.Number())) if err != nil { return err } diff --git a/core/blockchain_test.go b/core/blockchain_test.go index e626dbb5f7..1b4b569575 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -176,7 +176,7 @@ func testBlockChainImport(chain types.Blocks, blockchain *BlockChain) error { blockchain.chainmu.MustLock() rawdb.WriteTd(blockchain.db, block.Hash(), block.NumberU64(), new(big.Int).Add(block.Difficulty(), blockchain.GetTd(block.ParentHash(), block.NumberU64()-1))) rawdb.WriteBlock(blockchain.db, block) - statedb.Commit(false) + statedb.Commit(block.NumberU64(), false) blockchain.chainmu.Unlock() } return nil diff --git a/core/chain_makers.go b/core/chain_makers.go index 61d0098af3..99e9fe513c 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -321,7 +321,7 @@ func GenerateChain(config *params.ChainConfig, parent *types.Block, engine conse } // Write state changes to db - root, err := statedb.Commit(config.IsEIP158(b.header.Number)) + root, err := statedb.Commit(b.header.Number.Uint64(), config.IsEIP158(b.header.Number)) if err != nil { panic(fmt.Sprintf("state write error: %v", err)) } diff --git a/core/genesis.go b/core/genesis.go index 0b255a8d37..d55c2cb672 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -137,7 +137,7 @@ func (ga *GenesisAlloc) deriveHash() (common.Hash, error) { statedb.SetState(addr, key, value) } } - return statedb.Commit(false) + return statedb.Commit(0, false) } // flush is very similar with deriveHash, but the main difference is @@ -156,7 +156,7 @@ func (ga *GenesisAlloc) flush(db ethdb.Database, triedb *trie.Database, blockhas statedb.SetState(addr, key, value) } } - root, err := statedb.Commit(false) + root, err := statedb.Commit(0, false) if err != nil { return err } diff --git a/core/state/dump.go b/core/state/dump.go index 3ea987731f..9ce6cd394b 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -173,11 +173,11 @@ func (s *StateDB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey [] } obj := newObject(s, addr, &data) if !conf.SkipCode { - account.Code = obj.Code(s.db) + account.Code = obj.Code() } if !conf.SkipStorage { account.Storage = make(map[common.Hash]string) - tr, err := obj.getTrie(s.db) + tr, err := obj.getTrie() if err != nil { log.Error("Failed to load storage trie", "err", err) continue diff --git a/core/state/snapshot/generate.go b/core/state/snapshot/generate.go index f80525fcd2..0a85f0006d 100644 --- a/core/state/snapshot/generate.go +++ b/core/state/snapshot/generate.go @@ -366,7 +366,7 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, trieId *trie.ID, prefi return false, nil, err } if nodes != nil { - tdb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) + tdb.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) tdb.Commit(root, false) } resolver = func(owner common.Hash, path []byte, hash common.Hash) []byte { diff --git a/core/state/snapshot/generate_test.go b/core/state/snapshot/generate_test.go index 8996877ceb..c503676241 100644 --- a/core/state/snapshot/generate_test.go +++ b/core/state/snapshot/generate_test.go @@ -203,7 +203,7 @@ func (t *testHelper) Commit() common.Hash { if nodes != nil { t.nodes.Merge(nodes) } - t.triedb.Update(root, types.EmptyRootHash, t.nodes, nil) + t.triedb.Update(root, types.EmptyRootHash, 0, t.nodes, nil) t.triedb.Commit(root, false) return root } diff --git a/core/state/state_object.go b/core/state/state_object.go index 6fea47f05b..c14eeaf0a1 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -137,7 +137,7 @@ func (s *stateObject) touch() { // getTrie returns the associated storage trie. The trie will be opened // if it's not loaded previously. An error will be returned if trie can't // be loaded. -func (s *stateObject) getTrie(db Database) (Trie, error) { +func (s *stateObject) getTrie() (Trie, error) { if s.trie == nil { // Try fetching from prefetcher first if s.data.Root != types.EmptyRootHash && s.db.prefetcher != nil { @@ -145,7 +145,7 @@ func (s *stateObject) getTrie(db Database) (Trie, error) { s.trie = s.db.prefetcher.trie(s.addrHash, s.data.Root) } if s.trie == nil { - tr, err := db.OpenStorageTrie(s.db.originalRoot, s.address, s.data.Root) + tr, err := s.db.db.OpenStorageTrie(s.db.originalRoot, s.address, s.data.Root) if err != nil { return nil, err } @@ -156,18 +156,18 @@ func (s *stateObject) getTrie(db Database) (Trie, error) { } // GetState retrieves a value from the account storage trie. -func (s *stateObject) GetState(db Database, key common.Hash) common.Hash { +func (s *stateObject) GetState(key common.Hash) common.Hash { // If we have a dirty value for this state entry, return it value, dirty := s.dirtyStorage[key] if dirty { return value } // Otherwise return the entry's original value - return s.GetCommittedState(db, key) + return s.GetCommittedState(key) } // GetCommittedState retrieves a value from the committed account storage trie. -func (s *stateObject) GetCommittedState(db Database, key common.Hash) common.Hash { +func (s *stateObject) GetCommittedState(key common.Hash) common.Hash { // If we have a pending write or clean cached, return that if value, pending := s.pendingStorage[key]; pending { return value @@ -207,7 +207,7 @@ func (s *stateObject) GetCommittedState(db Database, key common.Hash) common.Has // If the snapshot is unavailable or reading from it fails, load from the database. if s.db.snap == nil || err != nil { start := time.Now() - tr, err := s.getTrie(db) + tr, err := s.getTrie() if err != nil { s.db.setError(err) return common.Hash{} @@ -227,9 +227,9 @@ func (s *stateObject) GetCommittedState(db Database, key common.Hash) common.Has } // SetState updates a value in account storage. -func (s *stateObject) SetState(db Database, key, value common.Hash) { +func (s *stateObject) SetState(key, value common.Hash) { // If the new value is the same as old, don't set - prev := s.GetState(db, key) + prev := s.GetState(key) if prev == value { return } @@ -267,7 +267,7 @@ func (s *stateObject) finalise(prefetch bool) { // updateTrie writes cached storage modifications into the object's storage trie. // It will return nil if the trie has not been loaded and no changes have been // made. An error will be returned if the trie can't be loaded/updated correctly. -func (s *stateObject) updateTrie(db Database) (Trie, error) { +func (s *stateObject) updateTrie() (Trie, error) { // Make sure all dirty slots are finalized into the pending storage area s.finalise(false) // Don't prefetch anymore, pull directly if need be if len(s.pendingStorage) == 0 { @@ -283,7 +283,7 @@ func (s *stateObject) updateTrie(db Database) (Trie, error) { origin map[common.Hash][]byte hasher = s.db.hasher ) - tr, err := s.getTrie(db) + tr, err := s.getTrie() if err != nil { s.db.setError(err) return nil, err @@ -357,8 +357,8 @@ func (s *stateObject) updateTrie(db Database) (Trie, error) { // UpdateRoot sets the trie root to the current root hash of. An error // will be returned if trie root hash is not computed correctly. -func (s *stateObject) updateRoot(db Database) { - tr, err := s.updateTrie(db) +func (s *stateObject) updateRoot() { + tr, err := s.updateTrie() if err != nil { return } @@ -374,8 +374,8 @@ func (s *stateObject) updateRoot(db Database) { } // commit returns the changes made in storage trie and updates the account data. -func (s *stateObject) commit(db Database) (*trienode.NodeSet, error) { - tr, err := s.updateTrie(db) +func (s *stateObject) commit() (*trienode.NodeSet, error) { + tr, err := s.updateTrie() if err != nil { return nil, err } @@ -465,14 +465,14 @@ func (s *stateObject) Address() common.Address { } // Code returns the contract code associated with this object, if any. -func (s *stateObject) Code(db Database) []byte { +func (s *stateObject) Code() []byte { if s.code != nil { return s.code } if bytes.Equal(s.CodeHash(), types.EmptyCodeHash.Bytes()) { return nil } - code, err := db.ContractCode(s.address, common.BytesToHash(s.CodeHash())) + code, err := s.db.db.ContractCode(s.address, common.BytesToHash(s.CodeHash())) if err != nil { s.db.setError(fmt.Errorf("can't load code hash %x: %v", s.CodeHash(), err)) } @@ -483,14 +483,14 @@ func (s *stateObject) Code(db Database) []byte { // CodeSize returns the size of the contract code associated with this object, // or zero if none. This method is an almost mirror of Code, but uses a cache // inside the database to avoid loading codes seen recently. -func (s *stateObject) CodeSize(db Database) int { +func (s *stateObject) CodeSize() int { if s.code != nil { return len(s.code) } if bytes.Equal(s.CodeHash(), types.EmptyCodeHash.Bytes()) { return 0 } - size, err := db.ContractCodeSize(s.address, common.BytesToHash(s.CodeHash())) + size, err := s.db.db.ContractCodeSize(s.address, common.BytesToHash(s.CodeHash())) if err != nil { s.db.setError(fmt.Errorf("can't load code size %x: %v", s.CodeHash(), err)) } @@ -498,7 +498,7 @@ func (s *stateObject) CodeSize(db Database) int { } func (s *stateObject) SetCode(codeHash common.Hash, code []byte) { - prevcode := s.Code(s.db.db) + prevcode := s.Code() s.db.journal.append(codeChange{ account: &s.address, prevhash: s.CodeHash(), diff --git a/core/state/state_test.go b/core/state/state_test.go index dc9df371d3..2553133dea 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -58,7 +58,7 @@ func TestDump(t *testing.T) { // write some of them to the trie s.state.updateStateObject(obj1) s.state.updateStateObject(obj2) - root, _ := s.state.Commit(false) + root, _ := s.state.Commit(0, false) // check that DumpToCollector contains the state objects that are in trie s.state, _ = New(root, tdb, nil) @@ -114,7 +114,7 @@ func TestIterativeDump(t *testing.T) { // write some of them to the trie s.state.updateStateObject(obj1) s.state.updateStateObject(obj2) - root, _ := s.state.Commit(false) + root, _ := s.state.Commit(0, false) s.state, _ = New(root, tdb, nil) b := &bytes.Buffer{} @@ -140,7 +140,7 @@ func TestNull(t *testing.T) { var value common.Hash s.state.SetState(address, common.Hash{}, value) - s.state.Commit(false) + s.state.Commit(0, false) if value := s.state.GetState(address, common.Hash{}); value != (common.Hash{}) { t.Errorf("expected empty current value, got %x", value) @@ -212,7 +212,7 @@ func TestSnapshot2(t *testing.T) { so0.deleted = false state.setStateObject(so0) - root, _ := state.Commit(false) + root, _ := state.Commit(0, false) state, _ = New(root, state.db, state.snaps) // and one with deleted == true @@ -234,8 +234,8 @@ func TestSnapshot2(t *testing.T) { so0Restored := state.getStateObject(stateobjaddr0) // Update lazily-loaded values before comparing. - so0Restored.GetState(state.db, storageaddr) - so0Restored.Code(state.db) + so0Restored.GetState(storageaddr) + so0Restored.Code() // non-deleted is equal (restored) compareStateObjects(so0Restored, so0, t) diff --git a/core/state/statedb.go b/core/state/statedb.go index fdcc76310f..7bf573a51a 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -309,7 +309,7 @@ func (s *StateDB) TxIndex() int { func (s *StateDB) GetCode(addr common.Address) []byte { stateObject := s.getStateObject(addr) if stateObject != nil { - return stateObject.Code(s.db) + return stateObject.Code() } return nil } @@ -317,7 +317,7 @@ func (s *StateDB) GetCode(addr common.Address) []byte { func (s *StateDB) GetCodeSize(addr common.Address) int { stateObject := s.getStateObject(addr) if stateObject != nil { - return stateObject.CodeSize(s.db) + return stateObject.CodeSize() } return 0 } @@ -334,7 +334,7 @@ func (s *StateDB) GetCodeHash(addr common.Address) common.Hash { func (s *StateDB) GetState(addr common.Address, hash common.Hash) common.Hash { stateObject := s.getStateObject(addr) if stateObject != nil { - return stateObject.GetState(s.db, hash) + return stateObject.GetState(hash) } return common.Hash{} } @@ -372,7 +372,7 @@ func (s *StateDB) GetStorageProof(a common.Address, key common.Hash) ([][]byte, func (s *StateDB) GetCommittedState(addr common.Address, hash common.Hash) common.Hash { stateObject := s.getStateObject(addr) if stateObject != nil { - return stateObject.GetCommittedState(s.db, hash) + return stateObject.GetCommittedState(hash) } return common.Hash{} } @@ -391,10 +391,10 @@ func (s *StateDB) StorageTrie(addr common.Address) (Trie, error) { return nil, nil } cpy := stateObject.deepCopy(s) - if _, err := cpy.updateTrie(s.db); err != nil { + if _, err := cpy.updateTrie(); err != nil { return nil, err } - return cpy.getTrie(s.db) + return cpy.getTrie() } func (s *StateDB) HasSelfDestructed(addr common.Address) bool { @@ -449,7 +449,7 @@ func (s *StateDB) SetCode(addr common.Address, code []byte) { func (s *StateDB) SetState(addr common.Address, key, value common.Hash) { stateObject := s.GetOrNewStateObject(addr) if stateObject != nil { - stateObject.SetState(s.db, key, value) + stateObject.SetState(key, value) } } @@ -470,7 +470,7 @@ func (s *StateDB) SetStorage(addr common.Address, storage map[common.Hash]common } stateObject := s.GetOrNewStateObject(addr) for k, v := range storage { - stateObject.SetState(s.db, k, v) + stateObject.SetState(k, v) } } @@ -724,7 +724,7 @@ func (s *StateDB) ForEachStorage(addr common.Address, cb func(key, value common. if so == nil { return nil } - tr, err := so.getTrie(s.db) + tr, err := so.getTrie() if err != nil { return err } @@ -968,7 +968,7 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { // to pull useful data from disk. for addr := range s.stateObjectsPending { if obj := s.stateObjects[addr]; !obj.deleted { - obj.updateRoot(s.db) + obj.updateRoot() } } // Now we're about to start to write changes to the trie. The trie is so far @@ -1155,7 +1155,10 @@ func (s *StateDB) handleDestruction(nodes *trienode.MergedNodeSet) (map[common.H // trie, storage tries) will no longer be functional. A new state instance // must be created with new root and updated database for accessing post- // commit states. -func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { +// +// The associated block number of the state transition is also provided +// for more chain context. +func (s *StateDB) Commit(block uint64, deleteEmptyObjects bool) (common.Hash, error) { // Short circuit in case any database failure occurred earlier. if s.dbErr != nil { return common.Hash{}, fmt.Errorf("commit aborted due to earlier error: %v", s.dbErr) @@ -1190,7 +1193,7 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { obj.dirtyCode = false } // Write any storage changes in the state object to its storage trie - set, err := obj.commit(s.db) + set, err := obj.commit() if err != nil { return common.Hash{}, err } @@ -1276,7 +1279,7 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { Storages: s.storagesOrigin, Incomplete: incomplete, } - if err := s.db.TrieDB().Update(root, origin, nodes, set); err != nil { + if err := s.db.TrieDB().Update(root, origin, block, nodes, set); err != nil { return common.Hash{}, err } s.originalRoot = root diff --git a/core/state/statedb_fuzz_test.go b/core/state/statedb_fuzz_test.go index 4dca96efd3..168f804b4f 100644 --- a/core/state/statedb_fuzz_test.go +++ b/core/state/statedb_fuzz_test.go @@ -206,7 +206,7 @@ func (test *stateTest) run() bool { } else { state.IntermediateRoot(true) // call intermediateRoot at the transaction boundary } - nroot, err := state.Commit(true) // call commit at the block boundary + nroot, err := state.Commit(0, true) // call commit at the block boundary if err != nil { panic(err) } diff --git a/core/state/statedb_test.go b/core/state/statedb_test.go index 6848b01cb1..66dda238e1 100644 --- a/core/state/statedb_test.go +++ b/core/state/statedb_test.go @@ -106,7 +106,7 @@ func TestIntermediateLeaks(t *testing.T) { } // Commit and cross check the databases. - transRoot, err := transState.Commit(false) + transRoot, err := transState.Commit(0, false) if err != nil { t.Fatalf("failed to commit transition state: %v", err) } @@ -114,7 +114,7 @@ func TestIntermediateLeaks(t *testing.T) { t.Errorf("can not commit trie %v to persistent database", transRoot.Hex()) } - finalRoot, err := finalState.Commit(false) + finalRoot, err := finalState.Commit(0, false) if err != nil { t.Fatalf("failed to commit final state: %v", err) } @@ -487,7 +487,7 @@ func (test *snapshotTest) checkEqual(state, checkstate *StateDB) error { func TestTouchDelete(t *testing.T) { s := newStateEnv() s.state.GetOrNewStateObject(common.Address{}) - root, _ := s.state.Commit(false) + root, _ := s.state.Commit(0, false) s.state, _ = New(root, s.state.db, s.state.snaps) snapshot := s.state.Snapshot() @@ -575,7 +575,7 @@ func TestCopyCommitCopy(t *testing.T) { t.Fatalf("second copy committed storage slot mismatch: have %x, want %x", val, sval) } // Commit state, ensure states can be loaded from disk - root, _ := state.Commit(false) + root, _ := state.Commit(0, false) state, _ = New(root, tdb, nil) if balance := state.GetBalance(addr); balance.Cmp(big.NewInt(42)) != 0 { t.Fatalf("state post-commit balance mismatch: have %v, want %v", balance, 42) @@ -689,7 +689,7 @@ func TestCommitCopy(t *testing.T) { t.Fatalf("initial committed storage slot mismatch: have %x, want %x", val, common.Hash{}) } // Copy the committed state database, the copied one is not functional. - state.Commit(true) + state.Commit(0, true) copied := state.Copy() if balance := copied.GetBalance(addr); balance.Cmp(big.NewInt(0)) != 0 { t.Fatalf("unexpected balance: have %v", balance) @@ -723,7 +723,7 @@ func TestDeleteCreateRevert(t *testing.T) { addr := common.BytesToAddress([]byte("so")) state.SetBalance(addr, big.NewInt(1)) - root, _ := state.Commit(false) + root, _ := state.Commit(0, false) state, _ = New(root, state.db, state.snaps) // Simulate self-destructing in one transaction, then create-reverting in another @@ -735,7 +735,7 @@ func TestDeleteCreateRevert(t *testing.T) { state.RevertToSnapshot(id) // Commit the entire state and make sure we don't crash and have the correct state - root, _ = state.Commit(true) + root, _ = state.Commit(0, true) state, _ = New(root, state.db, state.snaps) if state.getStateObject(addr) != nil { @@ -759,7 +759,7 @@ func TestMissingTrieNodes(t *testing.T) { a2 := common.BytesToAddress([]byte("another")) state.SetBalance(a2, big.NewInt(100)) state.SetCode(a2, []byte{1, 2, 4}) - root, _ = state.Commit(false) + root, _ = state.Commit(0, false) t.Logf("root: %x", root) // force-flush state.Database().TrieDB().Cap(0) @@ -783,7 +783,7 @@ func TestMissingTrieNodes(t *testing.T) { } // Modify the state state.SetBalance(addr, big.NewInt(2)) - root, err := state.Commit(false) + root, err := state.Commit(0, false) if err == nil { t.Fatalf("expected error, got root :%x", root) } @@ -978,7 +978,7 @@ func TestFlushOrderDataLoss(t *testing.T) { state.SetState(common.Address{a}, common.Hash{a, s}, common.Hash{a, s}) } } - root, err := state.Commit(false) + root, err := state.Commit(0, false) if err != nil { t.Fatalf("failed to commit state trie: %v", err) } @@ -1057,7 +1057,7 @@ func TestResetObject(t *testing.T) { state.CreateAccount(addr) state.SetBalance(addr, big.NewInt(2)) state.SetState(addr, slotB, common.BytesToHash([]byte{0x2})) - root, _ := state.Commit(true) + root, _ := state.Commit(0, true) // Ensure the original account is wiped properly snap := snaps.Snapshot(root) diff --git a/core/state/sync_test.go b/core/state/sync_test.go index 7de69e0aa8..b065ad8355 100644 --- a/core/state/sync_test.go +++ b/core/state/sync_test.go @@ -64,13 +64,13 @@ func makeTestState() (ethdb.Database, Database, common.Hash, []*testAccount) { if i%5 == 0 { for j := byte(0); j < 5; j++ { hash := crypto.Keccak256Hash([]byte{i, i, i, i, i, j, j}) - obj.SetState(sdb, hash, hash) + obj.SetState(hash, hash) } } state.updateStateObject(obj) accounts = append(accounts, acc) } - root, _ := state.Commit(false) + root, _ := state.Commit(0, false) // Return the generated state return db, sdb, root, accounts diff --git a/eth/api_debug_test.go b/eth/api_debug_test.go index ee54e4c3c8..4b08bd4fc5 100644 --- a/eth/api_debug_test.go +++ b/eth/api_debug_test.go @@ -79,7 +79,7 @@ func TestAccountRange(t *testing.T) { m[addr] = true } } - root, _ := sdb.Commit(true) + root, _ := sdb.Commit(0, true) sdb, _ = state.New(root, statedb, nil) trie, err := statedb.OpenTrie(root) @@ -137,7 +137,7 @@ func TestEmptyAccountRange(t *testing.T) { st, _ = state.New(types.EmptyRootHash, statedb, nil) ) // Commit(although nothing to flush) and re-init the statedb - st.Commit(true) + st.Commit(0, true) st, _ = state.New(types.EmptyRootHash, statedb, nil) results := st.IteratorDump(&state.DumpConfig{ diff --git a/eth/protocols/snap/sync_test.go b/eth/protocols/snap/sync_test.go index 1c5f103cb5..79b1cad25b 100644 --- a/eth/protocols/snap/sync_test.go +++ b/eth/protocols/snap/sync_test.go @@ -1387,7 +1387,7 @@ func makeAccountTrieNoStorage(n int) (string, *trie.Trie, []*kv) { // Commit the state changes into db and re-create the trie // for accessing later. root, nodes, _ := accTrie.Commit(false) - db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) + db.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) accTrie, _ = trie.New(trie.StateTrieID(root), db) return db.Scheme(), accTrie, entries @@ -1449,7 +1449,7 @@ func makeBoundaryAccountTrie(n int) (string, *trie.Trie, []*kv) { // Commit the state changes into db and re-create the trie // for accessing later. root, nodes, _ := accTrie.Commit(false) - db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) + db.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) accTrie, _ = trie.New(trie.StateTrieID(root), db) return db.Scheme(), accTrie, entries @@ -1498,7 +1498,7 @@ func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) nodes.Merge(set) // Commit gathered dirty nodes into database - db.Update(root, types.EmptyRootHash, nodes, nil) + db.Update(root, types.EmptyRootHash, 0, nodes, nil) // Re-create tries with new root accTrie, _ = trie.New(trie.StateTrieID(root), db) @@ -1563,7 +1563,7 @@ func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (strin nodes.Merge(set) // Commit gathered dirty nodes into database - db.Update(root, types.EmptyRootHash, nodes, nil) + db.Update(root, types.EmptyRootHash, 0, nodes, nil) // Re-create tries with new root accTrie, err := trie.New(trie.StateTrieID(root), db) diff --git a/eth/state_accessor.go b/eth/state_accessor.go index 1b0ab0c545..a725585e71 100644 --- a/eth/state_accessor.go +++ b/eth/state_accessor.go @@ -164,7 +164,7 @@ func (eth *Ethereum) StateAtBlock(ctx context.Context, block *types.Block, reexe return nil, nil, fmt.Errorf("processing block %d failed: %v", current.NumberU64(), err) } // Finalize the state so any modifications are written to the trie - root, err := statedb.Commit(eth.blockchain.Config().IsEIP158(current.Number())) + root, err := statedb.Commit(current.NumberU64(), eth.blockchain.Config().IsEIP158(current.Number())) if err != nil { return nil, nil, fmt.Errorf("stateAtBlock commit failed, number %d root %v: %w", current.NumberU64(), current.Root().Hex(), err) diff --git a/light/postprocess.go b/light/postprocess.go index 8905694211..567814e2bf 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -220,7 +220,7 @@ func (c *ChtIndexerBackend) Commit() error { } // Commit trie changes into trie database in case it's not nil. if nodes != nil { - if err := c.triedb.Update(root, c.originRoot, trienode.NewWithNodeSet(nodes), nil); err != nil { + if err := c.triedb.Update(root, c.originRoot, 0, trienode.NewWithNodeSet(nodes), nil); err != nil { return err } if err := c.triedb.Commit(root, false); err != nil { @@ -473,7 +473,7 @@ func (b *BloomTrieIndexerBackend) Commit() error { } // Commit trie changes into trie database in case it's not nil. if nodes != nil { - if err := b.triedb.Update(root, b.originRoot, trienode.NewWithNodeSet(nodes), nil); err != nil { + if err := b.triedb.Update(root, b.originRoot, 0, trienode.NewWithNodeSet(nodes), nil); err != nil { return err } if err := b.triedb.Commit(root, false); err != nil { diff --git a/tests/fuzzers/stacktrie/trie_fuzzer.go b/tests/fuzzers/stacktrie/trie_fuzzer.go index deb0c92c27..1eb639f644 100644 --- a/tests/fuzzers/stacktrie/trie_fuzzer.go +++ b/tests/fuzzers/stacktrie/trie_fuzzer.go @@ -176,7 +176,7 @@ func (f *fuzzer) fuzz() int { panic(err) } if nodes != nil { - dbA.Update(rootA, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) + dbA.Update(rootA, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) } // Flush memdb -> disk (sponge) dbA.Commit(rootA, false) diff --git a/tests/fuzzers/trie/trie-fuzzer.go b/tests/fuzzers/trie/trie-fuzzer.go index 2f9b97390b..fe9bf3d0fd 100644 --- a/tests/fuzzers/trie/trie-fuzzer.go +++ b/tests/fuzzers/trie/trie-fuzzer.go @@ -170,7 +170,7 @@ func runRandTest(rt randTest) error { return err } if nodes != nil { - if err := triedb.Update(hash, origin, trienode.NewWithNodeSet(nodes), nil); err != nil { + if err := triedb.Update(hash, origin, 0, trienode.NewWithNodeSet(nodes), nil); err != nil { return err } } diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 0958e248b4..721fe8f5c8 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -283,7 +283,7 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh // the coinbase gets no txfee, so isn't created, and thus needs to be touched statedb.AddBalance(block.Coinbase(), new(big.Int)) // Commit block - root, _ := statedb.Commit(config.IsEIP158(block.Number())) + root, _ := statedb.Commit(block.NumberU64(), config.IsEIP158(block.Number())) return snaps, statedb, root, err } @@ -303,7 +303,7 @@ func MakePreState(db ethdb.Database, accounts core.GenesisAlloc, snapshotter boo } } // Commit and re-open to start with a clean state. - root, _ := statedb.Commit(false) + root, _ := statedb.Commit(0, false) var snaps *snapshot.Tree if snapshotter { diff --git a/trie/database.go b/trie/database.go index 85681d7597..08ef5d07e1 100644 --- a/trie/database.go +++ b/trie/database.go @@ -118,7 +118,7 @@ func (db *Database) Reader(blockRoot common.Hash) (Reader, error) { // given set in order to update state from the specified parent to the specified // root. The held pre-images accumulated up to this point will be flushed in case // the size exceeds the threshold. -func (db *Database) Update(root common.Hash, parent common.Hash, nodes *trienode.MergedNodeSet, states *triestate.Set) error { +func (db *Database) Update(root common.Hash, parent common.Hash, block uint64, nodes *trienode.MergedNodeSet, states *triestate.Set) error { if db.config != nil && db.config.OnCommit != nil { db.config.OnCommit(states) } diff --git a/trie/iterator_test.go b/trie/iterator_test.go index 1ffb4bfbe8..abe9cf1b11 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -63,7 +63,7 @@ func TestIterator(t *testing.T) { trie.MustUpdate([]byte(val.k), []byte(val.v)) } root, nodes, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) + db.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) trie, _ = New(TrieID(root), db) found := make(map[string]string) @@ -255,7 +255,7 @@ func TestDifferenceIterator(t *testing.T) { triea.MustUpdate([]byte(val.k), []byte(val.v)) } rootA, nodesA, _ := triea.Commit(false) - dba.Update(rootA, types.EmptyRootHash, trienode.NewWithNodeSet(nodesA), nil) + dba.Update(rootA, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodesA), nil) triea, _ = New(TrieID(rootA), dba) dbb := NewDatabase(rawdb.NewMemoryDatabase()) @@ -264,7 +264,7 @@ func TestDifferenceIterator(t *testing.T) { trieb.MustUpdate([]byte(val.k), []byte(val.v)) } rootB, nodesB, _ := trieb.Commit(false) - dbb.Update(rootB, types.EmptyRootHash, trienode.NewWithNodeSet(nodesB), nil) + dbb.Update(rootB, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodesB), nil) trieb, _ = New(TrieID(rootB), dbb) found := make(map[string]string) @@ -297,7 +297,7 @@ func TestUnionIterator(t *testing.T) { triea.MustUpdate([]byte(val.k), []byte(val.v)) } rootA, nodesA, _ := triea.Commit(false) - dba.Update(rootA, types.EmptyRootHash, trienode.NewWithNodeSet(nodesA), nil) + dba.Update(rootA, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodesA), nil) triea, _ = New(TrieID(rootA), dba) dbb := NewDatabase(rawdb.NewMemoryDatabase()) @@ -306,7 +306,7 @@ func TestUnionIterator(t *testing.T) { trieb.MustUpdate([]byte(val.k), []byte(val.v)) } rootB, nodesB, _ := trieb.Commit(false) - dbb.Update(rootB, types.EmptyRootHash, trienode.NewWithNodeSet(nodesB), nil) + dbb.Update(rootB, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodesB), nil) trieb, _ = New(TrieID(rootB), dbb) di, _ := NewUnionIterator([]NodeIterator{triea.MustNodeIterator(nil), trieb.MustNodeIterator(nil)}) @@ -368,7 +368,7 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool, scheme string) { tr.MustUpdate([]byte(val.k), []byte(val.v)) } root, nodes, _ := tr.Commit(false) - tdb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) + tdb.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) if !memonly { tdb.Commit(root, false) } @@ -484,7 +484,7 @@ func testIteratorContinueAfterSeekError(t *testing.T, memonly bool, scheme strin break } } - triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) + triedb.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) if !memonly { triedb.Commit(root, false) } @@ -605,7 +605,7 @@ func makeLargeTestTrie() (*Database, *StateTrie, *loggingDb) { trie.MustUpdate(key, val) } root, nodes, _ := trie.Commit(false) - triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) + triedb.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) triedb.Commit(root, false) // Return the generated trie @@ -648,7 +648,7 @@ func testIteratorNodeBlob(t *testing.T, scheme string) { trie.MustUpdate([]byte(val.k), []byte(val.v)) } root, nodes, _ := trie.Commit(false) - triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) + triedb.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) triedb.Commit(root, false) var found = make(map[common.Hash][]byte) diff --git a/trie/secure_trie_test.go b/trie/secure_trie_test.go index d756cb5223..a610ca2fd6 100644 --- a/trie/secure_trie_test.go +++ b/trie/secure_trie_test.go @@ -61,7 +61,7 @@ func makeTestStateTrie() (*Database, *StateTrie, map[string][]byte) { } } root, nodes, _ := trie.Commit(false) - if err := triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil); err != nil { + if err := triedb.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil); err != nil { panic(fmt.Errorf("failed to commit db %v", err)) } // Re-create the trie based on the new state diff --git a/trie/sync_test.go b/trie/sync_test.go index 437cca7312..589e2858fb 100644 --- a/trie/sync_test.go +++ b/trie/sync_test.go @@ -57,7 +57,7 @@ func makeTestTrie(scheme string) (ethdb.Database, *Database, *StateTrie, map[str } } root, nodes, _ := trie.Commit(false) - if err := triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil); err != nil { + if err := triedb.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil); err != nil { panic(fmt.Errorf("failed to commit db %v", err)) } if err := triedb.Commit(root, false); err != nil { @@ -740,7 +740,7 @@ func testSyncMovingTarget(t *testing.T, scheme string) { diff[string(key)] = val } root, nodes, _ := srcTrie.Commit(false) - if err := srcDb.Update(root, preRoot, trienode.NewWithNodeSet(nodes), nil); err != nil { + if err := srcDb.Update(root, preRoot, 0, trienode.NewWithNodeSet(nodes), nil); err != nil { panic(err) } if err := srcDb.Commit(root, false); err != nil { @@ -765,7 +765,7 @@ func testSyncMovingTarget(t *testing.T, scheme string) { reverted[k] = val } root, nodes, _ = srcTrie.Commit(false) - if err := srcDb.Update(root, preRoot, trienode.NewWithNodeSet(nodes), nil); err != nil { + if err := srcDb.Update(root, preRoot, 0, trienode.NewWithNodeSet(nodes), nil); err != nil { panic(err) } if err := srcDb.Commit(root, false); err != nil { diff --git a/trie/tracer_test.go b/trie/tracer_test.go index a68cf3abaf..86daec6d20 100644 --- a/trie/tracer_test.go +++ b/trie/tracer_test.go @@ -71,7 +71,7 @@ func testTrieTracer(t *testing.T, vals []struct{ k, v string }) { insertSet := copySet(trie.tracer.inserts) // copy before commit deleteSet := copySet(trie.tracer.deletes) // copy before commit root, nodes, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) + db.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) seen := setKeys(iterNodes(db, root)) if !compareSet(insertSet, seen) { @@ -137,7 +137,7 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) { trie.MustUpdate([]byte(val.k), []byte(val.v)) } root, nodes, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) + db.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) trie, _ = New(TrieID(root), db) if err := verifyAccessList(orig, trie, nodes); err != nil { @@ -152,7 +152,7 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) { trie.MustUpdate([]byte(val.k), randBytes(32)) } root, nodes, _ = trie.Commit(false) - db.Update(root, parent, trienode.NewWithNodeSet(nodes), nil) + db.Update(root, parent, 0, trienode.NewWithNodeSet(nodes), nil) trie, _ = New(TrieID(root), db) if err := verifyAccessList(orig, trie, nodes); err != nil { @@ -170,7 +170,7 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) { trie.MustUpdate(key, randBytes(32)) } root, nodes, _ = trie.Commit(false) - db.Update(root, parent, trienode.NewWithNodeSet(nodes), nil) + db.Update(root, parent, 0, trienode.NewWithNodeSet(nodes), nil) trie, _ = New(TrieID(root), db) if err := verifyAccessList(orig, trie, nodes); err != nil { @@ -185,7 +185,7 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) { trie.MustUpdate([]byte(key), nil) } root, nodes, _ = trie.Commit(false) - db.Update(root, parent, trienode.NewWithNodeSet(nodes), nil) + db.Update(root, parent, 0, trienode.NewWithNodeSet(nodes), nil) trie, _ = New(TrieID(root), db) if err := verifyAccessList(orig, trie, nodes); err != nil { @@ -200,7 +200,7 @@ func testAccessList(t *testing.T, vals []struct{ k, v string }) { trie.MustUpdate([]byte(val.k), nil) } root, nodes, _ = trie.Commit(false) - db.Update(root, parent, trienode.NewWithNodeSet(nodes), nil) + db.Update(root, parent, 0, trienode.NewWithNodeSet(nodes), nil) trie, _ = New(TrieID(root), db) if err := verifyAccessList(orig, trie, nodes); err != nil { @@ -219,7 +219,7 @@ func TestAccessListLeak(t *testing.T) { trie.MustUpdate([]byte(val.k), []byte(val.v)) } root, nodes, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) + db.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) var cases = []struct { op func(tr *Trie) @@ -269,7 +269,7 @@ func TestTinyTree(t *testing.T) { trie.MustUpdate([]byte(val.k), randBytes(32)) } root, set, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(set), nil) + db.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(set), nil) parent := root trie, _ = New(TrieID(root), db) @@ -278,7 +278,7 @@ func TestTinyTree(t *testing.T) { trie.MustUpdate([]byte(val.k), []byte(val.v)) } root, set, _ = trie.Commit(false) - db.Update(root, parent, trienode.NewWithNodeSet(set), nil) + db.Update(root, parent, 0, trienode.NewWithNodeSet(set), nil) trie, _ = New(TrieID(root), db) if err := verifyAccessList(orig, trie, set); err != nil { diff --git a/trie/trie_test.go b/trie/trie_test.go index 8e7066eade..cc86e7aee2 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -89,7 +89,7 @@ func testMissingNode(t *testing.T, memonly bool, scheme string) { updateString(trie, "120000", "qwerqwerqwerqwerqwerqwerqwerqwer") updateString(trie, "123456", "asdfasdfasdfasdfasdfasdfasdfasdf") root, nodes, _ := trie.Commit(false) - triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) + triedb.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) if !memonly { triedb.Commit(root, false) @@ -203,7 +203,7 @@ func TestGet(t *testing.T) { return } root, nodes, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) + db.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) trie, _ = New(TrieID(root), db) } } @@ -275,7 +275,7 @@ func TestReplication(t *testing.T) { updateString(trie, val.k, val.v) } root, nodes, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) + db.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) // create a new trie on top of the database and check that lookups work. trie2, err := New(TrieID(root), db) @@ -294,7 +294,7 @@ func TestReplication(t *testing.T) { // recreate the trie after commit if nodes != nil { - db.Update(hash, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) + db.Update(hash, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) } trie2, err = New(TrieID(hash), db) if err != nil { @@ -503,7 +503,7 @@ func runRandTest(rt randTest) bool { case opCommit: root, nodes, _ := tr.Commit(true) if nodes != nil { - triedb.Update(root, origin, trienode.NewWithNodeSet(nodes), nil) + triedb.Update(root, origin, 0, trienode.NewWithNodeSet(nodes), nil) } newtr, err := New(TrieID(root), triedb) if err != nil { @@ -838,7 +838,7 @@ func TestCommitSequence(t *testing.T) { } // Flush trie -> database root, nodes, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) + db.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) // Flush memdb -> disk (sponge) db.Commit(root, false) if got, exp := s.sponge.Sum(nil), tc.expWriteSeqHash; !bytes.Equal(got, exp) { @@ -879,7 +879,7 @@ func TestCommitSequenceRandomBlobs(t *testing.T) { } // Flush trie -> database root, nodes, _ := trie.Commit(false) - db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) + db.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) // Flush memdb -> disk (sponge) db.Commit(root, false) if got, exp := s.sponge.Sum(nil), tc.expWriteSeqHash; !bytes.Equal(got, exp) { @@ -919,7 +919,7 @@ func TestCommitSequenceStackTrie(t *testing.T) { // Flush trie -> database root, nodes, _ := trie.Commit(false) // Flush memdb -> disk (sponge) - db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) + db.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) db.Commit(root, false) // And flush stacktrie -> disk stRoot, err := stTrie.Commit() @@ -967,7 +967,7 @@ func TestCommitSequenceSmallRoot(t *testing.T) { // Flush trie -> database root, nodes, _ := trie.Commit(false) // Flush memdb -> disk (sponge) - db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) + db.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) db.Commit(root, false) // And flush stacktrie -> disk stRoot, err := stTrie.Commit() @@ -1139,7 +1139,7 @@ func benchmarkDerefRootFixedSize(b *testing.B, addresses [][20]byte, accounts [] } h := trie.Hash() root, nodes, _ := trie.Commit(false) - triedb.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes), nil) + triedb.Update(root, types.EmptyRootHash, 0, trienode.NewWithNodeSet(nodes), nil) b.StartTimer() triedb.Dereference(h) b.StopTimer() From 7a1fba1a020a61a864a69d119ae0119c70e22b2b Mon Sep 17 00:00:00 2001 From: ucwong Date: Mon, 24 Jul 2023 11:22:54 +0100 Subject: [PATCH 88/90] cmd/geth: atomic types used (#27756) --- cmd/geth/attach_test.go | 6 +++--- cmd/geth/chaincmd.go | 14 +++++++------- cmd/geth/les_test.go | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/cmd/geth/attach_test.go b/cmd/geth/attach_test.go index 7c5f951750..91007ccf65 100644 --- a/cmd/geth/attach_test.go +++ b/cmd/geth/attach_test.go @@ -61,7 +61,7 @@ func TestRemoteDbWithHeaders(t *testing.T) { } func testReceiveHeaders(t *testing.T, ln net.Listener, gethArgs ...string) { - var ok uint32 + var ok atomic.Uint32 server := &http.Server{ Addr: "localhost:0", Handler: &testHandler{func(w http.ResponseWriter, r *http.Request) { @@ -72,12 +72,12 @@ func testReceiveHeaders(t *testing.T, ln net.Listener, gethArgs ...string) { if have, want := r.Header.Get("second"), "two"; have != want { t.Fatalf("missing header, have %v want %v", have, want) } - atomic.StoreUint32(&ok, 1) + ok.Store(1) }}} go server.Serve(ln) defer server.Close() runGeth(t, gethArgs...).WaitExit() - if atomic.LoadUint32(&ok) != 1 { + if ok.Load() != 1 { t.Fatal("Test fail, expected invocation to succeed") } } diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index b27ce55710..41591ac138 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -261,16 +261,16 @@ func importChain(ctx *cli.Context) error { defer db.Close() // Start periodically gathering memory profiles - var peakMemAlloc, peakMemSys uint64 + var peakMemAlloc, peakMemSys atomic.Uint64 go func() { stats := new(runtime.MemStats) for { runtime.ReadMemStats(stats) - if atomic.LoadUint64(&peakMemAlloc) < stats.Alloc { - atomic.StoreUint64(&peakMemAlloc, stats.Alloc) + if peakMemAlloc.Load() < stats.Alloc { + peakMemAlloc.Store(stats.Alloc) } - if atomic.LoadUint64(&peakMemSys) < stats.Sys { - atomic.StoreUint64(&peakMemSys, stats.Sys) + if peakMemSys.Load() < stats.Sys { + peakMemSys.Store(stats.Sys) } time.Sleep(5 * time.Second) } @@ -303,8 +303,8 @@ func importChain(ctx *cli.Context) error { mem := new(runtime.MemStats) runtime.ReadMemStats(mem) - fmt.Printf("Object memory: %.3f MB current, %.3f MB peak\n", float64(mem.Alloc)/1024/1024, float64(atomic.LoadUint64(&peakMemAlloc))/1024/1024) - fmt.Printf("System memory: %.3f MB current, %.3f MB peak\n", float64(mem.Sys)/1024/1024, float64(atomic.LoadUint64(&peakMemSys))/1024/1024) + fmt.Printf("Object memory: %.3f MB current, %.3f MB peak\n", float64(mem.Alloc)/1024/1024, float64(peakMemAlloc.Load())/1024/1024) + fmt.Printf("System memory: %.3f MB current, %.3f MB peak\n", float64(mem.Sys)/1024/1024, float64(peakMemSys.Load())/1024/1024) fmt.Printf("Allocations: %.3f million\n", float64(mem.Mallocs)/1000000) fmt.Printf("GC pause: %v\n\n", time.Duration(mem.PauseTotalNs)) diff --git a/cmd/geth/les_test.go b/cmd/geth/les_test.go index 746f12c765..b36c3265a3 100644 --- a/cmd/geth/les_test.go +++ b/cmd/geth/les_test.go @@ -107,10 +107,10 @@ func ipcEndpoint(ipcPath, datadir string) string { // but windows require pipes to sit in "\\.\pipe\". Therefore, to run several // nodes simultaneously, we need to distinguish between them, which we do by // the pipe filename instead of folder. -var nextIPC = uint32(0) +var nextIPC atomic.Uint32 func startGethWithIpc(t *testing.T, name string, args ...string) *gethrpc { - ipcName := fmt.Sprintf("geth-%d.ipc", atomic.AddUint32(&nextIPC, 1)) + ipcName := fmt.Sprintf("geth-%d.ipc", nextIPC.Add(1)) args = append([]string{"--networkid=42", "--port=0", "--authrpc.port", "0", "--ipcpath", ipcName}, args...) t.Logf("Starting %v with rpc: %v", name, args) From a196f3e8a22b6ad22ced5c2e3baf32bc3ebd4ec9 Mon Sep 17 00:00:00 2001 From: ucwong Date: Mon, 24 Jul 2023 11:23:38 +0100 Subject: [PATCH 89/90] cmd/devp2p: atomic types used (#27755) --- cmd/devp2p/crawl.go | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/cmd/devp2p/crawl.go b/cmd/devp2p/crawl.go index 1b964164d8..8c0defff6d 100644 --- a/cmd/devp2p/crawl.go +++ b/cmd/devp2p/crawl.go @@ -87,11 +87,11 @@ func (c *crawler) run(timeout time.Duration, nthreads int) nodeSet { go c.runIterator(doneCh, it) } var ( - added uint64 - updated uint64 - skipped uint64 - recent uint64 - removed uint64 + added atomic.Uint64 + updated atomic.Uint64 + skipped atomic.Uint64 + recent atomic.Uint64 + removed atomic.Uint64 wg sync.WaitGroup ) wg.Add(nthreads) @@ -103,15 +103,15 @@ func (c *crawler) run(timeout time.Duration, nthreads int) nodeSet { case n := <-c.ch: switch c.updateNode(n) { case nodeSkipIncompat: - atomic.AddUint64(&skipped, 1) + skipped.Add(1) case nodeSkipRecent: - atomic.AddUint64(&recent, 1) + recent.Add(1) case nodeRemoved: - atomic.AddUint64(&removed, 1) + removed.Add(1) case nodeAdded: - atomic.AddUint64(&added, 1) + added.Add(1) default: - atomic.AddUint64(&updated, 1) + updated.Add(1) } case <-c.closed: return @@ -138,11 +138,11 @@ loop: break loop case <-statusTicker.C: log.Info("Crawling in progress", - "added", atomic.LoadUint64(&added), - "updated", atomic.LoadUint64(&updated), - "removed", atomic.LoadUint64(&removed), - "ignored(recent)", atomic.LoadUint64(&recent), - "ignored(incompatible)", atomic.LoadUint64(&skipped)) + "added", added.Load(), + "updated", updated.Load(), + "removed", removed.Load(), + "ignored(recent)", recent.Load(), + "ignored(incompatible)", skipped.Load()) } } From 2274a03e339f213361453590b54917bbfd0a0c31 Mon Sep 17 00:00:00 2001 From: Justin Traglia <95511699+jtraglia@users.noreply.github.com> Date: Mon, 24 Jul 2023 18:13:34 +0200 Subject: [PATCH 90/90] crypto/kzg4844: do lazy init in all ckzg funcs (#27679) * crypto/kzg4844: remove unnecessary init call & fix typo * Fix kzg4844 tests/benchmarks * Make init lazy & revert changes to tests --- crypto/kzg4844/kzg4844.go | 2 +- crypto/kzg4844/kzg4844_ckzg_cgo.go | 8 ++++++++ crypto/kzg4844/kzg4844_test.go | 12 ++++++++++-- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/crypto/kzg4844/kzg4844.go b/crypto/kzg4844/kzg4844.go index 3bd814159d..5969d1c2ce 100644 --- a/crypto/kzg4844/kzg4844.go +++ b/crypto/kzg4844/kzg4844.go @@ -54,7 +54,7 @@ func UseCKZG(use bool) error { useCKZG.Store(use) // Initializing the library can take 2-4 seconds - and can potentially crash - // on CKZG and non-ADX CPUs - so might as well so it now and don't wait until + // on CKZG and non-ADX CPUs - so might as well do it now and don't wait until // a crypto operation is actually needed live. if use { ckzgIniter.Do(ckzgInit) diff --git a/crypto/kzg4844/kzg4844_ckzg_cgo.go b/crypto/kzg4844/kzg4844_ckzg_cgo.go index d62ca3fad6..5400285698 100644 --- a/crypto/kzg4844/kzg4844_ckzg_cgo.go +++ b/crypto/kzg4844/kzg4844_ckzg_cgo.go @@ -74,6 +74,8 @@ func ckzgBlobToCommitment(blob Blob) (Commitment, error) { // ckzgComputeProof computes the KZG proof at the given point for the polynomial // represented by the blob. func ckzgComputeProof(blob Blob, point Point) (Proof, Claim, error) { + ckzgIniter.Do(ckzgInit) + proof, claim, err := ckzg4844.ComputeKZGProof((ckzg4844.Blob)(blob), (ckzg4844.Bytes32)(point)) if err != nil { return Proof{}, Claim{}, err @@ -84,6 +86,8 @@ func ckzgComputeProof(blob Blob, point Point) (Proof, Claim, error) { // ckzgVerifyProof verifies the KZG proof that the polynomial represented by the blob // evaluated at the given point is the claimed value. func ckzgVerifyProof(commitment Commitment, point Point, claim Claim, proof Proof) error { + ckzgIniter.Do(ckzgInit) + valid, err := ckzg4844.VerifyKZGProof((ckzg4844.Bytes48)(commitment), (ckzg4844.Bytes32)(point), (ckzg4844.Bytes32)(claim), (ckzg4844.Bytes48)(proof)) if err != nil { return err @@ -99,6 +103,8 @@ func ckzgVerifyProof(commitment Commitment, point Point, claim Claim, proof Proo // // This method does not verify that the commitment is correct with respect to blob. func ckzgComputeBlobProof(blob Blob, commitment Commitment) (Proof, error) { + ckzgIniter.Do(ckzgInit) + proof, err := ckzg4844.ComputeBlobKZGProof((ckzg4844.Blob)(blob), (ckzg4844.Bytes48)(commitment)) if err != nil { return Proof{}, err @@ -108,6 +114,8 @@ func ckzgComputeBlobProof(blob Blob, commitment Commitment) (Proof, error) { // ckzgVerifyBlobProof verifies that the blob data corresponds to the provided commitment. func ckzgVerifyBlobProof(blob Blob, commitment Commitment, proof Proof) error { + ckzgIniter.Do(ckzgInit) + valid, err := ckzg4844.VerifyBlobKZGProof((ckzg4844.Blob)(blob), (ckzg4844.Bytes48)(commitment), (ckzg4844.Bytes48)(proof)) if err != nil { return err diff --git a/crypto/kzg4844/kzg4844_test.go b/crypto/kzg4844/kzg4844_test.go index 0d35667a9c..fae8a7a76e 100644 --- a/crypto/kzg4844/kzg4844_test.go +++ b/crypto/kzg4844/kzg4844_test.go @@ -47,7 +47,6 @@ func randBlob() Blob { func TestCKZGWithPoint(t *testing.T) { testKZGWithPoint(t, true) } func TestGoKZGWithPoint(t *testing.T) { testKZGWithPoint(t, false) } - func testKZGWithPoint(t *testing.T, ckzg bool) { if ckzg && !ckzgAvailable { t.Skip("CKZG unavailable in this test build") @@ -73,7 +72,6 @@ func testKZGWithPoint(t *testing.T, ckzg bool) { func TestCKZGWithBlob(t *testing.T) { testKZGWithBlob(t, true) } func TestGoKZGWithBlob(t *testing.T) { testKZGWithBlob(t, false) } - func testKZGWithBlob(t *testing.T, ckzg bool) { if ckzg && !ckzgAvailable { t.Skip("CKZG unavailable in this test build") @@ -106,6 +104,8 @@ func benchmarkBlobToCommitment(b *testing.B, ckzg bool) { useCKZG.Store(ckzg) blob := randBlob() + + b.ResetTimer() for i := 0; i < b.N; i++ { BlobToCommitment(blob) } @@ -124,6 +124,8 @@ func benchmarkComputeProof(b *testing.B, ckzg bool) { blob = randBlob() point = randFieldElement() ) + + b.ResetTimer() for i := 0; i < b.N; i++ { ComputeProof(blob, point) } @@ -144,6 +146,8 @@ func benchmarkVerifyProof(b *testing.B, ckzg bool) { commitment, _ = BlobToCommitment(blob) proof, claim, _ = ComputeProof(blob, point) ) + + b.ResetTimer() for i := 0; i < b.N; i++ { VerifyProof(commitment, point, claim, proof) } @@ -162,6 +166,8 @@ func benchmarkComputeBlobProof(b *testing.B, ckzg bool) { blob = randBlob() commitment, _ = BlobToCommitment(blob) ) + + b.ResetTimer() for i := 0; i < b.N; i++ { ComputeBlobProof(blob, commitment) } @@ -181,6 +187,8 @@ func benchmarkVerifyBlobProof(b *testing.B, ckzg bool) { commitment, _ = BlobToCommitment(blob) proof, _ = ComputeBlobProof(blob, commitment) ) + + b.ResetTimer() for i := 0; i < b.N; i++ { VerifyBlobProof(blob, commitment, proof) }