From 1296cdb7484d3b6236f1d1d95381cc8ffde7d4d9 Mon Sep 17 00:00:00 2001 From: Gabriel-Trintinalia Date: Sat, 19 Apr 2025 21:42:54 +1000 Subject: [PATCH 1/8] core: fail execution if system call fails to execute (#31639) see: https://github.com/ethereum/pm/issues/1450#issuecomment-2800911584 --- cmd/evm/internal/t8ntool/execution.go | 8 ++++++-- core/chain_makers.go | 8 ++++++-- core/state_processor.go | 29 +++++++++++++++++---------- internal/ethapi/simulate.go | 8 ++++++-- miner/worker.go | 8 ++++++-- 5 files changed, 42 insertions(+), 19 deletions(-) diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index 7de1eb6949..b2e5f70714 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -363,9 +363,13 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, return nil, nil, nil, NewError(ErrorEVM, fmt.Errorf("could not parse requests logs: %v", err)) } // EIP-7002 - core.ProcessWithdrawalQueue(&requests, evm) + if err := core.ProcessWithdrawalQueue(&requests, evm); err != nil { + return nil, nil, nil, NewError(ErrorEVM, fmt.Errorf("could not process withdrawal requests: %v", err)) + } // EIP-7251 - core.ProcessConsolidationQueue(&requests, evm) + if err := core.ProcessConsolidationQueue(&requests, evm); err != nil { + return nil, nil, nil, NewError(ErrorEVM, fmt.Errorf("could not process consolidation requests: %v", err)) + } } // Commit block diff --git a/core/chain_makers.go b/core/chain_makers.go index 7a258dc05f..37bddcfda5 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -328,9 +328,13 @@ func (b *BlockGen) collectRequests(readonly bool) (requests [][]byte) { blockContext := NewEVMBlockContext(b.header, b.cm, &b.header.Coinbase) evm := vm.NewEVM(blockContext, statedb, b.cm.config, vm.Config{}) // EIP-7002 - ProcessWithdrawalQueue(&requests, evm) + if err := ProcessWithdrawalQueue(&requests, evm); err != nil { + panic(fmt.Sprintf("could not process withdrawal requests: %v", err)) + } // EIP-7251 - ProcessConsolidationQueue(&requests, evm) + if err := ProcessConsolidationQueue(&requests, evm); err != nil { + panic(fmt.Sprintf("could not process consolidation requests: %v", err)) + } } return requests } diff --git a/core/state_processor.go b/core/state_processor.go index 9241d091ad..322bd24f41 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -113,9 +113,13 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg return nil, err } // EIP-7002 - ProcessWithdrawalQueue(&requests, evm) + if err := ProcessWithdrawalQueue(&requests, evm); err != nil { + return nil, err + } // EIP-7251 - ProcessConsolidationQueue(&requests, evm) + if err := ProcessConsolidationQueue(&requests, evm); err != nil { + return nil, err + } } // Finalize the block, applying any consensus engine specific extras (e.g. block rewards) @@ -265,17 +269,17 @@ func ProcessParentBlockHash(prevHash common.Hash, evm *vm.EVM) { // ProcessWithdrawalQueue calls the EIP-7002 withdrawal queue contract. // It returns the opaque request data returned by the contract. -func ProcessWithdrawalQueue(requests *[][]byte, evm *vm.EVM) { - processRequestsSystemCall(requests, evm, 0x01, params.WithdrawalQueueAddress) +func ProcessWithdrawalQueue(requests *[][]byte, evm *vm.EVM) error { + return processRequestsSystemCall(requests, evm, 0x01, params.WithdrawalQueueAddress) } // ProcessConsolidationQueue calls the EIP-7251 consolidation queue contract. // It returns the opaque request data returned by the contract. -func ProcessConsolidationQueue(requests *[][]byte, evm *vm.EVM) { - processRequestsSystemCall(requests, evm, 0x02, params.ConsolidationQueueAddress) +func ProcessConsolidationQueue(requests *[][]byte, evm *vm.EVM) error { + return processRequestsSystemCall(requests, evm, 0x02, params.ConsolidationQueueAddress) } -func processRequestsSystemCall(requests *[][]byte, evm *vm.EVM, requestType byte, addr common.Address) { +func processRequestsSystemCall(requests *[][]byte, evm *vm.EVM, requestType byte, addr common.Address) error { if tracer := evm.Config.Tracer; tracer != nil { onSystemCallStart(tracer, evm.GetVMContext()) if tracer.OnSystemCallEnd != nil { @@ -292,17 +296,20 @@ func processRequestsSystemCall(requests *[][]byte, evm *vm.EVM, requestType byte } evm.SetTxContext(NewEVMTxContext(msg)) evm.StateDB.AddAddressToAccessList(addr) - ret, _, _ := evm.Call(msg.From, *msg.To, msg.Data, 30_000_000, common.U2560) + ret, _, err := evm.Call(msg.From, *msg.To, msg.Data, 30_000_000, common.U2560) evm.StateDB.Finalise(true) - if len(ret) == 0 { - return // skip empty output + if err != nil { + return fmt.Errorf("system call failed to execute: %v", err) + } + if len(ret) == 0 { + return nil // skip empty output } - // Append prefixed requestsData to the requests list. requestsData := make([]byte, len(ret)+1) requestsData[0] = requestType copy(requestsData[1:], ret) *requests = append(*requests, requestsData) + return nil } var depositTopic = common.HexToHash("0x649bbc62d0e31342afea4e5cd82d4049e7e1ee912fc0889aa790803be39038c5") diff --git a/internal/ethapi/simulate.go b/internal/ethapi/simulate.go index ba346b132f..9241b509da 100644 --- a/internal/ethapi/simulate.go +++ b/internal/ethapi/simulate.go @@ -314,9 +314,13 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, return nil, nil, err } // EIP-7002 - core.ProcessWithdrawalQueue(&requests, evm) + if err := core.ProcessWithdrawalQueue(&requests, evm); err != nil { + return nil, nil, err + } // EIP-7251 - core.ProcessConsolidationQueue(&requests, evm) + if err := core.ProcessConsolidationQueue(&requests, evm); err != nil { + return nil, nil, err + } } if requests != nil { reqHash := types.CalcRequestsHash(requests) diff --git a/miner/worker.go b/miner/worker.go index 8fb42e31bc..d80cb8913b 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -127,9 +127,13 @@ func (miner *Miner) generateWork(params *generateParams, witness bool) *newPaylo return &newPayloadResult{err: err} } // EIP-7002 - core.ProcessWithdrawalQueue(&requests, work.evm) + if err := core.ProcessWithdrawalQueue(&requests, work.evm); err != nil { + return &newPayloadResult{err: err} + } // EIP-7251 consolidations - core.ProcessConsolidationQueue(&requests, work.evm) + if err := core.ProcessConsolidationQueue(&requests, work.evm); err != nil { + return &newPayloadResult{err: err} + } } if requests != nil { reqHash := types.CalcRequestsHash(requests) From bf6da20012f63573fff4dad19634c5bf5dbef964 Mon Sep 17 00:00:00 2001 From: Morty <70688412+yiweichi@users.noreply.github.com> Date: Sat, 19 Apr 2025 22:02:31 +0800 Subject: [PATCH 2/8] eth/gasprice: fix eth_feeHistory blobGasUsedRatio divide zero (#31663) The API `eth_feeHistory` returns `{"jsonrpc":"2.0","id":1,"error":{"code":-32603,"message":"json: unsupported value: NaN"}}`, when we query `eth_feeHistory` with a old block that without a blob, or when the field `config.blobSchedule.cancun.max` in genesis.config is 0 (that happens for some projects fork geth but they don't have blob). So here we specially handle the case when maxBlobGas == 0 to prevent this issue from happening. --- eth/gasprice/feehistory.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/eth/gasprice/feehistory.go b/eth/gasprice/feehistory.go index 59830e9fe8..e5a197f640 100644 --- a/eth/gasprice/feehistory.go +++ b/eth/gasprice/feehistory.go @@ -108,7 +108,9 @@ func (oracle *Oracle) processBlock(bf *blockFees, percentiles []float64) { bf.results.gasUsedRatio = float64(bf.header.GasUsed) / float64(bf.header.GasLimit) if blobGasUsed := bf.header.BlobGasUsed; blobGasUsed != nil { maxBlobGas := eip4844.MaxBlobGasPerBlock(config, bf.header.Time) - bf.results.blobGasUsedRatio = float64(*blobGasUsed) / float64(maxBlobGas) + if maxBlobGas != 0 { + bf.results.blobGasUsedRatio = float64(*blobGasUsed) / float64(maxBlobGas) + } } if len(percentiles) == 0 { From 7f574372d5cc3b8d37b59dd0da53e9f779bbfdd1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Sun, 20 Apr 2025 09:48:49 +0200 Subject: [PATCH 3/8] eth/filters, core/filtermaps: safe chain view update (#31590) This PR changes the chain view update mechanism of the log filter. Previously the head updates were all wired through the indexer, even in unindexed mode. This was both a bit weird and also unsafe as the indexer's chain view was updates asynchronously with some delay, making some log related tests flaky. Also, the reorg safety of the indexed search was integrated with unindexed search in a weird way, relying on `syncRange.ValidBlocks` in the unindexed case too, with a special condition added to only consider the head of the valid range but not the tail in the unindexed case. In this PR the current chain view is directly accessible through the filter backend and unindexed search is also chain view based, making it inherently safe. The matcher sync mechanism is now only used for indexed search as originally intended, removing a few ugly special conditions. The PR is currently based on top of https://github.com/ethereum/go-ethereum/pull/31642 Together they fix https://github.com/ethereum/go-ethereum/issues/31518 and replace https://github.com/ethereum/go-ethereum/pull/31542 --------- Co-authored-by: Gary Rong --- accounts/abi/abigen/bind_test.go | 5 +- core/filtermaps/chain_view.go | 61 ++++-- core/filtermaps/filtermaps.go | 8 +- core/filtermaps/indexer.go | 8 +- core/filtermaps/map_renderer.go | 32 +-- core/filtermaps/matcher.go | 2 +- core/filtermaps/matcher_backend.go | 6 +- eth/api_backend.go | 8 + eth/filters/filter.go | 250 +++++++++++++---------- eth/filters/filter_system.go | 1 + eth/filters/filter_system_test.go | 5 + eth/filters/filter_test.go | 137 ++++++++----- internal/ethapi/api_test.go | 3 + internal/ethapi/backend.go | 1 + internal/ethapi/transaction_args_test.go | 1 + 15 files changed, 327 insertions(+), 201 deletions(-) diff --git a/accounts/abi/abigen/bind_test.go b/accounts/abi/abigen/bind_test.go index 195064fb7a..b3c52e81e5 100644 --- a/accounts/abi/abigen/bind_test.go +++ b/accounts/abi/abigen/bind_test.go @@ -939,6 +939,7 @@ var bindTests = []struct { if _, err := eventer.RaiseSimpleEvent(auth, common.Address{byte(j)}, [32]byte{byte(j)}, true, big.NewInt(int64(10*i+j))); err != nil { t.Fatalf("block %d, event %d: raise failed: %v", i, j, err) } + time.Sleep(time.Millisecond * 200) } sim.Commit() } @@ -1495,7 +1496,7 @@ var bindTests = []struct { if n != 3 { t.Fatalf("Invalid bar0 event") } - case <-time.NewTimer(3 * time.Second).C: + case <-time.NewTimer(10 * time.Second).C: t.Fatalf("Wait bar0 event timeout") } @@ -1506,7 +1507,7 @@ var bindTests = []struct { if n != 1 { t.Fatalf("Invalid bar event") } - case <-time.NewTimer(3 * time.Second).C: + case <-time.NewTimer(10 * time.Second).C: t.Fatalf("Wait bar event timeout") } close(stopCh) diff --git a/core/filtermaps/chain_view.go b/core/filtermaps/chain_view.go index d6f0a727bf..aa74f3901a 100644 --- a/core/filtermaps/chain_view.go +++ b/core/filtermaps/chain_view.go @@ -58,47 +58,75 @@ func NewChainView(chain blockchain, number uint64, hash common.Hash) *ChainView return cv } -// getBlockHash returns the block hash belonging to the given block number. +// HeadNumber returns the head block number of the chain view. +func (cv *ChainView) HeadNumber() uint64 { + return cv.headNumber +} + +// BlockHash returns the block hash belonging to the given block number. // Note that the hash of the head block is not returned because ChainView might // represent a view where the head block is currently being created. -func (cv *ChainView) getBlockHash(number uint64) common.Hash { - if number >= cv.headNumber { +func (cv *ChainView) BlockHash(number uint64) common.Hash { + cv.lock.Lock() + defer cv.lock.Unlock() + + if number > cv.headNumber { panic("invalid block number") } return cv.blockHash(number) } -// getBlockId returns the unique block id belonging to the given block number. +// BlockId returns the unique block id belonging to the given block number. // Note that it is currently equal to the block hash. In the future it might // be a different id for future blocks if the log index root becomes part of // consensus and therefore rendering the index with the new head will happen // before the hash of that new head is available. -func (cv *ChainView) getBlockId(number uint64) common.Hash { +func (cv *ChainView) BlockId(number uint64) common.Hash { + cv.lock.Lock() + defer cv.lock.Unlock() + if number > cv.headNumber { panic("invalid block number") } return cv.blockHash(number) } -// getReceipts returns the set of receipts belonging to the block at the given +// Header returns the block header at the given block number. +func (cv *ChainView) Header(number uint64) *types.Header { + return cv.chain.GetHeader(cv.BlockHash(number), number) +} + +// Receipts returns the set of receipts belonging to the block at the given // block number. -func (cv *ChainView) getReceipts(number uint64) types.Receipts { - if number > cv.headNumber { - panic("invalid block number") - } - blockHash := cv.blockHash(number) +func (cv *ChainView) Receipts(number uint64) types.Receipts { + blockHash := cv.BlockHash(number) if blockHash == (common.Hash{}) { log.Error("Chain view: block hash unavailable", "number", number, "head", cv.headNumber) } return cv.chain.GetReceiptsByHash(blockHash) } +// SharedRange returns the block range shared by two chain views. +func (cv *ChainView) SharedRange(cv2 *ChainView) common.Range[uint64] { + cv.lock.Lock() + defer cv.lock.Unlock() + + if cv == nil || cv2 == nil || !cv.extendNonCanonical() || !cv2.extendNonCanonical() { + return common.Range[uint64]{} + } + var sharedLen uint64 + for n := min(cv.headNumber+1-uint64(len(cv.hashes)), cv2.headNumber+1-uint64(len(cv2.hashes))); n <= cv.headNumber && n <= cv2.headNumber && cv.blockHash(n) == cv2.blockHash(n); n++ { + sharedLen = n + 1 + } + return common.NewRange(0, sharedLen) +} + // limitedView returns a new chain view that is a truncated version of the parent view. func (cv *ChainView) limitedView(newHead uint64) *ChainView { if newHead >= cv.headNumber { return cv } - return NewChainView(cv.chain, newHead, cv.blockHash(newHead)) + return NewChainView(cv.chain, newHead, cv.BlockHash(newHead)) } // equalViews returns true if the two chain views are equivalent. @@ -106,7 +134,7 @@ func equalViews(cv1, cv2 *ChainView) bool { if cv1 == nil || cv2 == nil { return false } - return cv1.headNumber == cv2.headNumber && cv1.getBlockId(cv1.headNumber) == cv2.getBlockId(cv2.headNumber) + return cv1.headNumber == cv2.headNumber && cv1.BlockId(cv1.headNumber) == cv2.BlockId(cv2.headNumber) } // matchViews returns true if the two chain views are equivalent up until the @@ -120,9 +148,9 @@ func matchViews(cv1, cv2 *ChainView, number uint64) bool { return false } if number == cv1.headNumber || number == cv2.headNumber { - return cv1.getBlockId(number) == cv2.getBlockId(number) + return cv1.BlockId(number) == cv2.BlockId(number) } - return cv1.getBlockHash(number) == cv2.getBlockHash(number) + return cv1.BlockHash(number) == cv2.BlockHash(number) } // extendNonCanonical checks whether the previously known reverse list of head @@ -150,9 +178,6 @@ func (cv *ChainView) extendNonCanonical() bool { // blockHash returns the given block hash without doing the head number check. func (cv *ChainView) blockHash(number uint64) common.Hash { - cv.lock.Lock() - defer cv.lock.Unlock() - if number+uint64(len(cv.hashes)) <= cv.headNumber { hash := cv.chain.GetCanonicalHash(number) if !cv.extendNonCanonical() { diff --git a/core/filtermaps/filtermaps.go b/core/filtermaps/filtermaps.go index fa2d6e3ffb..a617de8968 100644 --- a/core/filtermaps/filtermaps.go +++ b/core/filtermaps/filtermaps.go @@ -262,7 +262,7 @@ func NewFilterMaps(db ethdb.KeyValueStore, initView *ChainView, historyCutoff, f f.targetView = initView if f.indexedRange.initialized { f.indexedView = f.initChainView(f.targetView) - f.indexedRange.headIndexed = f.indexedRange.blocks.AfterLast() == f.indexedView.headNumber+1 + f.indexedRange.headIndexed = f.indexedRange.blocks.AfterLast() == f.indexedView.HeadNumber()+1 if !f.indexedRange.headIndexed { f.indexedRange.headDelimiter = 0 } @@ -313,7 +313,7 @@ func (f *FilterMaps) initChainView(chainView *ChainView) *ChainView { log.Error("Could not initialize indexed chain view", "error", err) break } - if lastBlockNumber <= chainView.headNumber && chainView.getBlockId(lastBlockNumber) == lastBlockId { + if lastBlockNumber <= chainView.HeadNumber() && chainView.BlockId(lastBlockNumber) == lastBlockId { return chainView.limitedView(lastBlockNumber) } } @@ -370,7 +370,7 @@ func (f *FilterMaps) init() error { for min < max { mid := (min + max + 1) / 2 cp := checkpointList[mid-1] - if cp.BlockNumber <= f.targetView.headNumber && f.targetView.getBlockId(cp.BlockNumber) == cp.BlockId { + if cp.BlockNumber <= f.targetView.HeadNumber() && f.targetView.BlockId(cp.BlockNumber) == cp.BlockId { min = mid } else { max = mid - 1 @@ -512,7 +512,7 @@ func (f *FilterMaps) getLogByLvIndex(lvIndex uint64) (*types.Log, error) { } } // get block receipts - receipts := f.indexedView.getReceipts(firstBlockNumber) + receipts := f.indexedView.Receipts(firstBlockNumber) if receipts == nil { return nil, fmt.Errorf("failed to retrieve receipts for block %d containing searched log value index %d: %v", firstBlockNumber, lvIndex, err) } diff --git a/core/filtermaps/indexer.go b/core/filtermaps/indexer.go index 9a5424da4a..383ec078c9 100644 --- a/core/filtermaps/indexer.go +++ b/core/filtermaps/indexer.go @@ -44,7 +44,7 @@ func (f *FilterMaps) indexerLoop() { for !f.stop { if !f.indexedRange.initialized { - if f.targetView.headNumber == 0 { + if f.targetView.HeadNumber() == 0 { // initialize when chain head is available f.processSingleEvent(true) continue @@ -249,7 +249,7 @@ func (f *FilterMaps) tryIndexHead() error { log.Info("Log index head rendering in progress", "first block", f.indexedRange.blocks.First(), "last block", f.indexedRange.blocks.Last(), "processed", f.indexedRange.blocks.AfterLast()-f.ptrHeadIndex, - "remaining", f.indexedView.headNumber-f.indexedRange.blocks.Last(), + "remaining", f.indexedView.HeadNumber()-f.indexedRange.blocks.Last(), "elapsed", common.PrettyDuration(time.Since(f.startedHeadIndexAt))) f.loggedHeadIndex = true f.lastLogHeadIndex = time.Now() @@ -418,10 +418,10 @@ func (f *FilterMaps) needTailEpoch(epoch uint32) bool { // tailTargetBlock returns the target value for the tail block number according // to the log history parameter and the current index head. func (f *FilterMaps) tailTargetBlock() uint64 { - if f.history == 0 || f.indexedView.headNumber < f.history { + if f.history == 0 || f.indexedView.HeadNumber() < f.history { return 0 } - return f.indexedView.headNumber + 1 - f.history + return f.indexedView.HeadNumber() + 1 - f.history } // tailPartialBlocks returns the number of rendered blocks in the partially diff --git a/core/filtermaps/map_renderer.go b/core/filtermaps/map_renderer.go index 23d84f0eca..7c2aa8dc32 100644 --- a/core/filtermaps/map_renderer.go +++ b/core/filtermaps/map_renderer.go @@ -143,8 +143,8 @@ func (f *FilterMaps) lastCanonicalSnapshotOfMap(mapIndex uint32) *renderedMap { var best *renderedMap for _, blockNumber := range f.renderSnapshots.Keys() { if cp, _ := f.renderSnapshots.Get(blockNumber); cp != nil && blockNumber < f.indexedRange.blocks.AfterLast() && - blockNumber <= f.indexedView.headNumber && f.indexedView.getBlockId(blockNumber) == cp.lastBlockId && - blockNumber <= f.targetView.headNumber && f.targetView.getBlockId(blockNumber) == cp.lastBlockId && + blockNumber <= f.indexedView.HeadNumber() && f.indexedView.BlockId(blockNumber) == cp.lastBlockId && + blockNumber <= f.targetView.HeadNumber() && f.targetView.BlockId(blockNumber) == cp.lastBlockId && cp.mapIndex == mapIndex && (best == nil || blockNumber > best.lastBlock) { best = cp } @@ -173,7 +173,7 @@ func (f *FilterMaps) lastCanonicalMapBoundaryBefore(renderBefore uint32) (nextMa return 0, 0, 0, fmt.Errorf("failed to retrieve last block of reverse iterated map %d: %v", mapIndex, err) } if (f.indexedRange.headIndexed && mapIndex >= f.indexedRange.maps.Last()) || - lastBlock >= f.targetView.headNumber || lastBlockId != f.targetView.getBlockId(lastBlock) { + lastBlock >= f.targetView.HeadNumber() || lastBlockId != f.targetView.BlockId(lastBlock) { continue // map is not full or inconsistent with targetView; roll back } lvPtr, err := f.getBlockLvPointer(lastBlock) @@ -247,7 +247,7 @@ func (f *FilterMaps) loadHeadSnapshot() error { filterMap: fm, mapIndex: f.indexedRange.maps.Last(), lastBlock: f.indexedRange.blocks.Last(), - lastBlockId: f.indexedView.getBlockId(f.indexedRange.blocks.Last()), + lastBlockId: f.indexedView.BlockId(f.indexedRange.blocks.Last()), blockLvPtrs: lvPtrs, finished: true, headDelimiter: f.indexedRange.headDelimiter, @@ -264,7 +264,7 @@ func (r *mapRenderer) makeSnapshot() { filterMap: r.currentMap.filterMap.fastCopy(), mapIndex: r.currentMap.mapIndex, lastBlock: r.currentMap.lastBlock, - lastBlockId: r.iterator.chainView.getBlockId(r.currentMap.lastBlock), + lastBlockId: r.iterator.chainView.BlockId(r.currentMap.lastBlock), blockLvPtrs: r.currentMap.blockLvPtrs, finished: true, headDelimiter: r.iterator.lvIndex, @@ -370,7 +370,7 @@ func (r *mapRenderer) renderCurrentMap(stopCb func() bool) (bool, error) { r.currentMap.finished = true r.currentMap.headDelimiter = r.iterator.lvIndex } - r.currentMap.lastBlockId = r.f.targetView.getBlockId(r.currentMap.lastBlock) + r.currentMap.lastBlockId = r.f.targetView.BlockId(r.currentMap.lastBlock) totalTime += time.Since(start) mapRenderTimer.Update(totalTime) mapLogValueMeter.Mark(logValuesProcessed) @@ -566,8 +566,8 @@ func (r *mapRenderer) getUpdatedRange() (filterMapsRange, error) { lm := r.finishedMaps[r.finished.Last()] newRange.headIndexed = lm.finished if lm.finished { - newRange.blocks.SetLast(r.f.targetView.headNumber) - if lm.lastBlock != r.f.targetView.headNumber { + newRange.blocks.SetLast(r.f.targetView.HeadNumber()) + if lm.lastBlock != r.f.targetView.HeadNumber() { panic("map rendering finished but last block != head block") } newRange.headDelimiter = lm.headDelimiter @@ -665,13 +665,13 @@ var errUnindexedRange = errors.New("unindexed range") // given block's first log value entry (the block delimiter), according to the // current targetView. func (f *FilterMaps) newLogIteratorFromBlockDelimiter(blockNumber, lvIndex uint64) (*logIterator, error) { - if blockNumber > f.targetView.headNumber { - return nil, fmt.Errorf("iterator entry point %d after target chain head block %d", blockNumber, f.targetView.headNumber) + if blockNumber > f.targetView.HeadNumber() { + return nil, fmt.Errorf("iterator entry point %d after target chain head block %d", blockNumber, f.targetView.HeadNumber()) } if !f.indexedRange.blocks.Includes(blockNumber) { return nil, errUnindexedRange } - finished := blockNumber == f.targetView.headNumber + finished := blockNumber == f.targetView.HeadNumber() l := &logIterator{ chainView: f.targetView, params: &f.Params, @@ -687,11 +687,11 @@ func (f *FilterMaps) newLogIteratorFromBlockDelimiter(blockNumber, lvIndex uint6 // newLogIteratorFromMapBoundary creates a logIterator starting at the given // map boundary, according to the current targetView. func (f *FilterMaps) newLogIteratorFromMapBoundary(mapIndex uint32, startBlock, startLvPtr uint64) (*logIterator, error) { - if startBlock > f.targetView.headNumber { - return nil, fmt.Errorf("iterator entry point %d after target chain head block %d", startBlock, f.targetView.headNumber) + if startBlock > f.targetView.HeadNumber() { + return nil, fmt.Errorf("iterator entry point %d after target chain head block %d", startBlock, f.targetView.HeadNumber()) } // get block receipts - receipts := f.targetView.getReceipts(startBlock) + receipts := f.targetView.Receipts(startBlock) if receipts == nil { return nil, fmt.Errorf("receipts not found for start block %d", startBlock) } @@ -758,7 +758,7 @@ func (l *logIterator) next() error { if l.delimiter { l.delimiter = false l.blockNumber++ - l.receipts = l.chainView.getReceipts(l.blockNumber) + l.receipts = l.chainView.Receipts(l.blockNumber) if l.receipts == nil { return fmt.Errorf("receipts not found for block %d", l.blockNumber) } @@ -795,7 +795,7 @@ func (l *logIterator) enforceValidState() { } l.logIndex = 0 } - if l.blockNumber == l.chainView.headNumber { + if l.blockNumber == l.chainView.HeadNumber() { l.finished = true } else { l.delimiter = true diff --git a/core/filtermaps/matcher.go b/core/filtermaps/matcher.go index 5738bf166a..a5eeaaa5f0 100644 --- a/core/filtermaps/matcher.go +++ b/core/filtermaps/matcher.go @@ -57,7 +57,7 @@ type MatcherBackend interface { // all states of the chain since the previous SyncLogIndex or the creation of // the matcher backend. type SyncRange struct { - HeadNumber uint64 + IndexedView *ChainView // block range where the index has not changed since the last matcher sync // and therefore the set of matches found in this region is guaranteed to // be valid and complete. diff --git a/core/filtermaps/matcher_backend.go b/core/filtermaps/matcher_backend.go index 335ac84551..ee18a0694c 100644 --- a/core/filtermaps/matcher_backend.go +++ b/core/filtermaps/matcher_backend.go @@ -128,7 +128,7 @@ func (fm *FilterMapsMatcherBackend) synced() { indexedBlocks.SetAfterLast(indexedBlocks.Last()) // remove partially indexed last block } fm.syncCh <- SyncRange{ - HeadNumber: fm.f.targetView.headNumber, + IndexedView: fm.f.indexedView, ValidBlocks: fm.validBlocks, IndexedBlocks: indexedBlocks, } @@ -154,7 +154,7 @@ func (fm *FilterMapsMatcherBackend) SyncLogIndex(ctx context.Context) (SyncRange case <-ctx.Done(): return SyncRange{}, ctx.Err() case <-fm.f.disabledCh: - return SyncRange{HeadNumber: fm.f.targetView.headNumber}, nil + return SyncRange{IndexedView: fm.f.indexedView}, nil } select { case vr := <-syncCh: @@ -162,7 +162,7 @@ func (fm *FilterMapsMatcherBackend) SyncLogIndex(ctx context.Context) (SyncRange case <-ctx.Done(): return SyncRange{}, ctx.Err() case <-fm.f.disabledCh: - return SyncRange{HeadNumber: fm.f.targetView.headNumber}, nil + return SyncRange{IndexedView: fm.f.indexedView}, nil } } diff --git a/eth/api_backend.go b/eth/api_backend.go index 64fb58a1fd..10f7ffcbce 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -443,6 +443,14 @@ func (b *EthAPIBackend) RPCTxFeeCap() float64 { return b.eth.config.RPCTxFeeCap } +func (b *EthAPIBackend) CurrentView() *filtermaps.ChainView { + head := b.eth.blockchain.CurrentBlock() + if head == nil { + return nil + } + return filtermaps.NewChainView(b.eth.blockchain, head.Number.Uint64(), head.Hash()) +} + func (b *EthAPIBackend) NewMatcherBackend() filtermaps.MatcherBackend { return b.eth.filterMaps.NewMatcherBackend() } diff --git a/eth/filters/filter.go b/eth/filters/filter.go index 78c80d8f26..dd6643c59e 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -146,25 +146,29 @@ func (f *Filter) Logs(ctx context.Context) ([]*types.Log, error) { } const ( - rangeLogsTestSync = iota - rangeLogsTestTrimmed - rangeLogsTestIndexed - rangeLogsTestUnindexed - rangeLogsTestDone + rangeLogsTestDone = iota // zero range + rangeLogsTestSync // before sync; zero range + rangeLogsTestSynced // after sync; valid blocks range + rangeLogsTestIndexed // individual search range + rangeLogsTestUnindexed // individual search range + rangeLogsTestResults // results range after search iteration + rangeLogsTestReorg // results range trimmed by reorg ) type rangeLogsTestEvent struct { - event int - begin, end uint64 + event int + blocks common.Range[uint64] } // searchSession represents a single search session. type searchSession struct { - ctx context.Context - filter *Filter - mb filtermaps.MatcherBackend - syncRange filtermaps.SyncRange // latest synchronized state with the matcher - firstBlock, lastBlock uint64 // specified search range; each can be MaxUint64 + ctx context.Context + filter *Filter + mb filtermaps.MatcherBackend + syncRange filtermaps.SyncRange // latest synchronized state with the matcher + chainView *filtermaps.ChainView // can be more recent than the indexed view in syncRange + // block ranges always refer to the current chainView + firstBlock, lastBlock uint64 // specified search range; MaxUint64 means latest block searchRange common.Range[uint64] // actual search range; end trimmed to latest head matchRange common.Range[uint64] // range in which we have results (subset of searchRange) matches []*types.Log // valid set of matches in matchRange @@ -182,84 +186,99 @@ func newSearchSession(ctx context.Context, filter *Filter, mb filtermaps.Matcher } // enforce a consistent state before starting the search in order to be able // to determine valid range later - if err := s.syncMatcher(0); err != nil { + var err error + s.syncRange, err = s.mb.SyncLogIndex(s.ctx) + if err != nil { + return nil, err + } + if err := s.updateChainView(); err != nil { return nil, err } return s, nil } -// syncMatcher performs a synchronization step with the matcher. The resulting -// syncRange structure holds information about the latest range of indexed blocks -// and the guaranteed valid blocks whose log index have not been changed since -// the previous synchronization. -// The function also performs trimming of the match set in order to always keep -// it consistent with the synced matcher state. -// Tail trimming is only performed if the first block of the valid log index range -// is higher than trimTailThreshold. This is useful because unindexed log search -// is not affected by the valid tail (on the other hand, valid head is taken into -// account in order to provide reorg safety, even though the log index is not used). -// In case of indexed search the tail is only trimmed if the first part of the -// recently obtained results might be invalid. If guaranteed valid new results -// have been added at the head of previously validated results then there is no -// need to discard those even if the index tail have been unindexed since that. -func (s *searchSession) syncMatcher(trimTailThreshold uint64) error { - if s.filter.rangeLogsTestHook != nil && !s.matchRange.IsEmpty() { - s.filter.rangeLogsTestHook <- rangeLogsTestEvent{event: rangeLogsTestSync, begin: s.matchRange.First(), end: s.matchRange.Last()} - } - var err error - s.syncRange, err = s.mb.SyncLogIndex(s.ctx) - if err != nil { - return err +// updateChainView updates to the latest view of the underlying chain and sets +// searchRange by replacing MaxUint64 (meaning latest block) with actual head +// number in the specified search range. +// If the session already had an existing chain view and set of matches then +// it also trims part of the match set that a chain reorg might have invalidated. +func (s *searchSession) updateChainView() error { + // update chain view based on current chain head (might be more recent than + // the indexed view of syncRange as the indexer updates it asynchronously + // with some delay + newChainView := s.filter.sys.backend.CurrentView() + if newChainView == nil { + return errors.New("head block not available") } + head := newChainView.HeadNumber() + // update actual search range based on current head number - first := min(s.firstBlock, s.syncRange.HeadNumber) - last := min(s.lastBlock, s.syncRange.HeadNumber) - s.searchRange = common.NewRange(first, last+1-first) - // discard everything that is not needed or might be invalid - trimRange := s.syncRange.ValidBlocks - if trimRange.First() <= trimTailThreshold { - // everything before this point is already known to be valid; if this is - // valid then keep everything before - trimRange.SetFirst(0) + firstBlock, lastBlock := s.firstBlock, s.lastBlock + if firstBlock == math.MaxUint64 { + firstBlock = head } - trimRange = trimRange.Intersection(s.searchRange) - s.trimMatches(trimRange) - if s.filter.rangeLogsTestHook != nil { - if !s.matchRange.IsEmpty() { - s.filter.rangeLogsTestHook <- rangeLogsTestEvent{event: rangeLogsTestTrimmed, begin: s.matchRange.First(), end: s.matchRange.Last()} - } else { - s.filter.rangeLogsTestHook <- rangeLogsTestEvent{event: rangeLogsTestTrimmed, begin: 0, end: 0} - } + if lastBlock == math.MaxUint64 { + lastBlock = head } + if firstBlock > lastBlock || lastBlock > head { + return errInvalidBlockRange + } + s.searchRange = common.NewRange(firstBlock, lastBlock+1-firstBlock) + + // Trim existing match set in case a reorg may have invalidated some results + if !s.matchRange.IsEmpty() { + trimRange := newChainView.SharedRange(s.chainView).Intersection(s.searchRange) + s.matchRange, s.matches = s.trimMatches(trimRange, s.matchRange, s.matches) + } + s.chainView = newChainView return nil } -// trimMatches removes any entries from the current set of matches that is outside -// the given range. -func (s *searchSession) trimMatches(trimRange common.Range[uint64]) { - s.matchRange = s.matchRange.Intersection(trimRange) - if s.matchRange.IsEmpty() { - s.matches = nil - return +// trimMatches removes any entries from the specified set of matches that is +// outside the given range. +func (s *searchSession) trimMatches(trimRange, matchRange common.Range[uint64], matches []*types.Log) (common.Range[uint64], []*types.Log) { + newRange := matchRange.Intersection(trimRange) + if newRange == matchRange { + return matchRange, matches } - for len(s.matches) > 0 && s.matches[0].BlockNumber < s.matchRange.First() { - s.matches = s.matches[1:] + if newRange.IsEmpty() { + return newRange, nil } - for len(s.matches) > 0 && s.matches[len(s.matches)-1].BlockNumber > s.matchRange.Last() { - s.matches = s.matches[:len(s.matches)-1] + for len(matches) > 0 && matches[0].BlockNumber < newRange.First() { + matches = matches[1:] } + for len(matches) > 0 && matches[len(matches)-1].BlockNumber > newRange.Last() { + matches = matches[:len(matches)-1] + } + return newRange, matches } // searchInRange performs a single range search, either indexed or unindexed. -func (s *searchSession) searchInRange(r common.Range[uint64], indexed bool) ([]*types.Log, error) { - first, last := r.First(), r.Last() +func (s *searchSession) searchInRange(r common.Range[uint64], indexed bool) (common.Range[uint64], []*types.Log, error) { if indexed { if s.filter.rangeLogsTestHook != nil { - s.filter.rangeLogsTestHook <- rangeLogsTestEvent{rangeLogsTestIndexed, first, last} + s.filter.rangeLogsTestHook <- rangeLogsTestEvent{rangeLogsTestIndexed, r} } - results, err := s.filter.indexedLogs(s.ctx, s.mb, first, last) - if err != filtermaps.ErrMatchAll { - return results, err + results, err := s.filter.indexedLogs(s.ctx, s.mb, r.First(), r.Last()) + if err != nil && !errors.Is(err, filtermaps.ErrMatchAll) { + return common.Range[uint64]{}, nil, err + } + if err == nil { + // sync with filtermaps matcher + if s.filter.rangeLogsTestHook != nil { + s.filter.rangeLogsTestHook <- rangeLogsTestEvent{rangeLogsTestSync, common.Range[uint64]{}} + } + var syncErr error + if s.syncRange, syncErr = s.mb.SyncLogIndex(s.ctx); syncErr != nil { + return common.Range[uint64]{}, nil, syncErr + } + if s.filter.rangeLogsTestHook != nil { + s.filter.rangeLogsTestHook <- rangeLogsTestEvent{rangeLogsTestSynced, s.syncRange.ValidBlocks} + } + // discard everything that might be invalid + trimRange := s.syncRange.ValidBlocks.Intersection(s.chainView.SharedRange(s.syncRange.IndexedView)) + matchRange, matches := s.trimMatches(trimRange, r, results) + return matchRange, matches, nil } // "match all" filters are not supported by filtermaps; fall back to // unindexed search which is the most efficient in this case @@ -267,79 +286,85 @@ func (s *searchSession) searchInRange(r common.Range[uint64], indexed bool) ([]* // fall through to unindexed case } if s.filter.rangeLogsTestHook != nil { - s.filter.rangeLogsTestHook <- rangeLogsTestEvent{rangeLogsTestUnindexed, first, last} + s.filter.rangeLogsTestHook <- rangeLogsTestEvent{rangeLogsTestUnindexed, r} } - return s.filter.unindexedLogs(s.ctx, first, last) + matches, err := s.filter.unindexedLogs(s.ctx, s.chainView, r.First(), r.Last()) + if err != nil { + return common.Range[uint64]{}, nil, err + } + return r, matches, nil } // doSearchIteration performs a search on a range missing from an incomplete set // of results, adds the new section and removes invalidated entries. func (s *searchSession) doSearchIteration() error { switch { - case s.syncRange.IndexedBlocks.IsEmpty(): - // indexer is not ready; fallback to completely unindexed search, do not check valid range - var err error - s.matchRange = s.searchRange - s.matches, err = s.searchInRange(s.searchRange, false) - return err - case s.matchRange.IsEmpty(): // no results yet; try search in entire range indexedSearchRange := s.searchRange.Intersection(s.syncRange.IndexedBlocks) - var err error if s.forceUnindexed = indexedSearchRange.IsEmpty(); !s.forceUnindexed { // indexed search on the intersection of indexed and searched range - s.matchRange = indexedSearchRange - s.matches, err = s.searchInRange(indexedSearchRange, true) + matchRange, matches, err := s.searchInRange(indexedSearchRange, true) if err != nil { return err } - return s.syncMatcher(0) // trim everything that the matcher considers potentially invalid + s.matchRange = matchRange + s.matches = matches + return nil } else { - // no intersection of indexed and searched range; unindexed search on the whole searched range - s.matchRange = s.searchRange - s.matches, err = s.searchInRange(s.searchRange, false) + // no intersection of indexed and searched range; unindexed search on + // the whole searched range + matchRange, matches, err := s.searchInRange(s.searchRange, false) if err != nil { return err } - return s.syncMatcher(math.MaxUint64) // unindexed search is not affected by the tail of valid range + s.matchRange = matchRange + s.matches = matches + return nil } case !s.matchRange.IsEmpty() && s.matchRange.First() > s.searchRange.First(): - // we have results but tail section is missing; do unindexed search for - // the tail part but still allow indexed search for missing head section + // Results are available, but the tail section is missing. Perform an unindexed + // search for the missing tail, while still allowing indexed search for the head. + // + // The unindexed search is necessary because the tail portion of the indexes + // has been pruned. tailRange := common.NewRange(s.searchRange.First(), s.matchRange.First()-s.searchRange.First()) - tailMatches, err := s.searchInRange(tailRange, false) + _, tailMatches, err := s.searchInRange(tailRange, false) if err != nil { return err } s.matches = append(tailMatches, s.matches...) s.matchRange = tailRange.Union(s.matchRange) - return s.syncMatcher(math.MaxUint64) // unindexed search is not affected by the tail of valid range + return nil case !s.matchRange.IsEmpty() && s.matchRange.First() == s.searchRange.First() && s.searchRange.AfterLast() > s.matchRange.AfterLast(): - // we have results but head section is missing + // Results are available, but the head section is missing. Try to perform + // the indexed search for the missing head, or fallback to unindexed search + // if the tail portion of indexed range has been pruned. headRange := common.NewRange(s.matchRange.AfterLast(), s.searchRange.AfterLast()-s.matchRange.AfterLast()) if !s.forceUnindexed { indexedHeadRange := headRange.Intersection(s.syncRange.IndexedBlocks) if !indexedHeadRange.IsEmpty() && indexedHeadRange.First() == headRange.First() { - // indexed head range search is possible headRange = indexedHeadRange } else { + // The tail portion of the indexes has been pruned, falling back + // to unindexed search. s.forceUnindexed = true } } - headMatches, err := s.searchInRange(headRange, !s.forceUnindexed) + headMatchRange, headMatches, err := s.searchInRange(headRange, !s.forceUnindexed) if err != nil { return err } - s.matches = append(s.matches, headMatches...) - s.matchRange = s.matchRange.Union(headRange) - if s.forceUnindexed { - return s.syncMatcher(math.MaxUint64) // unindexed search is not affected by the tail of valid range - } else { - return s.syncMatcher(headRange.First()) // trim if the tail of latest head search results might be invalid + if headMatchRange.First() != s.matchRange.AfterLast() { + // improbable corner case, first part of new head range invalidated by tail unindexing + s.matches, s.matchRange = headMatches, headMatchRange + return nil } + s.matches = append(s.matches, headMatches...) + s.matchRange = s.matchRange.Union(headMatchRange) + return nil default: panic("invalid search session state") @@ -349,7 +374,7 @@ func (s *searchSession) doSearchIteration() error { func (f *Filter) rangeLogs(ctx context.Context, firstBlock, lastBlock uint64) ([]*types.Log, error) { if f.rangeLogsTestHook != nil { defer func() { - f.rangeLogsTestHook <- rangeLogsTestEvent{rangeLogsTestDone, 0, 0} + f.rangeLogsTestHook <- rangeLogsTestEvent{rangeLogsTestDone, common.Range[uint64]{}} close(f.rangeLogsTestHook) }() } @@ -366,7 +391,17 @@ func (f *Filter) rangeLogs(ctx context.Context, firstBlock, lastBlock uint64) ([ } for session.searchRange != session.matchRange { if err := session.doSearchIteration(); err != nil { - return session.matches, err + return nil, err + } + if f.rangeLogsTestHook != nil { + f.rangeLogsTestHook <- rangeLogsTestEvent{rangeLogsTestResults, session.matchRange} + } + mr := session.matchRange + if err := session.updateChainView(); err != nil { + return nil, err + } + if f.rangeLogsTestHook != nil && session.matchRange != mr { + f.rangeLogsTestHook <- rangeLogsTestEvent{rangeLogsTestReorg, session.matchRange} } } return session.matches, nil @@ -382,7 +417,7 @@ func (f *Filter) indexedLogs(ctx context.Context, mb filtermaps.MatcherBackend, // unindexedLogs returns the logs matching the filter criteria based on raw block // iteration and bloom matching. -func (f *Filter) unindexedLogs(ctx context.Context, begin, end uint64) ([]*types.Log, error) { +func (f *Filter) unindexedLogs(ctx context.Context, chainView *filtermaps.ChainView, begin, end uint64) ([]*types.Log, error) { start := time.Now() log.Debug("Performing unindexed log search", "begin", begin, "end", end) var matches []*types.Log @@ -392,9 +427,14 @@ func (f *Filter) unindexedLogs(ctx context.Context, begin, end uint64) ([]*types return matches, ctx.Err() default: } - header, err := f.sys.backend.HeaderByNumber(ctx, rpc.BlockNumber(blockNumber)) - if header == nil || err != nil { - return matches, err + if blockNumber > chainView.HeadNumber() { + // check here so that we can return matches up until head along with + // the error + return matches, errInvalidBlockRange + } + header := chainView.Header(blockNumber) + if header == nil { + return matches, errors.New("header not found") } found, err := f.blockLogs(ctx, header) if err != nil { diff --git a/eth/filters/filter_system.go b/eth/filters/filter_system.go index b787f1067b..10e433f09b 100644 --- a/eth/filters/filter_system.go +++ b/eth/filters/filter_system.go @@ -71,6 +71,7 @@ type Backend interface { SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription + CurrentView() *filtermaps.ChainView NewMatcherBackend() filtermaps.MatcherBackend } diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index 3bb019d105..fa5d4fe897 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -154,6 +154,11 @@ func (b *testBackend) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subsc return b.chainFeed.Subscribe(ch) } +func (b *testBackend) CurrentView() *filtermaps.ChainView { + head := b.CurrentBlock() + return filtermaps.NewChainView(b, head.Number.Uint64(), head.Hash()) +} + func (b *testBackend) NewMatcherBackend() filtermaps.MatcherBackend { return b.fm.NewMatcherBackend() } diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go index 4026c03e89..d6065230f8 100644 --- a/eth/filters/filter_test.go +++ b/eth/filters/filter_test.go @@ -453,7 +453,8 @@ func TestRangeLogs(t *testing.T) { addresses = []common.Address{{}} ) - expEvent := func(exp rangeLogsTestEvent) { + expEvent := func(expEvent int, expFirst, expAfterLast uint64) { + exp := rangeLogsTestEvent{expEvent, common.NewRange[uint64](expFirst, expAfterLast-expFirst)} event++ ev := <-filter.rangeLogsTestHook if ev != exp { @@ -472,7 +473,6 @@ func TestRangeLogs(t *testing.T) { for range filter.rangeLogsTestHook { } }(filter) - expEvent(rangeLogsTestEvent{rangeLogsTestTrimmed, 0, 0}) } updateHead := func() { @@ -483,81 +483,122 @@ func TestRangeLogs(t *testing.T) { // test case #1 newFilter(300, 500) - expEvent(rangeLogsTestEvent{rangeLogsTestIndexed, 401, 500}) - expEvent(rangeLogsTestEvent{rangeLogsTestSync, 401, 500}) - expEvent(rangeLogsTestEvent{rangeLogsTestTrimmed, 401, 500}) - expEvent(rangeLogsTestEvent{rangeLogsTestUnindexed, 300, 400}) + expEvent(rangeLogsTestIndexed, 401, 501) + expEvent(rangeLogsTestSync, 0, 0) + expEvent(rangeLogsTestSynced, 401, 601) + expEvent(rangeLogsTestResults, 401, 501) + expEvent(rangeLogsTestUnindexed, 300, 401) if _, err := bc.InsertChain(chain[600:700]); err != nil { t.Fatal(err) } updateHead() - expEvent(rangeLogsTestEvent{rangeLogsTestSync, 300, 500}) - expEvent(rangeLogsTestEvent{rangeLogsTestTrimmed, 300, 500}) // unindexed search is not affected by trimmed tail - expEvent(rangeLogsTestEvent{rangeLogsTestDone, 0, 0}) + expEvent(rangeLogsTestResults, 300, 501) + expEvent(rangeLogsTestDone, 0, 0) // test case #2 newFilter(400, int64(rpc.LatestBlockNumber)) - expEvent(rangeLogsTestEvent{rangeLogsTestIndexed, 501, 700}) + expEvent(rangeLogsTestIndexed, 501, 701) if _, err := bc.InsertChain(chain[700:800]); err != nil { t.Fatal(err) } updateHead() - expEvent(rangeLogsTestEvent{rangeLogsTestSync, 501, 700}) - expEvent(rangeLogsTestEvent{rangeLogsTestTrimmed, 601, 698}) - expEvent(rangeLogsTestEvent{rangeLogsTestUnindexed, 400, 600}) - expEvent(rangeLogsTestEvent{rangeLogsTestSync, 400, 698}) - expEvent(rangeLogsTestEvent{rangeLogsTestTrimmed, 400, 698}) - expEvent(rangeLogsTestEvent{rangeLogsTestIndexed, 699, 800}) - if err := bc.SetHead(750); err != nil { + expEvent(rangeLogsTestSync, 0, 0) + expEvent(rangeLogsTestSynced, 601, 699) + expEvent(rangeLogsTestResults, 601, 699) + expEvent(rangeLogsTestUnindexed, 400, 601) + expEvent(rangeLogsTestResults, 400, 699) + expEvent(rangeLogsTestIndexed, 699, 801) + if _, err := bc.SetCanonical(chain[749]); err != nil { // set head to block 750 t.Fatal(err) } updateHead() - expEvent(rangeLogsTestEvent{rangeLogsTestSync, 400, 800}) - expEvent(rangeLogsTestEvent{rangeLogsTestTrimmed, 400, 748}) - expEvent(rangeLogsTestEvent{rangeLogsTestIndexed, 749, 750}) - expEvent(rangeLogsTestEvent{rangeLogsTestSync, 400, 750}) - expEvent(rangeLogsTestEvent{rangeLogsTestTrimmed, 400, 750}) - expEvent(rangeLogsTestEvent{rangeLogsTestDone, 0, 0}) + expEvent(rangeLogsTestSync, 0, 0) + expEvent(rangeLogsTestSynced, 601, 749) + expEvent(rangeLogsTestResults, 400, 749) + expEvent(rangeLogsTestIndexed, 749, 751) + expEvent(rangeLogsTestSync, 0, 0) + expEvent(rangeLogsTestSynced, 551, 751) + expEvent(rangeLogsTestResults, 400, 751) + expEvent(rangeLogsTestDone, 0, 0) // test case #3 newFilter(int64(rpc.LatestBlockNumber), int64(rpc.LatestBlockNumber)) - expEvent(rangeLogsTestEvent{rangeLogsTestIndexed, 750, 750}) - if err := bc.SetHead(740); err != nil { + expEvent(rangeLogsTestIndexed, 750, 751) + if _, err := bc.SetCanonical(chain[739]); err != nil { t.Fatal(err) } updateHead() - expEvent(rangeLogsTestEvent{rangeLogsTestSync, 750, 750}) - expEvent(rangeLogsTestEvent{rangeLogsTestTrimmed, 0, 0}) - expEvent(rangeLogsTestEvent{rangeLogsTestIndexed, 740, 740}) + expEvent(rangeLogsTestSync, 0, 0) + expEvent(rangeLogsTestSynced, 551, 739) + expEvent(rangeLogsTestResults, 0, 0) + expEvent(rangeLogsTestIndexed, 740, 741) if _, err := bc.InsertChain(chain[740:750]); err != nil { t.Fatal(err) } updateHead() - expEvent(rangeLogsTestEvent{rangeLogsTestSync, 740, 740}) - expEvent(rangeLogsTestEvent{rangeLogsTestTrimmed, 0, 0}) - expEvent(rangeLogsTestEvent{rangeLogsTestIndexed, 750, 750}) - expEvent(rangeLogsTestEvent{rangeLogsTestSync, 750, 750}) - expEvent(rangeLogsTestEvent{rangeLogsTestTrimmed, 750, 750}) - expEvent(rangeLogsTestEvent{rangeLogsTestDone, 0, 0}) + expEvent(rangeLogsTestSync, 0, 0) + expEvent(rangeLogsTestSynced, 551, 739) + expEvent(rangeLogsTestResults, 0, 0) + expEvent(rangeLogsTestIndexed, 750, 751) + expEvent(rangeLogsTestSync, 0, 0) + expEvent(rangeLogsTestSynced, 551, 751) + expEvent(rangeLogsTestResults, 750, 751) + expEvent(rangeLogsTestDone, 0, 0) // test case #4 + if _, err := bc.SetCanonical(chain[499]); err != nil { + t.Fatal(err) + } + updateHead() newFilter(400, int64(rpc.LatestBlockNumber)) - expEvent(rangeLogsTestEvent{rangeLogsTestIndexed, 551, 750}) - expEvent(rangeLogsTestEvent{rangeLogsTestSync, 551, 750}) - expEvent(rangeLogsTestEvent{rangeLogsTestTrimmed, 551, 750}) - expEvent(rangeLogsTestEvent{rangeLogsTestUnindexed, 400, 550}) + expEvent(rangeLogsTestIndexed, 400, 501) + if _, err := bc.InsertChain(chain[500:650]); err != nil { + t.Fatal(err) + } + updateHead() + expEvent(rangeLogsTestSync, 0, 0) + expEvent(rangeLogsTestSynced, 451, 499) + expEvent(rangeLogsTestResults, 451, 499) + expEvent(rangeLogsTestUnindexed, 400, 451) + expEvent(rangeLogsTestResults, 400, 499) + // indexed head extension seems possible + expEvent(rangeLogsTestIndexed, 499, 651) + // further head extension causes tail unindexing in searched range + if _, err := bc.InsertChain(chain[650:750]); err != nil { + t.Fatal(err) + } + updateHead() + expEvent(rangeLogsTestSync, 0, 0) + expEvent(rangeLogsTestSynced, 551, 649) + // tail trimmed to 551; cannot merge with existing results + expEvent(rangeLogsTestResults, 551, 649) + expEvent(rangeLogsTestUnindexed, 400, 551) + expEvent(rangeLogsTestResults, 400, 649) + expEvent(rangeLogsTestIndexed, 649, 751) + expEvent(rangeLogsTestSync, 0, 0) + expEvent(rangeLogsTestSynced, 551, 751) + expEvent(rangeLogsTestResults, 400, 751) + expEvent(rangeLogsTestDone, 0, 0) + + // test case #5 + newFilter(400, int64(rpc.LatestBlockNumber)) + expEvent(rangeLogsTestIndexed, 551, 751) + expEvent(rangeLogsTestSync, 0, 0) + expEvent(rangeLogsTestSynced, 551, 751) + expEvent(rangeLogsTestResults, 551, 751) + expEvent(rangeLogsTestUnindexed, 400, 551) if _, err := bc.InsertChain(chain[750:1000]); err != nil { t.Fatal(err) } updateHead() - expEvent(rangeLogsTestEvent{rangeLogsTestSync, 400, 750}) - // indexed range affected by tail pruning so we have to discard the entire - // match set - expEvent(rangeLogsTestEvent{rangeLogsTestTrimmed, 0, 0}) - expEvent(rangeLogsTestEvent{rangeLogsTestIndexed, 801, 1000}) - expEvent(rangeLogsTestEvent{rangeLogsTestSync, 801, 1000}) - expEvent(rangeLogsTestEvent{rangeLogsTestTrimmed, 801, 1000}) - expEvent(rangeLogsTestEvent{rangeLogsTestUnindexed, 400, 800}) - expEvent(rangeLogsTestEvent{rangeLogsTestSync, 400, 1000}) - expEvent(rangeLogsTestEvent{rangeLogsTestTrimmed, 400, 1000}) + expEvent(rangeLogsTestResults, 400, 751) + // indexed tail already beyond results head; revert to unindexed head search + expEvent(rangeLogsTestUnindexed, 751, 1001) + if _, err := bc.SetCanonical(chain[899]); err != nil { + t.Fatal(err) + } + updateHead() + expEvent(rangeLogsTestResults, 400, 1001) + expEvent(rangeLogsTestReorg, 400, 901) + expEvent(rangeLogsTestDone, 0, 0) } diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go index 3fbf32e22e..aff051937d 100644 --- a/internal/ethapi/api_test.go +++ b/internal/ethapi/api_test.go @@ -619,6 +619,9 @@ func (b testBackend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) func (b testBackend) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription { panic("implement me") } +func (b testBackend) CurrentView() *filtermaps.ChainView { + panic("implement me") +} func (b testBackend) NewMatcherBackend() filtermaps.MatcherBackend { panic("implement me") } diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index c4bf2e0591..e28cb93296 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -95,6 +95,7 @@ type Backend interface { SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription + CurrentView() *filtermaps.ChainView NewMatcherBackend() filtermaps.MatcherBackend } diff --git a/internal/ethapi/transaction_args_test.go b/internal/ethapi/transaction_args_test.go index b4d11a3033..9dd6a54729 100644 --- a/internal/ethapi/transaction_args_test.go +++ b/internal/ethapi/transaction_args_test.go @@ -401,6 +401,7 @@ func (b *backendMock) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) func (b *backendMock) Engine() consensus.Engine { return nil } +func (b *backendMock) CurrentView() *filtermaps.ChainView { return nil } func (b *backendMock) NewMatcherBackend() filtermaps.MatcherBackend { return nil } func (b *backendMock) HistoryPruningCutoff() uint64 { return 0 } From 5a7bbb423feb88ec8997dff57c3b644cc6c4f143 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Sun, 20 Apr 2025 12:54:40 +0200 Subject: [PATCH 4/8] beacon/params, core/filtermaps: update checkpoints (#31674) This PR updates checkpoints for blsync and filtermaps. --- beacon/params/checkpoint_holesky.hex | 2 +- beacon/params/checkpoint_mainnet.hex | 2 +- beacon/params/checkpoint_sepolia.hex | 2 +- core/filtermaps/checkpoints_holesky.json | 3 ++- core/filtermaps/checkpoints_mainnet.json | 9 ++++++++- core/filtermaps/checkpoints_sepolia.json | 6 +++++- 6 files changed, 18 insertions(+), 6 deletions(-) diff --git a/beacon/params/checkpoint_holesky.hex b/beacon/params/checkpoint_holesky.hex index f1e27b48f6..740d7aba21 100644 --- a/beacon/params/checkpoint_holesky.hex +++ b/beacon/params/checkpoint_holesky.hex @@ -1 +1 @@ -0xf5606a019f0f1006e9ec2070695045f4334450362a48da4c5965314510e0b4c3 \ No newline at end of file +0xd60e5310c5d52ced44cfb13be4e9f22a1e6a6dc56964c3cccd429182d26d72d0 \ No newline at end of file diff --git a/beacon/params/checkpoint_mainnet.hex b/beacon/params/checkpoint_mainnet.hex index a11c3394bb..45f065ca15 100644 --- a/beacon/params/checkpoint_mainnet.hex +++ b/beacon/params/checkpoint_mainnet.hex @@ -1 +1 @@ -0x3c0cb4aa83beded1803d262664ba4392b1023f334d9cca02dc3a6925987ebe91 \ No newline at end of file +0x02f0bb348b0d45f95a9b7e2bb5705768ad06548876cee03d880a2c9dabb1ff88 \ No newline at end of file diff --git a/beacon/params/checkpoint_sepolia.hex b/beacon/params/checkpoint_sepolia.hex index e4f2f96733..3d1b2885b3 100644 --- a/beacon/params/checkpoint_sepolia.hex +++ b/beacon/params/checkpoint_sepolia.hex @@ -1 +1 @@ -0xa8d56457aa414523d93659aef1f7409bbfb72ad75e94d917c8c0b1baa38153ef \ No newline at end of file +0xa0dad451a230c01be6f2492980ec5bb412d8cf33351a75e8b172b5b84a5fd03a \ No newline at end of file diff --git a/core/filtermaps/checkpoints_holesky.json b/core/filtermaps/checkpoints_holesky.json index b8f6be9caf..a56611cc8e 100644 --- a/core/filtermaps/checkpoints_holesky.json +++ b/core/filtermaps/checkpoints_holesky.json @@ -17,5 +17,6 @@ {"blockNumber": 3101669, "blockId": "0xa6e66a18fed64d33178c7088f273c404be597662c34f6e6884cf2e24f3ea4ace", "firstIndex": 1073741353}, {"blockNumber": 3221725, "blockId": "0xe771f897dece48b1583cc1d1d10de8015da57407eb1fdf239fdbe46eaab85143", "firstIndex": 1140850137}, {"blockNumber": 3357164, "blockId": "0x6252d0aa54c79623b0680069c88d7b5c47983f0d5c4845b6c811b8d9b5e8ff3c", "firstIndex": 1207959453}, -{"blockNumber": 3447019, "blockId": "0xeb7d585e1e063f3cc05ed399fbf6c2df63c271f62f030acb804e9fb1e74b6dc1", "firstIndex": 1275067542} +{"blockNumber": 3447019, "blockId": "0xeb7d585e1e063f3cc05ed399fbf6c2df63c271f62f030acb804e9fb1e74b6dc1", "firstIndex": 1275067542}, +{"blockNumber": 3546397, "blockId": "0xdabdef7defa4281180a57c5af121877b82274f15ccf074ea0096146f4c246df2", "firstIndex": 1342176778} ] diff --git a/core/filtermaps/checkpoints_mainnet.json b/core/filtermaps/checkpoints_mainnet.json index ca30280e03..70a08b1aaf 100644 --- a/core/filtermaps/checkpoints_mainnet.json +++ b/core/filtermaps/checkpoints_mainnet.json @@ -260,5 +260,12 @@ {"blockNumber": 21959188, "blockId": "0x6b9c482cc4822af2ad62a359b923062556ab6a046d1a39a8243b764848690601", "firstIndex": 17381193886}, {"blockNumber": 21994911, "blockId": "0x0d99ef9dd42dd1c62fc5249eb4f618e7672ab93fa0ba7545c77360371cf972e5", "firstIndex": 17448302795}, {"blockNumber": 22026007, "blockId": "0xf7cdc7f6694f2c2ed31fa8a607f65cfa59d0dd7d7424ab5c017f959ae2982c71", "firstIndex": 17515413036}, -{"blockNumber": 22059890, "blockId": "0x82b768a0dddefda2eefd3a33596ea2be075312f1dd4b01f6b0d377faca2af98b", "firstIndex": 17582521768} +{"blockNumber": 22059890, "blockId": "0x82b768a0dddefda2eefd3a33596ea2be075312f1dd4b01f6b0d377faca2af98b", "firstIndex": 17582521768}, +{"blockNumber": 22090784, "blockId": "0xf97c2eaf9a550360ac24000c0ff17ffa388a2bdd6f73f2f36718e332edfa107a", "firstIndex": 17649630983}, +{"blockNumber": 22121157, "blockId": "0xa790025235db782e899f23d8b09663ec2d74ec149e4125d62989f98829b08e2d", "firstIndex": 17716724973}, +{"blockNumber": 22148056, "blockId": "0xbe25ac4f1bdd89a7db5782bf1157e6c4378d80220d67a029397853ef16cd1a4c", "firstIndex": 17783838366}, +{"blockNumber": 22168652, "blockId": "0x6ae43618c915e636794e2cc2d75dde9992766881c7405fe6479c045ed4bee57e", "firstIndex": 17850956277}, +{"blockNumber": 22190975, "blockId": "0x9437121647899a4b7b84d67fbea7cc6ff967481c2eab4328ccd86e2cefe19420", "firstIndex": 17918066140}, +{"blockNumber": 22234357, "blockId": "0x036030830134f9224160d5a0b62da35ec7813dc8855d554bd22e9d38545243ed", "firstIndex": 17985175075}, +{"blockNumber": 22276736, "blockId": "0x5ceb96d98aa1b4c1c2f2fa253ae9cdb1b04e0420c11bf795400e8762c0a1635c", "firstIndex": 18052284344} ] diff --git a/core/filtermaps/checkpoints_sepolia.json b/core/filtermaps/checkpoints_sepolia.json index c6d3cd9086..8d799daefd 100644 --- a/core/filtermaps/checkpoints_sepolia.json +++ b/core/filtermaps/checkpoints_sepolia.json @@ -64,5 +64,9 @@ {"blockNumber": 7730079, "blockId": "0x08e0d511a93e2a9c004b3456d77d687ae86247417ada1cdd1f85332a62dfed1d", "firstIndex": 4227846795}, {"blockNumber": 7777515, "blockId": "0xeba8faed2e12bb9ed5e7fad19dd82662f15f6f4f512a0d232eedf1e676712428", "firstIndex": 4294966082}, {"blockNumber": 7828860, "blockId": "0x62c054bd24be351dc4777460ee922a75fdcea5c9830455cbac6fa29552719c85", "firstIndex": 4362076087}, -{"blockNumber": 7869890, "blockId": "0x5664bcfa6151f798781e22bea4f9a2c796d097402350cb131e4e3c78e13e461a", "firstIndex": 4429183267} +{"blockNumber": 7869890, "blockId": "0x5664bcfa6151f798781e22bea4f9a2c796d097402350cb131e4e3c78e13e461a", "firstIndex": 4429183267}, +{"blockNumber": 7911722, "blockId": "0x9a85e48e3135c97c51fc1786f2af0596c802e021b6c53cfca65a129cafcd23ed", "firstIndex": 4496287265}, +{"blockNumber": 7960147, "blockId": "0xc9359cc76d7090e1c8a031108f0ab7a8935d971efd4325fe53612a1d99562f6f", "firstIndex": 4563402388}, +{"blockNumber": 8030418, "blockId": "0x21867e68cd8327aed2da2601399d60f7f9e41dca4a4f2f9be982e5a2b9304a88", "firstIndex": 4630511616}, +{"blockNumber": 8087701, "blockId": "0x0fa8c8d7549cc9a8d308262706fe248efe759f8b63511efb1e7f3926e9af2dcb", "firstIndex": 4697614758} ] From 14f15430bb99646f516e239271f21e4a52584666 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Mon, 21 Apr 2025 09:27:24 +0200 Subject: [PATCH 5/8] core/filtermaps: clone cached slices, fix tempRange (#31680) This PR ensures that caching a slice or a slice of slices will never affect the original version by always cloning a slice fetched from cache if it is not used in a guaranteed read only way. --- core/filtermaps/filtermaps.go | 9 ++++-- core/filtermaps/indexer.go | 3 ++ core/filtermaps/indexer_test.go | 52 +++++++++++++++++++++++++++++++++ core/filtermaps/map_renderer.go | 6 ++-- 4 files changed, 65 insertions(+), 5 deletions(-) diff --git a/core/filtermaps/filtermaps.go b/core/filtermaps/filtermaps.go index a617de8968..920167ca8d 100644 --- a/core/filtermaps/filtermaps.go +++ b/core/filtermaps/filtermaps.go @@ -128,6 +128,7 @@ type FilterMaps struct { // test hooks testDisableSnapshots, testSnapshotUsed bool + testProcessEventsHook func() } // filterMap is a full or partial in-memory representation of a filter map where @@ -573,7 +574,7 @@ func (f *FilterMaps) getFilterMapRow(mapIndex, rowIndex uint32, baseLayerOnly bo } f.baseRowsCache.Add(baseMapRowIndex, baseRows) } - baseRow := baseRows[mapIndex&(f.baseRowGroupLength-1)] + baseRow := slices.Clone(baseRows[mapIndex&(f.baseRowGroupLength-1)]) if baseLayerOnly { return baseRow, nil } @@ -610,7 +611,9 @@ func (f *FilterMaps) storeFilterMapRowsOfGroup(batch ethdb.Batch, mapIndices []u if uint32(len(mapIndices)) != f.baseRowGroupLength { // skip base rows read if all rows are replaced var ok bool baseRows, ok = f.baseRowsCache.Get(baseMapRowIndex) - if !ok { + if ok { + baseRows = slices.Clone(baseRows) + } else { var err error baseRows, err = rawdb.ReadFilterMapBaseRows(f.db, baseMapRowIndex, f.baseRowGroupLength, f.logMapWidth) if err != nil { @@ -656,7 +659,7 @@ func (f *FilterMaps) mapRowIndex(mapIndex, rowIndex uint32) uint64 { // called from outside the indexerLoop goroutine. func (f *FilterMaps) getBlockLvPointer(blockNumber uint64) (uint64, error) { if blockNumber >= f.indexedRange.blocks.AfterLast() && f.indexedRange.headIndexed { - return f.indexedRange.headDelimiter, nil + return f.indexedRange.headDelimiter + 1, nil } if lvPointer, ok := f.lvPointerCache.Get(blockNumber); ok { return lvPointer, nil diff --git a/core/filtermaps/indexer.go b/core/filtermaps/indexer.go index 383ec078c9..787197345a 100644 --- a/core/filtermaps/indexer.go +++ b/core/filtermaps/indexer.go @@ -165,6 +165,9 @@ func (f *FilterMaps) waitForNewHead() { // processEvents processes all events, blocking only if a block processing is // happening and indexing should be suspended. func (f *FilterMaps) processEvents() { + if f.testProcessEventsHook != nil { + f.testProcessEventsHook() + } for f.processSingleEvent(f.blockProcessing) { } } diff --git a/core/filtermaps/indexer_test.go b/core/filtermaps/indexer_test.go index 4dddd27087..e60130ba4b 100644 --- a/core/filtermaps/indexer_test.go +++ b/core/filtermaps/indexer_test.go @@ -219,6 +219,58 @@ func testIndexerMatcherView(t *testing.T, concurrentRead bool) { } } +func TestLogsByIndex(t *testing.T) { + ts := newTestSetup(t) + defer func() { + ts.fm.testProcessEventsHook = nil + ts.close() + }() + + ts.chain.addBlocks(1000, 10, 3, 4, true) + ts.setHistory(0, false) + ts.fm.WaitIdle() + firstLog := make([]uint64, 1001) // first valid log position per block + lastLog := make([]uint64, 1001) // last valid log position per block + for i := uint64(0); i <= ts.fm.indexedRange.headDelimiter; i++ { + log, err := ts.fm.getLogByLvIndex(i) + if err != nil { + t.Fatalf("Error getting log by index %d: %v", i, err) + } + if log != nil { + if firstLog[log.BlockNumber] == 0 { + firstLog[log.BlockNumber] = i + } + lastLog[log.BlockNumber] = i + } + } + var failed bool + ts.fm.testProcessEventsHook = func() { + if ts.fm.indexedRange.blocks.IsEmpty() { + return + } + if lvi := firstLog[ts.fm.indexedRange.blocks.First()]; lvi != 0 { + log, err := ts.fm.getLogByLvIndex(lvi) + if log == nil || err != nil { + t.Errorf("Error getting first log of indexed block range: %v", err) + failed = true + } + } + if lvi := lastLog[ts.fm.indexedRange.blocks.Last()]; lvi != 0 { + log, err := ts.fm.getLogByLvIndex(lvi) + if log == nil || err != nil { + t.Errorf("Error getting last log of indexed block range: %v", err) + failed = true + } + } + } + chain := ts.chain.getCanonicalChain() + for i := 0; i < 1000 && !failed; i++ { + head := rand.Intn(len(chain)) + ts.chain.setCanonicalChain(chain[:head+1]) + ts.fm.WaitIdle() + } +} + func TestIndexerCompareDb(t *testing.T) { ts := newTestSetup(t) defer ts.close() diff --git a/core/filtermaps/map_renderer.go b/core/filtermaps/map_renderer.go index 7c2aa8dc32..f59a01c032 100644 --- a/core/filtermaps/map_renderer.go +++ b/core/filtermaps/map_renderer.go @@ -20,6 +20,7 @@ import ( "errors" "fmt" "math" + "slices" "sort" "time" @@ -107,7 +108,7 @@ func (f *FilterMaps) renderMapsFromSnapshot(cp *renderedMap) (*mapRenderer, erro filterMap: cp.filterMap.fullCopy(), mapIndex: cp.mapIndex, lastBlock: cp.lastBlock, - blockLvPtrs: cp.blockLvPtrs, + blockLvPtrs: slices.Clone(cp.blockLvPtrs), }, finishedMaps: make(map[uint32]*renderedMap), finished: common.NewRange(cp.mapIndex, 0), @@ -244,7 +245,7 @@ func (f *FilterMaps) loadHeadSnapshot() error { } } f.renderSnapshots.Add(f.indexedRange.blocks.Last(), &renderedMap{ - filterMap: fm, + filterMap: fm.fullCopy(), mapIndex: f.indexedRange.maps.Last(), lastBlock: f.indexedRange.blocks.Last(), lastBlockId: f.indexedView.BlockId(f.indexedRange.blocks.Last()), @@ -536,6 +537,7 @@ func (r *mapRenderer) getTempRange() (filterMapsRange, error) { } else { tempRange.blocks.SetAfterLast(0) } + tempRange.headIndexed = false tempRange.headDelimiter = 0 } return tempRange, nil From 74165a8fe31847fa4161929c6a9b00ac07251c5f Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 21 Apr 2025 14:56:16 +0200 Subject: [PATCH 6/8] version: release go-ethereum v1.15.9 stable --- version/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/version/version.go b/version/version.go index c969ab479e..53a1bae280 100644 --- a/version/version.go +++ b/version/version.go @@ -17,8 +17,8 @@ package version const ( - Major = 1 // Major version component of the current release - Minor = 15 // Minor version component of the current release - Patch = 9 // Patch version component of the current release - Meta = "unstable" // Version metadata to append to the version string + Major = 1 // Major version component of the current release + Minor = 15 // Minor version component of the current release + Patch = 9 // Patch version component of the current release + Meta = "stable" // Version metadata to append to the version string ) From 263aef9cc41135a9207c8e788beabc39506430c0 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 21 Apr 2025 14:57:25 +0200 Subject: [PATCH 7/8] version: begin v1.15.10 release cycle --- version/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/version/version.go b/version/version.go index 53a1bae280..2bbd7e7b17 100644 --- a/version/version.go +++ b/version/version.go @@ -17,8 +17,8 @@ package version const ( - Major = 1 // Major version component of the current release - Minor = 15 // Minor version component of the current release - Patch = 9 // Patch version component of the current release - Meta = "stable" // Version metadata to append to the version string + Major = 1 // Major version component of the current release + Minor = 15 // Minor version component of the current release + Patch = 10 // Patch version component of the current release + Meta = "unstable" // Version metadata to append to the version string ) From 1591d165c44e027c961d174c0f4ee1bb45a82520 Mon Sep 17 00:00:00 2001 From: georgehao Date: Tue, 22 Apr 2025 12:57:17 +0800 Subject: [PATCH 8/8] internal/debug: add debug_setMemoryLimit (#31441) --- internal/debug/api.go | 19 +++++++++++++++++++ internal/web3ext/web3ext.go | 5 +++++ 2 files changed, 24 insertions(+) diff --git a/internal/debug/api.go b/internal/debug/api.go index 5e93585bf8..1bac36e908 100644 --- a/internal/debug/api.go +++ b/internal/debug/api.go @@ -35,6 +35,7 @@ import ( "sync" "time" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" "github.com/hashicorp/go-bexpr" ) @@ -237,6 +238,24 @@ func (*HandlerT) SetGCPercent(v int) int { return debug.SetGCPercent(v) } +// SetMemoryLimit sets the GOMEMLIMIT for the process. It returns the previous limit. +// Note: +// +// - The input limit is provided as bytes. A negative input does not adjust the limit +// +// - A zero limit or a limit that's lower than the amount of memory used by the Go +// runtime may cause the garbage collector to run nearly continuously. However, +// the application may still make progress. +// +// - Setting the limit too low will cause Geth to become unresponsive. +// +// - Geth also allocates memory off-heap, particularly for fastCache and Pebble, +// which can be non-trivial (a few gigabytes by default). +func (*HandlerT) SetMemoryLimit(limit int64) int64 { + log.Info("Setting memory limit", "size", common.PrettyDuration(limit)) + return debug.SetMemoryLimit(limit) +} + func writeProfile(name, file string) error { p := pprof.Lookup(name) log.Info("Writing profile records", "count", p.Count(), "type", name, "dump", file) diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index 8ac8f44958..aed6bbbdb9 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -269,6 +269,11 @@ web3._extend({ call: 'debug_setGCPercent', params: 1, }), + new web3._extend.Method({ + name: 'setMemoryLimit', + call: 'debug_setMemoryLimit', + params: 1, + }), new web3._extend.Method({ name: 'memStats', call: 'debug_memStats',