From 7cbf934488b510206b8d0c9e1453fe9fad83692a Mon Sep 17 00:00:00 2001 From: Pierre Rousset Date: Fri, 21 Feb 2025 11:28:11 +0900 Subject: [PATCH 01/42] Fix flakey behavior in simulated backend Rollback --- core/txpool/txpool.go | 1 + 1 file changed, 1 insertion(+) diff --git a/core/txpool/txpool.go b/core/txpool/txpool.go index 0ebf4c7e4b..e09c920779 100644 --- a/core/txpool/txpool.go +++ b/core/txpool/txpool.go @@ -486,6 +486,7 @@ func (p *TxPool) Sync() error { // Clear removes all tracked txs from the subpools. func (p *TxPool) Clear() { + p.Sync() for _, subpool := range p.subpools { subpool.Clear() } From 18faa251b3cdf9ead91d861c524d7a9e3dec0af7 Mon Sep 17 00:00:00 2001 From: Pierre R Date: Tue, 11 Mar 2025 18:26:16 +0900 Subject: [PATCH 02/42] Update core/txpool/txpool.go Co-authored-by: jwasinger --- core/txpool/txpool.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/txpool/txpool.go b/core/txpool/txpool.go index e09c920779..9b1136756b 100644 --- a/core/txpool/txpool.go +++ b/core/txpool/txpool.go @@ -486,6 +486,8 @@ func (p *TxPool) Sync() error { // Clear removes all tracked txs from the subpools. func (p *TxPool) Clear() { + // Invoke Sync to ensure that txs pending addition don't get added to the pool after + // the subpools are subsequently cleared p.Sync() for _, subpool := range p.subpools { subpool.Clear() From 07cca7ab9f9c26e435acbf81432f61102d3ed1d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Fri, 21 Mar 2025 10:47:58 +0100 Subject: [PATCH 03/42] core/bloombits: remove old bloombits logic and chain indexer (#31081) This PR is #3 of a 3-part series that implements the new log index intended to replace core/bloombits. Based on https://github.com/ethereum/go-ethereum/pull/31079 and https://github.com/ethereum/go-ethereum/pull/31080 Replaces https://github.com/ethereum/go-ethereum/pull/30370 This part removes the old bloombits package and the chain indexer that was only used by bloombits. Deletes the old bloombits database. FilterMaps data structure explanation: https://gist.github.com/zsfelfoldi/a60795f9da7ae6422f28c7a34e02a07e Log index generator code overview: https://gist.github.com/zsfelfoldi/97105dff0b1a4f5ed557924a24b9b9e7 Search pattern matcher code overview: https://gist.github.com/zsfelfoldi/5981735641c956afb18065e84f8aff34 Note that the possibility of a tree hashing scheme and remote proof protocol are mentioned in the documents above but they are not exactly specified yet. These specs are WIP and will be finalized after the local log indexer/filter code is finalized and merged. --------- Co-authored-by: Felix Lange --- core/blockchain.go | 2 +- core/bloom_indexer.go | 92 ---- core/bloombits/doc.go | 18 - core/bloombits/generator.go | 98 ---- core/bloombits/generator_test.go | 100 ----- core/bloombits/matcher.go | 649 --------------------------- core/bloombits/matcher_test.go | 292 ------------ core/bloombits/scheduler.go | 181 -------- core/bloombits/scheduler_test.go | 103 ----- core/chain_indexer.go | 522 --------------------- core/chain_indexer_test.go | 246 ---------- core/filtermaps/filtermaps.go | 35 +- core/filtermaps/indexer.go | 33 +- core/rawdb/accessors_indexes.go | 56 +-- core/rawdb/accessors_indexes_test.go | 53 --- core/rawdb/database.go | 8 +- core/rawdb/schema.go | 24 +- eth/api_backend.go | 12 - eth/backend.go | 35 +- eth/bloombits.go | 74 --- params/network_params.go | 8 - 21 files changed, 80 insertions(+), 2561 deletions(-) delete mode 100644 core/bloom_indexer.go delete mode 100644 core/bloombits/doc.go delete mode 100644 core/bloombits/generator.go delete mode 100644 core/bloombits/generator_test.go delete mode 100644 core/bloombits/matcher.go delete mode 100644 core/bloombits/matcher_test.go delete mode 100644 core/bloombits/scheduler.go delete mode 100644 core/bloombits/scheduler_test.go delete mode 100644 core/chain_indexer.go delete mode 100644 core/chain_indexer_test.go delete mode 100644 eth/bloombits.go diff --git a/core/blockchain.go b/core/blockchain.go index 2024c63b80..2bf7fba427 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -906,7 +906,7 @@ func (bc *BlockChain) setHeadBeyondRoot(head uint64, time uint64, root common.Ha rawdb.DeleteBody(db, hash, num) rawdb.DeleteReceipts(db, hash, num) } - // Todo(rjl493456442) txlookup, bloombits, etc + // Todo(rjl493456442) txlookup, log index, etc } // If SetHead was only called as a chain reparation method, try to skip // touching the header chain altogether, unless the freezer is broken diff --git a/core/bloom_indexer.go b/core/bloom_indexer.go deleted file mode 100644 index 68a35d811e..0000000000 --- a/core/bloom_indexer.go +++ /dev/null @@ -1,92 +0,0 @@ -// Copyright 2021 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package core - -import ( - "context" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/bitutil" - "github.com/ethereum/go-ethereum/core/bloombits" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethdb" -) - -const ( - // bloomThrottling is the time to wait between processing two consecutive index - // sections. It's useful during chain upgrades to prevent disk overload. - bloomThrottling = 100 * time.Millisecond -) - -// BloomIndexer implements a core.ChainIndexer, building up a rotated bloom bits index -// for the Ethereum header bloom filters, permitting blazing fast filtering. -type BloomIndexer struct { - size uint64 // section size to generate bloombits for - db ethdb.Database // database instance to write index data and metadata into - gen *bloombits.Generator // generator to rotate the bloom bits crating the bloom index - section uint64 // Section is the section number being processed currently - head common.Hash // Head is the hash of the last header processed -} - -// NewBloomIndexer returns a chain indexer that generates bloom bits data for the -// canonical chain for fast logs filtering. -func NewBloomIndexer(db ethdb.Database, size, confirms uint64) *ChainIndexer { - backend := &BloomIndexer{ - db: db, - size: size, - } - table := rawdb.NewTable(db, string(rawdb.BloomBitsIndexPrefix)) - - return NewChainIndexer(db, table, backend, size, confirms, bloomThrottling, "bloombits") -} - -// Reset implements core.ChainIndexerBackend, starting a new bloombits index -// section. -func (b *BloomIndexer) Reset(ctx context.Context, section uint64, lastSectionHead common.Hash) error { - gen, err := bloombits.NewGenerator(uint(b.size)) - b.gen, b.section, b.head = gen, section, common.Hash{} - return err -} - -// Process implements core.ChainIndexerBackend, adding a new header's bloom into -// the index. -func (b *BloomIndexer) Process(ctx context.Context, header *types.Header) error { - b.gen.AddBloom(uint(header.Number.Uint64()-b.section*b.size), header.Bloom) - b.head = header.Hash() - return nil -} - -// Commit implements core.ChainIndexerBackend, finalizing the bloom section and -// writing it out into the database. -func (b *BloomIndexer) Commit() error { - batch := b.db.NewBatchWithSize((int(b.size) / 8) * types.BloomBitLength) - for i := 0; i < types.BloomBitLength; i++ { - bits, err := b.gen.Bitset(uint(i)) - if err != nil { - return err - } - rawdb.WriteBloomBits(batch, uint(i), b.section, b.head, bitutil.CompressBytes(bits)) - } - return batch.Write() -} - -// Prune returns an empty error since we don't support pruning here. -func (b *BloomIndexer) Prune(threshold uint64) error { - return nil -} diff --git a/core/bloombits/doc.go b/core/bloombits/doc.go deleted file mode 100644 index 3d159e74f7..0000000000 --- a/core/bloombits/doc.go +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Package bloombits implements bloom filtering on batches of data. -package bloombits diff --git a/core/bloombits/generator.go b/core/bloombits/generator.go deleted file mode 100644 index 646151db0b..0000000000 --- a/core/bloombits/generator.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package bloombits - -import ( - "errors" - - "github.com/ethereum/go-ethereum/core/types" -) - -var ( - // errSectionOutOfBounds is returned if the user tried to add more bloom filters - // to the batch than available space, or if tries to retrieve above the capacity. - errSectionOutOfBounds = errors.New("section out of bounds") - - // errBloomBitOutOfBounds is returned if the user tried to retrieve specified - // bit bloom above the capacity. - errBloomBitOutOfBounds = errors.New("bloom bit out of bounds") -) - -// Generator takes a number of bloom filters and generates the rotated bloom bits -// to be used for batched filtering. -type Generator struct { - blooms [types.BloomBitLength][]byte // Rotated blooms for per-bit matching - sections uint // Number of sections to batch together - nextSec uint // Next section to set when adding a bloom -} - -// NewGenerator creates a rotated bloom generator that can iteratively fill a -// batched bloom filter's bits. -func NewGenerator(sections uint) (*Generator, error) { - if sections%8 != 0 { - return nil, errors.New("section count not multiple of 8") - } - b := &Generator{sections: sections} - for i := 0; i < types.BloomBitLength; i++ { - b.blooms[i] = make([]byte, sections/8) - } - return b, nil -} - -// AddBloom takes a single bloom filter and sets the corresponding bit column -// in memory accordingly. -func (b *Generator) AddBloom(index uint, bloom types.Bloom) error { - // Make sure we're not adding more bloom filters than our capacity - if b.nextSec >= b.sections { - return errSectionOutOfBounds - } - if b.nextSec != index { - return errors.New("bloom filter with unexpected index") - } - // Rotate the bloom and insert into our collection - byteIndex := b.nextSec / 8 - bitIndex := byte(7 - b.nextSec%8) - for byt := 0; byt < types.BloomByteLength; byt++ { - bloomByte := bloom[types.BloomByteLength-1-byt] - if bloomByte == 0 { - continue - } - base := 8 * byt - b.blooms[base+7][byteIndex] |= ((bloomByte >> 7) & 1) << bitIndex - b.blooms[base+6][byteIndex] |= ((bloomByte >> 6) & 1) << bitIndex - b.blooms[base+5][byteIndex] |= ((bloomByte >> 5) & 1) << bitIndex - b.blooms[base+4][byteIndex] |= ((bloomByte >> 4) & 1) << bitIndex - b.blooms[base+3][byteIndex] |= ((bloomByte >> 3) & 1) << bitIndex - b.blooms[base+2][byteIndex] |= ((bloomByte >> 2) & 1) << bitIndex - b.blooms[base+1][byteIndex] |= ((bloomByte >> 1) & 1) << bitIndex - b.blooms[base][byteIndex] |= (bloomByte & 1) << bitIndex - } - b.nextSec++ - return nil -} - -// Bitset returns the bit vector belonging to the given bit index after all -// blooms have been added. -func (b *Generator) Bitset(idx uint) ([]byte, error) { - if b.nextSec != b.sections { - return nil, errors.New("bloom not fully generated yet") - } - if idx >= types.BloomBitLength { - return nil, errBloomBitOutOfBounds - } - return b.blooms[idx], nil -} diff --git a/core/bloombits/generator_test.go b/core/bloombits/generator_test.go deleted file mode 100644 index ac1aee0b25..0000000000 --- a/core/bloombits/generator_test.go +++ /dev/null @@ -1,100 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package bloombits - -import ( - "bytes" - crand "crypto/rand" - "math/rand" - "testing" - - "github.com/ethereum/go-ethereum/core/types" -) - -// Tests that batched bloom bits are correctly rotated from the input bloom -// filters. -func TestGenerator(t *testing.T) { - // Generate the input and the rotated output - var input, output [types.BloomBitLength][types.BloomByteLength]byte - - for i := 0; i < types.BloomBitLength; i++ { - for j := 0; j < types.BloomBitLength; j++ { - bit := byte(rand.Int() % 2) - - input[i][j/8] |= bit << byte(7-j%8) - output[types.BloomBitLength-1-j][i/8] |= bit << byte(7-i%8) - } - } - // Crunch the input through the generator and verify the result - gen, err := NewGenerator(types.BloomBitLength) - if err != nil { - t.Fatalf("failed to create bloombit generator: %v", err) - } - for i, bloom := range input { - if err := gen.AddBloom(uint(i), bloom); err != nil { - t.Fatalf("bloom %d: failed to add: %v", i, err) - } - } - for i, want := range output { - have, err := gen.Bitset(uint(i)) - if err != nil { - t.Fatalf("output %d: failed to retrieve bits: %v", i, err) - } - if !bytes.Equal(have, want[:]) { - t.Errorf("output %d: bit vector mismatch have %x, want %x", i, have, want) - } - } -} - -func BenchmarkGenerator(b *testing.B) { - var input [types.BloomBitLength][types.BloomByteLength]byte - b.Run("empty", func(b *testing.B) { - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - // Crunch the input through the generator and verify the result - gen, err := NewGenerator(types.BloomBitLength) - if err != nil { - b.Fatalf("failed to create bloombit generator: %v", err) - } - for j, bloom := range &input { - if err := gen.AddBloom(uint(j), bloom); err != nil { - b.Fatalf("bloom %d: failed to add: %v", i, err) - } - } - } - }) - for i := 0; i < types.BloomBitLength; i++ { - crand.Read(input[i][:]) - } - b.Run("random", func(b *testing.B) { - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - // Crunch the input through the generator and verify the result - gen, err := NewGenerator(types.BloomBitLength) - if err != nil { - b.Fatalf("failed to create bloombit generator: %v", err) - } - for j, bloom := range &input { - if err := gen.AddBloom(uint(j), bloom); err != nil { - b.Fatalf("bloom %d: failed to add: %v", i, err) - } - } - } - }) -} diff --git a/core/bloombits/matcher.go b/core/bloombits/matcher.go deleted file mode 100644 index 486581fe23..0000000000 --- a/core/bloombits/matcher.go +++ /dev/null @@ -1,649 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package bloombits - -import ( - "bytes" - "context" - "errors" - "math" - "sort" - "sync" - "sync/atomic" - "time" - - "github.com/ethereum/go-ethereum/common/bitutil" - "github.com/ethereum/go-ethereum/crypto" -) - -// bloomIndexes represents the bit indexes inside the bloom filter that belong -// to some key. -type bloomIndexes [3]uint - -// calcBloomIndexes returns the bloom filter bit indexes belonging to the given key. -func calcBloomIndexes(b []byte) bloomIndexes { - b = crypto.Keccak256(b) - - var idxs bloomIndexes - for i := 0; i < len(idxs); i++ { - idxs[i] = (uint(b[2*i])<<8)&2047 + uint(b[2*i+1]) - } - return idxs -} - -// partialMatches with a non-nil vector represents a section in which some sub- -// matchers have already found potential matches. Subsequent sub-matchers will -// binary AND their matches with this vector. If vector is nil, it represents a -// section to be processed by the first sub-matcher. -type partialMatches struct { - section uint64 - bitset []byte -} - -// Retrieval represents a request for retrieval task assignments for a given -// bit with the given number of fetch elements, or a response for such a request. -// It can also have the actual results set to be used as a delivery data struct. -// -// The context and error fields are used by the light client to terminate matching -// early if an error is encountered on some path of the pipeline. -type Retrieval struct { - Bit uint - Sections []uint64 - Bitsets [][]byte - - Context context.Context - Error error -} - -// Matcher is a pipelined system of schedulers and logic matchers which perform -// binary AND/OR operations on the bit-streams, creating a stream of potential -// blocks to inspect for data content. -type Matcher struct { - sectionSize uint64 // Size of the data batches to filter on - - filters [][]bloomIndexes // Filter the system is matching for - schedulers map[uint]*scheduler // Retrieval schedulers for loading bloom bits - - retrievers chan chan uint // Retriever processes waiting for bit allocations - counters chan chan uint // Retriever processes waiting for task count reports - retrievals chan chan *Retrieval // Retriever processes waiting for task allocations - deliveries chan *Retrieval // Retriever processes waiting for task response deliveries - - running atomic.Bool // Atomic flag whether a session is live or not -} - -// NewMatcher creates a new pipeline for retrieving bloom bit streams and doing -// address and topic filtering on them. Setting a filter component to `nil` is -// allowed and will result in that filter rule being skipped (OR 0x11...1). -func NewMatcher(sectionSize uint64, filters [][][]byte) *Matcher { - // Create the matcher instance - m := &Matcher{ - sectionSize: sectionSize, - schedulers: make(map[uint]*scheduler), - retrievers: make(chan chan uint), - counters: make(chan chan uint), - retrievals: make(chan chan *Retrieval), - deliveries: make(chan *Retrieval), - } - // Calculate the bloom bit indexes for the groups we're interested in - m.filters = nil - - for _, filter := range filters { - // Gather the bit indexes of the filter rule, special casing the nil filter - if len(filter) == 0 { - continue - } - bloomBits := make([]bloomIndexes, len(filter)) - for i, clause := range filter { - if clause == nil { - bloomBits = nil - break - } - bloomBits[i] = calcBloomIndexes(clause) - } - // Accumulate the filter rules if no nil rule was within - if bloomBits != nil { - m.filters = append(m.filters, bloomBits) - } - } - // For every bit, create a scheduler to load/download the bit vectors - for _, bloomIndexLists := range m.filters { - for _, bloomIndexList := range bloomIndexLists { - for _, bloomIndex := range bloomIndexList { - m.addScheduler(bloomIndex) - } - } - } - return m -} - -// addScheduler adds a bit stream retrieval scheduler for the given bit index if -// it has not existed before. If the bit is already selected for filtering, the -// existing scheduler can be used. -func (m *Matcher) addScheduler(idx uint) { - if _, ok := m.schedulers[idx]; ok { - return - } - m.schedulers[idx] = newScheduler(idx) -} - -// Start starts the matching process and returns a stream of bloom matches in -// a given range of blocks. If there are no more matches in the range, the result -// channel is closed. -func (m *Matcher) Start(ctx context.Context, begin, end uint64, results chan uint64) (*MatcherSession, error) { - // Make sure we're not creating concurrent sessions - if m.running.Swap(true) { - return nil, errors.New("matcher already running") - } - defer m.running.Store(false) - - // Initiate a new matching round - session := &MatcherSession{ - matcher: m, - quit: make(chan struct{}), - ctx: ctx, - } - for _, scheduler := range m.schedulers { - scheduler.reset() - } - sink := m.run(begin, end, cap(results), session) - - // Read the output from the result sink and deliver to the user - session.pend.Add(1) - go func() { - defer session.pend.Done() - defer close(results) - - for { - select { - case <-session.quit: - return - - case res, ok := <-sink: - // New match result found - if !ok { - return - } - // Calculate the first and last blocks of the section - sectionStart := res.section * m.sectionSize - - first := sectionStart - if begin > first { - first = begin - } - last := sectionStart + m.sectionSize - 1 - if end < last { - last = end - } - // Iterate over all the blocks in the section and return the matching ones - for i := first; i <= last; i++ { - // Skip the entire byte if no matches are found inside (and we're processing an entire byte!) - next := res.bitset[(i-sectionStart)/8] - if next == 0 { - if i%8 == 0 { - i += 7 - } - continue - } - // Some bit it set, do the actual submatching - if bit := 7 - i%8; next&(1<= req.section }) - requests[req.bit] = append(queue[:index], append([]uint64{req.section}, queue[index:]...)...) - - // If it's a new bit and we have waiting fetchers, allocate to them - if len(queue) == 0 { - assign(req.bit) - } - - case fetcher := <-retrievers: - // New retriever arrived, find the lowest section-ed bit to assign - bit, best := uint(0), uint64(math.MaxUint64) - for idx := range unallocs { - if requests[idx][0] < best { - bit, best = idx, requests[idx][0] - } - } - // Stop tracking this bit (and alloc notifications if no more work is available) - delete(unallocs, bit) - if len(unallocs) == 0 { - retrievers = nil - } - allocs++ - fetcher <- bit - - case fetcher := <-m.counters: - // New task count request arrives, return number of items - fetcher <- uint(len(requests[<-fetcher])) - - case fetcher := <-m.retrievals: - // New fetcher waiting for tasks to retrieve, assign - task := <-fetcher - if want := len(task.Sections); want >= len(requests[task.Bit]) { - task.Sections = requests[task.Bit] - delete(requests, task.Bit) - } else { - task.Sections = append(task.Sections[:0], requests[task.Bit][:want]...) - requests[task.Bit] = append(requests[task.Bit][:0], requests[task.Bit][want:]...) - } - fetcher <- task - - // If anything was left unallocated, try to assign to someone else - if len(requests[task.Bit]) > 0 { - assign(task.Bit) - } - - case result := <-m.deliveries: - // New retrieval task response from fetcher, split out missing sections and - // deliver complete ones - var ( - sections = make([]uint64, 0, len(result.Sections)) - bitsets = make([][]byte, 0, len(result.Bitsets)) - missing = make([]uint64, 0, len(result.Sections)) - ) - for i, bitset := range result.Bitsets { - if len(bitset) == 0 { - missing = append(missing, result.Sections[i]) - continue - } - sections = append(sections, result.Sections[i]) - bitsets = append(bitsets, bitset) - } - m.schedulers[result.Bit].deliver(sections, bitsets) - allocs-- - - // Reschedule missing sections and allocate bit if newly available - if len(missing) > 0 { - queue := requests[result.Bit] - for _, section := range missing { - index := sort.Search(len(queue), func(i int) bool { return queue[i] >= section }) - queue = append(queue[:index], append([]uint64{section}, queue[index:]...)...) - } - requests[result.Bit] = queue - - if len(queue) == len(missing) { - assign(result.Bit) - } - } - - // End the session when all pending deliveries have arrived. - if shutdown == nil && allocs == 0 { - return - } - } - } -} - -// MatcherSession is returned by a started matcher to be used as a terminator -// for the actively running matching operation. -type MatcherSession struct { - matcher *Matcher - - closer sync.Once // Sync object to ensure we only ever close once - quit chan struct{} // Quit channel to request pipeline termination - - ctx context.Context // Context used by the light client to abort filtering - err error // Global error to track retrieval failures deep in the chain - errLock sync.Mutex - - pend sync.WaitGroup -} - -// Close stops the matching process and waits for all subprocesses to terminate -// before returning. The timeout may be used for graceful shutdown, allowing the -// currently running retrievals to complete before this time. -func (s *MatcherSession) Close() { - s.closer.Do(func() { - // Signal termination and wait for all goroutines to tear down - close(s.quit) - s.pend.Wait() - }) -} - -// Error returns any failure encountered during the matching session. -func (s *MatcherSession) Error() error { - s.errLock.Lock() - defer s.errLock.Unlock() - - return s.err -} - -// allocateRetrieval assigns a bloom bit index to a client process that can either -// immediately request and fetch the section contents assigned to this bit or wait -// a little while for more sections to be requested. -func (s *MatcherSession) allocateRetrieval() (uint, bool) { - fetcher := make(chan uint) - - select { - case <-s.quit: - return 0, false - case s.matcher.retrievers <- fetcher: - bit, ok := <-fetcher - return bit, ok - } -} - -// pendingSections returns the number of pending section retrievals belonging to -// the given bloom bit index. -func (s *MatcherSession) pendingSections(bit uint) int { - fetcher := make(chan uint) - - select { - case <-s.quit: - return 0 - case s.matcher.counters <- fetcher: - fetcher <- bit - return int(<-fetcher) - } -} - -// allocateSections assigns all or part of an already allocated bit-task queue -// to the requesting process. -func (s *MatcherSession) allocateSections(bit uint, count int) []uint64 { - fetcher := make(chan *Retrieval) - - select { - case <-s.quit: - return nil - case s.matcher.retrievals <- fetcher: - task := &Retrieval{ - Bit: bit, - Sections: make([]uint64, count), - } - fetcher <- task - return (<-fetcher).Sections - } -} - -// deliverSections delivers a batch of section bit-vectors for a specific bloom -// bit index to be injected into the processing pipeline. -func (s *MatcherSession) deliverSections(bit uint, sections []uint64, bitsets [][]byte) { - s.matcher.deliveries <- &Retrieval{Bit: bit, Sections: sections, Bitsets: bitsets} -} - -// Multiplex polls the matcher session for retrieval tasks and multiplexes it into -// the requested retrieval queue to be serviced together with other sessions. -// -// This method will block for the lifetime of the session. Even after termination -// of the session, any request in-flight need to be responded to! Empty responses -// are fine though in that case. -func (s *MatcherSession) Multiplex(batch int, wait time.Duration, mux chan chan *Retrieval) { - waitTimer := time.NewTimer(wait) - defer waitTimer.Stop() - - for { - // Allocate a new bloom bit index to retrieve data for, stopping when done - bit, ok := s.allocateRetrieval() - if !ok { - return - } - // Bit allocated, throttle a bit if we're below our batch limit - if s.pendingSections(bit) < batch { - waitTimer.Reset(wait) - select { - case <-s.quit: - // Session terminating, we can't meaningfully service, abort - s.allocateSections(bit, 0) - s.deliverSections(bit, []uint64{}, [][]byte{}) - return - - case <-waitTimer.C: - // Throttling up, fetch whatever is available - } - } - // Allocate as much as we can handle and request servicing - sections := s.allocateSections(bit, batch) - request := make(chan *Retrieval) - - select { - case <-s.quit: - // Session terminating, we can't meaningfully service, abort - s.deliverSections(bit, sections, make([][]byte, len(sections))) - return - - case mux <- request: - // Retrieval accepted, something must arrive before we're aborting - request <- &Retrieval{Bit: bit, Sections: sections, Context: s.ctx} - - result := <-request - - // Deliver a result before s.Close() to avoid a deadlock - s.deliverSections(result.Bit, result.Sections, result.Bitsets) - - if result.Error != nil { - s.errLock.Lock() - s.err = result.Error - s.errLock.Unlock() - s.Close() - } - } - } -} diff --git a/core/bloombits/matcher_test.go b/core/bloombits/matcher_test.go deleted file mode 100644 index 7f3d5f279c..0000000000 --- a/core/bloombits/matcher_test.go +++ /dev/null @@ -1,292 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package bloombits - -import ( - "context" - "math/rand" - "sync/atomic" - "testing" - "time" - - "github.com/ethereum/go-ethereum/common" -) - -const testSectionSize = 4096 - -// Tests that wildcard filter rules (nil) can be specified and are handled well. -func TestMatcherWildcards(t *testing.T) { - t.Parallel() - matcher := NewMatcher(testSectionSize, [][][]byte{ - {common.Address{}.Bytes(), common.Address{0x01}.Bytes()}, // Default address is not a wildcard - {common.Hash{}.Bytes(), common.Hash{0x01}.Bytes()}, // Default hash is not a wildcard - {common.Hash{0x01}.Bytes()}, // Plain rule, sanity check - {common.Hash{0x01}.Bytes(), nil}, // Wildcard suffix, drop rule - {nil, common.Hash{0x01}.Bytes()}, // Wildcard prefix, drop rule - {nil, nil}, // Wildcard combo, drop rule - {}, // Inited wildcard rule, drop rule - nil, // Proper wildcard rule, drop rule - }) - if len(matcher.filters) != 3 { - t.Fatalf("filter system size mismatch: have %d, want %d", len(matcher.filters), 3) - } - if len(matcher.filters[0]) != 2 { - t.Fatalf("address clause size mismatch: have %d, want %d", len(matcher.filters[0]), 2) - } - if len(matcher.filters[1]) != 2 { - t.Fatalf("combo topic clause size mismatch: have %d, want %d", len(matcher.filters[1]), 2) - } - if len(matcher.filters[2]) != 1 { - t.Fatalf("singletone topic clause size mismatch: have %d, want %d", len(matcher.filters[2]), 1) - } -} - -// Tests the matcher pipeline on a single continuous workflow without interrupts. -func TestMatcherContinuous(t *testing.T) { - t.Parallel() - testMatcherDiffBatches(t, [][]bloomIndexes{{{10, 20, 30}}}, 0, 100000, false, 75) - testMatcherDiffBatches(t, [][]bloomIndexes{{{32, 3125, 100}}, {{40, 50, 10}}}, 0, 100000, false, 81) - testMatcherDiffBatches(t, [][]bloomIndexes{{{4, 8, 11}, {7, 8, 17}}, {{9, 9, 12}, {15, 20, 13}}, {{18, 15, 15}, {12, 10, 4}}}, 0, 10000, false, 36) -} - -// Tests the matcher pipeline on a constantly interrupted and resumed work pattern -// with the aim of ensuring data items are requested only once. -func TestMatcherIntermittent(t *testing.T) { - t.Parallel() - testMatcherDiffBatches(t, [][]bloomIndexes{{{10, 20, 30}}}, 0, 100000, true, 75) - testMatcherDiffBatches(t, [][]bloomIndexes{{{32, 3125, 100}}, {{40, 50, 10}}}, 0, 100000, true, 81) - testMatcherDiffBatches(t, [][]bloomIndexes{{{4, 8, 11}, {7, 8, 17}}, {{9, 9, 12}, {15, 20, 13}}, {{18, 15, 15}, {12, 10, 4}}}, 0, 10000, true, 36) -} - -// Tests the matcher pipeline on random input to hopefully catch anomalies. -func TestMatcherRandom(t *testing.T) { - t.Parallel() - for i := 0; i < 10; i++ { - testMatcherBothModes(t, makeRandomIndexes([]int{1}, 50), 0, 10000, 0) - testMatcherBothModes(t, makeRandomIndexes([]int{3}, 50), 0, 10000, 0) - testMatcherBothModes(t, makeRandomIndexes([]int{2, 2, 2}, 20), 0, 10000, 0) - testMatcherBothModes(t, makeRandomIndexes([]int{5, 5, 5}, 50), 0, 10000, 0) - testMatcherBothModes(t, makeRandomIndexes([]int{4, 4, 4}, 20), 0, 10000, 0) - } -} - -// Tests that the matcher can properly find matches if the starting block is -// shifted from a multiple of 8. This is needed to cover an optimisation with -// bitset matching https://github.com/ethereum/go-ethereum/issues/15309. -func TestMatcherShifted(t *testing.T) { - t.Parallel() - // Block 0 always matches in the tests, skip ahead of first 8 blocks with the - // start to get a potential zero byte in the matcher bitset. - - // To keep the second bitset byte zero, the filter must only match for the first - // time in block 16, so doing an all-16 bit filter should suffice. - - // To keep the starting block non divisible by 8, block number 9 is the first - // that would introduce a shift and not match block 0. - testMatcherBothModes(t, [][]bloomIndexes{{{16, 16, 16}}}, 9, 64, 0) -} - -// Tests that matching on everything doesn't crash (special case internally). -func TestWildcardMatcher(t *testing.T) { - t.Parallel() - testMatcherBothModes(t, nil, 0, 10000, 0) -} - -// makeRandomIndexes generates a random filter system, composed of multiple filter -// criteria, each having one bloom list component for the address and arbitrarily -// many topic bloom list components. -func makeRandomIndexes(lengths []int, max int) [][]bloomIndexes { - res := make([][]bloomIndexes, len(lengths)) - for i, topics := range lengths { - res[i] = make([]bloomIndexes, topics) - for j := 0; j < topics; j++ { - for k := 0; k < len(res[i][j]); k++ { - res[i][j][k] = uint(rand.Intn(max-1) + 2) - } - } - } - return res -} - -// testMatcherDiffBatches runs the given matches test in single-delivery and also -// in batches delivery mode, verifying that all kinds of deliveries are handled -// correctly within. -func testMatcherDiffBatches(t *testing.T, filter [][]bloomIndexes, start, blocks uint64, intermittent bool, retrievals uint32) { - singleton := testMatcher(t, filter, start, blocks, intermittent, retrievals, 1) - batched := testMatcher(t, filter, start, blocks, intermittent, retrievals, 16) - - if singleton != batched { - t.Errorf("filter = %v blocks = %v intermittent = %v: request count mismatch, %v in singleton vs. %v in batched mode", filter, blocks, intermittent, singleton, batched) - } -} - -// testMatcherBothModes runs the given matcher test in both continuous as well as -// in intermittent mode, verifying that the request counts match each other. -func testMatcherBothModes(t *testing.T, filter [][]bloomIndexes, start, blocks uint64, retrievals uint32) { - continuous := testMatcher(t, filter, start, blocks, false, retrievals, 16) - intermittent := testMatcher(t, filter, start, blocks, true, retrievals, 16) - - if continuous != intermittent { - t.Errorf("filter = %v blocks = %v: request count mismatch, %v in continuous vs. %v in intermittent mode", filter, blocks, continuous, intermittent) - } -} - -// testMatcher is a generic tester to run the given matcher test and return the -// number of requests made for cross validation between different modes. -func testMatcher(t *testing.T, filter [][]bloomIndexes, start, blocks uint64, intermittent bool, retrievals uint32, maxReqCount int) uint32 { - // Create a new matcher an simulate our explicit random bitsets - matcher := NewMatcher(testSectionSize, nil) - matcher.filters = filter - - for _, rule := range filter { - for _, topic := range rule { - for _, bit := range topic { - matcher.addScheduler(bit) - } - } - } - // Track the number of retrieval requests made - var requested atomic.Uint32 - - // Start the matching session for the filter and the retriever goroutines - quit := make(chan struct{}) - matches := make(chan uint64, 16) - - session, err := matcher.Start(context.Background(), start, blocks-1, matches) - if err != nil { - t.Fatalf("failed to stat matcher session: %v", err) - } - startRetrievers(session, quit, &requested, maxReqCount) - - // Iterate over all the blocks and verify that the pipeline produces the correct matches - for i := start; i < blocks; i++ { - if expMatch3(filter, i) { - match, ok := <-matches - if !ok { - t.Errorf("filter = %v blocks = %v intermittent = %v: expected #%v, results channel closed", filter, blocks, intermittent, i) - return 0 - } - if match != i { - t.Errorf("filter = %v blocks = %v intermittent = %v: expected #%v, got #%v", filter, blocks, intermittent, i, match) - } - // If we're testing intermittent mode, abort and restart the pipeline - if intermittent { - session.Close() - close(quit) - - quit = make(chan struct{}) - matches = make(chan uint64, 16) - - session, err = matcher.Start(context.Background(), i+1, blocks-1, matches) - if err != nil { - t.Fatalf("failed to stat matcher session: %v", err) - } - startRetrievers(session, quit, &requested, maxReqCount) - } - } - } - // Ensure the result channel is torn down after the last block - match, ok := <-matches - if ok { - t.Errorf("filter = %v blocks = %v intermittent = %v: expected closed channel, got #%v", filter, blocks, intermittent, match) - } - // Clean up the session and ensure we match the expected retrieval count - session.Close() - close(quit) - - if retrievals != 0 && requested.Load() != retrievals { - t.Errorf("filter = %v blocks = %v intermittent = %v: request count mismatch, have #%v, want #%v", filter, blocks, intermittent, requested.Load(), retrievals) - } - return requested.Load() -} - -// startRetrievers starts a batch of goroutines listening for section requests -// and serving them. -func startRetrievers(session *MatcherSession, quit chan struct{}, retrievals *atomic.Uint32, batch int) { - requests := make(chan chan *Retrieval) - - for i := 0; i < 10; i++ { - // Start a multiplexer to test multiple threaded execution - go session.Multiplex(batch, 100*time.Microsecond, requests) - - // Start a services to match the above multiplexer - go func() { - for { - // Wait for a service request or a shutdown - select { - case <-quit: - return - - case request := <-requests: - task := <-request - - task.Bitsets = make([][]byte, len(task.Sections)) - for i, section := range task.Sections { - if rand.Int()%4 != 0 { // Handle occasional missing deliveries - task.Bitsets[i] = generateBitset(task.Bit, section) - retrievals.Add(1) - } - } - request <- task - } - } - }() - } -} - -// generateBitset generates the rotated bitset for the given bloom bit and section -// numbers. -func generateBitset(bit uint, section uint64) []byte { - bitset := make([]byte, testSectionSize/8) - for i := 0; i < len(bitset); i++ { - for b := 0; b < 8; b++ { - blockIdx := section*testSectionSize + uint64(i*8+b) - bitset[i] += bitset[i] - if (blockIdx % uint64(bit)) == 0 { - bitset[i]++ - } - } - } - return bitset -} - -func expMatch1(filter bloomIndexes, i uint64) bool { - for _, ii := range filter { - if (i % uint64(ii)) != 0 { - return false - } - } - return true -} - -func expMatch2(filter []bloomIndexes, i uint64) bool { - for _, ii := range filter { - if expMatch1(ii, i) { - return true - } - } - return false -} - -func expMatch3(filter [][]bloomIndexes, i uint64) bool { - for _, ii := range filter { - if !expMatch2(ii, i) { - return false - } - } - return true -} diff --git a/core/bloombits/scheduler.go b/core/bloombits/scheduler.go deleted file mode 100644 index a523bc55ab..0000000000 --- a/core/bloombits/scheduler.go +++ /dev/null @@ -1,181 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package bloombits - -import ( - "sync" -) - -// request represents a bloom retrieval task to prioritize and pull from the local -// database or remotely from the network. -type request struct { - section uint64 // Section index to retrieve the bit-vector from - bit uint // Bit index within the section to retrieve the vector of -} - -// response represents the state of a requested bit-vector through a scheduler. -type response struct { - cached []byte // Cached bits to dedup multiple requests - done chan struct{} // Channel to allow waiting for completion -} - -// scheduler handles the scheduling of bloom-filter retrieval operations for -// entire section-batches belonging to a single bloom bit. Beside scheduling the -// retrieval operations, this struct also deduplicates the requests and caches -// the results to minimize network/database overhead even in complex filtering -// scenarios. -type scheduler struct { - bit uint // Index of the bit in the bloom filter this scheduler is responsible for - responses map[uint64]*response // Currently pending retrieval requests or already cached responses - lock sync.Mutex // Lock protecting the responses from concurrent access -} - -// newScheduler creates a new bloom-filter retrieval scheduler for a specific -// bit index. -func newScheduler(idx uint) *scheduler { - return &scheduler{ - bit: idx, - responses: make(map[uint64]*response), - } -} - -// run creates a retrieval pipeline, receiving section indexes from sections and -// returning the results in the same order through the done channel. Concurrent -// runs of the same scheduler are allowed, leading to retrieval task deduplication. -func (s *scheduler) run(sections chan uint64, dist chan *request, done chan []byte, quit chan struct{}, wg *sync.WaitGroup) { - // Create a forwarder channel between requests and responses of the same size as - // the distribution channel (since that will block the pipeline anyway). - pend := make(chan uint64, cap(dist)) - - // Start the pipeline schedulers to forward between user -> distributor -> user - wg.Add(2) - go s.scheduleRequests(sections, dist, pend, quit, wg) - go s.scheduleDeliveries(pend, done, quit, wg) -} - -// reset cleans up any leftovers from previous runs. This is required before a -// restart to ensure the no previously requested but never delivered state will -// cause a lockup. -func (s *scheduler) reset() { - s.lock.Lock() - defer s.lock.Unlock() - - for section, res := range s.responses { - if res.cached == nil { - delete(s.responses, section) - } - } -} - -// scheduleRequests reads section retrieval requests from the input channel, -// deduplicates the stream and pushes unique retrieval tasks into the distribution -// channel for a database or network layer to honour. -func (s *scheduler) scheduleRequests(reqs chan uint64, dist chan *request, pend chan uint64, quit chan struct{}, wg *sync.WaitGroup) { - // Clean up the goroutine and pipeline when done - defer wg.Done() - defer close(pend) - - // Keep reading and scheduling section requests - for { - select { - case <-quit: - return - - case section, ok := <-reqs: - // New section retrieval requested - if !ok { - return - } - // Deduplicate retrieval requests - unique := false - - s.lock.Lock() - if s.responses[section] == nil { - s.responses[section] = &response{ - done: make(chan struct{}), - } - unique = true - } - s.lock.Unlock() - - // Schedule the section for retrieval and notify the deliverer to expect this section - if unique { - select { - case <-quit: - return - case dist <- &request{bit: s.bit, section: section}: - } - } - select { - case <-quit: - return - case pend <- section: - } - } - } -} - -// scheduleDeliveries reads section acceptance notifications and waits for them -// to be delivered, pushing them into the output data buffer. -func (s *scheduler) scheduleDeliveries(pend chan uint64, done chan []byte, quit chan struct{}, wg *sync.WaitGroup) { - // Clean up the goroutine and pipeline when done - defer wg.Done() - defer close(done) - - // Keep reading notifications and scheduling deliveries - for { - select { - case <-quit: - return - - case idx, ok := <-pend: - // New section retrieval pending - if !ok { - return - } - // Wait until the request is honoured - s.lock.Lock() - res := s.responses[idx] - s.lock.Unlock() - - select { - case <-quit: - return - case <-res.done: - } - // Deliver the result - select { - case <-quit: - return - case done <- res.cached: - } - } - } -} - -// deliver is called by the request distributor when a reply to a request arrives. -func (s *scheduler) deliver(sections []uint64, data [][]byte) { - s.lock.Lock() - defer s.lock.Unlock() - - for i, section := range sections { - if res := s.responses[section]; res != nil && res.cached == nil { // Avoid non-requests and double deliveries - res.cached = data[i] - close(res.done) - } - } -} diff --git a/core/bloombits/scheduler_test.go b/core/bloombits/scheduler_test.go deleted file mode 100644 index dcaaa91525..0000000000 --- a/core/bloombits/scheduler_test.go +++ /dev/null @@ -1,103 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package bloombits - -import ( - "bytes" - "math/big" - "sync" - "sync/atomic" - "testing" -) - -// Tests that the scheduler can deduplicate and forward retrieval requests to -// underlying fetchers and serve responses back, irrelevant of the concurrency -// of the requesting clients or serving data fetchers. -func TestSchedulerSingleClientSingleFetcher(t *testing.T) { testScheduler(t, 1, 1, 5000) } -func TestSchedulerSingleClientMultiFetcher(t *testing.T) { testScheduler(t, 1, 10, 5000) } -func TestSchedulerMultiClientSingleFetcher(t *testing.T) { testScheduler(t, 10, 1, 5000) } -func TestSchedulerMultiClientMultiFetcher(t *testing.T) { testScheduler(t, 10, 10, 5000) } - -func testScheduler(t *testing.T, clients int, fetchers int, requests int) { - t.Parallel() - f := newScheduler(0) - - // Create a batch of handler goroutines that respond to bloom bit requests and - // deliver them to the scheduler. - var fetchPend sync.WaitGroup - fetchPend.Add(fetchers) - defer fetchPend.Wait() - - fetch := make(chan *request, 16) - defer close(fetch) - - var delivered atomic.Uint32 - for i := 0; i < fetchers; i++ { - go func() { - defer fetchPend.Done() - - for req := range fetch { - delivered.Add(1) - - f.deliver([]uint64{ - req.section + uint64(requests), // Non-requested data (ensure it doesn't go out of bounds) - req.section, // Requested data - req.section, // Duplicated data (ensure it doesn't double close anything) - }, [][]byte{ - {}, - new(big.Int).SetUint64(req.section).Bytes(), - new(big.Int).SetUint64(req.section).Bytes(), - }) - } - }() - } - // Start a batch of goroutines to concurrently run scheduling tasks - quit := make(chan struct{}) - - var pend sync.WaitGroup - pend.Add(clients) - - for i := 0; i < clients; i++ { - go func() { - defer pend.Done() - - in := make(chan uint64, 16) - out := make(chan []byte, 16) - - f.run(in, fetch, out, quit, &pend) - - go func() { - for j := 0; j < requests; j++ { - in <- uint64(j) - } - close(in) - }() - b := new(big.Int) - for j := 0; j < requests; j++ { - bits := <-out - if want := b.SetUint64(uint64(j)).Bytes(); !bytes.Equal(bits, want) { - t.Errorf("vector %d: delivered content mismatch: have %x, want %x", j, bits, want) - } - } - }() - } - pend.Wait() - - if have := delivered.Load(); int(have) != requests { - t.Errorf("request count mismatch: have %v, want %v", have, requests) - } -} diff --git a/core/chain_indexer.go b/core/chain_indexer.go deleted file mode 100644 index 2865daa1ff..0000000000 --- a/core/chain_indexer.go +++ /dev/null @@ -1,522 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package core - -import ( - "context" - "encoding/binary" - "errors" - "fmt" - "sync" - "sync/atomic" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/log" -) - -// ChainIndexerBackend defines the methods needed to process chain segments in -// the background and write the segment results into the database. These can be -// used to create filter blooms or CHTs. -type ChainIndexerBackend interface { - // Reset initiates the processing of a new chain segment, potentially terminating - // any partially completed operations (in case of a reorg). - Reset(ctx context.Context, section uint64, prevHead common.Hash) error - - // Process crunches through the next header in the chain segment. The caller - // will ensure a sequential order of headers. - Process(ctx context.Context, header *types.Header) error - - // Commit finalizes the section metadata and stores it into the database. - Commit() error - - // Prune deletes the chain index older than the given threshold. - Prune(threshold uint64) error -} - -// ChainIndexerChain interface is used for connecting the indexer to a blockchain -type ChainIndexerChain interface { - // CurrentHeader retrieves the latest locally known header. - CurrentHeader() *types.Header - - // SubscribeChainHeadEvent subscribes to new head header notifications. - SubscribeChainHeadEvent(ch chan<- ChainHeadEvent) event.Subscription -} - -// ChainIndexer does a post-processing job for equally sized sections of the -// canonical chain (like BlooomBits and CHT structures). A ChainIndexer is -// connected to the blockchain through the event system by starting a -// ChainHeadEventLoop in a goroutine. -// -// Further child ChainIndexers can be added which use the output of the parent -// section indexer. These child indexers receive new head notifications only -// after an entire section has been finished or in case of rollbacks that might -// affect already finished sections. -type ChainIndexer struct { - chainDb ethdb.Database // Chain database to index the data from - indexDb ethdb.Database // Prefixed table-view of the db to write index metadata into - backend ChainIndexerBackend // Background processor generating the index data content - children []*ChainIndexer // Child indexers to cascade chain updates to - - active atomic.Bool // Flag whether the event loop was started - update chan struct{} // Notification channel that headers should be processed - quit chan chan error // Quit channel to tear down running goroutines - ctx context.Context - ctxCancel func() - - sectionSize uint64 // Number of blocks in a single chain segment to process - confirmsReq uint64 // Number of confirmations before processing a completed segment - - storedSections uint64 // Number of sections successfully indexed into the database - knownSections uint64 // Number of sections known to be complete (block wise) - cascadedHead uint64 // Block number of the last completed section cascaded to subindexers - - checkpointSections uint64 // Number of sections covered by the checkpoint - checkpointHead common.Hash // Section head belonging to the checkpoint - - throttling time.Duration // Disk throttling to prevent a heavy upgrade from hogging resources - - log log.Logger - lock sync.Mutex -} - -// NewChainIndexer creates a new chain indexer to do background processing on -// chain segments of a given size after certain number of confirmations passed. -// The throttling parameter might be used to prevent database thrashing. -func NewChainIndexer(chainDb ethdb.Database, indexDb ethdb.Database, backend ChainIndexerBackend, section, confirm uint64, throttling time.Duration, kind string) *ChainIndexer { - c := &ChainIndexer{ - chainDb: chainDb, - indexDb: indexDb, - backend: backend, - update: make(chan struct{}, 1), - quit: make(chan chan error), - sectionSize: section, - confirmsReq: confirm, - throttling: throttling, - log: log.New("type", kind), - } - // Initialize database dependent fields and start the updater - c.loadValidSections() - c.ctx, c.ctxCancel = context.WithCancel(context.Background()) - - go c.updateLoop() - - return c -} - -// AddCheckpoint adds a checkpoint. Sections are never processed and the chain -// is not expected to be available before this point. The indexer assumes that -// the backend has sufficient information available to process subsequent sections. -// -// Note: knownSections == 0 and storedSections == checkpointSections until -// syncing reaches the checkpoint -func (c *ChainIndexer) AddCheckpoint(section uint64, shead common.Hash) { - c.lock.Lock() - defer c.lock.Unlock() - - // Short circuit if the given checkpoint is below than local's. - if c.checkpointSections >= section+1 || section < c.storedSections { - return - } - c.checkpointSections = section + 1 - c.checkpointHead = shead - - c.setSectionHead(section, shead) - c.setValidSections(section + 1) -} - -// Start creates a goroutine to feed chain head events into the indexer for -// cascading background processing. Children do not need to be started, they -// are notified about new events by their parents. -func (c *ChainIndexer) Start(chain ChainIndexerChain) { - events := make(chan ChainHeadEvent, 10) - sub := chain.SubscribeChainHeadEvent(events) - - go c.eventLoop(chain.CurrentHeader(), events, sub) -} - -// Close tears down all goroutines belonging to the indexer and returns any error -// that might have occurred internally. -func (c *ChainIndexer) Close() error { - var errs []error - - c.ctxCancel() - - // Tear down the primary update loop - errc := make(chan error) - c.quit <- errc - if err := <-errc; err != nil { - errs = append(errs, err) - } - // If needed, tear down the secondary event loop - if c.active.Load() { - c.quit <- errc - if err := <-errc; err != nil { - errs = append(errs, err) - } - } - // Close all children - for _, child := range c.children { - if err := child.Close(); err != nil { - errs = append(errs, err) - } - } - // Return any failures - switch { - case len(errs) == 0: - return nil - - case len(errs) == 1: - return errs[0] - - default: - return fmt.Errorf("%v", errs) - } -} - -// eventLoop is a secondary - optional - event loop of the indexer which is only -// started for the outermost indexer to push chain head events into a processing -// queue. -func (c *ChainIndexer) eventLoop(currentHeader *types.Header, events chan ChainHeadEvent, sub event.Subscription) { - // Mark the chain indexer as active, requiring an additional teardown - c.active.Store(true) - - defer sub.Unsubscribe() - - // Fire the initial new head event to start any outstanding processing - c.newHead(currentHeader.Number.Uint64(), false) - - var ( - prevHeader = currentHeader - prevHash = currentHeader.Hash() - ) - for { - select { - case errc := <-c.quit: - // Chain indexer terminating, report no failure and abort - errc <- nil - return - - case ev, ok := <-events: - // Received a new event, ensure it's not nil (closing) and update - if !ok { - errc := <-c.quit - errc <- nil - return - } - if ev.Header.ParentHash != prevHash { - // Reorg to the common ancestor if needed (might not exist in light sync mode, skip reorg then) - // TODO(karalabe, zsfelfoldi): This seems a bit brittle, can we detect this case explicitly? - - if rawdb.ReadCanonicalHash(c.chainDb, prevHeader.Number.Uint64()) != prevHash { - if h := rawdb.FindCommonAncestor(c.chainDb, prevHeader, ev.Header); h != nil { - c.newHead(h.Number.Uint64(), true) - } - } - } - c.newHead(ev.Header.Number.Uint64(), false) - - prevHeader, prevHash = ev.Header, ev.Header.Hash() - } - } -} - -// newHead notifies the indexer about new chain heads and/or reorgs. -func (c *ChainIndexer) newHead(head uint64, reorg bool) { - c.lock.Lock() - defer c.lock.Unlock() - - // If a reorg happened, invalidate all sections until that point - if reorg { - // Revert the known section number to the reorg point - known := (head + 1) / c.sectionSize - stored := known - if known < c.checkpointSections { - known = 0 - } - if stored < c.checkpointSections { - stored = c.checkpointSections - } - if known < c.knownSections { - c.knownSections = known - } - // Revert the stored sections from the database to the reorg point - if stored < c.storedSections { - c.setValidSections(stored) - } - // Update the new head number to the finalized section end and notify children - head = known * c.sectionSize - - if head < c.cascadedHead { - c.cascadedHead = head - for _, child := range c.children { - child.newHead(c.cascadedHead, true) - } - } - return - } - // No reorg, calculate the number of newly known sections and update if high enough - var sections uint64 - if head >= c.confirmsReq { - sections = (head + 1 - c.confirmsReq) / c.sectionSize - if sections < c.checkpointSections { - sections = 0 - } - if sections > c.knownSections { - if c.knownSections < c.checkpointSections { - // syncing reached the checkpoint, verify section head - syncedHead := rawdb.ReadCanonicalHash(c.chainDb, c.checkpointSections*c.sectionSize-1) - if syncedHead != c.checkpointHead { - c.log.Error("Synced chain does not match checkpoint", "number", c.checkpointSections*c.sectionSize-1, "expected", c.checkpointHead, "synced", syncedHead) - return - } - } - c.knownSections = sections - - select { - case c.update <- struct{}{}: - default: - } - } - } -} - -// updateLoop is the main event loop of the indexer which pushes chain segments -// down into the processing backend. -func (c *ChainIndexer) updateLoop() { - var ( - updating bool - updated time.Time - ) - - for { - select { - case errc := <-c.quit: - // Chain indexer terminating, report no failure and abort - errc <- nil - return - - case <-c.update: - // Section headers completed (or rolled back), update the index - c.lock.Lock() - if c.knownSections > c.storedSections { - // Periodically print an upgrade log message to the user - if time.Since(updated) > 8*time.Second { - if c.knownSections > c.storedSections+1 { - updating = true - c.log.Info("Upgrading chain index", "percentage", c.storedSections*100/c.knownSections) - } - updated = time.Now() - } - // Cache the current section count and head to allow unlocking the mutex - c.verifyLastHead() - section := c.storedSections - var oldHead common.Hash - if section > 0 { - oldHead = c.SectionHead(section - 1) - } - // Process the newly defined section in the background - c.lock.Unlock() - newHead, err := c.processSection(section, oldHead) - if err != nil { - select { - case <-c.ctx.Done(): - <-c.quit <- nil - return - default: - } - c.log.Error("Section processing failed", "error", err) - } - c.lock.Lock() - - // If processing succeeded and no reorgs occurred, mark the section completed - if err == nil && (section == 0 || oldHead == c.SectionHead(section-1)) { - c.setSectionHead(section, newHead) - c.setValidSections(section + 1) - if c.storedSections == c.knownSections && updating { - updating = false - c.log.Info("Finished upgrading chain index") - } - c.cascadedHead = c.storedSections*c.sectionSize - 1 - for _, child := range c.children { - c.log.Trace("Cascading chain index update", "head", c.cascadedHead) - child.newHead(c.cascadedHead, false) - } - } else { - // If processing failed, don't retry until further notification - c.log.Debug("Chain index processing failed", "section", section, "err", err) - c.verifyLastHead() - c.knownSections = c.storedSections - } - } - // If there are still further sections to process, reschedule - if c.knownSections > c.storedSections { - time.AfterFunc(c.throttling, func() { - select { - case c.update <- struct{}{}: - default: - } - }) - } - c.lock.Unlock() - } - } -} - -// processSection processes an entire section by calling backend functions while -// ensuring the continuity of the passed headers. Since the chain mutex is not -// held while processing, the continuity can be broken by a long reorg, in which -// case the function returns with an error. -func (c *ChainIndexer) processSection(section uint64, lastHead common.Hash) (common.Hash, error) { - c.log.Trace("Processing new chain section", "section", section) - - // Reset and partial processing - if err := c.backend.Reset(c.ctx, section, lastHead); err != nil { - c.setValidSections(0) - return common.Hash{}, err - } - - for number := section * c.sectionSize; number < (section+1)*c.sectionSize; number++ { - hash := rawdb.ReadCanonicalHash(c.chainDb, number) - if hash == (common.Hash{}) { - return common.Hash{}, fmt.Errorf("canonical block #%d unknown", number) - } - header := rawdb.ReadHeader(c.chainDb, hash, number) - if header == nil { - return common.Hash{}, fmt.Errorf("block #%d [%x..] not found", number, hash[:4]) - } else if header.ParentHash != lastHead { - return common.Hash{}, errors.New("chain reorged during section processing") - } - if err := c.backend.Process(c.ctx, header); err != nil { - return common.Hash{}, err - } - lastHead = header.Hash() - } - if err := c.backend.Commit(); err != nil { - return common.Hash{}, err - } - return lastHead, nil -} - -// verifyLastHead compares last stored section head with the corresponding block hash in the -// actual canonical chain and rolls back reorged sections if necessary to ensure that stored -// sections are all valid -func (c *ChainIndexer) verifyLastHead() { - for c.storedSections > 0 && c.storedSections > c.checkpointSections { - if c.SectionHead(c.storedSections-1) == rawdb.ReadCanonicalHash(c.chainDb, c.storedSections*c.sectionSize-1) { - return - } - c.setValidSections(c.storedSections - 1) - } -} - -// Sections returns the number of processed sections maintained by the indexer -// and also the information about the last header indexed for potential canonical -// verifications. -func (c *ChainIndexer) Sections() (uint64, uint64, common.Hash) { - c.lock.Lock() - defer c.lock.Unlock() - - c.verifyLastHead() - return c.storedSections, c.storedSections*c.sectionSize - 1, c.SectionHead(c.storedSections - 1) -} - -// AddChildIndexer adds a child ChainIndexer that can use the output of this one -func (c *ChainIndexer) AddChildIndexer(indexer *ChainIndexer) { - if indexer == c { - panic("can't add indexer as a child of itself") - } - c.lock.Lock() - defer c.lock.Unlock() - - c.children = append(c.children, indexer) - - // Cascade any pending updates to new children too - sections := c.storedSections - if c.knownSections < sections { - // if a section is "stored" but not "known" then it is a checkpoint without - // available chain data so we should not cascade it yet - sections = c.knownSections - } - if sections > 0 { - indexer.newHead(sections*c.sectionSize-1, false) - } -} - -// Prune deletes all chain data older than given threshold. -func (c *ChainIndexer) Prune(threshold uint64) error { - return c.backend.Prune(threshold) -} - -// loadValidSections reads the number of valid sections from the index database -// and caches is into the local state. -func (c *ChainIndexer) loadValidSections() { - data, _ := c.indexDb.Get([]byte("count")) - if len(data) == 8 { - c.storedSections = binary.BigEndian.Uint64(data) - } -} - -// setValidSections writes the number of valid sections to the index database -func (c *ChainIndexer) setValidSections(sections uint64) { - // Set the current number of valid sections in the database - var data [8]byte - binary.BigEndian.PutUint64(data[:], sections) - c.indexDb.Put([]byte("count"), data[:]) - - // Remove any reorged sections, caching the valids in the mean time - for c.storedSections > sections { - c.storedSections-- - c.removeSectionHead(c.storedSections) - } - c.storedSections = sections // needed if new > old -} - -// SectionHead retrieves the last block hash of a processed section from the -// index database. -func (c *ChainIndexer) SectionHead(section uint64) common.Hash { - var data [8]byte - binary.BigEndian.PutUint64(data[:], section) - - hash, _ := c.indexDb.Get(append([]byte("shead"), data[:]...)) - if len(hash) == len(common.Hash{}) { - return common.BytesToHash(hash) - } - return common.Hash{} -} - -// setSectionHead writes the last block hash of a processed section to the index -// database. -func (c *ChainIndexer) setSectionHead(section uint64, hash common.Hash) { - var data [8]byte - binary.BigEndian.PutUint64(data[:], section) - - c.indexDb.Put(append([]byte("shead"), data[:]...), hash.Bytes()) -} - -// removeSectionHead removes the reference to a processed section from the index -// database. -func (c *ChainIndexer) removeSectionHead(section uint64) { - var data [8]byte - binary.BigEndian.PutUint64(data[:], section) - - c.indexDb.Delete(append([]byte("shead"), data[:]...)) -} diff --git a/core/chain_indexer_test.go b/core/chain_indexer_test.go deleted file mode 100644 index bf3bde756c..0000000000 --- a/core/chain_indexer_test.go +++ /dev/null @@ -1,246 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package core - -import ( - "context" - "errors" - "fmt" - "math/big" - "math/rand" - "testing" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/types" -) - -// Runs multiple tests with randomized parameters. -func TestChainIndexerSingle(t *testing.T) { - for i := 0; i < 10; i++ { - testChainIndexer(t, 1) - } -} - -// Runs multiple tests with randomized parameters and different number of -// chain backends. -func TestChainIndexerWithChildren(t *testing.T) { - for i := 2; i < 8; i++ { - testChainIndexer(t, i) - } -} - -// testChainIndexer runs a test with either a single chain indexer or a chain of -// multiple backends. The section size and required confirmation count parameters -// are randomized. -func testChainIndexer(t *testing.T, count int) { - db := rawdb.NewMemoryDatabase() - defer db.Close() - - // Create a chain of indexers and ensure they all report empty - backends := make([]*testChainIndexBackend, count) - for i := 0; i < count; i++ { - var ( - sectionSize = uint64(rand.Intn(100) + 1) - confirmsReq = uint64(rand.Intn(10)) - ) - backends[i] = &testChainIndexBackend{t: t, processCh: make(chan uint64)} - backends[i].indexer = NewChainIndexer(db, rawdb.NewTable(db, string([]byte{byte(i)})), backends[i], sectionSize, confirmsReq, 0, fmt.Sprintf("indexer-%d", i)) - - if sections, _, _ := backends[i].indexer.Sections(); sections != 0 { - t.Fatalf("Canonical section count mismatch: have %v, want %v", sections, 0) - } - if i > 0 { - backends[i-1].indexer.AddChildIndexer(backends[i].indexer) - } - } - defer backends[0].indexer.Close() // parent indexer shuts down children - // notify pings the root indexer about a new head or reorg, then expect - // processed blocks if a section is processable - notify := func(headNum, failNum uint64, reorg bool) { - backends[0].indexer.newHead(headNum, reorg) - if reorg { - for _, backend := range backends { - headNum = backend.reorg(headNum) - backend.assertSections() - } - return - } - var cascade bool - for _, backend := range backends { - headNum, cascade = backend.assertBlocks(headNum, failNum) - if !cascade { - break - } - backend.assertSections() - } - } - // inject inserts a new random canonical header into the database directly - inject := func(number uint64) { - header := &types.Header{Number: big.NewInt(int64(number)), Extra: big.NewInt(rand.Int63()).Bytes()} - if number > 0 { - header.ParentHash = rawdb.ReadCanonicalHash(db, number-1) - } - rawdb.WriteHeader(db, header) - rawdb.WriteCanonicalHash(db, header.Hash(), number) - } - // Start indexer with an already existing chain - for i := uint64(0); i <= 100; i++ { - inject(i) - } - notify(100, 100, false) - - // Add new blocks one by one - for i := uint64(101); i <= 1000; i++ { - inject(i) - notify(i, i, false) - } - // Do a reorg - notify(500, 500, true) - - // Create new fork - for i := uint64(501); i <= 1000; i++ { - inject(i) - notify(i, i, false) - } - for i := uint64(1001); i <= 1500; i++ { - inject(i) - } - // Failed processing scenario where less blocks are available than notified - notify(2000, 1500, false) - - // Notify about a reorg (which could have caused the missing blocks if happened during processing) - notify(1500, 1500, true) - - // Create new fork - for i := uint64(1501); i <= 2000; i++ { - inject(i) - notify(i, i, false) - } -} - -// testChainIndexBackend implements ChainIndexerBackend -type testChainIndexBackend struct { - t *testing.T - indexer *ChainIndexer - section, headerCnt, stored uint64 - processCh chan uint64 -} - -// assertSections verifies if a chain indexer has the correct number of section. -func (b *testChainIndexBackend) assertSections() { - // Keep trying for 3 seconds if it does not match - var sections uint64 - for i := 0; i < 300; i++ { - sections, _, _ = b.indexer.Sections() - if sections == b.stored { - return - } - time.Sleep(10 * time.Millisecond) - } - b.t.Fatalf("Canonical section count mismatch: have %v, want %v", sections, b.stored) -} - -// assertBlocks expects processing calls after new blocks have arrived. If the -// failNum < headNum then we are simulating a scenario where a reorg has happened -// after the processing has started and the processing of a section fails. -func (b *testChainIndexBackend) assertBlocks(headNum, failNum uint64) (uint64, bool) { - var sections uint64 - if headNum >= b.indexer.confirmsReq { - sections = (headNum + 1 - b.indexer.confirmsReq) / b.indexer.sectionSize - if sections > b.stored { - // expect processed blocks - for expectd := b.stored * b.indexer.sectionSize; expectd < sections*b.indexer.sectionSize; expectd++ { - if expectd > failNum { - // rolled back after processing started, no more process calls expected - // wait until updating is done to make sure that processing actually fails - var updating bool - for i := 0; i < 300; i++ { - b.indexer.lock.Lock() - updating = b.indexer.knownSections > b.indexer.storedSections - b.indexer.lock.Unlock() - if !updating { - break - } - time.Sleep(10 * time.Millisecond) - } - if updating { - b.t.Fatalf("update did not finish") - } - sections = expectd / b.indexer.sectionSize - break - } - select { - case <-time.After(10 * time.Second): - b.t.Fatalf("Expected processed block #%d, got nothing", expectd) - case processed := <-b.processCh: - if processed != expectd { - b.t.Errorf("Expected processed block #%d, got #%d", expectd, processed) - } - } - } - b.stored = sections - } - } - if b.stored == 0 { - return 0, false - } - return b.stored*b.indexer.sectionSize - 1, true -} - -func (b *testChainIndexBackend) reorg(headNum uint64) uint64 { - firstChanged := (headNum + 1) / b.indexer.sectionSize - if firstChanged < b.stored { - b.stored = firstChanged - } - return b.stored * b.indexer.sectionSize -} - -func (b *testChainIndexBackend) Reset(ctx context.Context, section uint64, prevHead common.Hash) error { - b.section = section - b.headerCnt = 0 - return nil -} - -func (b *testChainIndexBackend) Process(ctx context.Context, header *types.Header) error { - b.headerCnt++ - if b.headerCnt > b.indexer.sectionSize { - b.t.Error("Processing too many headers") - } - //t.processCh <- header.Number.Uint64() - select { - case <-time.After(10 * time.Second): - b.t.Error("Unexpected call to Process") - // Can't use Fatal since this is not the test's goroutine. - // Returning error stops the chainIndexer's updateLoop - return errors.New("unexpected call to Process") - case b.processCh <- header.Number.Uint64(): - } - return nil -} - -func (b *testChainIndexBackend) Commit() error { - if b.headerCnt != b.indexer.sectionSize { - b.t.Error("Not enough headers processed") - } - return nil -} - -func (b *testChainIndexBackend) Prune(threshold uint64) error { - return nil -} diff --git a/core/filtermaps/filtermaps.go b/core/filtermaps/filtermaps.go index dfc20521f6..d74b11da04 100644 --- a/core/filtermaps/filtermaps.go +++ b/core/filtermaps/filtermaps.go @@ -17,7 +17,6 @@ package filtermaps import ( - "bytes" "errors" "fmt" "os" @@ -242,7 +241,8 @@ func (f *FilterMaps) Start() { log.Error("Could not load head filter map snapshot", "error", err) } } - f.closeWg.Add(1) + f.closeWg.Add(2) + go f.removeBloomBits() go f.indexerLoop() } @@ -292,7 +292,7 @@ func (f *FilterMaps) reset() bool { // deleting the range first ensures that resetDb will be called again at next // startup and any leftover data will be removed even if it cannot finish now. rawdb.DeleteFilterMapsRange(f.db) - return f.removeDbWithPrefix([]byte(rawdb.FilterMapsPrefix), "Resetting log index database") + return f.safeDeleteRange(rawdb.DeleteFilterMapsDb, "Resetting log index database") } // init initializes an empty log index according to the current targetView. @@ -335,24 +335,25 @@ func (f *FilterMaps) init() error { return batch.Write() } -// removeDbWithPrefix removes data with the given prefix from the database and -// returns true if everything was successfully removed. -func (f *FilterMaps) removeDbWithPrefix(prefix []byte, action string) bool { - it := f.db.NewIterator(prefix, nil) - hasData := it.Next() - it.Release() - if !hasData { - return true - } +// removeBloomBits removes old bloom bits data from the database. +func (f *FilterMaps) removeBloomBits() { + f.safeDeleteRange(rawdb.DeleteBloomBitsDb, "Removing old bloom bits database") + f.closeWg.Done() +} - end := bytes.Clone(prefix) - end[len(end)-1]++ +// safeDeleteRange calls the specified database range deleter function +// repeatedly as long as it returns leveldb.ErrTooManyKeys. +// This wrapper is necessary because of the leveldb fallback implementation +// of DeleteRange. +func (f *FilterMaps) safeDeleteRange(removeFn func(ethdb.KeyValueRangeDeleter) error, action string) bool { start := time.Now() var retry bool for { - err := f.db.DeleteRange(prefix, end) + err := removeFn(f.db) if err == nil { - log.Info(action+" finished", "elapsed", time.Since(start)) + if retry { + log.Info(action+" finished", "elapsed", time.Since(start)) + } return true } if err != leveldb.ErrTooManyKeys { @@ -365,7 +366,7 @@ func (f *FilterMaps) removeDbWithPrefix(prefix []byte, action string) bool { default: } if !retry { - log.Info(action + " in progress...") + log.Info(action+" in progress...", "elapsed", time.Since(start)) retry = true } } diff --git a/core/filtermaps/indexer.go b/core/filtermaps/indexer.go index 61a9bf0352..69f42d8b60 100644 --- a/core/filtermaps/indexer.go +++ b/core/filtermaps/indexer.go @@ -44,13 +44,18 @@ func (f *FilterMaps) indexerLoop() { if !f.indexedRange.initialized { if err := f.init(); err != nil { log.Error("Error initializing log index", "error", err) - f.waitForEvent() + // unexpected error; there is not a lot we can do here, maybe it + // recovers, maybe not. Calling event processing here ensures + // that we can still properly shutdown in case of an infinite loop. + f.processSingleEvent(true) continue } } if !f.targetHeadIndexed() { if !f.tryIndexHead() { - f.waitForEvent() + // either shutdown or unexpected error; in the latter case ensure + // that proper shutdown is still possible. + f.processSingleEvent(true) } } else { if f.finalBlock != f.lastFinal { @@ -60,7 +65,7 @@ func (f *FilterMaps) indexerLoop() { f.lastFinal = f.finalBlock } if f.tryIndexTail() && f.tryUnindexTail() { - f.waitForEvent() + f.waitForNewHead() } } } @@ -119,8 +124,9 @@ func (f *FilterMaps) WaitIdle() { } } -// waitForEvent blocks until an event happens that the indexer might react to. -func (f *FilterMaps) waitForEvent() { +// waitForNewHead blocks until there is a new target head to index and block +// processing has been finished. +func (f *FilterMaps) waitForNewHead() { for !f.stop && (f.blockProcessing || f.targetHeadIndexed()) { f.processSingleEvent(true) } @@ -129,13 +135,16 @@ func (f *FilterMaps) waitForEvent() { // processEvents processes all events, blocking only if a block processing is // happening and indexing should be suspended. func (f *FilterMaps) processEvents() { - for !f.stop && f.processSingleEvent(f.blockProcessing) { + for f.processSingleEvent(f.blockProcessing) { } } // processSingleEvent processes a single event either in a blocking or -// non-blocking manner. +// non-blocking manner. It returns true if it did process an event. func (f *FilterMaps) processSingleEvent(blocking bool) bool { + if f.stop { + return false + } if !f.hasTempRange { for _, mb := range f.matcherSyncRequests { mb.synced() @@ -356,15 +365,17 @@ func (f *FilterMaps) needTailEpoch(epoch uint32) bool { if epoch+1 < firstEpoch { return false } + var lastBlockOfPrevEpoch uint64 if epoch > 0 { - lastBlockOfPrevEpoch, _, err := f.getLastBlockOfMap(epoch<= firstEpoch } - if f.historyCutoff > lastBlockOfPrevEpoch { - return false - } + } + if f.historyCutoff > lastBlockOfPrevEpoch { + return false } lastBlockOfEpoch, _, err := f.getLastBlockOfMap((epoch+1)<= 0 { - break - } - if len(it.Key()) != len(bloomBitsPrefix)+2+8+32 { - continue - } - db.Delete(it.Key()) - } - if it.Error() != nil { - log.Crit("Failed to delete bloom bits", "err", it.Error()) - } -} - // ReadFilterMapRow retrieves a filter map row at the given mapRowIndex // (see filtermaps.mapRowIndex for the storage index encoding). // Note that zero length rows are not stored in the database and therefore all @@ -485,3 +450,24 @@ func DeleteFilterMapsRange(db ethdb.KeyValueWriter) { log.Crit("Failed to delete filter maps range", "err", err) } } + +// deletePrefixRange deletes everything with the given prefix from the database. +func deletePrefixRange(db ethdb.KeyValueRangeDeleter, prefix []byte) error { + end := bytes.Clone(prefix) + end[len(end)-1]++ + return db.DeleteRange(prefix, end) +} + +// DeleteFilterMapsDb removes the entire filter maps database +func DeleteFilterMapsDb(db ethdb.KeyValueRangeDeleter) error { + return deletePrefixRange(db, []byte(filterMapsPrefix)) +} + +// DeleteFilterMapsDb removes the old bloombits database and the associated +// chain indexer database. +func DeleteBloomBitsDb(db ethdb.KeyValueRangeDeleter) error { + if err := deletePrefixRange(db, bloomBitsPrefix); err != nil { + return err + } + return deletePrefixRange(db, bloomBitsIndexPrefix) +} diff --git a/core/rawdb/accessors_indexes_test.go b/core/rawdb/accessors_indexes_test.go index ea25bf1e2c..29b468fb2a 100644 --- a/core/rawdb/accessors_indexes_test.go +++ b/core/rawdb/accessors_indexes_test.go @@ -17,7 +17,6 @@ package rawdb import ( - "bytes" "math/big" "testing" @@ -25,7 +24,6 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/internal/blocktest" - "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" ) @@ -111,54 +109,3 @@ func TestLookupStorage(t *testing.T) { }) } } - -func TestDeleteBloomBits(t *testing.T) { - // Prepare testing data - db := NewMemoryDatabase() - for i := uint(0); i < 2; i++ { - for s := uint64(0); s < 2; s++ { - WriteBloomBits(db, i, s, params.MainnetGenesisHash, []byte{0x01, 0x02}) - WriteBloomBits(db, i, s, params.SepoliaGenesisHash, []byte{0x01, 0x02}) - WriteBloomBits(db, i, s, params.HoodiGenesisHash, []byte{0x01, 0x02}) - } - } - check := func(bit uint, section uint64, head common.Hash, exist bool) { - bits, _ := ReadBloomBits(db, bit, section, head) - if exist && !bytes.Equal(bits, []byte{0x01, 0x02}) { - t.Fatalf("Bloombits mismatch") - } - if !exist && len(bits) > 0 { - t.Fatalf("Bloombits should be removed") - } - } - // Check the existence of written data. - check(0, 0, params.MainnetGenesisHash, true) - check(0, 0, params.SepoliaGenesisHash, true) - check(0, 0, params.HoodiGenesisHash, true) - - // Check the existence of deleted data. - DeleteBloombits(db, 0, 0, 1) - check(0, 0, params.MainnetGenesisHash, false) - check(0, 0, params.SepoliaGenesisHash, false) - check(0, 0, params.HoodiGenesisHash, false) - check(0, 1, params.MainnetGenesisHash, true) - check(0, 1, params.SepoliaGenesisHash, true) - check(0, 1, params.HoodiGenesisHash, true) - - // Check the existence of deleted data. - DeleteBloombits(db, 0, 0, 2) - check(0, 0, params.MainnetGenesisHash, false) - check(0, 0, params.SepoliaGenesisHash, false) - check(0, 0, params.HoodiGenesisHash, false) - check(0, 1, params.MainnetGenesisHash, false) - check(0, 1, params.SepoliaGenesisHash, false) - check(0, 1, params.HoodiGenesisHash, false) - - // Bit1 shouldn't be affect. - check(1, 0, params.MainnetGenesisHash, true) - check(1, 0, params.SepoliaGenesisHash, true) - check(1, 0, params.HoodiGenesisHash, true) - check(1, 1, params.MainnetGenesisHash, true) - check(1, 1, params.SepoliaGenesisHash, true) - check(1, 1, params.HoodiGenesisHash, true) -} diff --git a/core/rawdb/database.go b/core/rawdb/database.go index bc29347324..4c87e66cfd 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -375,7 +375,6 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { accountSnaps stat storageSnaps stat preimages stat - bloomBits stat filterMaps stat beaconHeaders stat cliqueSnaps stat @@ -437,11 +436,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { metadata.Add(size) case bytes.HasPrefix(key, genesisPrefix) && len(key) == (len(genesisPrefix)+common.HashLength): metadata.Add(size) - case bytes.HasPrefix(key, bloomBitsPrefix) && len(key) == (len(bloomBitsPrefix)+10+common.HashLength): - bloomBits.Add(size) - case bytes.HasPrefix(key, BloomBitsIndexPrefix): - bloomBits.Add(size) - case bytes.HasPrefix(key, []byte(FilterMapsPrefix)): + case bytes.HasPrefix(key, []byte(filterMapsPrefix)): filterMaps.Add(size) case bytes.HasPrefix(key, skeletonHeaderPrefix) && len(key) == (len(skeletonHeaderPrefix)+8): beaconHeaders.Add(size) @@ -507,7 +502,6 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { {"Key-Value store", "Block number->hash", numHashPairings.Size(), numHashPairings.Count()}, {"Key-Value store", "Block hash->number", hashNumPairings.Size(), hashNumPairings.Count()}, {"Key-Value store", "Transaction index", txLookups.Size(), txLookups.Count()}, - {"Key-Value store", "Bloombit index", bloomBits.Size(), bloomBits.Count()}, {"Key-Value store", "Log search index", filterMaps.Size(), filterMaps.Count()}, {"Key-Value store", "Contract codes", codes.Size(), codes.Count()}, {"Key-Value store", "Hash trie nodes", legacyTries.Size(), legacyTries.Count()}, diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index b41951113c..c21a96bd24 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -128,8 +128,8 @@ var ( configPrefix = []byte("ethereum-config-") // config prefix for the db genesisPrefix = []byte("ethereum-genesis-") // genesis state prefix for the db - // BloomBitsIndexPrefix is the data table of a chain indexer to track its progress - BloomBitsIndexPrefix = []byte("iB") + // bloomBitsIndexPrefix is the data table of a chain indexer to track its progress + bloomBitsIndexPrefix = []byte("iB") ChtPrefix = []byte("chtRootV2-") // ChtPrefix + chtNum (uint64 big endian) -> trie root hash ChtTablePrefix = []byte("cht-") @@ -145,11 +145,11 @@ var ( FixedCommitteeRootKey = []byte("fixedRoot-") // bigEndian64(syncPeriod) -> committee root hash SyncCommitteeKey = []byte("committee-") // bigEndian64(syncPeriod) -> serialized committee - FilterMapsPrefix = "fm-" - filterMapsRangeKey = []byte(FilterMapsPrefix + "R") - filterMapRowPrefix = []byte(FilterMapsPrefix + "r") // filterMapRowPrefix + mapRowIndex (uint64 big endian) -> filter row - filterMapLastBlockPrefix = []byte(FilterMapsPrefix + "b") // filterMapLastBlockPrefix + mapIndex (uint32 big endian) -> block number (uint64 big endian) - filterMapBlockLVPrefix = []byte(FilterMapsPrefix + "p") // filterMapBlockLVPrefix + num (uint64 big endian) -> log value pointer (uint64 big endian) + filterMapsPrefix = "fm-" + filterMapsRangeKey = []byte(filterMapsPrefix + "R") + filterMapRowPrefix = []byte(filterMapsPrefix + "r") // filterMapRowPrefix + mapRowIndex (uint64 big endian) -> filter row + filterMapLastBlockPrefix = []byte(filterMapsPrefix + "b") // filterMapLastBlockPrefix + mapIndex (uint32 big endian) -> block number (uint64 big endian) + filterMapBlockLVPrefix = []byte(filterMapsPrefix + "p") // filterMapBlockLVPrefix + num (uint64 big endian) -> log value pointer (uint64 big endian) preimageCounter = metrics.NewRegisteredCounter("db/preimage/total", nil) preimageHitsCounter = metrics.NewRegisteredCounter("db/preimage/hits", nil) @@ -225,16 +225,6 @@ func storageSnapshotsKey(accountHash common.Hash) []byte { return append(SnapshotStoragePrefix, accountHash.Bytes()...) } -// bloomBitsKey = bloomBitsPrefix + bit (uint16 big endian) + section (uint64 big endian) + hash -func bloomBitsKey(bit uint, section uint64, hash common.Hash) []byte { - key := append(append(bloomBitsPrefix, make([]byte, 10)...), hash.Bytes()...) - - binary.BigEndian.PutUint16(key[1:], uint16(bit)) - binary.BigEndian.PutUint64(key[3:], section) - - return key -} - // skeletonHeaderKey = skeletonHeaderPrefix + num (uint64 big endian) func skeletonHeaderKey(number uint64) []byte { return append(skeletonHeaderPrefix, encodeBlockNumber(number)...) diff --git a/eth/api_backend.go b/eth/api_backend.go index b4b63556a9..b08e38afe5 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -28,7 +28,6 @@ import ( "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/misc/eip4844" "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/filtermaps" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" @@ -403,17 +402,6 @@ func (b *EthAPIBackend) RPCTxFeeCap() float64 { return b.eth.config.RPCTxFeeCap } -func (b *EthAPIBackend) BloomStatus() (uint64, uint64) { - sections, _, _ := b.eth.bloomIndexer.Sections() - return params.BloomBitsBlocks, sections -} - -func (b *EthAPIBackend) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) { - for i := 0; i < bloomFilterThreads; i++ { - go session.Multiplex(bloomRetrievalBatch, bloomRetrievalWait, b.eth.bloomRequests) - } -} - func (b *EthAPIBackend) NewMatcherBackend() filtermaps.MatcherBackend { return b.eth.filterMaps.NewMatcherBackend() } diff --git a/eth/backend.go b/eth/backend.go index 99cb3841c3..6deedab872 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -30,7 +30,6 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/filtermaps" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state/pruner" @@ -85,10 +84,6 @@ type Ethereum struct { engine consensus.Engine accountManager *accounts.Manager - bloomRequests chan chan *bloombits.Retrieval // Channel receiving bloom data retrieval requests - bloomIndexer *core.ChainIndexer // Bloom indexer operating during block imports - closeBloomHandler chan struct{} - filterMaps *filtermaps.FilterMaps closeFilterMaps chan chan struct{} @@ -176,19 +171,16 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { // Assemble the Ethereum object. eth := &Ethereum{ - config: config, - chainDb: chainDb, - eventMux: stack.EventMux(), - accountManager: stack.AccountManager(), - engine: engine, - closeBloomHandler: make(chan struct{}), - networkID: networkID, - gasPrice: config.Miner.GasPrice, - bloomRequests: make(chan chan *bloombits.Retrieval), - bloomIndexer: core.NewBloomIndexer(chainDb, params.BloomBitsBlocks, params.BloomConfirms), - p2pServer: stack.Server(), - discmix: enode.NewFairMix(0), - shutdownTracker: shutdowncheck.NewShutdownTracker(chainDb), + config: config, + chainDb: chainDb, + eventMux: stack.EventMux(), + accountManager: stack.AccountManager(), + engine: engine, + networkID: networkID, + gasPrice: config.Miner.GasPrice, + p2pServer: stack.Server(), + discmix: enode.NewFairMix(0), + shutdownTracker: shutdowncheck.NewShutdownTracker(chainDb), } bcVersion := rawdb.ReadDatabaseVersion(chainDb) var dbVer = "" @@ -247,7 +239,6 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { if err != nil { return nil, err } - eth.bloomIndexer.Start(eth.blockchain) fmConfig := filtermaps.Config{History: config.LogHistory, Disabled: config.LogNoHistory, ExportFileName: config.LogExportCheckpoints} chainView := eth.newChainView(eth.blockchain.CurrentBlock()) eth.filterMaps = filtermaps.NewFilterMaps(chainDb, chainView, 0, 0, filtermaps.DefaultParams, fmConfig) @@ -379,7 +370,6 @@ func (s *Ethereum) Downloader() *downloader.Downloader { return s.handler.downlo func (s *Ethereum) Synced() bool { return s.handler.synced.Load() } func (s *Ethereum) SetSynced() { s.handler.enableSyncedFeatures() } func (s *Ethereum) ArchiveMode() bool { return s.config.NoPruning } -func (s *Ethereum) BloomIndexer() *core.ChainIndexer { return s.bloomIndexer } // Protocols returns all the currently configured // network protocols to start. @@ -398,9 +388,6 @@ func (s *Ethereum) Start() error { return err } - // Start the bloom bits servicing goroutines - s.startBloomHandlers(params.BloomBitsBlocks) - // Regularly update shutdown marker s.shutdownTracker.Start() @@ -511,8 +498,6 @@ func (s *Ethereum) Stop() error { s.handler.Stop() // Then stop everything else. - s.bloomIndexer.Close() - close(s.closeBloomHandler) ch := make(chan struct{}) s.closeFilterMaps <- ch <-ch diff --git a/eth/bloombits.go b/eth/bloombits.go deleted file mode 100644 index 0cb7050d23..0000000000 --- a/eth/bloombits.go +++ /dev/null @@ -1,74 +0,0 @@ -// Copyright 2017 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package eth - -import ( - "time" - - "github.com/ethereum/go-ethereum/common/bitutil" - "github.com/ethereum/go-ethereum/core/rawdb" -) - -const ( - // bloomServiceThreads is the number of goroutines used globally by an Ethereum - // instance to service bloombits lookups for all running filters. - bloomServiceThreads = 16 - - // bloomFilterThreads is the number of goroutines used locally per filter to - // multiplex requests onto the global servicing goroutines. - bloomFilterThreads = 3 - - // bloomRetrievalBatch is the maximum number of bloom bit retrievals to service - // in a single batch. - bloomRetrievalBatch = 16 - - // bloomRetrievalWait is the maximum time to wait for enough bloom bit requests - // to accumulate request an entire batch (avoiding hysteresis). - bloomRetrievalWait = time.Duration(0) -) - -// startBloomHandlers starts a batch of goroutines to accept bloom bit database -// retrievals from possibly a range of filters and serving the data to satisfy. -func (eth *Ethereum) startBloomHandlers(sectionSize uint64) { - for i := 0; i < bloomServiceThreads; i++ { - go func() { - for { - select { - case <-eth.closeBloomHandler: - return - - case request := <-eth.bloomRequests: - task := <-request - task.Bitsets = make([][]byte, len(task.Sections)) - for i, section := range task.Sections { - head := rawdb.ReadCanonicalHash(eth.chainDb, (section+1)*sectionSize-1) - if compVector, err := rawdb.ReadBloomBits(eth.chainDb, task.Bit, section, head); err == nil { - if blob, err := bitutil.DecompressBytes(compVector, int(sectionSize/8)); err == nil { - task.Bitsets[i] = blob - } else { - task.Error = err - } - } else { - task.Error = err - } - } - request <- task - } - } - }() - } -} diff --git a/params/network_params.go b/params/network_params.go index 61bd6b2f42..c016e7fcf3 100644 --- a/params/network_params.go +++ b/params/network_params.go @@ -20,14 +20,6 @@ package params // aren't necessarily consensus related. const ( - // BloomBitsBlocks is the number of blocks a single bloom bit section vector - // contains on the server side. - BloomBitsBlocks uint64 = 4096 - - // BloomConfirms is the number of confirmation blocks before a bloom section is - // considered probably final and its rotated bits are calculated. - BloomConfirms = 256 - // FullImmutabilityThreshold is the number of blocks after which a chain segment is // considered immutable (i.e. soft finality). It is used by the downloader as a // hard limit against deep ancestors, by the blockchain against deep reorgs, by From 18869222646d1706168929515df83c442ebba2b0 Mon Sep 17 00:00:00 2001 From: Sina M <1591639+s1na@users.noreply.github.com> Date: Fri, 21 Mar 2025 11:29:51 +0100 Subject: [PATCH 04/42] core: respect history cutoff in txindexer (#31393) In #31384 we unindex TXes prior to the merge block. However when the node starts up it will try to re-index those back if the config is to index the whole chain. This change makes the indexer aware of the history cutoff block, avoiding reindexing in that segment. --------- Co-authored-by: Gary Rong Co-authored-by: Felix Lange --- core/rawdb/accessors_chain.go | 8 + core/rawdb/accessors_indexes.go | 48 ++- core/txindexer.go | 151 +++++++-- core/txindexer_test.go | 538 ++++++++++++++++++++++---------- 4 files changed, 543 insertions(+), 202 deletions(-) diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index 1a2b31b360..020d35619e 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -277,6 +277,14 @@ func WriteTxIndexTail(db ethdb.KeyValueWriter, number uint64) { } } +// DeleteTxIndexTail deletes the number of oldest indexed block +// from database. +func DeleteTxIndexTail(db ethdb.KeyValueWriter) { + if err := db.Delete(txIndexTailKey); err != nil { + log.Crit("Failed to delete the transaction index tail", "err", err) + } +} + // ReadHeaderRange returns the rlp-encoded headers, starting at 'number', and going // backwards towards genesis. This method assumes that the caller already has // placed a cap on count, to prevent DoS issues. diff --git a/core/rawdb/accessors_indexes.go b/core/rawdb/accessors_indexes.go index 1d5897cef1..342aedd8dc 100644 --- a/core/rawdb/accessors_indexes.go +++ b/core/rawdb/accessors_indexes.go @@ -30,13 +30,8 @@ import ( "github.com/ethereum/go-ethereum/rlp" ) -// ReadTxLookupEntry retrieves the positional metadata associated with a transaction -// hash to allow retrieving the transaction or receipt by hash. -func ReadTxLookupEntry(db ethdb.Reader, hash common.Hash) *uint64 { - data, _ := db.Get(txLookupKey(hash)) - if len(data) == 0 { - return nil - } +// DecodeTxLookupEntry decodes the supplied tx lookup data. +func DecodeTxLookupEntry(data []byte, db ethdb.Reader) *uint64 { // Database v6 tx lookup just stores the block number if len(data) < common.HashLength { number := new(big.Int).SetBytes(data).Uint64() @@ -49,12 +44,22 @@ func ReadTxLookupEntry(db ethdb.Reader, hash common.Hash) *uint64 { // Finally try database v3 tx lookup format var entry LegacyTxLookupEntry if err := rlp.DecodeBytes(data, &entry); err != nil { - log.Error("Invalid transaction lookup entry RLP", "hash", hash, "blob", data, "err", err) + log.Error("Invalid transaction lookup entry RLP", "blob", data, "err", err) return nil } return &entry.BlockIndex } +// ReadTxLookupEntry retrieves the positional metadata associated with a transaction +// hash to allow retrieving the transaction or receipt by hash. +func ReadTxLookupEntry(db ethdb.Reader, hash common.Hash) *uint64 { + data, _ := db.Get(txLookupKey(hash)) + if len(data) == 0 { + return nil + } + return DecodeTxLookupEntry(data, db) +} + // writeTxLookupEntry stores a positional metadata for a transaction, // enabling hash based transaction and receipt lookups. func writeTxLookupEntry(db ethdb.KeyValueWriter, hash common.Hash, numberBytes []byte) { @@ -95,6 +100,33 @@ func DeleteTxLookupEntries(db ethdb.KeyValueWriter, hashes []common.Hash) { } } +// DeleteAllTxLookupEntries purges all the transaction indexes in the database. +// If condition is specified, only the entry with condition as True will be +// removed; If condition is not specified, the entry is deleted. +func DeleteAllTxLookupEntries(db ethdb.KeyValueStore, condition func([]byte) bool) { + iter := NewKeyLengthIterator(db.NewIterator(txLookupPrefix, nil), common.HashLength+len(txLookupPrefix)) + defer iter.Release() + + batch := db.NewBatch() + for iter.Next() { + if condition == nil || condition(iter.Value()) { + batch.Delete(iter.Key()) + } + if batch.ValueSize() >= ethdb.IdealBatchSize { + if err := batch.Write(); err != nil { + log.Crit("Failed to delete transaction lookup entries", "err", err) + } + batch.Reset() + } + } + if batch.ValueSize() > 0 { + if err := batch.Write(); err != nil { + log.Crit("Failed to delete transaction lookup entries", "err", err) + } + batch.Reset() + } +} + // ReadTransaction retrieves a specific transaction from the database, along with // its added positional metadata. func ReadTransaction(db ethdb.Reader, hash common.Hash) (*types.Transaction, common.Hash, uint64, uint64) { diff --git a/core/txindexer.go b/core/txindexer.go index 293124f681..29e87905d5 100644 --- a/core/txindexer.go +++ b/core/txindexer.go @@ -44,7 +44,11 @@ type txIndexer struct { // * 0: means the entire chain should be indexed // * N: means the latest N blocks [HEAD-N+1, HEAD] should be indexed // and all others shouldn't. - limit uint64 + limit uint64 + + // cutoff denotes the block number before which the chain segment should + // be pruned and not available locally. + cutoff uint64 db ethdb.Database progress chan chan TxIndexProgress term chan chan struct{} @@ -55,6 +59,7 @@ type txIndexer struct { func newTxIndexer(limit uint64, chain *BlockChain) *txIndexer { indexer := &txIndexer{ limit: limit, + cutoff: chain.HistoryPruningCutoff(), db: chain.db, progress: make(chan chan TxIndexProgress), term: make(chan chan struct{}), @@ -64,7 +69,11 @@ func newTxIndexer(limit uint64, chain *BlockChain) *txIndexer { var msg string if limit == 0 { - msg = "entire chain" + if indexer.cutoff == 0 { + msg = "entire chain" + } else { + msg = fmt.Sprintf("blocks since #%d", indexer.cutoff) + } } else { msg = fmt.Sprintf("last %d blocks", limit) } @@ -74,23 +83,31 @@ func newTxIndexer(limit uint64, chain *BlockChain) *txIndexer { } // run executes the scheduled indexing/unindexing task in a separate thread. -// If the stop channel is closed, the task should be terminated as soon as -// possible, the done channel will be closed once the task is finished. -func (indexer *txIndexer) run(tail *uint64, head uint64, stop chan struct{}, done chan struct{}) { +// If the stop channel is closed, the task should terminate as soon as possible. +// The done channel will be closed once the task is complete. +// +// Existing transaction indexes are assumed to be valid, with both the head +// and tail above the configured cutoff. +func (indexer *txIndexer) run(head uint64, stop chan struct{}, done chan struct{}) { defer func() { close(done) }() - // Short circuit if chain is empty and nothing to index. - if head == 0 { + // Short circuit if the chain is either empty, or entirely below the + // cutoff point. + if head == 0 || head < indexer.cutoff { return } // The tail flag is not existent, it means the node is just initialized // and all blocks in the chain (part of them may from ancient store) are // not indexed yet, index the chain according to the configured limit. + tail := rawdb.ReadTxIndexTail(indexer.db) if tail == nil { + // Determine the first block for transaction indexing, taking the + // configured cutoff point into account. from := uint64(0) if indexer.limit != 0 && head >= indexer.limit { from = head - indexer.limit + 1 } + from = max(from, indexer.cutoff) rawdb.IndexTransactions(indexer.db, from, head+1, stop, true) return } @@ -98,25 +115,82 @@ func (indexer *txIndexer) run(tail *uint64, head uint64, stop chan struct{}, don // present), while the whole chain are requested for indexing. if indexer.limit == 0 || head < indexer.limit { if *tail > 0 { - // It can happen when chain is rewound to a historical point which - // is even lower than the indexes tail, recap the indexing target - // to new head to avoid reading non-existent block bodies. - end := *tail - if end > head+1 { - end = head + 1 - } - rawdb.IndexTransactions(indexer.db, 0, end, stop, true) + from := max(uint64(0), indexer.cutoff) + rawdb.IndexTransactions(indexer.db, from, *tail, stop, true) } return } // The tail flag is existent, adjust the index range according to configured // limit and the latest chain head. - if head-indexer.limit+1 < *tail { + from := head - indexer.limit + 1 + from = max(from, indexer.cutoff) + if from < *tail { // Reindex a part of missing indices and rewind index tail to HEAD-limit - rawdb.IndexTransactions(indexer.db, head-indexer.limit+1, *tail, stop, true) + rawdb.IndexTransactions(indexer.db, from, *tail, stop, true) } else { // Unindex a part of stale indices and forward index tail to HEAD-limit - rawdb.UnindexTransactions(indexer.db, *tail, head-indexer.limit+1, stop, false) + rawdb.UnindexTransactions(indexer.db, *tail, from, stop, false) + } +} + +// repair ensures that transaction indexes are in a valid state and invalidates +// them if they are not. The following cases are considered invalid: +// * The index tail is higher than the chain head. +// * The chain head is below the configured cutoff, but the index tail is not empty. +// * The index tail is below the configured cutoff, but it is not empty. +func (indexer *txIndexer) repair(head uint64) { + // If the transactions haven't been indexed yet, nothing to repair + tail := rawdb.ReadTxIndexTail(indexer.db) + if tail == nil { + return + } + // The transaction index tail is higher than the chain head, which may occur + // when the chain is rewound to a historical height below the index tail. + // Purge the transaction indexes from the database. **It's not a common case + // to rewind the chain head below the index tail**. + if *tail > head { + // A crash may occur between the two delete operations, + // potentially leaving dangling indexes in the database. + // However, this is considered acceptable. + rawdb.DeleteTxIndexTail(indexer.db) + rawdb.DeleteAllTxLookupEntries(indexer.db, nil) + log.Warn("Purge transaction indexes", "head", head, "tail", *tail) + return + } + + // If the entire chain is below the configured cutoff point, + // removing the tail of transaction indexing and purges the + // transaction indexes. **It's not a common case, as the cutoff + // is usually defined below the chain head**. + if head < indexer.cutoff { + // A crash may occur between the two delete operations, + // potentially leaving dangling indexes in the database. + // However, this is considered acceptable. + // + // The leftover indexes can't be unindexed by scanning + // the blocks as they are not guaranteed to be available. + // Traversing the database directly within the transaction + // index namespace might be slow and expensive, but we + // have no choice. + rawdb.DeleteTxIndexTail(indexer.db) + rawdb.DeleteAllTxLookupEntries(indexer.db, nil) + log.Warn("Purge transaction indexes", "head", head, "cutoff", indexer.cutoff) + return + } + + // The chain head is above the cutoff while the tail is below the + // cutoff. Shift the tail to the cutoff point and remove the indexes + // below. + if *tail < indexer.cutoff { + // A crash may occur between the two delete operations, + // potentially leaving dangling indexes in the database. + // However, this is considered acceptable. + rawdb.WriteTxIndexTail(indexer.db, indexer.cutoff) + rawdb.DeleteAllTxLookupEntries(indexer.db, func(blob []byte) bool { + n := rawdb.DecodeTxLookupEntry(blob, indexer.db) + return n != nil && *n < indexer.cutoff + }) + log.Warn("Purge transaction indexes below cutoff", "tail", *tail, "cutoff", indexer.cutoff) } } @@ -127,39 +201,39 @@ func (indexer *txIndexer) loop(chain *BlockChain) { // Listening to chain events and manipulate the transaction indexes. var ( - stop chan struct{} // Non-nil if background routine is active. - done chan struct{} // Non-nil if background routine is active. - lastHead uint64 // The latest announced chain head (whose tx indexes are assumed created) - lastTail = rawdb.ReadTxIndexTail(indexer.db) // The oldest indexed block, nil means nothing indexed + stop chan struct{} // Non-nil if background routine is active + done chan struct{} // Non-nil if background routine is active + head = rawdb.ReadHeadBlock(indexer.db).NumberU64() // The latest announced chain head headCh = make(chan ChainHeadEvent) sub = chain.SubscribeChainHeadEvent(headCh) ) defer sub.Unsubscribe() + // Validate the transaction indexes and repair if necessary + indexer.repair(head) + // Launch the initial processing if chain is not empty (head != genesis). // This step is useful in these scenarios that chain has no progress. - if head := rawdb.ReadHeadBlock(indexer.db); head != nil && head.Number().Uint64() != 0 { + if head != 0 { stop = make(chan struct{}) done = make(chan struct{}) - lastHead = head.Number().Uint64() - go indexer.run(rawdb.ReadTxIndexTail(indexer.db), head.NumberU64(), stop, done) + go indexer.run(head, stop, done) } for { select { - case head := <-headCh: + case h := <-headCh: if done == nil { stop = make(chan struct{}) done = make(chan struct{}) - go indexer.run(rawdb.ReadTxIndexTail(indexer.db), head.Header.Number.Uint64(), stop, done) + go indexer.run(h.Header.Number.Uint64(), stop, done) } - lastHead = head.Header.Number.Uint64() + head = h.Header.Number.Uint64() case <-done: stop = nil done = nil - lastTail = rawdb.ReadTxIndexTail(indexer.db) case ch := <-indexer.progress: - ch <- indexer.report(lastHead, lastTail) + ch <- indexer.report(head) case ch := <-indexer.term: if stop != nil { close(stop) @@ -175,12 +249,27 @@ func (indexer *txIndexer) loop(chain *BlockChain) { } // report returns the tx indexing progress. -func (indexer *txIndexer) report(head uint64, tail *uint64) TxIndexProgress { +func (indexer *txIndexer) report(head uint64) TxIndexProgress { + // Special case if the head is even below the cutoff, + // nothing to index. + if head < indexer.cutoff { + return TxIndexProgress{ + Indexed: 0, + Remaining: 0, + } + } + // Compute how many blocks are supposed to be indexed total := indexer.limit if indexer.limit == 0 || total > head { total = head + 1 // genesis included } + length := head - indexer.cutoff + 1 // all available chain for indexing + if total > length { + total = length + } + // Compute how many blocks have been indexed var indexed uint64 + tail := rawdb.ReadTxIndexTail(indexer.db) if tail != nil { indexed = head - *tail + 1 } diff --git a/core/txindexer_test.go b/core/txindexer_test.go index 4425f0d9a5..7a5688241f 100644 --- a/core/txindexer_test.go +++ b/core/txindexer_test.go @@ -29,6 +29,46 @@ import ( "github.com/ethereum/go-ethereum/params" ) +func verifyIndexes(t *testing.T, db ethdb.Database, block *types.Block, exist bool) { + for _, tx := range block.Transactions() { + lookup := rawdb.ReadTxLookupEntry(db, tx.Hash()) + if exist && lookup == nil { + t.Fatalf("missing %d %x", block.NumberU64(), tx.Hash().Hex()) + } + if !exist && lookup != nil { + t.Fatalf("unexpected %d %x", block.NumberU64(), tx.Hash().Hex()) + } + } +} + +func verify(t *testing.T, db ethdb.Database, blocks []*types.Block, expTail uint64) { + tail := rawdb.ReadTxIndexTail(db) + if tail == nil { + t.Fatal("Failed to write tx index tail") + return + } + if *tail != expTail { + t.Fatalf("Unexpected tx index tail, want %v, got %d", expTail, *tail) + } + for _, b := range blocks { + if b.Number().Uint64() < *tail { + verifyIndexes(t, db, b, false) + } else { + verifyIndexes(t, db, b, true) + } + } +} + +func verifyNoIndex(t *testing.T, db ethdb.Database, blocks []*types.Block) { + tail := rawdb.ReadTxIndexTail(db) + if tail != nil { + t.Fatalf("Unexpected tx index tail %d", *tail) + } + for _, b := range blocks { + verifyIndexes(t, db, b, false) + } +} + // TestTxIndexer tests the functionalities for managing transaction indexes. func TestTxIndexer(t *testing.T) { var ( @@ -50,163 +90,29 @@ func TestTxIndexer(t *testing.T) { gen.AddTx(tx) nonce += 1 }) - - // verifyIndexes checks if the transaction indexes are present or not - // of the specified block. - verifyIndexes := func(db ethdb.Database, number uint64, exist bool) { - if number == 0 { - return - } - block := blocks[number-1] - for _, tx := range block.Transactions() { - lookup := rawdb.ReadTxLookupEntry(db, tx.Hash()) - if exist && lookup == nil { - t.Fatalf("missing %d %x", number, tx.Hash().Hex()) - } - if !exist && lookup != nil { - t.Fatalf("unexpected %d %x", number, tx.Hash().Hex()) - } - } - } - verify := func(db ethdb.Database, expTail uint64, indexer *txIndexer) { - tail := rawdb.ReadTxIndexTail(db) - if tail == nil { - t.Fatal("Failed to write tx index tail") - } - if *tail != expTail { - t.Fatalf("Unexpected tx index tail, want %v, got %d", expTail, *tail) - } - if *tail != 0 { - for number := uint64(0); number < *tail; number += 1 { - verifyIndexes(db, number, false) - } - } - for number := *tail; number <= chainHead; number += 1 { - verifyIndexes(db, number, true) - } - progress := indexer.report(chainHead, tail) - if !progress.Done() { - t.Fatalf("Expect fully indexed") - } - } - var cases = []struct { - limitA uint64 - tailA uint64 - limitB uint64 - tailB uint64 - limitC uint64 - tailC uint64 + limits []uint64 + tails []uint64 }{ { - // LimitA: 0 - // TailA: 0 - // - // all blocks are indexed - limitA: 0, - tailA: 0, - - // LimitB: 1 - // TailB: 128 - // - // block-128 is indexed - limitB: 1, - tailB: 128, - - // LimitB: 64 - // TailB: 65 - // - // block [65, 128] are indexed - limitC: 64, - tailC: 65, + limits: []uint64{0, 1, 64, 129, 0}, + tails: []uint64{0, 128, 65, 0, 0}, }, { - // LimitA: 64 - // TailA: 65 - // - // block [65, 128] are indexed - limitA: 64, - tailA: 65, - - // LimitB: 1 - // TailB: 128 - // - // block-128 is indexed - limitB: 1, - tailB: 128, - - // LimitB: 64 - // TailB: 65 - // - // block [65, 128] are indexed - limitC: 64, - tailC: 65, + limits: []uint64{64, 1, 64, 0}, + tails: []uint64{65, 128, 65, 0}, }, { - // LimitA: 127 - // TailA: 2 - // - // block [2, 128] are indexed - limitA: 127, - tailA: 2, - - // LimitB: 1 - // TailB: 128 - // - // block-128 is indexed - limitB: 1, - tailB: 128, - - // LimitB: 64 - // TailB: 65 - // - // block [65, 128] are indexed - limitC: 64, - tailC: 65, + limits: []uint64{127, 1, 64, 0}, + tails: []uint64{2, 128, 65, 0}, }, { - // LimitA: 128 - // TailA: 1 - // - // block [2, 128] are indexed - limitA: 128, - tailA: 1, - - // LimitB: 1 - // TailB: 128 - // - // block-128 is indexed - limitB: 1, - tailB: 128, - - // LimitB: 64 - // TailB: 65 - // - // block [65, 128] are indexed - limitC: 64, - tailC: 65, + limits: []uint64{128, 1, 64, 0}, + tails: []uint64{1, 128, 65, 0}, }, { - // LimitA: 129 - // TailA: 0 - // - // block [0, 128] are indexed - limitA: 129, - tailA: 0, - - // LimitB: 1 - // TailB: 128 - // - // block-128 is indexed - limitB: 1, - tailB: 128, - - // LimitB: 64 - // TailB: 65 - // - // block [65, 128] are indexed - limitC: 64, - tailC: 65, + limits: []uint64{129, 1, 64, 0}, + tails: []uint64{0, 128, 65, 0}, }, } for _, c := range cases { @@ -215,26 +121,332 @@ func TestTxIndexer(t *testing.T) { // Index the initial blocks from ancient store indexer := &txIndexer{ - limit: c.limitA, + limit: 0, db: db, progress: make(chan chan TxIndexProgress), } - indexer.run(nil, 128, make(chan struct{}), make(chan struct{})) - verify(db, c.tailA, indexer) - - indexer.limit = c.limitB - indexer.run(rawdb.ReadTxIndexTail(db), 128, make(chan struct{}), make(chan struct{})) - verify(db, c.tailB, indexer) - - indexer.limit = c.limitC - indexer.run(rawdb.ReadTxIndexTail(db), 128, make(chan struct{}), make(chan struct{})) - verify(db, c.tailC, indexer) - - // Recover all indexes - indexer.limit = 0 - indexer.run(rawdb.ReadTxIndexTail(db), 128, make(chan struct{}), make(chan struct{})) - verify(db, 0, indexer) - + for i, limit := range c.limits { + indexer.limit = limit + indexer.run(chainHead, make(chan struct{}), make(chan struct{})) + verify(t, db, blocks, c.tails[i]) + } + db.Close() + } +} + +func TestTxIndexerRepair(t *testing.T) { + var ( + testBankKey, _ = crypto.GenerateKey() + testBankAddress = crypto.PubkeyToAddress(testBankKey.PublicKey) + testBankFunds = big.NewInt(1000000000000000000) + + gspec = &Genesis{ + Config: params.TestChainConfig, + Alloc: types.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}, + BaseFee: big.NewInt(params.InitialBaseFee), + } + engine = ethash.NewFaker() + nonce = uint64(0) + chainHead = uint64(128) + ) + _, blocks, receipts := GenerateChainWithGenesis(gspec, engine, int(chainHead), func(i int, gen *BlockGen) { + tx, _ := types.SignTx(types.NewTransaction(nonce, common.HexToAddress("0xdeadbeef"), big.NewInt(1000), params.TxGas, big.NewInt(10*params.InitialBaseFee), nil), types.HomesteadSigner{}, testBankKey) + gen.AddTx(tx) + nonce += 1 + }) + tailPointer := func(n uint64) *uint64 { + return &n + } + var cases = []struct { + limit uint64 + head uint64 + cutoff uint64 + expTail *uint64 + }{ + // if *tail > head => purge indexes + { + limit: 0, + head: chainHead / 2, + cutoff: 0, + expTail: tailPointer(0), + }, + { + limit: 1, // tail = 128 + head: chainHead / 2, // newhead = 64 + cutoff: 0, + expTail: nil, + }, + { + limit: 64, // tail = 65 + head: chainHead / 2, // newhead = 64 + cutoff: 0, + expTail: nil, + }, + { + limit: 65, // tail = 64 + head: chainHead / 2, // newhead = 64 + cutoff: 0, + expTail: tailPointer(64), + }, + { + limit: 66, // tail = 63 + head: chainHead / 2, // newhead = 64 + cutoff: 0, + expTail: tailPointer(63), + }, + + // if tail < cutoff => remove indexes below cutoff + { + limit: 0, // tail = 0 + head: chainHead, // head = 128 + cutoff: chainHead, // cutoff = 128 + expTail: tailPointer(chainHead), + }, + { + limit: 1, // tail = 128 + head: chainHead, // head = 128 + cutoff: chainHead, // cutoff = 128 + expTail: tailPointer(128), + }, + { + limit: 2, // tail = 127 + head: chainHead, // head = 128 + cutoff: chainHead, // cutoff = 128 + expTail: tailPointer(chainHead), + }, + { + limit: 2, // tail = 127 + head: chainHead, // head = 128 + cutoff: chainHead / 2, // cutoff = 64 + expTail: tailPointer(127), + }, + + // if head < cutoff => purge indexes + { + limit: 0, // tail = 0 + head: chainHead, // head = 128 + cutoff: 2 * chainHead, // cutoff = 256 + expTail: nil, + }, + { + limit: 64, // tail = 65 + head: chainHead, // head = 128 + cutoff: chainHead / 2, // cutoff = 64 + expTail: tailPointer(65), + }, + } + for _, c := range cases { + db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false) + rawdb.WriteAncientBlocks(db, append([]*types.Block{gspec.ToBlock()}, blocks...), append([]types.Receipts{{}}, receipts...)) + + // Index the initial blocks from ancient store + indexer := &txIndexer{ + limit: c.limit, + db: db, + progress: make(chan chan TxIndexProgress), + } + indexer.run(chainHead, make(chan struct{}), make(chan struct{})) + + indexer.cutoff = c.cutoff + indexer.repair(c.head) + + if c.expTail == nil { + verifyNoIndex(t, db, blocks) + } else { + verify(t, db, blocks, *c.expTail) + } + db.Close() + } +} + +func TestTxIndexerReport(t *testing.T) { + var ( + testBankKey, _ = crypto.GenerateKey() + testBankAddress = crypto.PubkeyToAddress(testBankKey.PublicKey) + testBankFunds = big.NewInt(1000000000000000000) + + gspec = &Genesis{ + Config: params.TestChainConfig, + Alloc: types.GenesisAlloc{testBankAddress: {Balance: testBankFunds}}, + BaseFee: big.NewInt(params.InitialBaseFee), + } + engine = ethash.NewFaker() + nonce = uint64(0) + chainHead = uint64(128) + ) + _, blocks, receipts := GenerateChainWithGenesis(gspec, engine, int(chainHead), func(i int, gen *BlockGen) { + tx, _ := types.SignTx(types.NewTransaction(nonce, common.HexToAddress("0xdeadbeef"), big.NewInt(1000), params.TxGas, big.NewInt(10*params.InitialBaseFee), nil), types.HomesteadSigner{}, testBankKey) + gen.AddTx(tx) + nonce += 1 + }) + tailPointer := func(n uint64) *uint64 { + return &n + } + var cases = []struct { + head uint64 + limit uint64 + cutoff uint64 + tail *uint64 + expIndexed uint64 + expRemaining uint64 + }{ + // The entire chain is supposed to be indexed + { + // head = 128, limit = 0, cutoff = 0 => all: 129 + head: chainHead, + limit: 0, + cutoff: 0, + + // tail = 0 + tail: tailPointer(0), + expIndexed: 129, + expRemaining: 0, + }, + { + // head = 128, limit = 0, cutoff = 0 => all: 129 + head: chainHead, + limit: 0, + cutoff: 0, + + // tail = 1 + tail: tailPointer(1), + expIndexed: 128, + expRemaining: 1, + }, + { + // head = 128, limit = 0, cutoff = 0 => all: 129 + head: chainHead, + limit: 0, + cutoff: 0, + + // tail = 128 + tail: tailPointer(chainHead), + expIndexed: 1, + expRemaining: 128, + }, + { + // head = 128, limit = 256, cutoff = 0 => all: 129 + head: chainHead, + limit: 256, + cutoff: 0, + + // tail = 0 + tail: tailPointer(0), + expIndexed: 129, + expRemaining: 0, + }, + + // The chain with specific range is supposed to be indexed + { + // head = 128, limit = 64, cutoff = 0 => index: [65, 128] + head: chainHead, + limit: 64, + cutoff: 0, + + // tail = 0, part of them need to be unindexed + tail: tailPointer(0), + expIndexed: 129, + expRemaining: 0, + }, + { + // head = 128, limit = 64, cutoff = 0 => index: [65, 128] + head: chainHead, + limit: 64, + cutoff: 0, + + // tail = 64, one of them needs to be unindexed + tail: tailPointer(64), + expIndexed: 65, + expRemaining: 0, + }, + { + // head = 128, limit = 64, cutoff = 0 => index: [65, 128] + head: chainHead, + limit: 64, + cutoff: 0, + + // tail = 65, all of them have been indexed + tail: tailPointer(65), + expIndexed: 64, + expRemaining: 0, + }, + { + // head = 128, limit = 64, cutoff = 0 => index: [65, 128] + head: chainHead, + limit: 64, + cutoff: 0, + + // tail = 66, one of them has to be indexed + tail: tailPointer(66), + expIndexed: 63, + expRemaining: 1, + }, + + // The chain with configured cutoff, the chain range could be capped + { + // head = 128, limit = 64, cutoff = 66 => index: [66, 128] + head: chainHead, + limit: 64, + cutoff: 66, + + // tail = 0, part of them need to be unindexed + tail: tailPointer(0), + expIndexed: 129, + expRemaining: 0, + }, + { + // head = 128, limit = 64, cutoff = 66 => index: [66, 128] + head: chainHead, + limit: 64, + cutoff: 66, + + // tail = 66, all of them have been indexed + tail: tailPointer(66), + expIndexed: 63, + expRemaining: 0, + }, + { + // head = 128, limit = 64, cutoff = 66 => index: [66, 128] + head: chainHead, + limit: 64, + cutoff: 66, + + // tail = 67, one of them has to be indexed + tail: tailPointer(67), + expIndexed: 62, + expRemaining: 1, + }, + { + // head = 128, limit = 64, cutoff = 256 => index: [66, 128] + head: chainHead, + limit: 0, + cutoff: 256, + tail: nil, + expIndexed: 0, + expRemaining: 0, + }, + } + for _, c := range cases { + db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false) + rawdb.WriteAncientBlocks(db, append([]*types.Block{gspec.ToBlock()}, blocks...), append([]types.Receipts{{}}, receipts...)) + + // Index the initial blocks from ancient store + indexer := &txIndexer{ + limit: c.limit, + cutoff: c.cutoff, + db: db, + progress: make(chan chan TxIndexProgress), + } + if c.tail != nil { + rawdb.WriteTxIndexTail(db, *c.tail) + } + p := indexer.report(c.head) + if p.Indexed != c.expIndexed { + t.Fatalf("Unexpected indexed: %d, expected: %d", p.Indexed, c.expIndexed) + } + if p.Remaining != c.expRemaining { + t.Fatalf("Unexpected remaining: %d, expected: %d", p.Remaining, c.expRemaining) + } db.Close() } } From 8fe09df54f55c2803f492a4be3bd16a43fa7b659 Mon Sep 17 00:00:00 2001 From: Sina M <1591639+s1na@users.noreply.github.com> Date: Fri, 21 Mar 2025 13:12:56 +0100 Subject: [PATCH 05/42] cmd/geth: add prune history command (#31384) This adds a new subcommand 'geth prune-history' that removes the pre-merge history on supported networks. Geth is not fully ready to work in this mode, please do not run this command on your production node. --------- Co-authored-by: Felix Lange --- cmd/geth/chaincmd.go | 61 ++++++++++++++++++++++ cmd/geth/main.go | 1 + core/blockchain.go | 3 +- core/rawdb/accessors_indexes.go | 5 +- core/rawdb/chain_iterator.go | 36 +++++++++++++ core/rawdb/chain_iterator_test.go | 85 +++++++++++++++++++++++-------- core/txindexer.go | 3 +- tests/testdata | 2 +- 8 files changed, 171 insertions(+), 25 deletions(-) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 95239bd640..05c8bc4c7c 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -35,6 +35,7 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/internal/era" "github.com/ethereum/go-ethereum/log" @@ -189,6 +190,18 @@ It's deprecated, please use "geth db import" instead. This command dumps out the state for a given block (or latest, if none provided). `, } + + pruneCommand = &cli.Command{ + Action: pruneHistory, + Name: "prune-history", + Usage: "Prune blockchain history (block bodies and receipts) up to the merge block", + ArgsUsage: "", + Flags: utils.DatabaseFlags, + Description: ` +The prune-history command removes historical block bodies and receipts from the +blockchain database up to the merge block, while preserving block headers. This +helps reduce storage requirements for nodes that don't need full historical data.`, + } ) // initGenesis will initialise the given JSON format genesis file and writes it as @@ -598,3 +611,51 @@ func hashish(x string) bool { _, err := strconv.Atoi(x) return err != nil } + +func pruneHistory(ctx *cli.Context) error { + stack, _ := makeConfigNode(ctx) + defer stack.Close() + + // Open the chain database + chain, chaindb := utils.MakeChain(ctx, stack, false) + defer chaindb.Close() + defer chain.Stop() + + // Determine the prune point. This will be the first PoS block. + prunePoint, ok := ethconfig.HistoryPrunePoints[chain.Genesis().Hash()] + if !ok || prunePoint == nil { + return errors.New("prune point not found") + } + var ( + mergeBlock = prunePoint.BlockNumber + mergeBlockHash = prunePoint.BlockHash.Hex() + ) + + // Check we're far enough past merge to ensure all data is in freezer + currentHeader := chain.CurrentHeader() + if currentHeader == nil { + return errors.New("current header not found") + } + if currentHeader.Number.Uint64() < mergeBlock+params.FullImmutabilityThreshold { + return fmt.Errorf("chain not far enough past merge block, need %d more blocks", + mergeBlock+params.FullImmutabilityThreshold-currentHeader.Number.Uint64()) + } + + // Double-check the prune block in db has the expected hash. + hash := rawdb.ReadCanonicalHash(chaindb, mergeBlock) + if hash != common.HexToHash(mergeBlockHash) { + return fmt.Errorf("merge block hash mismatch: got %s, want %s", hash.Hex(), mergeBlockHash) + } + + log.Info("Starting history pruning", "head", currentHeader.Number, "tail", mergeBlock, "tailHash", mergeBlockHash) + start := time.Now() + rawdb.PruneTransactionIndex(chaindb, mergeBlock) + if _, err := chaindb.TruncateTail(mergeBlock); err != nil { + return fmt.Errorf("failed to truncate ancient data: %v", err) + } + log.Info("History pruning completed", "tail", mergeBlock, "elapsed", common.PrettyDuration(time.Since(start))) + + // TODO(s1na): what if there is a crash between the two prune operations? + + return nil +} diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 07fbeaca5c..9c0c0d9dfc 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -226,6 +226,7 @@ func init() { removedbCommand, dumpCommand, dumpGenesisCommand, + pruneCommand, // See accountcmd.go: accountCommand, walletCommand, diff --git a/core/blockchain.go b/core/blockchain.go index 2bf7fba427..d80236c902 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -332,7 +332,8 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, genesis *Genesis bc.prefetcher = newStatePrefetcher(chainConfig, bc.hc) bc.processor = NewStateProcessor(chainConfig, bc.hc) - bc.genesisBlock = bc.GetBlockByNumber(0) + genesisHeader := bc.GetHeaderByNumber(0) + bc.genesisBlock = types.NewBlockWithHeader(genesisHeader) if bc.genesisBlock == nil { return nil, ErrNoGenesis } diff --git a/core/rawdb/accessors_indexes.go b/core/rawdb/accessors_indexes.go index 342aedd8dc..7bb96b1fa1 100644 --- a/core/rawdb/accessors_indexes.go +++ b/core/rawdb/accessors_indexes.go @@ -103,13 +103,14 @@ func DeleteTxLookupEntries(db ethdb.KeyValueWriter, hashes []common.Hash) { // DeleteAllTxLookupEntries purges all the transaction indexes in the database. // If condition is specified, only the entry with condition as True will be // removed; If condition is not specified, the entry is deleted. -func DeleteAllTxLookupEntries(db ethdb.KeyValueStore, condition func([]byte) bool) { +func DeleteAllTxLookupEntries(db ethdb.KeyValueStore, condition func(common.Hash, []byte) bool) { iter := NewKeyLengthIterator(db.NewIterator(txLookupPrefix, nil), common.HashLength+len(txLookupPrefix)) defer iter.Release() batch := db.NewBatch() for iter.Next() { - if condition == nil || condition(iter.Value()) { + txhash := common.Hash(iter.Key()[1:]) + if condition == nil || condition(txhash, iter.Value()) { batch.Delete(iter.Key()) } if batch.ValueSize() >= ethdb.IdealBatchSize { diff --git a/core/rawdb/chain_iterator.go b/core/rawdb/chain_iterator.go index 759e5913d1..ecbc44e1f1 100644 --- a/core/rawdb/chain_iterator.go +++ b/core/rawdb/chain_iterator.go @@ -17,6 +17,7 @@ package rawdb import ( + "encoding/binary" "runtime" "sync/atomic" "time" @@ -361,3 +362,38 @@ func UnindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt ch func unindexTransactionsForTesting(db ethdb.Database, from uint64, to uint64, interrupt chan struct{}, hook func(uint64) bool) { unindexTransactions(db, from, to, interrupt, hook, false) } + +// PruneTransactionIndex removes all tx index entries below a certain block number. +func PruneTransactionIndex(db ethdb.Database, pruneBlock uint64) { + tail := ReadTxIndexTail(db) + if tail == nil || *tail > pruneBlock { + return // no index, or index ends above pruneBlock + } + // There are blocks below pruneBlock in the index. Iterate the entire index to remove + // their entries. Note if this fails, the index is messed up, but tail still points to + // the old tail. + var count, removed int + DeleteAllTxLookupEntries(db, func(txhash common.Hash, v []byte) bool { + count++ + if count%10000000 == 0 { + log.Info("Pruning tx index", "count", count, "removed", removed) + } + if len(v) > 8 { + log.Error("Skipping legacy tx index entry", "hash", txhash) + return false + } + bn := decodeNumber(v) + if bn < pruneBlock { + removed++ + return true + } + return false + }) + WriteTxIndexTail(db, pruneBlock) +} + +func decodeNumber(b []byte) uint64 { + var numBuffer [8]byte + copy(numBuffer[8-len(b):], b) + return binary.BigEndian.Uint64(numBuffer[:]) +} diff --git a/core/rawdb/chain_iterator_test.go b/core/rawdb/chain_iterator_test.go index 390424f673..75bd5a9a94 100644 --- a/core/rawdb/chain_iterator_test.go +++ b/core/rawdb/chain_iterator_test.go @@ -25,6 +25,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" ) func TestChainIterator(t *testing.T) { @@ -102,19 +103,18 @@ func TestChainIterator(t *testing.T) { } } -func TestIndexTransactions(t *testing.T) { - // Construct test chain db - chainDb := NewMemoryDatabase() - - var block *types.Block +func initDatabaseWithTransactions(db ethdb.Database) ([]*types.Block, []*types.Transaction) { + var blocks []*types.Block var txs []*types.Transaction to := common.BytesToAddress([]byte{0x11}) // Write empty genesis block - block = types.NewBlock(&types.Header{Number: big.NewInt(int64(0))}, nil, nil, newTestHasher()) - WriteBlock(chainDb, block) - WriteCanonicalHash(chainDb, block.Hash(), block.NumberU64()) + block := types.NewBlock(&types.Header{Number: big.NewInt(int64(0))}, nil, nil, newTestHasher()) + WriteBlock(db, block) + WriteCanonicalHash(db, block.Hash(), block.NumberU64()) + blocks = append(blocks, block) + // Create transactions. for i := uint64(1); i <= 10; i++ { var tx *types.Transaction if i%2 == 0 { @@ -138,10 +138,21 @@ func TestIndexTransactions(t *testing.T) { }) } txs = append(txs, tx) - block = types.NewBlock(&types.Header{Number: big.NewInt(int64(i))}, &types.Body{Transactions: types.Transactions{tx}}, nil, newTestHasher()) - WriteBlock(chainDb, block) - WriteCanonicalHash(chainDb, block.Hash(), block.NumberU64()) + block := types.NewBlock(&types.Header{Number: big.NewInt(int64(i))}, &types.Body{Transactions: types.Transactions{tx}}, nil, newTestHasher()) + WriteBlock(db, block) + WriteCanonicalHash(db, block.Hash(), block.NumberU64()) + blocks = append(blocks, block) } + + return blocks, txs +} + +func TestIndexTransactions(t *testing.T) { + // Construct test chain db + chainDB := NewMemoryDatabase() + + _, txs := initDatabaseWithTransactions(chainDB) + // verify checks whether the tx indices in the range [from, to) // is expected. verify := func(from, to int, exist bool, tail uint64) { @@ -149,7 +160,7 @@ func TestIndexTransactions(t *testing.T) { if i == 0 { continue } - number := ReadTxLookupEntry(chainDb, txs[i-1].Hash()) + number := ReadTxLookupEntry(chainDB, txs[i-1].Hash()) if exist && number == nil { t.Fatalf("Transaction index %d missing", i) } @@ -157,29 +168,29 @@ func TestIndexTransactions(t *testing.T) { t.Fatalf("Transaction index %d is not deleted", i) } } - number := ReadTxIndexTail(chainDb) + number := ReadTxIndexTail(chainDB) if number == nil || *number != tail { t.Fatalf("Transaction tail mismatch") } } - IndexTransactions(chainDb, 5, 11, nil, false) + IndexTransactions(chainDB, 5, 11, nil, false) verify(5, 11, true, 5) verify(0, 5, false, 5) - IndexTransactions(chainDb, 0, 5, nil, false) + IndexTransactions(chainDB, 0, 5, nil, false) verify(0, 11, true, 0) - UnindexTransactions(chainDb, 0, 5, nil, false) + UnindexTransactions(chainDB, 0, 5, nil, false) verify(5, 11, true, 5) verify(0, 5, false, 5) - UnindexTransactions(chainDb, 5, 11, nil, false) + UnindexTransactions(chainDB, 5, 11, nil, false) verify(0, 11, false, 11) // Testing corner cases signal := make(chan struct{}) var once sync.Once - indexTransactionsForTesting(chainDb, 5, 11, signal, func(n uint64) bool { + indexTransactionsForTesting(chainDB, 5, 11, signal, func(n uint64) bool { if n <= 8 { once.Do(func() { close(signal) @@ -190,11 +201,11 @@ func TestIndexTransactions(t *testing.T) { }) verify(9, 11, true, 9) verify(0, 9, false, 9) - IndexTransactions(chainDb, 0, 9, nil, false) + IndexTransactions(chainDB, 0, 9, nil, false) signal = make(chan struct{}) var once2 sync.Once - unindexTransactionsForTesting(chainDb, 0, 11, signal, func(n uint64) bool { + unindexTransactionsForTesting(chainDB, 0, 11, signal, func(n uint64) bool { if n >= 8 { once2.Do(func() { close(signal) @@ -206,3 +217,37 @@ func TestIndexTransactions(t *testing.T) { verify(8, 11, true, 8) verify(0, 8, false, 8) } + +func TestPruneTransactionIndex(t *testing.T) { + chainDB := NewMemoryDatabase() + blocks, _ := initDatabaseWithTransactions(chainDB) + lastBlock := blocks[len(blocks)-1].NumberU64() + pruneBlock := lastBlock - 3 + + IndexTransactions(chainDB, 0, lastBlock+1, nil, false) + + // Check all transactions are in index. + for _, block := range blocks { + for _, tx := range block.Transactions() { + num := ReadTxLookupEntry(chainDB, tx.Hash()) + if num == nil || *num != block.NumberU64() { + t.Fatalf("wrong TxLookup entry: %x -> %v", tx.Hash(), num) + } + } + } + + PruneTransactionIndex(chainDB, pruneBlock) + + // Check transactions from old blocks not included. + for _, block := range blocks { + for _, tx := range block.Transactions() { + num := ReadTxLookupEntry(chainDB, tx.Hash()) + if block.NumberU64() < pruneBlock && num != nil { + t.Fatalf("TxLookup entry not removed: %x -> %v", tx.Hash(), num) + } + if block.NumberU64() >= pruneBlock && (num == nil || *num != block.NumberU64()) { + t.Fatalf("wrong TxLookup entry after pruning: %x -> %v", tx.Hash(), num) + } + } + } +} diff --git a/core/txindexer.go b/core/txindexer.go index 29e87905d5..d0fce302f3 100644 --- a/core/txindexer.go +++ b/core/txindexer.go @@ -20,6 +20,7 @@ import ( "errors" "fmt" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" @@ -186,7 +187,7 @@ func (indexer *txIndexer) repair(head uint64) { // potentially leaving dangling indexes in the database. // However, this is considered acceptable. rawdb.WriteTxIndexTail(indexer.db, indexer.cutoff) - rawdb.DeleteAllTxLookupEntries(indexer.db, func(blob []byte) bool { + rawdb.DeleteAllTxLookupEntries(indexer.db, func(txhash common.Hash, blob []byte) bool { n := rawdb.DecodeTxLookupEntry(blob, indexer.db) return n != nil && *n < indexer.cutoff }) diff --git a/tests/testdata b/tests/testdata index 81862e4848..faf33b4714 160000 --- a/tests/testdata +++ b/tests/testdata @@ -1 +1 @@ -Subproject commit 81862e4848585a438d64f911a19b3825f0f4cd95 +Subproject commit faf33b471465d3c6cdc3d04fbd690895f78d33f2 From 624a5d8b235961a53f3091f86570d71c1cb872c4 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Fri, 21 Mar 2025 21:08:51 +0800 Subject: [PATCH 06/42] eth/filter: downgrade log level (#31450) --- eth/filters/filter.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eth/filters/filter.go b/eth/filters/filter.go index cedb5b3519..b743c25994 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -372,7 +372,7 @@ func (f *Filter) indexedLogs(ctx context.Context, mb filtermaps.MatcherBackend, // iteration and bloom matching. func (f *Filter) unindexedLogs(ctx context.Context, begin, end uint64) ([]*types.Log, error) { start := time.Now() - log.Warn("Performing unindexed log search", "begin", begin, "end", end) + log.Debug("Performing unindexed log search", "begin", begin, "end", end) var matches []*types.Log for blockNumber := begin; blockNumber <= end; blockNumber++ { select { @@ -390,7 +390,7 @@ func (f *Filter) unindexedLogs(ctx context.Context, begin, end uint64) ([]*types } matches = append(matches, found...) } - log.Trace("Performed unindexed log search", "begin", begin, "end", end, "matches", len(matches), "elapsed", common.PrettyDuration(time.Since(start))) + log.Debug("Performed unindexed log search", "begin", begin, "end", end, "matches", len(matches), "elapsed", common.PrettyDuration(time.Since(start))) return matches, nil } From b0b2b765094ed4abc490ba26bc3be431c84c5b96 Mon Sep 17 00:00:00 2001 From: jwasinger Date: Sun, 23 Mar 2025 17:38:26 +0100 Subject: [PATCH 07/42] internal/ethapi: return code 3 from call/estimateGas even if a revert reason was not returned (#31456) --- internal/ethapi/api.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index f3975d35a0..979a7e822a 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -761,8 +761,7 @@ func (api *BlockChainAPI) Call(ctx context.Context, args TransactionArgs, blockN if err != nil { return nil, err } - // If the result contains a revert reason, try to unpack and return it. - if len(result.Revert()) > 0 { + if errors.Is(result.Err, vm.ErrExecutionReverted) { return nil, newRevertError(result.Revert()) } return result.Return(), result.Err @@ -842,7 +841,7 @@ func DoEstimateGas(ctx context.Context, b Backend, args TransactionArgs, blockNr // Run the gas estimation and wrap any revertals into a custom return estimate, revert, err := gasestimator.Estimate(ctx, call, opts, gasCap) if err != nil { - if len(revert) > 0 { + if errors.Is(err, vm.ErrExecutionReverted) { return 0, newRevertError(revert) } return 0, err From fd4049dc1e9d8e80368c9e1ae1da5c2fd81644c5 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 24 Mar 2025 10:07:38 +0100 Subject: [PATCH 08/42] core/rawdb: improve database stats output (#31463) Instead of reporting all filtermaps stuff in one line, I'm breaking it down into the three separate kinds of entries here. ``` +-----------------------+-----------------------------+------------+------------+ | DATABASE | CATEGORY | SIZE | ITEMS | +-----------------------+-----------------------------+------------+------------+ | Key-Value store | Log index filter-map rows | 59.21 GiB | 616077345 | | Key-Value store | Log index last-block-of-map | 12.35 MiB | 269755 | | Key-Value store | Log index block-lv | 421.70 MiB | 22109169 | ``` Also added some other changes to make it easier to debug: - restored bloombits into the inspect output, so we notice if it doesn't get deleted for some reason - tracking of unaccounted key examples --- core/rawdb/accessors_indexes.go | 2 +- core/rawdb/database.go | 123 +++++++++++++++++++++----------- core/rawdb/schema.go | 23 +++--- 3 files changed, 94 insertions(+), 54 deletions(-) diff --git a/core/rawdb/accessors_indexes.go b/core/rawdb/accessors_indexes.go index 7bb96b1fa1..297e339c83 100644 --- a/core/rawdb/accessors_indexes.go +++ b/core/rawdb/accessors_indexes.go @@ -502,5 +502,5 @@ func DeleteBloomBitsDb(db ethdb.KeyValueRangeDeleter) error { if err := deletePrefixRange(db, bloomBitsPrefix); err != nil { return err } - return deletePrefixRange(db, bloomBitsIndexPrefix) + return deletePrefixRange(db, bloomBitsMetaPrefix) } diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 4c87e66cfd..a394f46e0a 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -20,8 +20,10 @@ import ( "bytes" "errors" "fmt" + "maps" "os" "path/filepath" + "slices" "strings" "time" @@ -360,24 +362,27 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { logged = time.Now() // Key-value store statistics - headers stat - bodies stat - receipts stat - tds stat - numHashPairings stat - hashNumPairings stat - legacyTries stat - stateLookups stat - accountTries stat - storageTries stat - codes stat - txLookups stat - accountSnaps stat - storageSnaps stat - preimages stat - filterMaps stat - beaconHeaders stat - cliqueSnaps stat + headers stat + bodies stat + receipts stat + tds stat + numHashPairings stat + hashNumPairings stat + legacyTries stat + stateLookups stat + accountTries stat + storageTries stat + codes stat + txLookups stat + accountSnaps stat + storageSnaps stat + preimages stat + beaconHeaders stat + cliqueSnaps stat + bloomBits stat + filterMapRows stat + filterMapLastBlock stat + filterMapBlockLV stat // Verkle statistics verkleTries stat @@ -393,6 +398,11 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { // Totals total common.StorageSize + + // This map tracks example keys for unaccounted data. + // For each unique two-byte prefix, the first unaccounted key encountered + // by the iterator will be stored. + unaccountedKeys = make(map[[2]byte][]byte) ) // Inspect key-value database first. for it.Next() { @@ -436,19 +446,33 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { metadata.Add(size) case bytes.HasPrefix(key, genesisPrefix) && len(key) == (len(genesisPrefix)+common.HashLength): metadata.Add(size) - case bytes.HasPrefix(key, []byte(filterMapsPrefix)): - filterMaps.Add(size) case bytes.HasPrefix(key, skeletonHeaderPrefix) && len(key) == (len(skeletonHeaderPrefix)+8): beaconHeaders.Add(size) case bytes.HasPrefix(key, CliqueSnapshotPrefix) && len(key) == 7+common.HashLength: cliqueSnaps.Add(size) - case bytes.HasPrefix(key, ChtTablePrefix) || - bytes.HasPrefix(key, ChtIndexTablePrefix) || - bytes.HasPrefix(key, ChtPrefix): // Canonical hash trie + + // new log index + case bytes.HasPrefix(key, filterMapRowPrefix) && len(key) <= len(filterMapRowPrefix)+9: + filterMapRows.Add(size) + case bytes.HasPrefix(key, filterMapLastBlockPrefix) && len(key) == len(filterMapLastBlockPrefix)+4: + filterMapLastBlock.Add(size) + case bytes.HasPrefix(key, filterMapBlockLVPrefix) && len(key) == len(filterMapBlockLVPrefix)+8: + filterMapBlockLV.Add(size) + + // old log index (deprecated) + case bytes.HasPrefix(key, bloomBitsPrefix) && len(key) == (len(bloomBitsPrefix)+10+common.HashLength): + bloomBits.Add(size) + case bytes.HasPrefix(key, bloomBitsMetaPrefix) && len(key) < len(bloomBitsMetaPrefix)+8: + bloomBits.Add(size) + + // LES indexes (deprecated) + case bytes.HasPrefix(key, chtTablePrefix) || + bytes.HasPrefix(key, chtIndexTablePrefix) || + bytes.HasPrefix(key, chtPrefix): // Canonical hash trie chtTrieNodes.Add(size) - case bytes.HasPrefix(key, BloomTrieTablePrefix) || - bytes.HasPrefix(key, BloomTrieIndexPrefix) || - bytes.HasPrefix(key, BloomTriePrefix): // Bloomtrie sub + case bytes.HasPrefix(key, bloomTrieTablePrefix) || + bytes.HasPrefix(key, bloomTrieIndexPrefix) || + bytes.HasPrefix(key, bloomTriePrefix): // Bloomtrie sub bloomTrieNodes.Add(size) // Verkle trie data is detected, determine the sub-category @@ -468,24 +492,19 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { default: unaccounted.Add(size) } + + // Metadata keys + case slices.ContainsFunc(knownMetadataKeys, func(x []byte) bool { return bytes.Equal(x, key) }): + metadata.Add(size) + default: - var accounted bool - for _, meta := range [][]byte{ - databaseVersionKey, headHeaderKey, headBlockKey, headFastBlockKey, headFinalizedBlockKey, - lastPivotKey, fastTrieProgressKey, snapshotDisabledKey, SnapshotRootKey, snapshotJournalKey, - snapshotGeneratorKey, snapshotRecoveryKey, txIndexTailKey, fastTxLookupLimitKey, - uncleanShutdownKey, badBlockKey, transitionStatusKey, skeletonSyncStatusKey, - persistentStateIDKey, trieJournalKey, snapshotSyncStatusKey, snapSyncStatusFlagKey, - } { - if bytes.Equal(key, meta) { - metadata.Add(size) - accounted = true - break + unaccounted.Add(size) + if len(key) >= 2 { + prefix := [2]byte(key[:2]) + if _, ok := unaccountedKeys[prefix]; !ok { + unaccountedKeys[prefix] = bytes.Clone(key) } } - if !accounted { - unaccounted.Add(size) - } } count++ if count%1000 == 0 && time.Since(logged) > 8*time.Second { @@ -502,7 +521,10 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { {"Key-Value store", "Block number->hash", numHashPairings.Size(), numHashPairings.Count()}, {"Key-Value store", "Block hash->number", hashNumPairings.Size(), hashNumPairings.Count()}, {"Key-Value store", "Transaction index", txLookups.Size(), txLookups.Count()}, - {"Key-Value store", "Log search index", filterMaps.Size(), filterMaps.Count()}, + {"Key-Value store", "Log index filter-map rows", filterMapRows.Size(), filterMapRows.Count()}, + {"Key-Value store", "Log index last-block-of-map", filterMapLastBlock.Size(), filterMapLastBlock.Count()}, + {"Key-Value store", "Log index block-lv", filterMapBlockLV.Size(), filterMapBlockLV.Count()}, + {"Key-Value store", "Log bloombits (deprecated)", bloomBits.Size(), bloomBits.Count()}, {"Key-Value store", "Contract codes", codes.Size(), codes.Count()}, {"Key-Value store", "Hash trie nodes", legacyTries.Size(), legacyTries.Count()}, {"Key-Value store", "Path trie state lookups", stateLookups.Size(), stateLookups.Count()}, @@ -543,10 +565,23 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { if unaccounted.size > 0 { log.Error("Database contains unaccounted data", "size", unaccounted.size, "count", unaccounted.count) + for _, e := range slices.SortedFunc(maps.Values(unaccountedKeys), bytes.Compare) { + log.Error(fmt.Sprintf(" example key: %x", e)) + } } return nil } +// This is the list of known 'metadata' keys stored in the databasse. +var knownMetadataKeys = [][]byte{ + databaseVersionKey, headHeaderKey, headBlockKey, headFastBlockKey, headFinalizedBlockKey, + lastPivotKey, fastTrieProgressKey, snapshotDisabledKey, SnapshotRootKey, snapshotJournalKey, + snapshotGeneratorKey, snapshotRecoveryKey, txIndexTailKey, fastTxLookupLimitKey, + uncleanShutdownKey, badBlockKey, transitionStatusKey, skeletonSyncStatusKey, + persistentStateIDKey, trieJournalKey, snapshotSyncStatusKey, snapSyncStatusFlagKey, + filterMapsRangeKey, +} + // printChainMetadata prints out chain metadata to stderr. func printChainMetadata(db ethdb.KeyValueStore) { fmt.Fprintf(os.Stderr, "Chain metadata\n") @@ -566,6 +601,7 @@ func ReadChainMetadata(db ethdb.KeyValueStore) [][]string { } return fmt.Sprintf("%d (%#x)", *val, *val) } + data := [][]string{ {"databaseVersion", pp(ReadDatabaseVersion(db))}, {"headBlockHash", fmt.Sprintf("%v", ReadHeadBlockHash(db))}, @@ -582,5 +618,8 @@ func ReadChainMetadata(db ethdb.KeyValueStore) [][]string { if b := ReadSkeletonSyncStatus(db); b != nil { data = append(data, []string{"SkeletonSyncStatus", string(b)}) } + if fmr, ok, _ := ReadFilterMapsRange(db); ok { + data = append(data, []string{"filterMapsRange", fmt.Sprintf("%+v", fmr)}) + } return data } diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index c21a96bd24..31e80da079 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -128,29 +128,30 @@ var ( configPrefix = []byte("ethereum-config-") // config prefix for the db genesisPrefix = []byte("ethereum-genesis-") // genesis state prefix for the db - // bloomBitsIndexPrefix is the data table of a chain indexer to track its progress - bloomBitsIndexPrefix = []byte("iB") - - ChtPrefix = []byte("chtRootV2-") // ChtPrefix + chtNum (uint64 big endian) -> trie root hash - ChtTablePrefix = []byte("cht-") - ChtIndexTablePrefix = []byte("chtIndexV2-") - - BloomTriePrefix = []byte("bltRoot-") // BloomTriePrefix + bloomTrieNum (uint64 big endian) -> trie root hash - BloomTrieTablePrefix = []byte("blt-") - BloomTrieIndexPrefix = []byte("bltIndex-") - CliqueSnapshotPrefix = []byte("clique-") BestUpdateKey = []byte("update-") // bigEndian64(syncPeriod) -> RLP(types.LightClientUpdate) (nextCommittee only referenced by root hash) FixedCommitteeRootKey = []byte("fixedRoot-") // bigEndian64(syncPeriod) -> committee root hash SyncCommitteeKey = []byte("committee-") // bigEndian64(syncPeriod) -> serialized committee + // new log index filterMapsPrefix = "fm-" filterMapsRangeKey = []byte(filterMapsPrefix + "R") filterMapRowPrefix = []byte(filterMapsPrefix + "r") // filterMapRowPrefix + mapRowIndex (uint64 big endian) -> filter row filterMapLastBlockPrefix = []byte(filterMapsPrefix + "b") // filterMapLastBlockPrefix + mapIndex (uint32 big endian) -> block number (uint64 big endian) filterMapBlockLVPrefix = []byte(filterMapsPrefix + "p") // filterMapBlockLVPrefix + num (uint64 big endian) -> log value pointer (uint64 big endian) + // old log index + bloomBitsMetaPrefix = []byte("iB") + + // LES indexes + chtPrefix = []byte("chtRootV2-") // ChtPrefix + chtNum (uint64 big endian) -> trie root hash + chtTablePrefix = []byte("cht-") + chtIndexTablePrefix = []byte("chtIndexV2-") + bloomTriePrefix = []byte("bltRoot-") // BloomTriePrefix + bloomTrieNum (uint64 big endian) -> trie root hash + bloomTrieTablePrefix = []byte("blt-") + bloomTrieIndexPrefix = []byte("bltIndex-") + preimageCounter = metrics.NewRegisteredCounter("db/preimage/total", nil) preimageHitsCounter = metrics.NewRegisteredCounter("db/preimage/hits", nil) preimageMissCounter = metrics.NewRegisteredCounter("db/preimage/miss", nil) From cbe902d5dadd5fb356f54f85dee7bdf556fa608e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Mon, 24 Mar 2025 12:27:11 +0100 Subject: [PATCH 09/42] core/filtermaps: fix log indexer init conditions (#31455) This PR adds an extra condition to the log indexer initialization in order to avoid initializing with block 0 as target head. Previously this caused the indexer to initialize without a checkpoint. Later, when the real chain head was set, it indexed the entire history, then unindexed most of it if only the recent history was supposed to be indexed. Now the init only happens when there is an actual synced chain head and therefore the index is initialized at the most recent checkpoint and only the last year is indexed according to the default parameters. During checkpoint initialization the best available checkpoint is also checked against the history cutoff point and fails if the indexing would have to start from a block older than the cutoff. If initialization fails then the indexer reverts to unindexed mode instead of retrying because the the failure conditions cannot be expected to recover later. --- core/filtermaps/filtermaps.go | 29 +++++++++++++++++++++++++---- core/filtermaps/indexer.go | 15 ++++++++++----- core/filtermaps/matcher_backend.go | 8 ++++---- eth/backend.go | 7 ++++++- 4 files changed, 45 insertions(+), 14 deletions(-) diff --git a/core/filtermaps/filtermaps.go b/core/filtermaps/filtermaps.go index d74b11da04..db7ab0a426 100644 --- a/core/filtermaps/filtermaps.go +++ b/core/filtermaps/filtermaps.go @@ -53,7 +53,8 @@ type FilterMaps struct { // This is configured by the --history.logs.disable Geth flag. // We chose to implement disabling this way because it requires less special // case logic in eth/filters. - disabled bool + disabled bool + disabledCh chan struct{} // closed by indexer if disabled closeCh chan struct{} closeWg sync.WaitGroup @@ -196,6 +197,7 @@ func NewFilterMaps(db ethdb.KeyValueStore, initView *ChainView, historyCutoff, f blockProcessingCh: make(chan bool, 1), history: config.History, disabled: config.Disabled, + disabledCh: make(chan struct{}), exportFileName: config.ExportFileName, Params: params, indexedRange: filterMapsRange{ @@ -206,6 +208,8 @@ func NewFilterMaps(db ethdb.KeyValueStore, initView *ChainView, historyCutoff, f maps: common.NewRange(rs.MapsFirst, rs.MapsAfterLast-rs.MapsFirst), tailPartialEpoch: rs.TailPartialEpoch, }, + historyCutoff: historyCutoff, + finalBlock: finalBlock, matcherSyncCh: make(chan *FilterMapsMatcherBackend), matchers: make(map[*FilterMapsMatcherBackend]struct{}), filterMapCache: lru.NewCache[uint32, filterMap](cachedFilterMaps), @@ -278,8 +282,13 @@ func (f *FilterMaps) initChainView(chainView *ChainView) *ChainView { } // reset un-initializes the FilterMaps structure and removes all related data from -// the database. The function returns true if everything was successfully removed. -func (f *FilterMaps) reset() bool { +// the database. +// Note that in case of leveldb database the fallback implementation of DeleteRange +// might take a long time to finish and deleting the entire database may be +// interrupted by a shutdown. Deleting the filterMapsRange entry first does +// guarantee though that the next init() will not return successfully until the +// entire database has been cleaned. +func (f *FilterMaps) reset() { f.indexLock.Lock() f.indexedRange = filterMapsRange{} f.indexedView = nil @@ -292,11 +301,16 @@ func (f *FilterMaps) reset() bool { // deleting the range first ensures that resetDb will be called again at next // startup and any leftover data will be removed even if it cannot finish now. rawdb.DeleteFilterMapsRange(f.db) - return f.safeDeleteRange(rawdb.DeleteFilterMapsDb, "Resetting log index database") + f.safeDeleteRange(rawdb.DeleteFilterMapsDb, "Resetting log index database") } // init initializes an empty log index according to the current targetView. func (f *FilterMaps) init() error { + // ensure that there is no remaining data in the filter maps key range + if !f.safeDeleteRange(rawdb.DeleteFilterMapsDb, "Resetting log index database") { + return errors.New("could not reset log index database") + } + f.indexLock.Lock() defer f.indexLock.Unlock() @@ -317,6 +331,13 @@ func (f *FilterMaps) init() error { bestIdx, bestLen = idx, max } } + var initBlockNumber uint64 + if bestLen > 0 { + initBlockNumber = checkpoints[bestIdx][bestLen-1].BlockNumber + } + if initBlockNumber < f.historyCutoff { + return errors.New("cannot start indexing before history cutoff point") + } batch := f.db.NewBatch() for epoch := range bestLen { cp := checkpoints[bestIdx][epoch] diff --git a/core/filtermaps/indexer.go b/core/filtermaps/indexer.go index 69f42d8b60..026b3b4f38 100644 --- a/core/filtermaps/indexer.go +++ b/core/filtermaps/indexer.go @@ -36,20 +36,25 @@ func (f *FilterMaps) indexerLoop() { if f.disabled { f.reset() + close(f.disabledCh) return } log.Info("Started log indexer") for !f.stop { if !f.indexedRange.initialized { - if err := f.init(); err != nil { - log.Error("Error initializing log index", "error", err) - // unexpected error; there is not a lot we can do here, maybe it - // recovers, maybe not. Calling event processing here ensures - // that we can still properly shutdown in case of an infinite loop. + if f.targetView.headNumber == 0 { + // initialize when chain head is available f.processSingleEvent(true) continue } + if err := f.init(); err != nil { + log.Error("Error initializing log index; reverting to unindexed mode", "error", err) + f.reset() + f.disabled = true + close(f.disabledCh) + return + } } if !f.targetHeadIndexed() { if !f.tryIndexHead() { diff --git a/core/filtermaps/matcher_backend.go b/core/filtermaps/matcher_backend.go index f24e9706cb..01bae7bb22 100644 --- a/core/filtermaps/matcher_backend.go +++ b/core/filtermaps/matcher_backend.go @@ -141,10 +141,6 @@ func (fm *FilterMapsMatcherBackend) synced() { // range that has not been changed and has been consistent with all states of the // chain since the previous SyncLogIndex or the creation of the matcher backend. func (fm *FilterMapsMatcherBackend) SyncLogIndex(ctx context.Context) (SyncRange, error) { - if fm.f.disabled { - return SyncRange{HeadNumber: fm.f.targetView.headNumber}, nil - } - syncCh := make(chan SyncRange, 1) fm.f.matchersLock.Lock() fm.syncCh = syncCh @@ -154,12 +150,16 @@ func (fm *FilterMapsMatcherBackend) SyncLogIndex(ctx context.Context) (SyncRange case fm.f.matcherSyncCh <- fm: case <-ctx.Done(): return SyncRange{}, ctx.Err() + case <-fm.f.disabledCh: + return SyncRange{HeadNumber: fm.f.targetView.headNumber}, nil } select { case vr := <-syncCh: return vr, nil case <-ctx.Done(): return SyncRange{}, ctx.Err() + case <-fm.f.disabledCh: + return SyncRange{HeadNumber: fm.f.targetView.headNumber}, nil } } diff --git a/eth/backend.go b/eth/backend.go index 6deedab872..909d153a2b 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -241,7 +241,12 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { } fmConfig := filtermaps.Config{History: config.LogHistory, Disabled: config.LogNoHistory, ExportFileName: config.LogExportCheckpoints} chainView := eth.newChainView(eth.blockchain.CurrentBlock()) - eth.filterMaps = filtermaps.NewFilterMaps(chainDb, chainView, 0, 0, filtermaps.DefaultParams, fmConfig) + historyCutoff := eth.blockchain.HistoryPruningCutoff() + var finalBlock uint64 + if fb := eth.blockchain.CurrentFinalBlock(); fb != nil { + finalBlock = fb.Number.Uint64() + } + eth.filterMaps = filtermaps.NewFilterMaps(chainDb, chainView, historyCutoff, finalBlock, filtermaps.DefaultParams, fmConfig) eth.closeFilterMaps = make(chan chan struct{}) if config.BlobPool.Datadir != "" { From 28238b6b7e004d838762bf96aa4c5be1271e5954 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Mon, 24 Mar 2025 12:27:40 +0100 Subject: [PATCH 10/42] beacon/params: new checkpoints (#31470) This PR updates beacon checkpoints. The checkpoints are now stored as embedded hex files, in the same format that https://github.com/ethereum/go-ethereum/pull/31469 uses. --- beacon/params/checkpoint_holesky.hex | 1 + beacon/params/checkpoint_mainnet.hex | 1 + beacon/params/checkpoint_sepolia.hex | 1 + beacon/params/networks.go | 17 ++++++++++++++--- 4 files changed, 17 insertions(+), 3 deletions(-) create mode 100644 beacon/params/checkpoint_holesky.hex create mode 100644 beacon/params/checkpoint_mainnet.hex create mode 100644 beacon/params/checkpoint_sepolia.hex diff --git a/beacon/params/checkpoint_holesky.hex b/beacon/params/checkpoint_holesky.hex new file mode 100644 index 0000000000..f1e27b48f6 --- /dev/null +++ b/beacon/params/checkpoint_holesky.hex @@ -0,0 +1 @@ +0xf5606a019f0f1006e9ec2070695045f4334450362a48da4c5965314510e0b4c3 \ No newline at end of file diff --git a/beacon/params/checkpoint_mainnet.hex b/beacon/params/checkpoint_mainnet.hex new file mode 100644 index 0000000000..a11c3394bb --- /dev/null +++ b/beacon/params/checkpoint_mainnet.hex @@ -0,0 +1 @@ +0x3c0cb4aa83beded1803d262664ba4392b1023f334d9cca02dc3a6925987ebe91 \ No newline at end of file diff --git a/beacon/params/checkpoint_sepolia.hex b/beacon/params/checkpoint_sepolia.hex new file mode 100644 index 0000000000..e4f2f96733 --- /dev/null +++ b/beacon/params/checkpoint_sepolia.hex @@ -0,0 +1 @@ +0xa8d56457aa414523d93659aef1f7409bbfb72ad75e94d917c8c0b1baa38153ef \ No newline at end of file diff --git a/beacon/params/networks.go b/beacon/params/networks.go index c547986977..7e73a2c69c 100644 --- a/beacon/params/networks.go +++ b/beacon/params/networks.go @@ -17,14 +17,25 @@ package params import ( + _ "embed" + "github.com/ethereum/go-ethereum/common" ) +//go:embed checkpoint_mainnet.hex +var checkpointMainnet string + +//go:embed checkpoint_sepolia.hex +var checkpointSepolia string + +//go:embed checkpoint_holesky.hex +var checkpointHolesky string + var ( MainnetLightConfig = (&ChainConfig{ GenesisValidatorsRoot: common.HexToHash("0x4b363db94e286120d76eb905340fdd4e54bfe9f06bf33ff6cf5ad27f511bfe95"), GenesisTime: 1606824023, - Checkpoint: common.HexToHash("0x6509b691f4de4f7b083f2784938fd52f0e131675432b3fd85ea549af9aebd3d0"), + Checkpoint: common.HexToHash(checkpointMainnet), }). AddFork("GENESIS", 0, []byte{0, 0, 0, 0}). AddFork("ALTAIR", 74240, []byte{1, 0, 0, 0}). @@ -35,7 +46,7 @@ var ( SepoliaLightConfig = (&ChainConfig{ GenesisValidatorsRoot: common.HexToHash("0xd8ea171f3c94aea21ebc42a1ed61052acf3f9209c00e4efbaaddac09ed9b8078"), GenesisTime: 1655733600, - Checkpoint: common.HexToHash("0x456e85f5608afab3465a0580bff8572255f6d97af0c5f939e3f7536b5edb2d3f"), + Checkpoint: common.HexToHash(checkpointSepolia), }). AddFork("GENESIS", 0, []byte{144, 0, 0, 105}). AddFork("ALTAIR", 50, []byte{144, 0, 0, 112}). @@ -47,7 +58,7 @@ var ( HoleskyLightConfig = (&ChainConfig{ GenesisValidatorsRoot: common.HexToHash("0x9143aa7c615a7f7115e2b6aac319c03529df8242ae705fba9df39b79c59fa8b1"), GenesisTime: 1695902400, - Checkpoint: common.HexToHash("0x6456a1317f54d4b4f2cb5bc9d153b5af0988fe767ef0609f0236cf29030bcff7"), + Checkpoint: common.HexToHash(checkpointHolesky), }). AddFork("GENESIS", 0, []byte{1, 1, 112, 0}). AddFork("ALTAIR", 0, []byte{2, 1, 112, 0}). From 8e3b94da1e9437dc48ef945011f6ad21c03f3c51 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Mon, 24 Mar 2025 22:19:14 +0800 Subject: [PATCH 11/42] tests: update test submodule (#31479) This commit upgrades the test submodule to latest version: Latest: https://github.com/ethereum/tests/commit/81862e4848585a438d64f911a19b3825f0f4cd95 Old: https://github.com/ethereum/tests/commit/faf33b471465d3c6cdc3d04fbd690895f78d33f2 --- tests/testdata | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testdata b/tests/testdata index faf33b4714..81862e4848 160000 --- a/tests/testdata +++ b/tests/testdata @@ -1 +1 @@ -Subproject commit faf33b471465d3c6cdc3d04fbd690895f78d33f2 +Subproject commit 81862e4848585a438d64f911a19b3825f0f4cd95 From 71e9c9b8a732b33f50b4565627463fb252b4aeff Mon Sep 17 00:00:00 2001 From: Rez Date: Tue, 25 Mar 2025 05:08:53 +1100 Subject: [PATCH 12/42] internal/ethapi: support for beacon root and withdrawals in simulate api (#31304) Adds block override fields for beacon block root and withdrawals to the eth_simulateV1. Addresses https://github.com/ethereum/go-ethereum/issues/31264 --- eth/gasestimator/gasestimator.go | 4 +- eth/tracers/api.go | 4 +- internal/ethapi/api.go | 4 +- internal/ethapi/api_test.go | 18 +++++++ internal/ethapi/override/override.go | 14 ++++- internal/ethapi/simulate.go | 79 +++++++++++++++++++++++----- 6 files changed, 106 insertions(+), 17 deletions(-) diff --git a/eth/gasestimator/gasestimator.go b/eth/gasestimator/gasestimator.go index a6c4718cf4..fc8e3a2e42 100644 --- a/eth/gasestimator/gasestimator.go +++ b/eth/gasestimator/gasestimator.go @@ -223,7 +223,9 @@ func run(ctx context.Context, call *core.Message, opts *Options) (*core.Executio dirtyState = opts.State.Copy() ) if opts.BlockOverrides != nil { - opts.BlockOverrides.Apply(&evmContext) + if err := opts.BlockOverrides.Apply(&evmContext); err != nil { + return nil, err + } } // Lower the basefee to 0 to avoid breaking EVM // invariants (basefee < feecap). diff --git a/eth/tracers/api.go b/eth/tracers/api.go index 627dd487fc..3cb86c80e2 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -950,7 +950,9 @@ func (api *API) TraceCall(ctx context.Context, args ethapi.TransactionArgs, bloc vmctx := core.NewEVMBlockContext(block.Header(), api.chainContext(ctx), nil) // Apply the customization rules if required. if config != nil { - config.BlockOverrides.Apply(&vmctx) + if overrideErr := config.BlockOverrides.Apply(&vmctx); overrideErr != nil { + return nil, overrideErr + } rules := api.backend.ChainConfig().Rules(vmctx.BlockNumber, vmctx.Random != nil, vmctx.Time) precompiles = vm.ActivePrecompiledContracts(rules) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 979a7e822a..e0d5b32622 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -660,7 +660,9 @@ func (context *ChainContext) Config() *params.ChainConfig { func doCall(ctx context.Context, b Backend, args TransactionArgs, state *state.StateDB, header *types.Header, overrides *override.StateOverride, blockOverrides *override.BlockOverrides, timeout time.Duration, globalGasCap uint64) (*core.ExecutionResult, error) { blockCtx := core.NewEVMBlockContext(header, NewChainContext(ctx, b), nil) if blockOverrides != nil { - blockOverrides.Apply(&blockCtx) + if err := blockOverrides.Apply(&blockCtx); err != nil { + return nil, err + } } rules := b.ChainConfig().Rules(blockCtx.BlockNumber, blockCtx.Random != nil, blockCtx.Time) precompiles := vm.ActivePrecompiledContracts(rules) diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go index 88ff7b8af3..b086d1a6d5 100644 --- a/internal/ethapi/api_test.go +++ b/internal/ethapi/api_test.go @@ -1134,6 +1134,24 @@ func TestCall(t *testing.T) { }, want: "0x0000000000000000000000000000000000000000000000000000000000000000", }, + { + name: "unsupported block override beaconRoot", + blockNumber: rpc.LatestBlockNumber, + call: TransactionArgs{}, + blockOverrides: override.BlockOverrides{ + BeaconRoot: &common.Hash{0, 1, 2}, + }, + expectErr: errors.New(`block override "beaconRoot" is not supported for this RPC method`), + }, + { + name: "unsupported block override withdrawals", + blockNumber: rpc.LatestBlockNumber, + call: TransactionArgs{}, + blockOverrides: override.BlockOverrides{ + Withdrawals: &types.Withdrawals{}, + }, + expectErr: errors.New(`block override "withdrawals" is not supported for this RPC method`), + }, } for _, tc := range testSuite { result, err := api.Call(context.Background(), tc.call, &rpc.BlockNumberOrHash{BlockNumber: &tc.blockNumber}, &tc.overrides, &tc.blockOverrides) diff --git a/internal/ethapi/override/override.go b/internal/ethapi/override/override.go index f6a8a94ffd..0bcf3c444d 100644 --- a/internal/ethapi/override/override.go +++ b/internal/ethapi/override/override.go @@ -17,6 +17,7 @@ package override import ( + "errors" "fmt" "math/big" @@ -128,12 +129,20 @@ type BlockOverrides struct { PrevRandao *common.Hash BaseFeePerGas *hexutil.Big BlobBaseFee *hexutil.Big + BeaconRoot *common.Hash + Withdrawals *types.Withdrawals } // Apply overrides the given header fields into the given block context. -func (o *BlockOverrides) Apply(blockCtx *vm.BlockContext) { +func (o *BlockOverrides) Apply(blockCtx *vm.BlockContext) error { if o == nil { - return + return nil + } + if o.BeaconRoot != nil { + return errors.New(`block override "beaconRoot" is not supported for this RPC method`) + } + if o.Withdrawals != nil { + return errors.New(`block override "withdrawals" is not supported for this RPC method`) } if o.Number != nil { blockCtx.BlockNumber = o.Number.ToInt() @@ -159,6 +168,7 @@ func (o *BlockOverrides) Apply(blockCtx *vm.BlockContext) { if o.BlobBaseFee != nil { blockCtx.BlobBaseFee = o.BlobBaseFee.ToInt() } + return nil } // MakeHeader returns a new header object with the overridden diff --git a/internal/ethapi/simulate.go b/internal/ethapi/simulate.go index 097de8b0b0..ba346b132f 100644 --- a/internal/ethapi/simulate.go +++ b/internal/ethapi/simulate.go @@ -36,7 +36,6 @@ import ( "github.com/ethereum/go-ethereum/internal/ethapi/override" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" - "github.com/ethereum/go-ethereum/trie" ) const ( @@ -95,6 +94,47 @@ type simOpts struct { ReturnFullTransactions bool } +// simChainHeadReader implements ChainHeaderReader which is needed as input for FinalizeAndAssemble. +type simChainHeadReader struct { + context.Context + Backend +} + +func (m *simChainHeadReader) Config() *params.ChainConfig { + return m.Backend.ChainConfig() +} + +func (m *simChainHeadReader) CurrentHeader() *types.Header { + return m.Backend.CurrentHeader() +} + +func (m *simChainHeadReader) GetHeader(hash common.Hash, number uint64) *types.Header { + header, err := m.Backend.HeaderByNumber(m.Context, rpc.BlockNumber(number)) + if err != nil || header == nil { + return nil + } + if header.Hash() != hash { + return nil + } + return header +} + +func (m *simChainHeadReader) GetHeaderByNumber(number uint64) *types.Header { + header, err := m.Backend.HeaderByNumber(m.Context, rpc.BlockNumber(number)) + if err != nil { + return nil + } + return header +} + +func (m *simChainHeadReader) GetHeaderByHash(hash common.Hash) *types.Header { + header, err := m.Backend.HeaderByHash(m.Context, hash) + if err != nil { + return nil + } + return header +} + // simulator is a stateful object that simulates a series of blocks. // it is not safe for concurrent use. type simulator struct { @@ -209,6 +249,9 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, if sim.chainConfig.IsPrague(header.Number, header.Time) || sim.chainConfig.IsVerkle(header.Number, header.Time) { core.ProcessParentBlockHash(header.ParentHash, evm) } + if header.ParentBeaconRoot != nil { + core.ProcessBeaconBlockRoot(*header.ParentBeaconRoot, evm) + } var allLogs []*types.Log for i, call := range block.Calls { if err := ctx.Err(); err != nil { @@ -258,6 +301,10 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, } callResults[i] = callRes } + header.GasUsed = gasUsed + if sim.chainConfig.IsCancun(header.Number, header.Time) { + header.BlobGasUsed = &blobGasUsed + } var requests [][]byte // Process EIP-7685 requests if sim.chainConfig.IsPrague(header.Number, header.Time) { @@ -271,20 +318,16 @@ func (sim *simulator) processBlock(ctx context.Context, block *simBlock, header, // EIP-7251 core.ProcessConsolidationQueue(&requests, evm) } - header.Root = sim.state.IntermediateRoot(true) - header.GasUsed = gasUsed - if sim.chainConfig.IsCancun(header.Number, header.Time) { - header.BlobGasUsed = &blobGasUsed - } - var withdrawals types.Withdrawals - if sim.chainConfig.IsShanghai(header.Number, header.Time) { - withdrawals = make([]*types.Withdrawal, 0) - } if requests != nil { reqHash := types.CalcRequestsHash(requests) header.RequestsHash = &reqHash } - b := types.NewBlock(header, &types.Body{Transactions: txes, Withdrawals: withdrawals}, receipts, trie.NewStackTrie(nil)) + blockBody := &types.Body{Transactions: txes, Withdrawals: *block.BlockOverrides.Withdrawals} + chainHeadReader := &simChainHeadReader{ctx, sim.b} + b, err := sim.b.Engine().FinalizeAndAssemble(chainHeadReader, header, sim.state, blockBody, receipts) + if err != nil { + return nil, nil, err + } repairLogs(callResults, b.Hash()) return b, callResults, nil } @@ -346,6 +389,9 @@ func (sim *simulator) sanitizeChain(blocks []simBlock) ([]simBlock, error) { n := new(big.Int).Add(prevNumber, big.NewInt(1)) block.BlockOverrides.Number = (*hexutil.Big)(n) } + if block.BlockOverrides.Withdrawals == nil { + block.BlockOverrides.Withdrawals = &types.Withdrawals{} + } diff := new(big.Int).Sub(block.BlockOverrides.Number.ToInt(), prevNumber) if diff.Cmp(common.Big0) <= 0 { return nil, &invalidBlockNumberError{fmt.Sprintf("block numbers must be in order: %d <= %d", block.BlockOverrides.Number.ToInt().Uint64(), prevNumber)} @@ -360,7 +406,13 @@ func (sim *simulator) sanitizeChain(blocks []simBlock) ([]simBlock, error) { for i := uint64(0); i < gap.Uint64(); i++ { n := new(big.Int).Add(prevNumber, big.NewInt(int64(i+1))) t := prevTimestamp + timestampIncrement - b := simBlock{BlockOverrides: &override.BlockOverrides{Number: (*hexutil.Big)(n), Time: (*hexutil.Uint64)(&t)}} + b := simBlock{ + BlockOverrides: &override.BlockOverrides{ + Number: (*hexutil.Big)(n), + Time: (*hexutil.Uint64)(&t), + Withdrawals: &types.Withdrawals{}, + }, + } prevTimestamp = t res = append(res, b) } @@ -405,6 +457,9 @@ func (sim *simulator) makeHeaders(blocks []simBlock) ([]*types.Header, error) { var parentBeaconRoot *common.Hash if sim.chainConfig.IsCancun(overrides.Number.ToInt(), (uint64)(*overrides.Time)) { parentBeaconRoot = &common.Hash{} + if overrides.BeaconRoot != nil { + parentBeaconRoot = overrides.BeaconRoot + } } header = overrides.MakeHeader(&types.Header{ UncleHash: types.EmptyUncleHash, From a14b8eca04e9b7476f4a03c225c02e7bc2c32b71 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Tue, 25 Mar 2025 18:16:26 +0800 Subject: [PATCH 13/42] core/txpool: reject stale transaction for local tracking (#31473) Fixes https://github.com/ethereum/go-ethereum/issues/31451 --- core/txpool/locals/tx_tracker_test.go | 179 ++++++++++++++++++++++++++ core/txpool/txpool.go | 54 +++++++- 2 files changed, 230 insertions(+), 3 deletions(-) create mode 100644 core/txpool/locals/tx_tracker_test.go diff --git a/core/txpool/locals/tx_tracker_test.go b/core/txpool/locals/tx_tracker_test.go new file mode 100644 index 0000000000..cb6b9b3453 --- /dev/null +++ b/core/txpool/locals/tx_tracker_test.go @@ -0,0 +1,179 @@ +// Copyright 2025 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package locals + +import ( + "errors" + "math/big" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/txpool" + "github.com/ethereum/go-ethereum/core/txpool/legacypool" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/params" +) + +var ( + key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + address = crypto.PubkeyToAddress(key.PublicKey) + funds = big.NewInt(1000000000000000) + gspec = &core.Genesis{ + Config: params.TestChainConfig, + Alloc: types.GenesisAlloc{ + address: {Balance: funds}, + }, + BaseFee: big.NewInt(params.InitialBaseFee), + } + signer = types.LatestSigner(gspec.Config) +) + +type testEnv struct { + chain *core.BlockChain + pool *txpool.TxPool + tracker *TxTracker + genDb ethdb.Database +} + +func newTestEnv(t *testing.T, n int, gasTip uint64, journal string) *testEnv { + genDb, blocks, _ := core.GenerateChainWithGenesis(gspec, ethash.NewFaker(), n, func(i int, gen *core.BlockGen) { + tx, err := types.SignTx(types.NewTransaction(gen.TxNonce(address), common.Address{0x00}, big.NewInt(1000), params.TxGas, gen.BaseFee(), nil), signer, key) + if err != nil { + panic(err) + } + gen.AddTx(tx) + }) + + db := rawdb.NewMemoryDatabase() + chain, _ := core.NewBlockChain(db, nil, gspec, nil, ethash.NewFaker(), vm.Config{}, nil) + + legacyPool := legacypool.New(legacypool.DefaultConfig, chain) + pool, err := txpool.New(gasTip, chain, []txpool.SubPool{legacyPool}) + if err != nil { + t.Fatalf("Failed to create tx pool: %v", err) + } + if n, err := chain.InsertChain(blocks); err != nil { + t.Fatalf("Failed to process block %d: %v", n, err) + } + if err := pool.Sync(); err != nil { + t.Fatalf("Failed to sync the txpool, %v", err) + } + return &testEnv{ + chain: chain, + pool: pool, + tracker: New(journal, time.Minute, gspec.Config, pool), + genDb: genDb, + } +} + +func (env *testEnv) close() { + env.chain.Stop() +} + +func (env *testEnv) setGasTip(gasTip uint64) { + env.pool.SetGasTip(new(big.Int).SetUint64(gasTip)) +} + +func (env *testEnv) makeTx(nonce uint64, gasPrice *big.Int) *types.Transaction { + if nonce == 0 { + head := env.chain.CurrentHeader() + state, _ := env.chain.StateAt(head.Root) + nonce = state.GetNonce(address) + } + if gasPrice == nil { + gasPrice = big.NewInt(params.GWei) + } + tx, _ := types.SignTx(types.NewTransaction(nonce, common.Address{0x00}, big.NewInt(1000), params.TxGas, gasPrice, nil), signer, key) + return tx +} + +func (env *testEnv) commit() { + head := env.chain.CurrentBlock() + block := env.chain.GetBlock(head.Hash(), head.Number.Uint64()) + blocks, _ := core.GenerateChain(env.chain.Config(), block, ethash.NewFaker(), env.genDb, 1, func(i int, gen *core.BlockGen) { + tx, err := types.SignTx(types.NewTransaction(gen.TxNonce(address), common.Address{0x00}, big.NewInt(1000), params.TxGas, gen.BaseFee(), nil), signer, key) + if err != nil { + panic(err) + } + gen.AddTx(tx) + }) + env.chain.InsertChain(blocks) + if err := env.pool.Sync(); err != nil { + panic(err) + } +} + +func TestRejectInvalids(t *testing.T) { + env := newTestEnv(t, 10, 0, "") + defer env.close() + + var cases = []struct { + gasTip uint64 + tx *types.Transaction + expErr error + commit bool + }{ + { + tx: env.makeTx(5, nil), // stale + expErr: core.ErrNonceTooLow, + }, + { + tx: env.makeTx(11, nil), // future transaction + expErr: nil, + }, + { + gasTip: params.GWei, + tx: env.makeTx(0, new(big.Int).SetUint64(params.GWei/2)), // low price + expErr: txpool.ErrUnderpriced, + }, + { + tx: types.NewTransaction(10, common.Address{0x00}, big.NewInt(1000), params.TxGas, big.NewInt(params.GWei), nil), // invalid signature + expErr: types.ErrInvalidSig, + }, + { + commit: true, + tx: env.makeTx(10, nil), // stale + expErr: core.ErrNonceTooLow, + }, + { + tx: env.makeTx(11, nil), + expErr: nil, + }, + } + for i, c := range cases { + if c.gasTip != 0 { + env.setGasTip(c.gasTip) + } + if c.commit { + env.commit() + } + gotErr := env.tracker.Track(c.tx) + if c.expErr == nil && gotErr != nil { + t.Fatalf("%d, unexpected error: %v", i, gotErr) + } + if c.expErr != nil && !errors.Is(gotErr, c.expErr) { + t.Fatalf("%d, unexpected error, want: %v, got: %v", i, c.expErr, gotErr) + } + } +} diff --git a/core/txpool/txpool.go b/core/txpool/txpool.go index 042e3d36d9..3c00699dc7 100644 --- a/core/txpool/txpool.go +++ b/core/txpool/txpool.go @@ -24,11 +24,13 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto/kzg4844" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/params" ) // TxStatus is the current status of a transaction as seen by the pool. @@ -53,11 +55,17 @@ var ( // BlockChain defines the minimal set of methods needed to back a tx pool with // a chain. Exists to allow mocking the live chain out of tests. type BlockChain interface { + // Config retrieves the chain's fork configuration. + Config() *params.ChainConfig + // CurrentBlock returns the current head of the chain. CurrentBlock() *types.Header // SubscribeChainHeadEvent subscribes to new blocks being added to the chain. SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription + + // StateAt returns a state database for a given root hash (generally the head). + StateAt(root common.Hash) (*state.StateDB, error) } // TxPool is an aggregator for various transaction specific pools, collectively @@ -67,6 +75,11 @@ type BlockChain interface { // resource constraints. type TxPool struct { subpools []SubPool // List of subpools for specialized transaction handling + chain BlockChain + signer types.Signer + + stateLock sync.RWMutex // The lock for protecting state instance + state *state.StateDB // Current state at the blockchain head reservations map[common.Address]SubPool // Map with the account to pool reservations reserveLock sync.Mutex // Lock protecting the account reservations @@ -86,8 +99,21 @@ func New(gasTip uint64, chain BlockChain, subpools []SubPool) (*TxPool, error) { // during initialization. head := chain.CurrentBlock() + // Initialize the state with head block, or fallback to empty one in + // case the head state is not available (might occur when node is not + // fully synced). + statedb, err := chain.StateAt(head.Root) + if err != nil { + statedb, err = chain.StateAt(types.EmptyRootHash) + } + if err != nil { + return nil, err + } pool := &TxPool{ subpools: subpools, + chain: chain, + signer: types.LatestSigner(chain.Config()), + state: statedb, reservations: make(map[common.Address]SubPool), quit: make(chan chan error), term: make(chan struct{}), @@ -101,7 +127,7 @@ func New(gasTip uint64, chain BlockChain, subpools []SubPool) (*TxPool, error) { return nil, err } } - go pool.loop(head, chain) + go pool.loop(head) return pool, nil } @@ -179,14 +205,14 @@ func (p *TxPool) Close() error { // loop is the transaction pool's main event loop, waiting for and reacting to // outside blockchain events as well as for various reporting and transaction // eviction events. -func (p *TxPool) loop(head *types.Header, chain BlockChain) { +func (p *TxPool) loop(head *types.Header) { // Close the termination marker when the pool stops defer close(p.term) // Subscribe to chain head events to trigger subpool resets var ( newHeadCh = make(chan core.ChainHeadEvent) - newHeadSub = chain.SubscribeChainHeadEvent(newHeadCh) + newHeadSub = p.chain.SubscribeChainHeadEvent(newHeadCh) ) defer newHeadSub.Unsubscribe() @@ -219,6 +245,14 @@ func (p *TxPool) loop(head *types.Header, chain BlockChain) { // Try to inject a busy marker and start a reset if successful select { case resetBusy <- struct{}{}: + statedb, err := p.chain.StateAt(newHead.Root) + if err != nil { + log.Crit("Failed to reset txpool state", "err", err) + } + p.stateLock.Lock() + p.state = statedb + p.stateLock.Unlock() + // Busy marker injected, start a new subpool reset go func(oldHead, newHead *types.Header) { for _, subpool := range p.subpools { @@ -339,6 +373,20 @@ func (p *TxPool) GetBlobs(vhashes []common.Hash) ([]*kzg4844.Blob, []*kzg4844.Pr // ValidateTxBasics checks whether a transaction is valid according to the consensus // rules, but does not check state-dependent validation such as sufficient balance. func (p *TxPool) ValidateTxBasics(tx *types.Transaction) error { + addr, err := types.Sender(p.signer, tx) + if err != nil { + return err + } + // Reject transactions with stale nonce. Gapped-nonce future transactions + // are considered valid and will be handled by the subpool according to its + // internal policy. + p.stateLock.RLock() + nonce := p.state.GetNonce(addr) + p.stateLock.RUnlock() + + if nonce > tx.Nonce() { + return core.ErrNonceTooLow + } for _, subpool := range p.subpools { if subpool.Filter(tx) { return subpool.ValidateTxBasics(tx) From 19d2b4c88083463f4ff38ebe51962970b7b37fc3 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 25 Mar 2025 11:30:13 +0100 Subject: [PATCH 14/42] version: release v1.15.6 stable --- version/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/version/version.go b/version/version.go index 586bb870d1..44740849bb 100644 --- a/version/version.go +++ b/version/version.go @@ -17,8 +17,8 @@ package version const ( - Major = 1 // Major version component of the current release - Minor = 15 // Minor version component of the current release - Patch = 6 // Patch version component of the current release - Meta = "unstable" // Version metadata to append to the version string + Major = 1 // Major version component of the current release + Minor = 15 // Minor version component of the current release + Patch = 6 // Patch version component of the current release + Meta = "stable" // Version metadata to append to the version string ) From aa6d0ef5f27508ba0ba3d7a146e9a033759a8edb Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 25 Mar 2025 12:26:15 +0100 Subject: [PATCH 15/42] version: begin v1.15.7 release cycle --- version/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/version/version.go b/version/version.go index 44740849bb..8cd428fd10 100644 --- a/version/version.go +++ b/version/version.go @@ -17,8 +17,8 @@ package version const ( - Major = 1 // Major version component of the current release - Minor = 15 // Minor version component of the current release - Patch = 6 // Patch version component of the current release - Meta = "stable" // Version metadata to append to the version string + Major = 1 // Major version component of the current release + Minor = 15 // Minor version component of the current release + Patch = 7 // Patch version component of the current release + Meta = "unstable" // Version metadata to append to the version string ) From 21d36f7c3797c6c5b8b737d586a79fa5bb47115f Mon Sep 17 00:00:00 2001 From: nethoxa <135072738+nethoxa@users.noreply.github.com> Date: Tue, 25 Mar 2025 14:13:05 +0100 Subject: [PATCH 16/42] core: process EL requests in GenerateVerkleChain (#31175) --- core/chain_makers.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/core/chain_makers.go b/core/chain_makers.go index 8d09390b72..7a258dc05f 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -504,6 +504,13 @@ func GenerateVerkleChain(config *params.ChainConfig, parent *types.Block, engine if gen != nil { gen(i, b) } + + requests := b.collectRequests(false) + if requests != nil { + reqHash := types.CalcRequestsHash(requests) + b.header.RequestsHash = &reqHash + } + body := &types.Body{ Transactions: b.txs, Uncles: b.uncles, From 4ff5093df1dba2d5578f8eb4bfd44ca805d7f67f Mon Sep 17 00:00:00 2001 From: Shude Li Date: Tue, 25 Mar 2025 21:53:02 +0800 Subject: [PATCH 17/42] all: use fmt.Appendf instead of fmt.Sprintf where possible (#31301) --- cmd/utils/export_test.go | 14 ++++---- common/math/big.go | 2 +- common/math/integer.go | 2 +- core/state/snapshot/generate_test.go | 2 +- core/state/snapshot/iterator_test.go | 32 +++++++++---------- p2p/nat/nat.go | 2 +- p2p/nat/natpmp.go | 2 +- .../apitypes/signed_data_internal_test.go | 2 +- signer/core/signed_data_test.go | 2 +- triedb/pathdb/iterator_test.go | 32 +++++++++---------- 10 files changed, 46 insertions(+), 46 deletions(-) diff --git a/cmd/utils/export_test.go b/cmd/utils/export_test.go index b70d2451c6..d1a004d9b7 100644 --- a/cmd/utils/export_test.go +++ b/cmd/utils/export_test.go @@ -54,8 +54,8 @@ func (iter *testIterator) Next() (byte, []byte, []byte, bool) { if iter.index == 42 { iter.index += 1 } - return OpBatchAdd, []byte(fmt.Sprintf("key-%04d", iter.index)), - []byte(fmt.Sprintf("value %d", iter.index)), true + return OpBatchAdd, fmt.Appendf(nil, "key-%04d", iter.index), + fmt.Appendf(nil, "value %d", iter.index), true } func (iter *testIterator) Release() {} @@ -72,7 +72,7 @@ func testExport(t *testing.T, f string) { } // verify for i := 0; i < 1000; i++ { - v, err := db.Get([]byte(fmt.Sprintf("key-%04d", i))) + v, err := db.Get(fmt.Appendf(nil, "key-%04d", i)) if (i < 5 || i == 42) && err == nil { t.Fatalf("expected no element at idx %d, got '%v'", i, string(v)) } @@ -85,7 +85,7 @@ func testExport(t *testing.T, f string) { } } } - v, err := db.Get([]byte(fmt.Sprintf("key-%04d", 1000))) + v, err := db.Get(fmt.Appendf(nil, "key-%04d", 1000)) if err == nil { t.Fatalf("expected no element at idx %d, got '%v'", 1000, string(v)) } @@ -120,7 +120,7 @@ func (iter *deletionIterator) Next() (byte, []byte, []byte, bool) { if iter.index == 42 { iter.index += 1 } - return OpBatchDel, []byte(fmt.Sprintf("key-%04d", iter.index)), nil, true + return OpBatchDel, fmt.Appendf(nil, "key-%04d", iter.index), nil, true } func (iter *deletionIterator) Release() {} @@ -132,14 +132,14 @@ func testDeletion(t *testing.T, f string) { } db := rawdb.NewMemoryDatabase() for i := 0; i < 1000; i++ { - db.Put([]byte(fmt.Sprintf("key-%04d", i)), []byte(fmt.Sprintf("value %d", i))) + db.Put(fmt.Appendf(nil, "key-%04d", i), fmt.Appendf(nil, "value %d", i)) } err = ImportLDBData(db, f, 5, make(chan struct{})) if err != nil { t.Fatal(err) } for i := 0; i < 1000; i++ { - v, err := db.Get([]byte(fmt.Sprintf("key-%04d", i))) + v, err := db.Get(fmt.Appendf(nil, "key-%04d", i)) if i < 5 || i == 42 { if err != nil { t.Fatalf("expected element at idx %d, got '%v'", i, err) diff --git a/common/math/big.go b/common/math/big.go index 825f4baec9..493c2b7861 100644 --- a/common/math/big.go +++ b/common/math/big.go @@ -72,7 +72,7 @@ func (i *HexOrDecimal256) MarshalText() ([]byte, error) { if i == nil { return []byte("0x0"), nil } - return []byte(fmt.Sprintf("%#x", (*big.Int)(i))), nil + return fmt.Appendf(nil, "%#x", (*big.Int)(i)), nil } // Decimal256 unmarshals big.Int as a decimal string. When unmarshalling, diff --git a/common/math/integer.go b/common/math/integer.go index 25ced87053..dfcb0aecc4 100644 --- a/common/math/integer.go +++ b/common/math/integer.go @@ -48,7 +48,7 @@ func (i *HexOrDecimal64) UnmarshalText(input []byte) error { // MarshalText implements encoding.TextMarshaler. func (i HexOrDecimal64) MarshalText() ([]byte, error) { - return []byte(fmt.Sprintf("%#x", uint64(i))), nil + return fmt.Appendf(nil, "%#x", uint64(i)), nil } // ParseUint64 parses s as an integer in decimal or hexadecimal syntax. diff --git a/core/state/snapshot/generate_test.go b/core/state/snapshot/generate_test.go index 661610840a..3de7735ef8 100644 --- a/core/state/snapshot/generate_test.go +++ b/core/state/snapshot/generate_test.go @@ -655,7 +655,7 @@ func testGenerateWithManyExtraAccounts(t *testing.T, scheme string) { for i := 0; i < 1000; i++ { acc := &types.StateAccount{Balance: uint256.NewInt(uint64(i)), Root: types.EmptyRootHash, CodeHash: types.EmptyCodeHash.Bytes()} val, _ := rlp.EncodeToBytes(acc) - key := hashData([]byte(fmt.Sprintf("acc-%d", i))) + key := hashData(fmt.Appendf(nil, "acc-%d", i)) rawdb.WriteAccountSnapshot(helper.diskdb, key, val) } } diff --git a/core/state/snapshot/iterator_test.go b/core/state/snapshot/iterator_test.go index b9fe370b86..2e882f484e 100644 --- a/core/state/snapshot/iterator_test.go +++ b/core/state/snapshot/iterator_test.go @@ -329,27 +329,27 @@ func TestAccountIteratorTraversalValues(t *testing.T) { h = make(map[common.Hash][]byte) ) for i := byte(2); i < 0xff; i++ { - a[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 0, i)) + a[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 0, i) if i > 20 && i%2 == 0 { - b[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 1, i)) + b[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 1, i) } if i%4 == 0 { - c[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 2, i)) + c[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 2, i) } if i%7 == 0 { - d[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 3, i)) + d[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 3, i) } if i%8 == 0 { - e[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 4, i)) + e[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 4, i) } if i > 50 || i < 85 { - f[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 5, i)) + f[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 5, i) } if i%64 == 0 { - g[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 6, i)) + g[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 6, i) } if i%128 == 0 { - h[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 7, i)) + h[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 7, i) } } // Assemble a stack of snapshots from the account layers @@ -428,27 +428,27 @@ func TestStorageIteratorTraversalValues(t *testing.T) { h = make(map[common.Hash][]byte) ) for i := byte(2); i < 0xff; i++ { - a[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 0, i)) + a[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 0, i) if i > 20 && i%2 == 0 { - b[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 1, i)) + b[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 1, i) } if i%4 == 0 { - c[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 2, i)) + c[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 2, i) } if i%7 == 0 { - d[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 3, i)) + d[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 3, i) } if i%8 == 0 { - e[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 4, i)) + e[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 4, i) } if i > 50 || i < 85 { - f[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 5, i)) + f[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 5, i) } if i%64 == 0 { - g[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 6, i)) + g[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 6, i) } if i%128 == 0 { - h[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 7, i)) + h[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 7, i) } } // Assemble a stack of snapshots from the account layers diff --git a/p2p/nat/nat.go b/p2p/nat/nat.go index 5f7af7424d..a0ddb3b29b 100644 --- a/p2p/nat/nat.go +++ b/p2p/nat/nat.go @@ -140,7 +140,7 @@ type ExtIP net.IP func (n ExtIP) ExternalIP() (net.IP, error) { return net.IP(n), nil } func (n ExtIP) String() string { return fmt.Sprintf("ExtIP(%v)", net.IP(n)) } -func (n ExtIP) MarshalText() ([]byte, error) { return []byte(fmt.Sprintf("extip:%v", net.IP(n))), nil } +func (n ExtIP) MarshalText() ([]byte, error) { return fmt.Appendf(nil, "extip:%v", net.IP(n)), nil } // These do nothing. diff --git a/p2p/nat/natpmp.go b/p2p/nat/natpmp.go index b8f59ee890..4a9644ac1a 100644 --- a/p2p/nat/natpmp.go +++ b/p2p/nat/natpmp.go @@ -71,7 +71,7 @@ func (n *pmp) DeleteMapping(protocol string, extport, intport int) (err error) { } func (n *pmp) MarshalText() ([]byte, error) { - return []byte(fmt.Sprintf("natpmp:%v", n.gw)), nil + return fmt.Appendf(nil, "natpmp:%v", n.gw), nil } func discoverPMP() Interface { diff --git a/signer/core/apitypes/signed_data_internal_test.go b/signer/core/apitypes/signed_data_internal_test.go index 1a14b35ef2..bee9f1c351 100644 --- a/signer/core/apitypes/signed_data_internal_test.go +++ b/signer/core/apitypes/signed_data_internal_test.go @@ -282,7 +282,7 @@ func TestTypedDataArrayValidate(t *testing.T) { messageHash, tErr := td.HashStruct(td.PrimaryType, td.Message) assert.NoError(t, tErr, "failed to hash message: %v", tErr) - digest := crypto.Keccak256Hash([]byte(fmt.Sprintf("%s%s%s", "\x19\x01", string(domainSeparator), string(messageHash)))) + digest := crypto.Keccak256Hash(fmt.Appendf(nil, "%s%s%s", "\x19\x01", string(domainSeparator), string(messageHash))) assert.Equal(t, tc.Digest, digest.String(), "digest doesn't not match") assert.NoError(t, td.validate(), "validation failed", tErr) diff --git a/signer/core/signed_data_test.go b/signer/core/signed_data_test.go index b6c080736c..001f6b6838 100644 --- a/signer/core/signed_data_test.go +++ b/signer/core/signed_data_test.go @@ -369,7 +369,7 @@ func sign(typedData apitypes.TypedData) ([]byte, []byte, error) { if err != nil { return nil, nil, err } - rawData := []byte(fmt.Sprintf("\x19\x01%s%s", string(domainSeparator), string(typedDataHash))) + rawData := fmt.Appendf(nil, "\x19\x01%s%s", string(domainSeparator), string(typedDataHash)) sighash := crypto.Keccak256(rawData) return typedDataHash, sighash, nil } diff --git a/triedb/pathdb/iterator_test.go b/triedb/pathdb/iterator_test.go index 05a166d1b6..3894118034 100644 --- a/triedb/pathdb/iterator_test.go +++ b/triedb/pathdb/iterator_test.go @@ -371,27 +371,27 @@ func TestAccountIteratorTraversalValues(t *testing.T) { h = make(map[common.Hash][]byte) ) for i := byte(2); i < 0xff; i++ { - a[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 0, i)) + a[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 0, i) if i > 20 && i%2 == 0 { - b[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 1, i)) + b[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 1, i) } if i%4 == 0 { - c[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 2, i)) + c[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 2, i) } if i%7 == 0 { - d[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 3, i)) + d[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 3, i) } if i%8 == 0 { - e[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 4, i)) + e[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 4, i) } if i > 50 || i < 85 { - f[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 5, i)) + f[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 5, i) } if i%64 == 0 { - g[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 6, i)) + g[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 6, i) } if i%128 == 0 { - h[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 7, i)) + h[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 7, i) } } // Assemble a stack of snapshots from the account layers @@ -480,27 +480,27 @@ func TestStorageIteratorTraversalValues(t *testing.T) { h = make(map[common.Hash][]byte) ) for i := byte(2); i < 0xff; i++ { - a[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 0, i)) + a[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 0, i) if i > 20 && i%2 == 0 { - b[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 1, i)) + b[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 1, i) } if i%4 == 0 { - c[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 2, i)) + c[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 2, i) } if i%7 == 0 { - d[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 3, i)) + d[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 3, i) } if i%8 == 0 { - e[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 4, i)) + e[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 4, i) } if i > 50 || i < 85 { - f[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 5, i)) + f[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 5, i) } if i%64 == 0 { - g[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 6, i)) + g[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 6, i) } if i%128 == 0 { - h[common.Hash{i}] = []byte(fmt.Sprintf("layer-%d, key %d", 7, i)) + h[common.Hash{i}] = fmt.Appendf(nil, "layer-%d, key %d", 7, i) } } // Assemble a stack of snapshots from the account layers From 4dfec7e83e4634cc8ab254b4bdbe9e692142316b Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Tue, 25 Mar 2025 21:59:44 +0800 Subject: [PATCH 18/42] trie: optimize memory allocation (#30932) This pull request removes the node copy operation to reduce memory allocation. Key Changes as below: **(a) Use `decodeNodeUnsafe` for decoding nodes retrieved from the trie node reader** In the current implementation of the MPT, once a trie node blob is retrieved, it is passed to `decodeNode` for decoding. However, `decodeNode` assumes the supplied byte slice might be mutated later, so it performs a deep copy internally before parsing the node. Given that the node reader is implemented by the path database and the hash database, both of which guarantee the immutability of the returned byte slice. By restricting the node reader interface to explicitly guarantee that the returned byte slice will not be modified, we can safely replace `decodeNode` with `decodeNodeUnsafe`. This eliminates the need for a redundant byte copy during each node resolution. **(b) Modify the trie in place** In the current implementation of the MPT, a copy of a trie node is created before any modifications are made. These modifications include: - Node resolution: Converting the value from a hash to the actual node. - Node hashing: Tagging the hash into its cache. - Node commit: Replacing the children with its hash. - Structural changes: For example, adding a new child to a fullNode or replacing a child of a shortNode. This mechanism ensures that modifications only affect the live tree, leaving all previously created copies unaffected. Unfortunately, this property leads to a huge memory allocation requirement. For example, if we want to modify the fullNode for n times, the node will be copied for n times. In this pull request, all the trie modifications are made in place. In order to make sure all previously created copies are unaffected, the `Copy` function now will deep-copy all the live nodes rather than the root node itself. With this change, while the `Copy` function becomes more expensive, it's totally acceptable as it's not a frequently used one. For the normal trie operations (Get, GetNode, Hash, Commit, Insert, Delete), the node copy is not required anymore. --- trie/committer.go | 39 +++------ trie/hasher.go | 84 +++++++++--------- trie/node.go | 14 ++- trie/trie.go | 68 ++++++++++----- trie/trie_test.go | 168 ++++++++++++++++++++++++++++++++++++ triedb/database/database.go | 2 + 6 files changed, 279 insertions(+), 96 deletions(-) diff --git a/trie/committer.go b/trie/committer.go index 6c4374ccfd..0939a07abb 100644 --- a/trie/committer.go +++ b/trie/committer.go @@ -57,32 +57,26 @@ func (c *committer) commit(path []byte, n node, parallel bool) node { // Commit children, then parent, and remove the dirty flag. switch cn := n.(type) { case *shortNode: - // Commit child - collapsed := cn.copy() - // If the child is fullNode, recursively commit, // otherwise it can only be hashNode or valueNode. if _, ok := cn.Val.(*fullNode); ok { - collapsed.Val = c.commit(append(path, cn.Key...), cn.Val, false) + cn.Val = c.commit(append(path, cn.Key...), cn.Val, false) } // The key needs to be copied, since we're adding it to the // modified nodeset. - collapsed.Key = hexToCompact(cn.Key) - hashedNode := c.store(path, collapsed) + cn.Key = hexToCompact(cn.Key) + hashedNode := c.store(path, cn) if hn, ok := hashedNode.(hashNode); ok { return hn } - return collapsed + return cn case *fullNode: - hashedKids := c.commitChildren(path, cn, parallel) - collapsed := cn.copy() - collapsed.Children = hashedKids - - hashedNode := c.store(path, collapsed) + c.commitChildren(path, cn, parallel) + hashedNode := c.store(path, cn) if hn, ok := hashedNode.(hashNode); ok { return hn } - return collapsed + return cn case hashNode: return cn default: @@ -92,11 +86,10 @@ func (c *committer) commit(path []byte, n node, parallel bool) node { } // commitChildren commits the children of the given fullnode -func (c *committer) commitChildren(path []byte, n *fullNode, parallel bool) [17]node { +func (c *committer) commitChildren(path []byte, n *fullNode, parallel bool) { var ( - wg sync.WaitGroup - nodesMu sync.Mutex - children [17]node + wg sync.WaitGroup + nodesMu sync.Mutex ) for i := 0; i < 16; i++ { child := n.Children[i] @@ -106,22 +99,21 @@ func (c *committer) commitChildren(path []byte, n *fullNode, parallel bool) [17] // If it's the hashed child, save the hash value directly. // Note: it's impossible that the child in range [0, 15] // is a valueNode. - if hn, ok := child.(hashNode); ok { - children[i] = hn + if _, ok := child.(hashNode); ok { continue } // Commit the child recursively and store the "hashed" value. // Note the returned node can be some embedded nodes, so it's // possible the type is not hashNode. if !parallel { - children[i] = c.commit(append(path, byte(i)), child, false) + n.Children[i] = c.commit(append(path, byte(i)), child, false) } else { wg.Add(1) go func(index int) { p := append(path, byte(index)) childSet := trienode.NewNodeSet(c.nodes.Owner) childCommitter := newCommitter(childSet, c.tracer, c.collectLeaf) - children[index] = childCommitter.commit(p, child, false) + n.Children[index] = childCommitter.commit(p, child, false) nodesMu.Lock() c.nodes.MergeSet(childSet) nodesMu.Unlock() @@ -132,11 +124,6 @@ func (c *committer) commitChildren(path []byte, n *fullNode, parallel bool) [17] if parallel { wg.Wait() } - // For the 17th child, it's possible the type is valuenode. - if n.Children[16] != nil { - children[16] = n.Children[16] - } - return children } // store hashes the node n and adds it to the modified nodeset. If leaf collection diff --git a/trie/hasher.go b/trie/hasher.go index 28f7f3d0c3..614640ae3a 100644 --- a/trie/hasher.go +++ b/trie/hasher.go @@ -53,62 +53,56 @@ func returnHasherToPool(h *hasher) { hasherPool.Put(h) } -// hash collapses a node down into a hash node, also returning a copy of the -// original node initialized with the computed hash to replace the original one. -func (h *hasher) hash(n node, force bool) (hashed node, cached node) { +// hash collapses a node down into a hash node. +func (h *hasher) hash(n node, force bool) node { // Return the cached hash if it's available if hash, _ := n.cache(); hash != nil { - return hash, n + return hash } // Trie not processed yet, walk the children switch n := n.(type) { case *shortNode: - collapsed, cached := h.hashShortNodeChildren(n) + collapsed := h.hashShortNodeChildren(n) hashed := h.shortnodeToHash(collapsed, force) - // We need to retain the possibly _not_ hashed node, in case it was too - // small to be hashed if hn, ok := hashed.(hashNode); ok { - cached.flags.hash = hn + n.flags.hash = hn } else { - cached.flags.hash = nil + n.flags.hash = nil } - return hashed, cached + return hashed case *fullNode: - collapsed, cached := h.hashFullNodeChildren(n) - hashed = h.fullnodeToHash(collapsed, force) + collapsed := h.hashFullNodeChildren(n) + hashed := h.fullnodeToHash(collapsed, force) if hn, ok := hashed.(hashNode); ok { - cached.flags.hash = hn + n.flags.hash = hn } else { - cached.flags.hash = nil + n.flags.hash = nil } - return hashed, cached + return hashed default: // Value and hash nodes don't have children, so they're left as were - return n, n + return n } } -// hashShortNodeChildren collapses the short node. The returned collapsed node -// holds a live reference to the Key, and must not be modified. -func (h *hasher) hashShortNodeChildren(n *shortNode) (collapsed, cached *shortNode) { - // Hash the short node's child, caching the newly hashed subtree - collapsed, cached = n.copy(), n.copy() - // Previously, we did copy this one. We don't seem to need to actually - // do that, since we don't overwrite/reuse keys - // cached.Key = common.CopyBytes(n.Key) +// hashShortNodeChildren returns a copy of the supplied shortNode, with its child +// being replaced by either the hash or an embedded node if the child is small. +func (h *hasher) hashShortNodeChildren(n *shortNode) *shortNode { + var collapsed shortNode collapsed.Key = hexToCompact(n.Key) - // Unless the child is a valuenode or hashnode, hash it switch n.Val.(type) { case *fullNode, *shortNode: - collapsed.Val, cached.Val = h.hash(n.Val, false) + collapsed.Val = h.hash(n.Val, false) + default: + collapsed.Val = n.Val } - return collapsed, cached + return &collapsed } -func (h *hasher) hashFullNodeChildren(n *fullNode) (collapsed *fullNode, cached *fullNode) { - // Hash the full node's children, caching the newly hashed subtrees - cached = n.copy() - collapsed = n.copy() +// hashFullNodeChildren returns a copy of the supplied fullNode, with its child +// being replaced by either the hash or an embedded node if the child is small. +func (h *hasher) hashFullNodeChildren(n *fullNode) *fullNode { + var children [17]node if h.parallel { var wg sync.WaitGroup wg.Add(16) @@ -116,9 +110,9 @@ func (h *hasher) hashFullNodeChildren(n *fullNode) (collapsed *fullNode, cached go func(i int) { hasher := newHasher(false) if child := n.Children[i]; child != nil { - collapsed.Children[i], cached.Children[i] = hasher.hash(child, false) + children[i] = hasher.hash(child, false) } else { - collapsed.Children[i] = nilValueNode + children[i] = nilValueNode } returnHasherToPool(hasher) wg.Done() @@ -128,19 +122,21 @@ func (h *hasher) hashFullNodeChildren(n *fullNode) (collapsed *fullNode, cached } else { for i := 0; i < 16; i++ { if child := n.Children[i]; child != nil { - collapsed.Children[i], cached.Children[i] = h.hash(child, false) + children[i] = h.hash(child, false) } else { - collapsed.Children[i] = nilValueNode + children[i] = nilValueNode } } } - return collapsed, cached + if n.Children[16] != nil { + children[16] = n.Children[16] + } + return &fullNode{flags: nodeFlag{}, Children: children} } -// shortnodeToHash creates a hashNode from a shortNode. The supplied shortnode -// should have hex-type Key, which will be converted (without modification) -// into compact form for RLP encoding. -// If the rlp data is smaller than 32 bytes, `nil` is returned. +// shortNodeToHash computes the hash of the given shortNode. The shortNode must +// first be collapsed, with its key converted to compact form. If the RLP-encoded +// node data is smaller than 32 bytes, the node itself is returned. func (h *hasher) shortnodeToHash(n *shortNode, force bool) node { n.encode(h.encbuf) enc := h.encodedBytes() @@ -151,8 +147,8 @@ func (h *hasher) shortnodeToHash(n *shortNode, force bool) node { return h.hashData(enc) } -// fullnodeToHash is used to create a hashNode from a fullNode, (which -// may contain nil values) +// fullnodeToHash computes the hash of the given fullNode. If the RLP-encoded +// node data is smaller than 32 bytes, the node itself is returned. func (h *hasher) fullnodeToHash(n *fullNode, force bool) node { n.encode(h.encbuf) enc := h.encodedBytes() @@ -203,10 +199,10 @@ func (h *hasher) hashDataTo(dst, data []byte) { func (h *hasher) proofHash(original node) (collapsed, hashed node) { switch n := original.(type) { case *shortNode: - sn, _ := h.hashShortNodeChildren(n) + sn := h.hashShortNodeChildren(n) return sn, h.shortnodeToHash(sn, false) case *fullNode: - fn, _ := h.hashFullNodeChildren(n) + fn := h.hashFullNodeChildren(n) return fn, h.fullnodeToHash(fn, false) default: // Value and hash nodes don't have children, so they're left as were diff --git a/trie/node.go b/trie/node.go index ecc2de192d..96f077ebbb 100644 --- a/trie/node.go +++ b/trie/node.go @@ -79,15 +79,19 @@ func (n *fullNode) EncodeRLP(w io.Writer) error { return eb.Flush() } -func (n *fullNode) copy() *fullNode { copy := *n; return © } -func (n *shortNode) copy() *shortNode { copy := *n; return © } - // nodeFlag contains caching-related metadata about a node. type nodeFlag struct { hash hashNode // cached hash of the node (may be nil) dirty bool // whether the node has changes that must be written to the database } +func (n nodeFlag) copy() nodeFlag { + return nodeFlag{ + hash: common.CopyBytes(n.hash), + dirty: n.dirty, + } +} + func (n *fullNode) cache() (hashNode, bool) { return n.flags.hash, n.flags.dirty } func (n *shortNode) cache() (hashNode, bool) { return n.flags.hash, n.flags.dirty } func (n hashNode) cache() (hashNode, bool) { return nil, true } @@ -228,7 +232,9 @@ func decodeRef(buf []byte) (node, []byte, error) { err := fmt.Errorf("oversized embedded node (size is %d bytes, want size < %d)", size, hashLen) return nil, buf, err } - n, err := decodeNode(nil, buf) + // The buffer content has already been copied or is safe to use; + // no additional copy is required. + n, err := decodeNodeUnsafe(nil, buf) return n, rest, err case kind == rlp.String && len(val) == 0: // empty node diff --git a/trie/trie.go b/trie/trie.go index ae2a7b21a2..fdb4da9be4 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -29,11 +29,11 @@ import ( "github.com/ethereum/go-ethereum/triedb/database" ) -// Trie is a Merkle Patricia Trie. Use New to create a trie that sits on -// top of a database. Whenever trie performs a commit operation, the generated -// nodes will be gathered and returned in a set. Once the trie is committed, -// it's not usable anymore. Callers have to re-create the trie with new root -// based on the updated trie database. +// Trie represents a Merkle Patricia Trie. Use New to create a trie that operates +// on top of a node database. During a commit operation, the trie collects all +// modified nodes into a set for return. After committing, the trie becomes +// unusable, and callers must recreate it with the new root based on the updated +// trie database. // // Trie is not safe for concurrent use. type Trie struct { @@ -67,13 +67,13 @@ func (t *Trie) newFlag() nodeFlag { // Copy returns a copy of Trie. func (t *Trie) Copy() *Trie { return &Trie{ - root: t.root, + root: copyNode(t.root), owner: t.owner, committed: t.committed, + unhashed: t.unhashed, + uncommitted: t.uncommitted, reader: t.reader, tracer: t.tracer.copy(), - uncommitted: t.uncommitted, - unhashed: t.unhashed, } } @@ -169,14 +169,12 @@ func (t *Trie) get(origNode node, key []byte, pos int) (value []byte, newnode no } value, newnode, didResolve, err = t.get(n.Val, key, pos+len(n.Key)) if err == nil && didResolve { - n = n.copy() n.Val = newnode } return value, n, didResolve, err case *fullNode: value, newnode, didResolve, err = t.get(n.Children[key[pos]], key, pos+1) if err == nil && didResolve { - n = n.copy() n.Children[key[pos]] = newnode } return value, n, didResolve, err @@ -257,7 +255,6 @@ func (t *Trie) getNode(origNode node, path []byte, pos int) (item []byte, newnod } item, newnode, resolved, err = t.getNode(n.Val, path, pos+len(n.Key)) if err == nil && resolved > 0 { - n = n.copy() n.Val = newnode } return item, n, resolved, err @@ -265,7 +262,6 @@ func (t *Trie) getNode(origNode node, path []byte, pos int) (item []byte, newnod case *fullNode: item, newnode, resolved, err = t.getNode(n.Children[path[pos]], path, pos+1) if err == nil && resolved > 0 { - n = n.copy() n.Children[path[pos]] = newnode } return item, n, resolved, err @@ -375,7 +371,6 @@ func (t *Trie) insert(n node, prefix, key []byte, value node) (bool, node, error if !dirty || err != nil { return false, n, err } - n = n.copy() n.flags = t.newFlag() n.Children[key[0]] = nn return true, n, nil @@ -483,7 +478,6 @@ func (t *Trie) delete(n node, prefix, key []byte) (bool, node, error) { if !dirty || err != nil { return false, n, err } - n = n.copy() n.flags = t.newFlag() n.Children[key[0]] = nn @@ -576,6 +570,36 @@ func concat(s1 []byte, s2 ...byte) []byte { return r } +// copyNode deep-copies the supplied node along with its children recursively. +func copyNode(n node) node { + switch n := (n).(type) { + case nil: + return nil + case valueNode: + return valueNode(common.CopyBytes(n)) + + case *shortNode: + return &shortNode{ + flags: n.flags.copy(), + Key: common.CopyBytes(n.Key), + Val: copyNode(n.Val), + } + case *fullNode: + var children [17]node + for i, cn := range n.Children { + children[i] = copyNode(cn) + } + return &fullNode{ + flags: n.flags.copy(), + Children: children, + } + case hashNode: + return n + default: + panic(fmt.Sprintf("%T: unknown node type", n)) + } +} + func (t *Trie) resolve(n node, prefix []byte) (node, error) { if n, ok := n.(hashNode); ok { return t.resolveAndTrack(n, prefix) @@ -593,15 +617,16 @@ func (t *Trie) resolveAndTrack(n hashNode, prefix []byte) (node, error) { return nil, err } t.tracer.onRead(prefix, blob) - return mustDecodeNode(n, blob), nil + + // The returned node blob won't be changed afterward. No need to + // deep-copy the slice. + return decodeNodeUnsafe(n, blob) } // Hash returns the root hash of the trie. It does not write to the // database and can be used even if the trie doesn't have one. func (t *Trie) Hash() common.Hash { - hash, cached := t.hashRoot() - t.root = cached - return common.BytesToHash(hash.(hashNode)) + return common.BytesToHash(t.hashRoot().(hashNode)) } // Commit collects all dirty nodes in the trie and replaces them with the @@ -652,9 +677,9 @@ func (t *Trie) Commit(collectLeaf bool) (common.Hash, *trienode.NodeSet) { } // hashRoot calculates the root hash of the given trie -func (t *Trie) hashRoot() (node, node) { +func (t *Trie) hashRoot() node { if t.root == nil { - return hashNode(types.EmptyRootHash.Bytes()), nil + return hashNode(types.EmptyRootHash.Bytes()) } // If the number of changes is below 100, we let one thread handle it h := newHasher(t.unhashed >= 100) @@ -662,8 +687,7 @@ func (t *Trie) hashRoot() (node, node) { returnHasherToPool(h) t.unhashed = 0 }() - hashed, cached := h.hash(t.root, true) - return hashed, cached + return h.hash(t.root, true) } // Witness returns a set containing all trie nodes that have been accessed. diff --git a/trie/trie_test.go b/trie/trie_test.go index 77234d9d9b..54d1b083d8 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -1330,3 +1330,171 @@ func printSet(set *trienode.NodeSet) string { } return out.String() } + +func TestTrieCopy(t *testing.T) { + testTrieCopy(t, []kv{ + {k: []byte("do"), v: []byte("verb")}, + {k: []byte("ether"), v: []byte("wookiedoo")}, + {k: []byte("horse"), v: []byte("stallion")}, + {k: []byte("shaman"), v: []byte("horse")}, + {k: []byte("doge"), v: []byte("coin")}, + {k: []byte("dog"), v: []byte("puppy")}, + }) + + var entries []kv + for i := 0; i < 256; i++ { + entries = append(entries, kv{k: testrand.Bytes(32), v: testrand.Bytes(32)}) + } + testTrieCopy(t, entries) +} + +func testTrieCopy(t *testing.T, entries []kv) { + tr := NewEmpty(nil) + for _, entry := range entries { + tr.Update(entry.k, entry.v) + } + trCpy := tr.Copy() + + if tr.Hash() != trCpy.Hash() { + t.Errorf("Hash mismatch: old %v, copy %v", tr.Hash(), trCpy.Hash()) + } + + // Check iterator + it, _ := tr.NodeIterator(nil) + itCpy, _ := trCpy.NodeIterator(nil) + + for it.Next(false) { + hasNext := itCpy.Next(false) + if !hasNext { + t.Fatal("Iterator is not matched") + } + if !bytes.Equal(it.Path(), itCpy.Path()) { + t.Fatal("Iterator is not matched") + } + if it.Leaf() != itCpy.Leaf() { + t.Fatal("Iterator is not matched") + } + if it.Leaf() && !bytes.Equal(it.LeafBlob(), itCpy.LeafBlob()) { + t.Fatal("Iterator is not matched") + } + } + + // Check commit + root, nodes := tr.Commit(false) + rootCpy, nodesCpy := trCpy.Commit(false) + if root != rootCpy { + t.Fatal("root mismatch") + } + if len(nodes.Nodes) != len(nodesCpy.Nodes) { + t.Fatal("commit node mismatch") + } + for p, n := range nodes.Nodes { + nn, exists := nodesCpy.Nodes[p] + if !exists { + t.Fatalf("node not exists: %v", p) + } + if !reflect.DeepEqual(n, nn) { + t.Fatalf("node mismatch: %v", p) + } + } +} + +func TestTrieCopyOldTrie(t *testing.T) { + testTrieCopyOldTrie(t, []kv{ + {k: []byte("do"), v: []byte("verb")}, + {k: []byte("ether"), v: []byte("wookiedoo")}, + {k: []byte("horse"), v: []byte("stallion")}, + {k: []byte("shaman"), v: []byte("horse")}, + {k: []byte("doge"), v: []byte("coin")}, + {k: []byte("dog"), v: []byte("puppy")}, + }) + + var entries []kv + for i := 0; i < 256; i++ { + entries = append(entries, kv{k: testrand.Bytes(32), v: testrand.Bytes(32)}) + } + testTrieCopyOldTrie(t, entries) +} + +func testTrieCopyOldTrie(t *testing.T, entries []kv) { + tr := NewEmpty(nil) + for _, entry := range entries { + tr.Update(entry.k, entry.v) + } + hash := tr.Hash() + + trCpy := tr.Copy() + for _, val := range entries { + if rand.Intn(2) == 0 { + trCpy.Delete(val.k) + } else { + trCpy.Update(val.k, testrand.Bytes(32)) + } + } + for i := 0; i < 10; i++ { + trCpy.Update(testrand.Bytes(32), testrand.Bytes(32)) + } + trCpy.Hash() + trCpy.Commit(false) + + // Traverse the original tree, the changes made on the copy one shouldn't + // affect the old one + for _, entry := range entries { + d, _ := tr.Get(entry.k) + if !bytes.Equal(d, entry.v) { + t.Errorf("Unexpected data, key: %v, want: %v, got: %v", entry.k, entry.v, d) + } + } + if tr.Hash() != hash { + t.Errorf("Hash mismatch: old %v, new %v", hash, tr.Hash()) + } +} + +func TestTrieCopyNewTrie(t *testing.T) { + testTrieCopyNewTrie(t, []kv{ + {k: []byte("do"), v: []byte("verb")}, + {k: []byte("ether"), v: []byte("wookiedoo")}, + {k: []byte("horse"), v: []byte("stallion")}, + {k: []byte("shaman"), v: []byte("horse")}, + {k: []byte("doge"), v: []byte("coin")}, + {k: []byte("dog"), v: []byte("puppy")}, + }) + + var entries []kv + for i := 0; i < 256; i++ { + entries = append(entries, kv{k: testrand.Bytes(32), v: testrand.Bytes(32)}) + } + testTrieCopyNewTrie(t, entries) +} + +func testTrieCopyNewTrie(t *testing.T, entries []kv) { + tr := NewEmpty(nil) + for _, entry := range entries { + tr.Update(entry.k, entry.v) + } + trCpy := tr.Copy() + hash := trCpy.Hash() + + for _, val := range entries { + if rand.Intn(2) == 0 { + tr.Delete(val.k) + } else { + tr.Update(val.k, testrand.Bytes(32)) + } + } + for i := 0; i < 10; i++ { + tr.Update(testrand.Bytes(32), testrand.Bytes(32)) + } + + // Traverse the original tree, the changes made on the copy one shouldn't + // affect the old one + for _, entry := range entries { + d, _ := trCpy.Get(entry.k) + if !bytes.Equal(d, entry.v) { + t.Errorf("Unexpected data, key: %v, want: %v, got: %v", entry.k, entry.v, d) + } + } + if trCpy.Hash() != hash { + t.Errorf("Hash mismatch: old %v, new %v", hash, tr.Hash()) + } +} diff --git a/triedb/database/database.go b/triedb/database/database.go index cd7ec1d931..8c61ea0293 100644 --- a/triedb/database/database.go +++ b/triedb/database/database.go @@ -27,6 +27,8 @@ type NodeReader interface { // node path and the corresponding node hash. No error will be returned // if the node is not found. // + // The returned node content won't be changed after the call. + // // Don't modify the returned byte slice since it's not deep-copied and // still be referenced by database. Node(owner common.Hash, path []byte, hash common.Hash) ([]byte, error) From c49aadc4b628c55da9948b0a1f37f487129ce28a Mon Sep 17 00:00:00 2001 From: sashabeton <23243361+sashabeton@users.noreply.github.com> Date: Tue, 25 Mar 2025 15:01:21 +0100 Subject: [PATCH 19/42] internal/ethapi: exclude 7702 authorities from result in eth_createAccessList (#31336) closes https://github.com/ethereum/go-ethereum/issues/31335 --------- Co-authored-by: sashabeton --- eth/tracers/logger/access_list_tracer.go | 12 +++------- internal/ethapi/api.go | 29 +++++++++++++++++++++--- 2 files changed, 29 insertions(+), 12 deletions(-) diff --git a/eth/tracers/logger/access_list_tracer.go b/eth/tracers/logger/access_list_tracer.go index e8231461b0..0d51f40522 100644 --- a/eth/tracers/logger/access_list_tracer.go +++ b/eth/tracers/logger/access_list_tracer.go @@ -103,16 +103,10 @@ type AccessListTracer struct { // NewAccessListTracer creates a new tracer that can generate AccessLists. // An optional AccessList can be specified to occupy slots and addresses in // the resulting accesslist. -func NewAccessListTracer(acl types.AccessList, from, to common.Address, precompiles []common.Address) *AccessListTracer { - excl := map[common.Address]struct{}{ - from: {}, to: {}, - } - for _, addr := range precompiles { - excl[addr] = struct{}{} - } +func NewAccessListTracer(acl types.AccessList, addressesToExclude map[common.Address]struct{}) *AccessListTracer { list := newAccessList() for _, al := range acl { - if _, ok := excl[al.Address]; !ok { + if _, ok := addressesToExclude[al.Address]; !ok { list.addAddress(al.Address) } for _, slot := range al.StorageKeys { @@ -120,7 +114,7 @@ func NewAccessListTracer(acl types.AccessList, from, to common.Address, precompi } } return &AccessListTracer{ - excl: excl, + excl: addressesToExclude, list: list, } } diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index e0d5b32622..203834fe38 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1166,10 +1166,33 @@ func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrH // Retrieve the precompiles since they don't need to be added to the access list precompiles := vm.ActivePrecompiles(b.ChainConfig().Rules(header.Number, isPostMerge, header.Time)) + // addressesToExclude contains sender, receiver, precompiles and valid authorizations + addressesToExclude := map[common.Address]struct{}{args.from(): {}, to: {}} + for _, addr := range precompiles { + addressesToExclude[addr] = struct{}{} + } + + // Prevent redundant operations if args contain more authorizations than EVM may handle + maxAuthorizations := uint64(*args.Gas) / params.CallNewAccountGas + if uint64(len(args.AuthorizationList)) > maxAuthorizations { + return nil, 0, nil, errors.New("insufficient gas to process all authorizations") + } + + for _, auth := range args.AuthorizationList { + // Duplicating stateTransition.validateAuthorization() logic + if (!auth.ChainID.IsZero() && auth.ChainID.CmpBig(b.ChainConfig().ChainID) != 0) || auth.Nonce+1 < auth.Nonce { + continue + } + + if authority, err := auth.Authority(); err == nil { + addressesToExclude[authority] = struct{}{} + } + } + // Create an initial tracer - prevTracer := logger.NewAccessListTracer(nil, args.from(), to, precompiles) + prevTracer := logger.NewAccessListTracer(nil, addressesToExclude) if args.AccessList != nil { - prevTracer = logger.NewAccessListTracer(*args.AccessList, args.from(), to, precompiles) + prevTracer = logger.NewAccessListTracer(*args.AccessList, addressesToExclude) } for { if err := ctx.Err(); err != nil { @@ -1186,7 +1209,7 @@ func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrH msg := args.ToMessage(header.BaseFee, true, true) // Apply the transaction with the access list tracer - tracer := logger.NewAccessListTracer(accessList, args.from(), to, precompiles) + tracer := logger.NewAccessListTracer(accessList, addressesToExclude) config := vm.Config{Tracer: tracer.Hooks(), NoBaseFee: true} evm := b.GetEVM(ctx, statedb, header, &config, nil) From c1ff2d8ba973f9f7ebfbf45e3c36f8d3299846ba Mon Sep 17 00:00:00 2001 From: Delweng Date: Wed, 26 Mar 2025 12:59:40 +0800 Subject: [PATCH 20/42] core/state: fix double-increment of accountLoaded counter (#31493) --- core/state/statedb.go | 1 - 1 file changed, 1 deletion(-) diff --git a/core/state/statedb.go b/core/state/statedb.go index efafdc1aa2..e3f5b9e1a0 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -600,7 +600,6 @@ func (s *StateDB) getStateObject(addr common.Address) *stateObject { // Insert into the live set obj := newObject(s, addr, acct) s.setStateObject(obj) - s.AccountLoaded++ return obj } From 5b4a74349372402fac545db7a7f80812a40b1b2b Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Wed, 26 Mar 2025 19:48:04 +0800 Subject: [PATCH 21/42] core/rawdb: remove LES database stats (#31495) This removes DB schema for LES related db entries. LES has been non-functional since the merge. --- core/rawdb/database.go | 16 ---------------- core/rawdb/schema.go | 8 -------- 2 files changed, 24 deletions(-) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index a394f46e0a..7fca822155 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -388,10 +388,6 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { verkleTries stat verkleStateLookups stat - // Les statistic - chtTrieNodes stat - bloomTrieNodes stat - // Meta- and unaccounted data metadata stat unaccounted stat @@ -465,16 +461,6 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { case bytes.HasPrefix(key, bloomBitsMetaPrefix) && len(key) < len(bloomBitsMetaPrefix)+8: bloomBits.Add(size) - // LES indexes (deprecated) - case bytes.HasPrefix(key, chtTablePrefix) || - bytes.HasPrefix(key, chtIndexTablePrefix) || - bytes.HasPrefix(key, chtPrefix): // Canonical hash trie - chtTrieNodes.Add(size) - case bytes.HasPrefix(key, bloomTrieTablePrefix) || - bytes.HasPrefix(key, bloomTrieIndexPrefix) || - bytes.HasPrefix(key, bloomTriePrefix): // Bloomtrie sub - bloomTrieNodes.Add(size) - // Verkle trie data is detected, determine the sub-category case bytes.HasPrefix(key, VerklePrefix): remain := key[len(VerklePrefix):] @@ -538,8 +524,6 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { {"Key-Value store", "Beacon sync headers", beaconHeaders.Size(), beaconHeaders.Count()}, {"Key-Value store", "Clique snapshots", cliqueSnaps.Size(), cliqueSnaps.Count()}, {"Key-Value store", "Singleton metadata", metadata.Size(), metadata.Count()}, - {"Light client", "CHT trie nodes", chtTrieNodes.Size(), chtTrieNodes.Count()}, - {"Light client", "Bloom trie nodes", bloomTrieNodes.Size(), bloomTrieNodes.Count()}, } // Inspect all registered append-only file store then. ancients, err := inspectFreezers(db) diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 31e80da079..fa125cecc0 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -144,14 +144,6 @@ var ( // old log index bloomBitsMetaPrefix = []byte("iB") - // LES indexes - chtPrefix = []byte("chtRootV2-") // ChtPrefix + chtNum (uint64 big endian) -> trie root hash - chtTablePrefix = []byte("cht-") - chtIndexTablePrefix = []byte("chtIndexV2-") - bloomTriePrefix = []byte("bltRoot-") // BloomTriePrefix + bloomTrieNum (uint64 big endian) -> trie root hash - bloomTrieTablePrefix = []byte("blt-") - bloomTrieIndexPrefix = []byte("bltIndex-") - preimageCounter = metrics.NewRegisteredCounter("db/preimage/total", nil) preimageHitsCounter = metrics.NewRegisteredCounter("db/preimage/hits", nil) preimageMissCounter = metrics.NewRegisteredCounter("db/preimage/miss", nil) From a775e68421595d9c3807e68cce7ff2037991a781 Mon Sep 17 00:00:00 2001 From: VolodymyrBg Date: Wed, 26 Mar 2025 13:57:08 +0200 Subject: [PATCH 22/42] eth: downgrade peer removal error to warning level (#31492) --- eth/handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/handler.go b/eth/handler.go index 4c83f5613c..7179c9980b 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -403,7 +403,7 @@ func (h *handler) unregisterPeer(id string) { // Abort if the peer does not exist peer := h.peers.peer(id) if peer == nil { - logger.Error("Ethereum peer removal failed", "err", errPeerNotRegistered) + logger.Warn("Ethereum peer removal failed", "err", errPeerNotRegistered) return } // Remove the `eth` peer if it exists From a82303f4e3cedcebe31540a53dde4f24fc93da80 Mon Sep 17 00:00:00 2001 From: protolambda Date: Wed, 26 Mar 2025 16:14:17 +0100 Subject: [PATCH 23/42] accounts/abi: include access-list in gas estimation (#31394) Simple bugfix to include the access-list in the gas-estimation step of the ABI bindings code. --- accounts/abi/bind/v2/base.go | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/accounts/abi/bind/v2/base.go b/accounts/abi/bind/v2/base.go index a9b667e697..535c0ed4fd 100644 --- a/accounts/abi/bind/v2/base.go +++ b/accounts/abi/bind/v2/base.go @@ -383,13 +383,14 @@ func (c *BoundContract) estimateGasLimit(opts *TransactOpts, contract *common.Ad } } msg := ethereum.CallMsg{ - From: opts.From, - To: contract, - GasPrice: gasPrice, - GasTipCap: gasTipCap, - GasFeeCap: gasFeeCap, - Value: value, - Data: input, + From: opts.From, + To: contract, + GasPrice: gasPrice, + GasTipCap: gasTipCap, + GasFeeCap: gasFeeCap, + Value: value, + Data: input, + AccessList: opts.AccessList, } return c.transactor.EstimateGas(ensureContext(opts.Context), msg) } From 6143c350ae1ecf3330678be02b4c2745bb6b8134 Mon Sep 17 00:00:00 2001 From: georgehao Date: Thu, 27 Mar 2025 19:22:17 +0800 Subject: [PATCH 24/42] internal/ethapi: CreateAccessList with stateOverrides (#31497) Add support for state overrides in eth_createAccessList. This will make the method consistent with other execution methods. --------- Co-authored-by: Sina Mahmoodi --- internal/ethapi/api.go | 16 ++++++-- internal/ethapi/api_test.go | 73 +++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 3 deletions(-) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 203834fe38..36a5df8087 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1117,12 +1117,13 @@ type accessListResult struct { // CreateAccessList creates an EIP-2930 type AccessList for the given transaction. // Reexec and BlockNrOrHash can be specified to create the accessList on top of a certain state. -func (api *BlockChainAPI) CreateAccessList(ctx context.Context, args TransactionArgs, blockNrOrHash *rpc.BlockNumberOrHash) (*accessListResult, error) { +// StateOverrides can be used to create the accessList while taking into account state changes from previous transactions. +func (api *BlockChainAPI) CreateAccessList(ctx context.Context, args TransactionArgs, blockNrOrHash *rpc.BlockNumberOrHash, stateOverrides *override.StateOverride) (*accessListResult, error) { bNrOrHash := rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber) if blockNrOrHash != nil { bNrOrHash = *blockNrOrHash } - acl, gasUsed, vmerr, err := AccessList(ctx, api.b, bNrOrHash, args) + acl, gasUsed, vmerr, err := AccessList(ctx, api.b, bNrOrHash, args, stateOverrides) if err != nil { return nil, err } @@ -1136,13 +1137,22 @@ func (api *BlockChainAPI) CreateAccessList(ctx context.Context, args Transaction // AccessList creates an access list for the given transaction. // If the accesslist creation fails an error is returned. // If the transaction itself fails, an vmErr is returned. -func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrHash, args TransactionArgs) (acl types.AccessList, gasUsed uint64, vmErr error, err error) { +func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrHash, args TransactionArgs, stateOverrides *override.StateOverride) (acl types.AccessList, gasUsed uint64, vmErr error, err error) { // Retrieve the execution context db, header, err := b.StateAndHeaderByNumberOrHash(ctx, blockNrOrHash) if db == nil || err != nil { return nil, 0, nil, err } + // Apply state overrides immediately after StateAndHeaderByNumberOrHash. + // If not applied here, there could be cases where user-specified overrides (e.g., nonce) + // may conflict with default values from the database, leading to inconsistencies. + if stateOverrides != nil { + if err := stateOverrides.Apply(db, nil); err != nil { + return nil, 0, nil, err + } + } + // Ensure any missing fields are filled, extract the recipient and input data if err = args.setFeeDefaults(ctx, b, header); err != nil { return nil, 0, nil, err diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go index b086d1a6d5..1ed1a8c8d8 100644 --- a/internal/ethapi/api_test.go +++ b/internal/ethapi/api_test.go @@ -3529,3 +3529,76 @@ func testRPCResponseWithFile(t *testing.T, testid int, result interface{}, rpc s func addressToHash(a common.Address) common.Hash { return common.BytesToHash(a.Bytes()) } + +func TestCreateAccessListWithStateOverrides(t *testing.T) { + // Initialize test backend + genesis := &core.Genesis{ + Config: params.TestChainConfig, + Alloc: types.GenesisAlloc{ + common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7"): {Balance: big.NewInt(1000000000000000000)}, + }, + } + backend := newTestBackend(t, 1, genesis, ethash.NewFaker(), nil) + + // Create a new BlockChainAPI instance + api := NewBlockChainAPI(backend) + + // Create test contract code - a simple storage contract + // + // SPDX-License-Identifier: MIT + // pragma solidity ^0.8.0; + // + // contract SimpleStorage { + // uint256 private value; + // + // function retrieve() public view returns (uint256) { + // return value; + // } + // } + var ( + contractCode = hexutil.Bytes(common.Hex2Bytes("6080604052348015600f57600080fd5b506004361060285760003560e01c80632e64cec114602d575b600080fd5b60336047565b604051603e91906067565b60405180910390f35b60008054905090565b6000819050919050565b6061816050565b82525050565b6000602082019050607a6000830184605a565b9291505056")) + // Create state overrides with more complete state + contractAddr = common.HexToAddress("0x1234567890123456789012345678901234567890") + nonce = hexutil.Uint64(1) + overrides = &override.StateOverride{ + contractAddr: override.OverrideAccount{ + Code: &contractCode, + Balance: (*hexutil.Big)(big.NewInt(1000000000000000000)), + Nonce: &nonce, + State: map[common.Hash]common.Hash{ + common.Hash{}: common.HexToHash("0x000000000000000000000000000000000000000000000000000000000000002a"), + }, + }, + } + ) + + // Create transaction arguments with gas and value + var ( + from = common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7") + data = hexutil.Bytes(common.Hex2Bytes("2e64cec1")) // retrieve() + gas = hexutil.Uint64(100000) + args = TransactionArgs{ + From: &from, + To: &contractAddr, + Data: &data, + Gas: &gas, + Value: new(hexutil.Big), + } + ) + // Call CreateAccessList + result, err := api.CreateAccessList(context.Background(), args, nil, overrides) + if err != nil { + t.Fatalf("Failed to create access list: %v", err) + } + if err != nil || result == nil { + t.Fatalf("Failed to create access list: %v", err) + } + require.NotNil(t, result.Accesslist) + + // Verify access list contains the contract address and storage slot + expected := &types.AccessList{{ + Address: contractAddr, + StorageKeys: []common.Hash{{}}, + }} + require.Equal(t, expected, result.Accesslist) +} From 714fa4f2e69a53ad714d6a0a259ab1c8b2032845 Mon Sep 17 00:00:00 2001 From: Delweng Date: Fri, 28 Mar 2025 15:15:13 +0800 Subject: [PATCH 25/42] cmd/geth: update geth subcommand arguments (#31293) --- cmd/geth/chaincmd.go | 35 ++++++++--------- cmd/geth/dbcmd.go | 94 +++++++++++++++++--------------------------- cmd/geth/main.go | 1 - 3 files changed, 53 insertions(+), 77 deletions(-) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 05c8bc4c7c..708dd8fb7a 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -37,7 +37,9 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/internal/debug" "github.com/ethereum/go-ethereum/internal/era" + "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/urfave/cli/v2" @@ -66,7 +68,7 @@ It expects the genesis file as argument.`, Name: "dumpgenesis", Usage: "Dumps genesis block JSON configuration to stdout", ArgsUsage: "", - Flags: append([]cli.Flag{utils.DataDirFlag}, utils.NetworkFlags...), + Flags: slices.Concat([]cli.Flag{utils.DataDirFlag}, utils.NetworkFlags), Description: ` The dumpgenesis command prints the genesis configuration of the network preset if one is set. Otherwise it prints the genesis from the datadir.`, @@ -78,11 +80,11 @@ if one is set. Otherwise it prints the genesis from the datadir.`, ArgsUsage: " ( ... ) ", Flags: slices.Concat([]cli.Flag{ utils.CacheFlag, - utils.SyncModeFlag, utils.GCModeFlag, utils.SnapshotFlag, utils.CacheDatabaseFlag, utils.CacheGCFlag, + utils.NoCompactionFlag, utils.MetricsEnabledFlag, utils.MetricsEnabledExpensiveFlag, utils.MetricsHTTPFlag, @@ -105,7 +107,11 @@ if one is set. Otherwise it prints the genesis from the datadir.`, utils.LogNoHistoryFlag, utils.LogExportCheckpointsFlag, utils.StateHistoryFlag, - }, utils.DatabaseFlags), + }, utils.DatabaseFlags, debug.Flags), + Before: func(ctx *cli.Context) error { + flags.MigrateGlobalFlags(ctx) + return debug.Setup(ctx) + }, Description: ` The import command allows the import of blocks from an RLP-encoded format. This format can be a single file containing multiple RLP-encoded blocks, or multiple files can be given. @@ -119,10 +125,7 @@ to import successfully.`, Name: "export", Usage: "Export blockchain into file", ArgsUsage: " [ ]", - Flags: slices.Concat([]cli.Flag{ - utils.CacheFlag, - utils.SyncModeFlag, - }, utils.DatabaseFlags), + Flags: slices.Concat([]cli.Flag{utils.CacheFlag}, utils.DatabaseFlags), Description: ` Requires a first argument of the file to write to. Optional second and third arguments control the first and @@ -135,12 +138,7 @@ be gzipped.`, Name: "import-history", Usage: "Import an Era archive", ArgsUsage: "", - Flags: slices.Concat([]cli.Flag{ - utils.TxLookupLimitFlag, - }, - utils.DatabaseFlags, - utils.NetworkFlags, - ), + Flags: slices.Concat([]cli.Flag{utils.TxLookupLimitFlag, utils.TransactionHistoryFlag}, utils.DatabaseFlags, utils.NetworkFlags), Description: ` The import-history command will import blocks and their corresponding receipts from Era archives. @@ -151,7 +149,7 @@ from Era archives. Name: "export-history", Usage: "Export blockchain history to Era archives", ArgsUsage: " ", - Flags: slices.Concat(utils.DatabaseFlags), + Flags: utils.DatabaseFlags, Description: ` The export-history command will export blocks and their corresponding receipts into Era archives. Eras are typically packaged in steps of 8192 blocks. @@ -162,10 +160,7 @@ into Era archives. Eras are typically packaged in steps of 8192 blocks. Name: "import-preimages", Usage: "Import the preimage database from an RLP stream", ArgsUsage: "", - Flags: slices.Concat([]cli.Flag{ - utils.CacheFlag, - utils.SyncModeFlag, - }, utils.DatabaseFlags), + Flags: slices.Concat([]cli.Flag{utils.CacheFlag}, utils.DatabaseFlags), Description: ` The import-preimages command imports hash preimages from an RLP encoded stream. It's deprecated, please use "geth db import" instead. @@ -435,6 +430,10 @@ func importHistory(ctx *cli.Context) error { network = "mainnet" case ctx.Bool(utils.SepoliaFlag.Name): network = "sepolia" + case ctx.Bool(utils.HoleskyFlag.Name): + network = "holesky" + case ctx.Bool(utils.HoodiFlag.Name): + network = "hoodi" } } else { // No network flag set, try to determine network based on files diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index cd41c57c75..44a52521f0 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -86,12 +86,10 @@ Remove blockchain and state databases`, }, } dbInspectCmd = &cli.Command{ - Action: inspect, - Name: "inspect", - ArgsUsage: " ", - Flags: slices.Concat([]cli.Flag{ - utils.SyncModeFlag, - }, utils.NetworkFlags, utils.DatabaseFlags), + Action: inspect, + Name: "inspect", + ArgsUsage: " ", + Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags), Usage: "Inspect the storage size for each type of data in the database", Description: `This commands iterates the entire database. If the optional 'prefix' and 'start' arguments are provided, then the iteration is limited to the given subset of data.`, } @@ -109,16 +107,13 @@ a data corruption.`, Action: dbStats, Name: "stats", Usage: "Print leveldb statistics", - Flags: slices.Concat([]cli.Flag{ - utils.SyncModeFlag, - }, utils.NetworkFlags, utils.DatabaseFlags), + Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags), } dbCompactCmd = &cli.Command{ Action: dbCompact, Name: "compact", Usage: "Compact leveldb database. WARNING: May take a very long time", Flags: slices.Concat([]cli.Flag{ - utils.SyncModeFlag, utils.CacheFlag, utils.CacheDatabaseFlag, }, utils.NetworkFlags, utils.DatabaseFlags), @@ -127,13 +122,11 @@ WARNING: This operation may take a very long time to finish, and may cause datab corruption if it is aborted during execution'!`, } dbGetCmd = &cli.Command{ - Action: dbGet, - Name: "get", - Usage: "Show the value of a database key", - ArgsUsage: "", - Flags: slices.Concat([]cli.Flag{ - utils.SyncModeFlag, - }, utils.NetworkFlags, utils.DatabaseFlags), + Action: dbGet, + Name: "get", + Usage: "Show the value of a database key", + ArgsUsage: "", + Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags), Description: "This command looks up the specified database key from the database.", } dbDeleteCmd = &cli.Command{ @@ -141,9 +134,7 @@ corruption if it is aborted during execution'!`, Name: "delete", Usage: "Delete a database key (WARNING: may corrupt your database)", ArgsUsage: "", - Flags: slices.Concat([]cli.Flag{ - utils.SyncModeFlag, - }, utils.NetworkFlags, utils.DatabaseFlags), + Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags), Description: `This command deletes the specified database key from the database. WARNING: This is a low-level operation which may cause database corruption!`, } @@ -152,59 +143,47 @@ WARNING: This is a low-level operation which may cause database corruption!`, Name: "put", Usage: "Set the value of a database key (WARNING: may corrupt your database)", ArgsUsage: " ", - Flags: slices.Concat([]cli.Flag{ - utils.SyncModeFlag, - }, utils.NetworkFlags, utils.DatabaseFlags), + Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags), Description: `This command sets a given database key to the given value. WARNING: This is a low-level operation which may cause database corruption!`, } dbGetSlotsCmd = &cli.Command{ - Action: dbDumpTrie, - Name: "dumptrie", - Usage: "Show the storage key/values of a given storage trie", - ArgsUsage: " ", - Flags: slices.Concat([]cli.Flag{ - utils.SyncModeFlag, - }, utils.NetworkFlags, utils.DatabaseFlags), + Action: dbDumpTrie, + Name: "dumptrie", + Usage: "Show the storage key/values of a given storage trie", + ArgsUsage: " ", + Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags), Description: "This command looks up the specified database key from the database.", } dbDumpFreezerIndex = &cli.Command{ - Action: freezerInspect, - Name: "freezer-index", - Usage: "Dump out the index of a specific freezer table", - ArgsUsage: " ", - Flags: slices.Concat([]cli.Flag{ - utils.SyncModeFlag, - }, utils.NetworkFlags, utils.DatabaseFlags), + Action: freezerInspect, + Name: "freezer-index", + Usage: "Dump out the index of a specific freezer table", + ArgsUsage: " ", + Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags), Description: "This command displays information about the freezer index.", } dbImportCmd = &cli.Command{ - Action: importLDBdata, - Name: "import", - Usage: "Imports leveldb-data from an exported RLP dump.", - ArgsUsage: " has .gz suffix, gzip compression will be used.", - ArgsUsage: " ", - Flags: slices.Concat([]cli.Flag{ - utils.SyncModeFlag, - }, utils.NetworkFlags, utils.DatabaseFlags), + Action: exportChaindata, + Name: "export", + Usage: "Exports the chain data into an RLP dump. If the has .gz suffix, gzip compression will be used.", + ArgsUsage: " ", + Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags), Description: "Exports the specified chain data to an RLP encoded stream, optionally gzip-compressed.", } dbMetadataCmd = &cli.Command{ - Action: showMetaData, - Name: "metadata", - Usage: "Shows metadata about the chain status.", - Flags: slices.Concat([]cli.Flag{ - utils.SyncModeFlag, - }, utils.NetworkFlags, utils.DatabaseFlags), + Action: showMetaData, + Name: "metadata", + Usage: "Shows metadata about the chain status.", + Flags: slices.Concat(utils.NetworkFlags, utils.DatabaseFlags), Description: "Shows metadata about the chain status.", } dbInspectHistoryCmd = &cli.Command{ @@ -213,7 +192,6 @@ WARNING: This is a low-level operation which may cause database corruption!`, Usage: "Inspect the state history within block range", ArgsUsage: "
[OPTIONAL ]", Flags: slices.Concat([]cli.Flag{ - utils.SyncModeFlag, &cli.Uint64Flag{ Name: "start", Usage: "block number of the range start, zero means earliest history", diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 9c0c0d9dfc..cd74fb7b6a 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -142,7 +142,6 @@ var ( utils.VMTraceJsonConfigFlag, utils.NetworkIdFlag, utils.EthStatsURLFlag, - utils.NoCompactionFlag, utils.GpoBlocksFlag, utils.GpoPercentileFlag, utils.GpoMaxGasPriceFlag, From 141968a48b45bf343c5921057eca158df5d3c96e Mon Sep 17 00:00:00 2001 From: rekyyang <511965710@qq.com> Date: Fri, 28 Mar 2025 15:16:37 +0800 Subject: [PATCH 26/42] core/txpool/legacypool: fix data race in checkDelegationLimit (#31475) --- core/txpool/legacypool/legacypool.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/core/txpool/legacypool/legacypool.go b/core/txpool/legacypool/legacypool.go index 7a0095a5ad..dafd185836 100644 --- a/core/txpool/legacypool/legacypool.go +++ b/core/txpool/legacypool/legacypool.go @@ -618,7 +618,7 @@ func (pool *LegacyPool) checkDelegationLimit(tx *types.Transaction) error { from, _ := types.Sender(pool.signer, tx) // validated // Short circuit if the sender has neither delegation nor pending delegation. - if pool.currentState.GetCodeHash(from) == types.EmptyCodeHash && len(pool.all.auths[from]) == 0 { + if pool.currentState.GetCodeHash(from) == types.EmptyCodeHash && pool.all.delegationTxsCount(from) == 0 { return nil } pending := pool.pending[from] @@ -1849,6 +1849,13 @@ func (t *lookup) removeAuthorities(tx *types.Transaction) { } } +// delegationTxsCount returns the number of pending authorizations for the specified address. +func (t *lookup) delegationTxsCount(addr common.Address) int { + t.lock.RLock() + defer t.lock.RUnlock() + return len(t.auths[addr]) +} + // numSlots calculates the number of slots needed for a single transaction. func numSlots(tx *types.Transaction) int { return int((tx.Size() + txSlotSize - 1) / txSlotSize) From 32f36a674990b5e2316321415d595c67b2f00b4f Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Fri, 28 Mar 2025 19:32:24 +0800 Subject: [PATCH 27/42] core/txpool: fix nonce assignment in local tracker (#31496) Fixes #31494 --- core/txpool/locals/tx_tracker_test.go | 39 +++++++++++++++++++++++++++ core/txpool/txpool.go | 13 +++++++-- eth/api_backend.go | 2 +- 3 files changed, 51 insertions(+), 3 deletions(-) diff --git a/core/txpool/locals/tx_tracker_test.go b/core/txpool/locals/tx_tracker_test.go index cb6b9b3453..5585589b6c 100644 --- a/core/txpool/locals/tx_tracker_test.go +++ b/core/txpool/locals/tx_tracker_test.go @@ -108,6 +108,19 @@ func (env *testEnv) makeTx(nonce uint64, gasPrice *big.Int) *types.Transaction { return tx } +func (env *testEnv) makeTxs(n int) []*types.Transaction { + head := env.chain.CurrentHeader() + state, _ := env.chain.StateAt(head.Root) + nonce := state.GetNonce(address) + + var txs []*types.Transaction + for i := 0; i < n; i++ { + tx, _ := types.SignTx(types.NewTransaction(nonce+uint64(i), common.Address{0x00}, big.NewInt(1000), params.TxGas, big.NewInt(params.GWei), nil), signer, key) + txs = append(txs, tx) + } + return txs +} + func (env *testEnv) commit() { head := env.chain.CurrentBlock() block := env.chain.GetBlock(head.Hash(), head.Number.Uint64()) @@ -177,3 +190,29 @@ func TestRejectInvalids(t *testing.T) { } } } + +func TestResubmit(t *testing.T) { + env := newTestEnv(t, 10, 0, "") + defer env.close() + + txs := env.makeTxs(10) + txsA := txs[:len(txs)/2] + txsB := txs[len(txs)/2:] + env.pool.Add(txsA, true) + pending, queued := env.pool.ContentFrom(address) + if len(pending) != len(txsA) || len(queued) != 0 { + t.Fatalf("Unexpected txpool content: %d, %d", len(pending), len(queued)) + } + env.tracker.TrackAll(txs) + + resubmit, all := env.tracker.recheck(true) + if len(resubmit) != len(txsB) { + t.Fatalf("Unexpected transactions to resubmit, got: %d, want: %d", len(resubmit), len(txsB)) + } + if len(all) == 0 || len(all[address]) == 0 { + t.Fatalf("Unexpected transactions being tracked, got: %d, want: %d", 0, len(txs)) + } + if len(all[address]) != len(txs) { + t.Fatalf("Unexpected transactions being tracked, got: %d, want: %d", len(all[address]), len(txs)) + } +} diff --git a/core/txpool/txpool.go b/core/txpool/txpool.go index c5f7718057..649c5d1a78 100644 --- a/core/txpool/txpool.go +++ b/core/txpool/txpool.go @@ -466,9 +466,9 @@ func (p *TxPool) SubscribeTransactions(ch chan<- core.NewTxsEvent, reorgs bool) return p.subs.Track(event.JoinSubscriptions(subs...)) } -// Nonce returns the next nonce of an account, with all transactions executable +// PoolNonce returns the next nonce of an account, with all transactions executable // by the pool already applied on top. -func (p *TxPool) Nonce(addr common.Address) uint64 { +func (p *TxPool) PoolNonce(addr common.Address) uint64 { // Since (for now) accounts are unique to subpools, only one pool will have // (at max) a non-state nonce. To avoid stateful lookups, just return the // highest nonce for now. @@ -481,6 +481,15 @@ func (p *TxPool) Nonce(addr common.Address) uint64 { return nonce } +// Nonce returns the next nonce of an account at the current chain head. Unlike +// PoolNonce, this function does not account for pending executable transactions. +func (p *TxPool) Nonce(addr common.Address) uint64 { + p.stateLock.RLock() + defer p.stateLock.RUnlock() + + return p.state.GetNonce(addr) +} + // Stats retrieves the current pool stats, namely the number of pending and the // number of queued (non-executable) transactions. func (p *TxPool) Stats() (int, int) { diff --git a/eth/api_backend.go b/eth/api_backend.go index b08e38afe5..c8c5ca707f 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -327,7 +327,7 @@ func (b *EthAPIBackend) GetTransaction(ctx context.Context, txHash common.Hash) } func (b *EthAPIBackend) GetPoolNonce(ctx context.Context, addr common.Address) (uint64, error) { - return b.eth.txPool.Nonce(addr), nil + return b.eth.txPool.PoolNonce(addr), nil } func (b *EthAPIBackend) Stats() (runnable int, blocked int) { From c8a9a9c0917dd57d077a79044e65dbbdd421458b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Fri, 28 Mar 2025 16:17:28 +0100 Subject: [PATCH 28/42] core/filtermaps: revert to unindexed mode in case of indexing error (#31500) This PR changes log indexer error handling so that if an indexing error happens then it disables the indexer and reverts to unindexed more without resetting the database (except in case of a failed database init). Resetting the database on the first error would probably be overkill as a client update might fix this without having to reindex the entire history. It would also make debugging very hard. On the other hand, these errors do not resolve themselves automatically so constantly retrying makes no sense either. With these changes a new attempt to resume indexing is made every time the client is restarted. The PR also fixes https://github.com/ethereum/go-ethereum/issues/31491 which originated from the tail indexer trying to resume processing a failed map renderer. --------- Co-authored-by: Felix Lange --- core/filtermaps/chain_view.go | 6 ++- core/filtermaps/indexer.go | 97 ++++++++++++++++++++++------------- 2 files changed, 65 insertions(+), 38 deletions(-) diff --git a/core/filtermaps/chain_view.go b/core/filtermaps/chain_view.go index bc0de47ded..a8cf53b1c0 100644 --- a/core/filtermaps/chain_view.go +++ b/core/filtermaps/chain_view.go @@ -83,7 +83,11 @@ func (cv *ChainView) getReceipts(number uint64) types.Receipts { if number > cv.headNumber { panic("invalid block number") } - return cv.chain.GetReceiptsByHash(cv.blockHash(number)) + blockHash := cv.blockHash(number) + if blockHash == (common.Hash{}) { + log.Error("Chain view: block hash unavailable", "number", number, "head", cv.headNumber) + } + return cv.chain.GetReceiptsByHash(blockHash) } // limitedView returns a new chain view that is a truncated version of the parent view. diff --git a/core/filtermaps/indexer.go b/core/filtermaps/indexer.go index 026b3b4f38..6732dc85ea 100644 --- a/core/filtermaps/indexer.go +++ b/core/filtermaps/indexer.go @@ -17,6 +17,7 @@ package filtermaps import ( + "errors" "math" "time" @@ -49,18 +50,15 @@ func (f *FilterMaps) indexerLoop() { continue } if err := f.init(); err != nil { - log.Error("Error initializing log index; reverting to unindexed mode", "error", err) - f.reset() - f.disabled = true - close(f.disabledCh) + f.disableForError("initialization", err) + f.reset() // remove broken index from DB return } } if !f.targetHeadIndexed() { - if !f.tryIndexHead() { - // either shutdown or unexpected error; in the latter case ensure - // that proper shutdown is still possible. - f.processSingleEvent(true) + if err := f.tryIndexHead(); err != nil && err != errChainUpdate { + f.disableForError("head rendering", err) + return } } else { if f.finalBlock != f.lastFinal { @@ -69,13 +67,36 @@ func (f *FilterMaps) indexerLoop() { } f.lastFinal = f.finalBlock } - if f.tryIndexTail() && f.tryUnindexTail() { - f.waitForNewHead() + if done, err := f.tryIndexTail(); err != nil { + f.disableForError("tail rendering", err) + return + } else if !done { + continue } + if done, err := f.tryUnindexTail(); err != nil { + f.disableForError("tail unindexing", err) + return + } else if !done { + continue + } + // tail indexing/unindexing is done; if head is also indexed then + // wait here until there is a new head + f.waitForNewHead() } } } +// disableForError is called when the indexer encounters a database error, for example a +// missing receipt. We can't continue operating when the database is broken, so the +// indexer goes into disabled state. +// Note that the partial index is left in disk; maybe a client update can fix the +// issue without reindexing. +func (f *FilterMaps) disableForError(op string, err error) { + log.Error("Log index "+op+" failed, reverting to unindexed mode", "error", err) + f.disabled = true + close(f.disabledCh) +} + type targetUpdate struct { targetView *ChainView historyCutoff, finalBlock uint64 @@ -116,14 +137,15 @@ func (f *FilterMaps) SetBlockProcessing(blockProcessing bool) { // WaitIdle blocks until the indexer is in an idle state while synced up to the // latest targetView. func (f *FilterMaps) WaitIdle() { - if f.disabled { - f.closeWg.Wait() - return - } for { ch := make(chan bool) - f.waitIdleCh <- ch - if <-ch { + select { + case f.waitIdleCh <- ch: + if <-ch { + return + } + case <-f.disabledCh: + f.closeWg.Wait() return } } @@ -196,16 +218,16 @@ func (f *FilterMaps) setTarget(target targetUpdate) { f.finalBlock = target.finalBlock } -// tryIndexHead tries to render head maps according to the current targetView -// and returns true if successful. -func (f *FilterMaps) tryIndexHead() bool { +// tryIndexHead tries to render head maps according to the current targetView. +// Should be called when targetHeadIndexed returns false. If this function +// returns no error then either stop is true or head indexing is finished. +func (f *FilterMaps) tryIndexHead() error { headRenderer, err := f.renderMapsBefore(math.MaxUint32) if err != nil { - log.Error("Error creating log index head renderer", "error", err) - return false + return err } if headRenderer == nil { - return true + return errors.New("head indexer has nothing to do") // tryIndexHead should be called when head is not indexed } if !f.startedHeadIndex { f.lastLogHeadIndex = time.Now() @@ -230,8 +252,7 @@ func (f *FilterMaps) tryIndexHead() bool { f.lastLogHeadIndex = time.Now() } }); err != nil { - log.Error("Log index head rendering failed", "error", err) - return false + return err } if f.loggedHeadIndex && f.indexedRange.hasIndexedBlocks() { log.Info("Log index head rendering finished", @@ -240,7 +261,7 @@ func (f *FilterMaps) tryIndexHead() bool { "elapsed", common.PrettyDuration(time.Since(f.startedHeadIndexAt))) } f.loggedHeadIndex, f.startedHeadIndex = false, false - return true + return nil } // tryIndexTail tries to render tail epochs until the tail target block is @@ -248,7 +269,7 @@ func (f *FilterMaps) tryIndexHead() bool { // Note that tail indexing is only started if the log index head is fully // rendered according to targetView and is suspended as soon as the targetView // is changed. -func (f *FilterMaps) tryIndexTail() bool { +func (f *FilterMaps) tryIndexTail() (bool, error) { for { firstEpoch := f.indexedRange.maps.First() >> f.logMapsPerEpoch if firstEpoch == 0 || !f.needTailEpoch(firstEpoch-1) { @@ -256,7 +277,7 @@ func (f *FilterMaps) tryIndexTail() bool { } f.processEvents() if f.stop || !f.targetHeadIndexed() { - return false + return false, nil } // resume process if tail rendering was interrupted because of head rendering tailRenderer := f.tailRenderer @@ -268,8 +289,7 @@ func (f *FilterMaps) tryIndexTail() bool { var err error tailRenderer, err = f.renderMapsBefore(f.indexedRange.maps.First()) if err != nil { - log.Error("Error creating log index tail renderer", "error", err) - return false + return false, err } } if tailRenderer == nil { @@ -302,13 +322,16 @@ func (f *FilterMaps) tryIndexTail() bool { f.lastLogTailIndex = time.Now() } }) - if err != nil && f.needTailEpoch(firstEpoch-1) { + if err != nil && !f.needTailEpoch(firstEpoch-1) { // stop silently if cutoff point has move beyond epoch boundary while rendering - log.Error("Log index tail rendering failed", "error", err) + return true, nil + } + if err != nil { + return false, err } if !done { f.tailRenderer = tailRenderer // only keep tail renderer if interrupted by stopCb - return false + return false, nil } } if f.loggedTailIndex && f.indexedRange.hasIndexedBlocks() { @@ -318,14 +341,14 @@ func (f *FilterMaps) tryIndexTail() bool { "elapsed", common.PrettyDuration(time.Since(f.startedTailIndexAt))) f.loggedTailIndex = false } - return true + return true, nil } // tryUnindexTail removes entire epochs of log index data as long as the first // fully indexed block is at least as old as the tail target. // Note that unindexing is very quick as it only removes continuous ranges of // data from the database and is also called while running head indexing. -func (f *FilterMaps) tryUnindexTail() bool { +func (f *FilterMaps) tryUnindexTail() (bool, error) { for { firstEpoch := (f.indexedRange.maps.First() - f.indexedRange.tailPartialEpoch) >> f.logMapsPerEpoch if f.needTailEpoch(firstEpoch) { @@ -333,7 +356,7 @@ func (f *FilterMaps) tryUnindexTail() bool { } f.processEvents() if f.stop { - return false + return false, nil } if !f.startedTailUnindex { f.startedTailUnindexAt = time.Now() @@ -343,7 +366,7 @@ func (f *FilterMaps) tryUnindexTail() bool { } if err := f.deleteTailEpoch(firstEpoch); err != nil { log.Error("Log index tail epoch unindexing failed", "error", err) - return false + return false, err } } if f.startedTailUnindex && f.indexedRange.hasIndexedBlocks() { @@ -354,7 +377,7 @@ func (f *FilterMaps) tryUnindexTail() bool { "elapsed", common.PrettyDuration(time.Since(f.startedTailUnindexAt))) f.startedTailUnindex = false } - return true + return true, nil } // needTailEpoch returns true if the given tail epoch needs to be kept From ffa315f7460b6f050e89b63ca63876d974fb19bf Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Mon, 31 Mar 2025 09:49:19 +0200 Subject: [PATCH 29/42] .gitignore: ignore binaries (#31531) Ignores all hand-built binaries (built with go build, everything built with make is already ignored) --- .gitignore | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.gitignore b/.gitignore index 7000fedd25..269455db7a 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,16 @@ profile.cov .vscode tests/spec-tests/ + +# binaries +cmd/abidump/abidump +cmd/abigen/abigen +cmd/blsync/blsync +cmd/clef/clef +cmd/devp2p/devp2p +cmd/era/era +cmd/ethkey/ethkey +cmd/evm/evm +cmd/geth/geth +cmd/rlpdump/rlpdump +cmd/workload/workload \ No newline at end of file From 14d576c002309e38864f9afd95e7305e35a68035 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Mon, 31 Mar 2025 14:47:56 +0200 Subject: [PATCH 30/42] core/filtermaps: hashdb safe delete range (#31525) This PR adds `rawdb.SafeDeleteRange` and uses it for range deletion in `core/filtermaps`. This includes deleting the old bloombits database, resetting the log index database and removing index data for unindexed tail epochs (which previously weren't properly implemented for the fallback case). `SafeDeleteRange` either calls `ethdb.DeleteRange` if the node uses the new path based state scheme or uses an iterator based fallback method that safely skips trie nodes in the range if the old hash based state scheme is used. Note that `ethdb.DeleteRange` also has its own iterator based fallback implementation in `ethdb/leveldb`. If a path based state scheme is used and the backing db is pebble (as it is on the majority of new nodes) then `rawdb.SafeDeleteRange` uses the fast native range delete. Also note that `rawdb.SafeDeleteRange` has different semantics from `ethdb.DeleteRange`, it does not automatically return if the operation takes a long time. Instead it receives a `stopCallback` that can interrupt the process if necessary. This is because in the safe mode potentially a lot of entries are iterated without being deleted (this is definitely the case when deleting the old bloombits database which has a single byte prefix) and therefore restarting the process every time a fixed number of entries have been iterated would result in a quadratic run time in the number of skipped entries. When running in safe mode, unindexing an epoch takes about a second, removing bloombits takes around 10s while resetting a full log index might take a few minutes. If a range delete operation takes a significant amount of time then log messages are printed. Also, any range delete operation can be interrupted by shutdown (tail uinindexing can also be interrupted by head indexing, similarly to how tail indexing works). If the last unindexed epoch might have "dirty" index data left then the indexed map range points to the first valid epoch and `cleanedEpochsBefore` points to the previous, potentially dirty one. At startup it is always assumed that the epoch before the first fully indexed one might be dirty. New tail maps are never rendered and also no further maps are unindexed before the previous unindexing is properly cleaned up. --------- Co-authored-by: Gary Rong Co-authored-by: Felix Lange --- core/filtermaps/filtermaps.go | 210 ++++++++++++++++++++------------ core/filtermaps/indexer.go | 32 ++--- core/rawdb/accessors_indexes.go | 38 +++--- core/rawdb/database.go | 73 +++++++++++ eth/backend.go | 7 +- ethdb/database.go | 9 +- ethdb/leveldb/leveldb.go | 4 +- 7 files changed, 256 insertions(+), 117 deletions(-) diff --git a/core/filtermaps/filtermaps.go b/core/filtermaps/filtermaps.go index db7ab0a426..5722f17daa 100644 --- a/core/filtermaps/filtermaps.go +++ b/core/filtermaps/filtermaps.go @@ -29,7 +29,6 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/ethdb/leveldb" "github.com/ethereum/go-ethereum/log" ) @@ -59,6 +58,7 @@ type FilterMaps struct { closeCh chan struct{} closeWg sync.WaitGroup history uint64 + hashScheme bool // use hashdb-safe delete range method exportFileName string Params @@ -67,10 +67,11 @@ type FilterMaps struct { // fields written by the indexer and read by matcher backend. Indexer can // read them without a lock and write them under indexLock write lock. // Matcher backend can read them under indexLock read lock. - indexLock sync.RWMutex - indexedRange filterMapsRange - indexedView *ChainView // always consistent with the log index - hasTempRange bool + indexLock sync.RWMutex + indexedRange filterMapsRange + cleanedEpochsBefore uint32 // all unindexed data cleaned before this point + indexedView *ChainView // always consistent with the log index + hasTempRange bool // also accessed by indexer and matcher backend but no locking needed. filterMapCache *lru.Cache[uint32, filterMap] @@ -180,6 +181,10 @@ type Config struct { // This option enables the checkpoint JSON file generator. // If set, the given file will be updated with checkpoint information. ExportFileName string + + // expect trie nodes of hash based state scheme in the filtermaps key range; + // use safe iterator based implementation of DeleteRange that skips them + HashScheme bool } // NewFilterMaps creates a new FilterMaps and starts the indexer. @@ -197,6 +202,7 @@ func NewFilterMaps(db ethdb.KeyValueStore, initView *ChainView, historyCutoff, f blockProcessingCh: make(chan bool, 1), history: config.History, disabled: config.Disabled, + hashScheme: config.HashScheme, disabledCh: make(chan struct{}), exportFileName: config.ExportFileName, Params: params, @@ -208,15 +214,17 @@ func NewFilterMaps(db ethdb.KeyValueStore, initView *ChainView, historyCutoff, f maps: common.NewRange(rs.MapsFirst, rs.MapsAfterLast-rs.MapsFirst), tailPartialEpoch: rs.TailPartialEpoch, }, - historyCutoff: historyCutoff, - finalBlock: finalBlock, - matcherSyncCh: make(chan *FilterMapsMatcherBackend), - matchers: make(map[*FilterMapsMatcherBackend]struct{}), - filterMapCache: lru.NewCache[uint32, filterMap](cachedFilterMaps), - lastBlockCache: lru.NewCache[uint32, lastBlockOfMap](cachedLastBlocks), - lvPointerCache: lru.NewCache[uint64, uint64](cachedLvPointers), - baseRowsCache: lru.NewCache[uint64, [][]uint32](cachedBaseRows), - renderSnapshots: lru.NewCache[uint64, *renderedMap](cachedRenderSnapshots), + // deleting last unindexed epoch might have been interrupted by shutdown + cleanedEpochsBefore: max(rs.MapsFirst>>params.logMapsPerEpoch, 1) - 1, + historyCutoff: historyCutoff, + finalBlock: finalBlock, + matcherSyncCh: make(chan *FilterMapsMatcherBackend), + matchers: make(map[*FilterMapsMatcherBackend]struct{}), + filterMapCache: lru.NewCache[uint32, filterMap](cachedFilterMaps), + lastBlockCache: lru.NewCache[uint32, lastBlockOfMap](cachedLastBlocks), + lvPointerCache: lru.NewCache[uint64, uint64](cachedLvPointers), + baseRowsCache: lru.NewCache[uint64, [][]uint32](cachedBaseRows), + renderSnapshots: lru.NewCache[uint64, *renderedMap](cachedRenderSnapshots), } // Set initial indexer target. @@ -301,14 +309,24 @@ func (f *FilterMaps) reset() { // deleting the range first ensures that resetDb will be called again at next // startup and any leftover data will be removed even if it cannot finish now. rawdb.DeleteFilterMapsRange(f.db) - f.safeDeleteRange(rawdb.DeleteFilterMapsDb, "Resetting log index database") + f.safeDeleteWithLogs(rawdb.DeleteFilterMapsDb, "Resetting log index database", f.isShuttingDown) +} + +// isShuttingDown returns true if FilterMaps is shutting down. +func (f *FilterMaps) isShuttingDown() bool { + select { + case <-f.closeCh: + return true + default: + return false + } } // init initializes an empty log index according to the current targetView. func (f *FilterMaps) init() error { // ensure that there is no remaining data in the filter maps key range - if !f.safeDeleteRange(rawdb.DeleteFilterMapsDb, "Resetting log index database") { - return errors.New("could not reset log index database") + if err := f.safeDeleteWithLogs(rawdb.DeleteFilterMapsDb, "Resetting log index database", f.isShuttingDown); err != nil { + return err } f.indexLock.Lock() @@ -358,38 +376,37 @@ func (f *FilterMaps) init() error { // removeBloomBits removes old bloom bits data from the database. func (f *FilterMaps) removeBloomBits() { - f.safeDeleteRange(rawdb.DeleteBloomBitsDb, "Removing old bloom bits database") + f.safeDeleteWithLogs(rawdb.DeleteBloomBitsDb, "Removing old bloom bits database", f.isShuttingDown) f.closeWg.Done() } -// safeDeleteRange calls the specified database range deleter function -// repeatedly as long as it returns leveldb.ErrTooManyKeys. -// This wrapper is necessary because of the leveldb fallback implementation -// of DeleteRange. -func (f *FilterMaps) safeDeleteRange(removeFn func(ethdb.KeyValueRangeDeleter) error, action string) bool { - start := time.Now() - var retry bool - for { - err := removeFn(f.db) - if err == nil { - if retry { - log.Info(action+" finished", "elapsed", time.Since(start)) - } - return true +// safeDeleteWithLogs is a wrapper for a function that performs a safe range +// delete operation using rawdb.SafeDeleteRange. It emits log messages if the +// process takes long enough to call the stop callback. +func (f *FilterMaps) safeDeleteWithLogs(deleteFn func(db ethdb.KeyValueStore, hashScheme bool, stopCb func(bool) bool) error, action string, stopCb func() bool) error { + var ( + start = time.Now() + logPrinted bool + lastLogPrinted = start + ) + switch err := deleteFn(f.db, f.hashScheme, func(deleted bool) bool { + if deleted && !logPrinted || time.Since(lastLogPrinted) > time.Second*10 { + log.Info(action+" in progress...", "elapsed", common.PrettyDuration(time.Since(start))) + logPrinted, lastLogPrinted = true, time.Now() } - if err != leveldb.ErrTooManyKeys { - log.Error(action+" failed", "error", err) - return false - } - select { - case <-f.closeCh: - return false - default: - } - if !retry { - log.Info(action+" in progress...", "elapsed", time.Since(start)) - retry = true + return stopCb() + }); { + case err == nil: + if logPrinted { + log.Info(action+" finished", "elapsed", common.PrettyDuration(time.Since(start))) } + return nil + case errors.Is(err, rawdb.ErrDeleteRangeInterrupted): + log.Warn(action+" interrupted", "elapsed", common.PrettyDuration(time.Since(start))) + return err + default: + log.Error(action+" failed", "error", err) + return err } } @@ -658,54 +675,97 @@ func (f *FilterMaps) deleteLastBlockOfMap(batch ethdb.Batch, mapIndex uint32) { rawdb.DeleteFilterMapLastBlock(batch, mapIndex) } -// deleteTailEpoch deletes index data from the earliest, either fully or partially -// indexed epoch. The last block pointer for the last map of the epoch and the -// corresponding block log value pointer are retained as these are always assumed -// to be available for each epoch. -func (f *FilterMaps) deleteTailEpoch(epoch uint32) error { +// deleteTailEpoch deletes index data from the specified epoch. The last block +// pointer for the last map of the epoch and the corresponding block log value +// pointer are retained as these are always assumed to be available for each +// epoch as boundary markers. +// The function returns true if all index data related to the epoch (except for +// the boundary markers) has been fully removed. +func (f *FilterMaps) deleteTailEpoch(epoch uint32) (bool, error) { f.indexLock.Lock() defer f.indexLock.Unlock() + // determine epoch boundaries firstMap := epoch << f.logMapsPerEpoch lastBlock, _, err := f.getLastBlockOfMap(firstMap + f.mapsPerEpoch - 1) if err != nil { - return fmt.Errorf("failed to retrieve last block of deleted epoch %d: %v", epoch, err) + return false, fmt.Errorf("failed to retrieve last block of deleted epoch %d: %v", epoch, err) } var firstBlock uint64 if epoch > 0 { firstBlock, _, err = f.getLastBlockOfMap(firstMap - 1) if err != nil { - return fmt.Errorf("failed to retrieve last block before deleted epoch %d: %v", epoch, err) + return false, fmt.Errorf("failed to retrieve last block before deleted epoch %d: %v", epoch, err) } firstBlock++ } - fmr := f.indexedRange - if f.indexedRange.maps.First() == firstMap && - f.indexedRange.maps.AfterLast() > firstMap+f.mapsPerEpoch && - f.indexedRange.tailPartialEpoch == 0 { - fmr.maps.SetFirst(firstMap + f.mapsPerEpoch) - fmr.blocks.SetFirst(lastBlock + 1) - } else if f.indexedRange.maps.First() == firstMap+f.mapsPerEpoch { + // update rendered range if necessary + var ( + fmr = f.indexedRange + firstEpoch = f.indexedRange.maps.First() >> f.logMapsPerEpoch + afterLastEpoch = (f.indexedRange.maps.AfterLast() + f.mapsPerEpoch - 1) >> f.logMapsPerEpoch + ) + if f.indexedRange.tailPartialEpoch != 0 && firstEpoch > 0 { + firstEpoch-- + } + switch { + case epoch < firstEpoch: + // cleanup of already unindexed epoch; range not affected + case epoch == firstEpoch && epoch+1 < afterLastEpoch: + // first fully or partially rendered epoch and there is at least one + // rendered map in the next epoch; remove from indexed range fmr.tailPartialEpoch = 0 + fmr.maps.SetFirst((epoch + 1) << f.logMapsPerEpoch) + fmr.blocks.SetFirst(lastBlock + 1) + f.setRange(f.db, f.indexedView, fmr, false) + default: + // cannot be cleaned or unindexed; return with error + return false, errors.New("invalid tail epoch number") + } + // remove index data + if err := f.safeDeleteWithLogs(func(db ethdb.KeyValueStore, hashScheme bool, stopCb func(bool) bool) error { + first := f.mapRowIndex(firstMap, 0) + count := f.mapRowIndex(firstMap+f.mapsPerEpoch, 0) - first + if err := rawdb.DeleteFilterMapRows(f.db, common.NewRange(first, count), hashScheme, stopCb); err != nil { + return err + } + for mapIndex := firstMap; mapIndex < firstMap+f.mapsPerEpoch; mapIndex++ { + f.filterMapCache.Remove(mapIndex) + } + delMapRange := common.NewRange(firstMap, f.mapsPerEpoch-1) // keep last entry + if err := rawdb.DeleteFilterMapLastBlocks(f.db, delMapRange, hashScheme, stopCb); err != nil { + return err + } + for mapIndex := firstMap; mapIndex < firstMap+f.mapsPerEpoch-1; mapIndex++ { + f.lastBlockCache.Remove(mapIndex) + } + delBlockRange := common.NewRange(firstBlock, lastBlock-firstBlock) // keep last entry + if err := rawdb.DeleteBlockLvPointers(f.db, delBlockRange, hashScheme, stopCb); err != nil { + return err + } + for blockNumber := firstBlock; blockNumber < lastBlock; blockNumber++ { + f.lvPointerCache.Remove(blockNumber) + } + return nil + }, fmt.Sprintf("Deleting tail epoch #%d", epoch), func() bool { + f.processEvents() + return f.stop || !f.targetHeadIndexed() + }); err == nil { + // everything removed; mark as cleaned and report success + if f.cleanedEpochsBefore == epoch { + f.cleanedEpochsBefore = epoch + 1 + } + return true, nil } else { - return errors.New("invalid tail epoch number") + // more data left in epoch range; mark as dirty and report unfinished + if f.cleanedEpochsBefore > epoch { + f.cleanedEpochsBefore = epoch + } + if errors.Is(err, rawdb.ErrDeleteRangeInterrupted) { + return false, nil + } + return false, err } - f.setRange(f.db, f.indexedView, fmr, false) - first := f.mapRowIndex(firstMap, 0) - count := f.mapRowIndex(firstMap+f.mapsPerEpoch, 0) - first - rawdb.DeleteFilterMapRows(f.db, common.NewRange(first, count)) - for mapIndex := firstMap; mapIndex < firstMap+f.mapsPerEpoch; mapIndex++ { - f.filterMapCache.Remove(mapIndex) - } - rawdb.DeleteFilterMapLastBlocks(f.db, common.NewRange(firstMap, f.mapsPerEpoch-1)) // keep last enrty - for mapIndex := firstMap; mapIndex < firstMap+f.mapsPerEpoch-1; mapIndex++ { - f.lastBlockCache.Remove(mapIndex) - } - rawdb.DeleteBlockLvPointers(f.db, common.NewRange(firstBlock, lastBlock-firstBlock)) // keep last enrty - for blockNumber := firstBlock; blockNumber < lastBlock; blockNumber++ { - f.lvPointerCache.Remove(blockNumber) - } - return nil } // exportCheckpoints exports epoch checkpoints in the format used by checkpoints.go. diff --git a/core/filtermaps/indexer.go b/core/filtermaps/indexer.go index 6732dc85ea..9a5424da4a 100644 --- a/core/filtermaps/indexer.go +++ b/core/filtermaps/indexer.go @@ -67,14 +67,17 @@ func (f *FilterMaps) indexerLoop() { } f.lastFinal = f.finalBlock } - if done, err := f.tryIndexTail(); err != nil { - f.disableForError("tail rendering", err) + // always attempt unindexing before indexing the tail in order to + // ensure that a potentially dirty previously unindexed epoch is + // always cleaned up before any new maps are rendered. + if done, err := f.tryUnindexTail(); err != nil { + f.disableForError("tail unindexing", err) return } else if !done { continue } - if done, err := f.tryUnindexTail(); err != nil { - f.disableForError("tail unindexing", err) + if done, err := f.tryIndexTail(); err != nil { + f.disableForError("tail rendering", err) return } else if !done { continue @@ -349,25 +352,24 @@ func (f *FilterMaps) tryIndexTail() (bool, error) { // Note that unindexing is very quick as it only removes continuous ranges of // data from the database and is also called while running head indexing. func (f *FilterMaps) tryUnindexTail() (bool, error) { - for { - firstEpoch := (f.indexedRange.maps.First() - f.indexedRange.tailPartialEpoch) >> f.logMapsPerEpoch - if f.needTailEpoch(firstEpoch) { - break - } - f.processEvents() - if f.stop { - return false, nil - } + firstEpoch := f.indexedRange.maps.First() >> f.logMapsPerEpoch + if f.indexedRange.tailPartialEpoch > 0 && firstEpoch > 0 { + firstEpoch-- + } + for epoch := min(firstEpoch, f.cleanedEpochsBefore); !f.needTailEpoch(epoch); epoch++ { if !f.startedTailUnindex { f.startedTailUnindexAt = time.Now() f.startedTailUnindex = true f.ptrTailUnindexMap = f.indexedRange.maps.First() - f.indexedRange.tailPartialEpoch f.ptrTailUnindexBlock = f.indexedRange.blocks.First() - f.tailPartialBlocks() } - if err := f.deleteTailEpoch(firstEpoch); err != nil { - log.Error("Log index tail epoch unindexing failed", "error", err) + if done, err := f.deleteTailEpoch(epoch); !done { return false, err } + f.processEvents() + if f.stop || !f.targetHeadIndexed() { + return false, nil + } } if f.startedTailUnindex && f.indexedRange.hasIndexedBlocks() { log.Info("Log index tail unindexing finished", diff --git a/core/rawdb/accessors_indexes.go b/core/rawdb/accessors_indexes.go index 297e339c83..c413839b7b 100644 --- a/core/rawdb/accessors_indexes.go +++ b/core/rawdb/accessors_indexes.go @@ -354,10 +354,8 @@ func WriteFilterMapBaseRows(db ethdb.KeyValueWriter, mapRowIndex uint64, rows [] } } -func DeleteFilterMapRows(db ethdb.KeyValueRangeDeleter, mapRows common.Range[uint64]) { - if err := db.DeleteRange(filterMapRowKey(mapRows.First(), false), filterMapRowKey(mapRows.AfterLast(), false)); err != nil { - log.Crit("Failed to delete range of filter map rows", "err", err) - } +func DeleteFilterMapRows(db ethdb.KeyValueStore, mapRows common.Range[uint64], hashScheme bool, stopCallback func(bool) bool) error { + return SafeDeleteRange(db, filterMapRowKey(mapRows.First(), false), filterMapRowKey(mapRows.AfterLast(), false), hashScheme, stopCallback) } // ReadFilterMapLastBlock retrieves the number of the block that generated the @@ -368,7 +366,7 @@ func ReadFilterMapLastBlock(db ethdb.KeyValueReader, mapIndex uint32) (uint64, c return 0, common.Hash{}, err } if len(enc) != 40 { - return 0, common.Hash{}, errors.New("Invalid block number and id encoding") + return 0, common.Hash{}, errors.New("invalid block number and id encoding") } var id common.Hash copy(id[:], enc[8:]) @@ -394,10 +392,8 @@ func DeleteFilterMapLastBlock(db ethdb.KeyValueWriter, mapIndex uint32) { } } -func DeleteFilterMapLastBlocks(db ethdb.KeyValueRangeDeleter, maps common.Range[uint32]) { - if err := db.DeleteRange(filterMapLastBlockKey(maps.First()), filterMapLastBlockKey(maps.AfterLast())); err != nil { - log.Crit("Failed to delete range of filter map last block pointers", "err", err) - } +func DeleteFilterMapLastBlocks(db ethdb.KeyValueStore, maps common.Range[uint32], hashScheme bool, stopCallback func(bool) bool) error { + return SafeDeleteRange(db, filterMapLastBlockKey(maps.First()), filterMapLastBlockKey(maps.AfterLast()), hashScheme, stopCallback) } // ReadBlockLvPointer retrieves the starting log value index where the log values @@ -408,7 +404,7 @@ func ReadBlockLvPointer(db ethdb.KeyValueReader, blockNumber uint64) (uint64, er return 0, err } if len(encPtr) != 8 { - return 0, errors.New("Invalid log value pointer encoding") + return 0, errors.New("invalid log value pointer encoding") } return binary.BigEndian.Uint64(encPtr), nil } @@ -431,10 +427,8 @@ func DeleteBlockLvPointer(db ethdb.KeyValueWriter, blockNumber uint64) { } } -func DeleteBlockLvPointers(db ethdb.KeyValueRangeDeleter, blocks common.Range[uint64]) { - if err := db.DeleteRange(filterMapBlockLVKey(blocks.First()), filterMapBlockLVKey(blocks.AfterLast())); err != nil { - log.Crit("Failed to delete range of block log value pointers", "err", err) - } +func DeleteBlockLvPointers(db ethdb.KeyValueStore, blocks common.Range[uint64], hashScheme bool, stopCallback func(bool) bool) error { + return SafeDeleteRange(db, filterMapBlockLVKey(blocks.First()), filterMapBlockLVKey(blocks.AfterLast()), hashScheme, stopCallback) } // FilterMapsRange is a storage representation of the block range covered by the @@ -485,22 +479,22 @@ func DeleteFilterMapsRange(db ethdb.KeyValueWriter) { } // deletePrefixRange deletes everything with the given prefix from the database. -func deletePrefixRange(db ethdb.KeyValueRangeDeleter, prefix []byte) error { +func deletePrefixRange(db ethdb.KeyValueStore, prefix []byte, hashScheme bool, stopCallback func(bool) bool) error { end := bytes.Clone(prefix) end[len(end)-1]++ - return db.DeleteRange(prefix, end) + return SafeDeleteRange(db, prefix, end, hashScheme, stopCallback) } // DeleteFilterMapsDb removes the entire filter maps database -func DeleteFilterMapsDb(db ethdb.KeyValueRangeDeleter) error { - return deletePrefixRange(db, []byte(filterMapsPrefix)) +func DeleteFilterMapsDb(db ethdb.KeyValueStore, hashScheme bool, stopCallback func(bool) bool) error { + return deletePrefixRange(db, []byte(filterMapsPrefix), hashScheme, stopCallback) } -// DeleteFilterMapsDb removes the old bloombits database and the associated +// DeleteBloomBitsDb removes the old bloombits database and the associated // chain indexer database. -func DeleteBloomBitsDb(db ethdb.KeyValueRangeDeleter) error { - if err := deletePrefixRange(db, bloomBitsPrefix); err != nil { +func DeleteBloomBitsDb(db ethdb.KeyValueStore, hashScheme bool, stopCallback func(bool) bool) error { + if err := deletePrefixRange(db, bloomBitsPrefix, hashScheme, stopCallback); err != nil { return err } - return deletePrefixRange(db, bloomBitsMetaPrefix) + return deletePrefixRange(db, bloomBitsMetaPrefix, hashScheme, stopCallback) } diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 7fca822155..2a50e3f9ee 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -28,12 +28,15 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb/memorydb" "github.com/ethereum/go-ethereum/log" "github.com/olekukonko/tablewriter" ) +var ErrDeleteRangeInterrupted = errors.New("safe delete range operation interrupted") + // freezerdb is a database wrapper that enables ancient chain segment freezing. type freezerdb struct { ethdb.KeyValueStore @@ -607,3 +610,73 @@ func ReadChainMetadata(db ethdb.KeyValueStore) [][]string { } return data } + +// SafeDeleteRange deletes all of the keys (and values) in the range +// [start,end) (inclusive on start, exclusive on end). +// If hashScheme is true then it always uses an iterator and skips hashdb trie +// node entries. If it is false and the backing db is pebble db then it uses +// the fast native range delete. +// In case of fallback mode (hashdb or leveldb) the range deletion might be +// very slow depending on the number of entries. In this case stopCallback +// is periodically called and if it returns an error then SafeDeleteRange +// stops and also returns that error. The callback is not called if native +// range delete is used or there are a small number of keys only. The bool +// argument passed to the callback is true if enrties have actually been +// deleted already. +func SafeDeleteRange(db ethdb.KeyValueStore, start, end []byte, hashScheme bool, stopCallback func(bool) bool) error { + if !hashScheme { + // delete entire range; use fast native range delete on pebble db + for { + switch err := db.DeleteRange(start, end); { + case err == nil: + return nil + case errors.Is(err, ethdb.ErrTooManyKeys): + if stopCallback(true) { + return ErrDeleteRangeInterrupted + } + default: + return err + } + } + } + + var ( + count, deleted, skipped int + buff = crypto.NewKeccakState() + startTime = time.Now() + ) + + batch := db.NewBatch() + it := db.NewIterator(nil, start) + defer func() { + it.Release() // it might be replaced during the process + log.Debug("SafeDeleteRange finished", "deleted", deleted, "skipped", skipped, "elapsed", common.PrettyDuration(time.Since(startTime))) + }() + + for it.Next() && bytes.Compare(end, it.Key()) > 0 { + // Prevent deletion for trie nodes in hash mode + if len(it.Key()) != 32 || crypto.HashData(buff, it.Value()) != common.BytesToHash(it.Key()) { + if err := batch.Delete(it.Key()); err != nil { + return err + } + deleted++ + } else { + skipped++ + } + count++ + if count > 10000 { // should not block for more than a second + if err := batch.Write(); err != nil { + return err + } + if stopCallback(deleted != 0) { + return ErrDeleteRangeInterrupted + } + start = append(bytes.Clone(it.Key()), 0) // appending a zero gives us the next possible key + it.Release() + batch = db.NewBatch() + it = db.NewIterator(nil, start) + count = 0 + } + } + return batch.Write() +} diff --git a/eth/backend.go b/eth/backend.go index 909d153a2b..ab612b1de7 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -239,7 +239,12 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { if err != nil { return nil, err } - fmConfig := filtermaps.Config{History: config.LogHistory, Disabled: config.LogNoHistory, ExportFileName: config.LogExportCheckpoints} + fmConfig := filtermaps.Config{ + History: config.LogHistory, + Disabled: config.LogNoHistory, + ExportFileName: config.LogExportCheckpoints, + HashScheme: scheme == rawdb.HashScheme, + } chainView := eth.newChainView(eth.blockchain.CurrentBlock()) historyCutoff := eth.blockchain.HistoryPruningCutoff() var finalBlock uint64 diff --git a/ethdb/database.go b/ethdb/database.go index b1577512f3..f2d458b85f 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -17,7 +17,10 @@ // Package ethdb defines the interfaces for an Ethereum data store. package ethdb -import "io" +import ( + "errors" + "io" +) // KeyValueReader wraps the Has and Get method of a backing data store. type KeyValueReader interface { @@ -37,10 +40,14 @@ type KeyValueWriter interface { Delete(key []byte) error } +var ErrTooManyKeys = errors.New("too many keys in deleted range") + // KeyValueRangeDeleter wraps the DeleteRange method of a backing data store. type KeyValueRangeDeleter interface { // DeleteRange deletes all of the keys (and values) in the range [start,end) // (inclusive on start, exclusive on end). + // Some implementations of DeleteRange may return ErrTooManyKeys after + // partially deleting entries in the given range. DeleteRange(start, end []byte) error } diff --git a/ethdb/leveldb/leveldb.go b/ethdb/leveldb/leveldb.go index 7f47523b82..ef02e91822 100644 --- a/ethdb/leveldb/leveldb.go +++ b/ethdb/leveldb/leveldb.go @@ -207,8 +207,6 @@ func (db *Database) Delete(key []byte) error { return db.db.Delete(key, nil) } -var ErrTooManyKeys = errors.New("too many keys in deleted range") - // DeleteRange deletes all of the keys (and values) in the range [start,end) // (inclusive on start, exclusive on end). // Note that this is a fallback implementation as leveldb does not natively @@ -228,7 +226,7 @@ func (db *Database) DeleteRange(start, end []byte) error { if err := batch.Write(); err != nil { return err } - return ErrTooManyKeys + return ethdb.ErrTooManyKeys } if err := batch.Delete(it.Key()); err != nil { return err From 82fc77a8658dc7adf53ba8c55e8329bf89d47544 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 31 Mar 2025 15:29:03 +0200 Subject: [PATCH 31/42] version: release go-ethereum v1.15.7 stable --- version/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/version/version.go b/version/version.go index 8cd428fd10..2098af17b5 100644 --- a/version/version.go +++ b/version/version.go @@ -17,8 +17,8 @@ package version const ( - Major = 1 // Major version component of the current release - Minor = 15 // Minor version component of the current release - Patch = 7 // Patch version component of the current release - Meta = "unstable" // Version metadata to append to the version string + Major = 1 // Major version component of the current release + Minor = 15 // Minor version component of the current release + Patch = 7 // Patch version component of the current release + Meta = "stable" // Version metadata to append to the version string ) From 9c970d80a28d1eb98ea101cdb6209cc3be9962bc Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 31 Mar 2025 15:30:31 +0200 Subject: [PATCH 32/42] version: begin v1.15.8 release cycle --- version/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/version/version.go b/version/version.go index 2098af17b5..72fd903a1f 100644 --- a/version/version.go +++ b/version/version.go @@ -17,8 +17,8 @@ package version const ( - Major = 1 // Major version component of the current release - Minor = 15 // Minor version component of the current release - Patch = 7 // Patch version component of the current release - Meta = "stable" // Version metadata to append to the version string + Major = 1 // Major version component of the current release + Minor = 15 // Minor version component of the current release + Patch = 8 // Patch version component of the current release + Meta = "unstable" // Version metadata to append to the version string ) From 9af88d1100e17da0764081acb76b0d2154fc1893 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 31 Mar 2025 18:26:56 +0200 Subject: [PATCH 33/42] version: back to v1.15.7, to fix the build --- version/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/version/version.go b/version/version.go index 72fd903a1f..2098af17b5 100644 --- a/version/version.go +++ b/version/version.go @@ -17,8 +17,8 @@ package version const ( - Major = 1 // Major version component of the current release - Minor = 15 // Minor version component of the current release - Patch = 8 // Patch version component of the current release - Meta = "unstable" // Version metadata to append to the version string + Major = 1 // Major version component of the current release + Minor = 15 // Minor version component of the current release + Patch = 7 // Patch version component of the current release + Meta = "stable" // Version metadata to append to the version string ) From 827d3fccf72b69324ba00f7050afd59cf956348d Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 31 Mar 2025 18:27:43 +0200 Subject: [PATCH 34/42] .travis.yml: remove macos build --- .travis.yml | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5e8a493d03..43f8ced19c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,12 +2,6 @@ language: go go_import_path: github.com/ethereum/go-ethereum sudo: false jobs: - allow_failures: - - stage: build - os: osx - env: - - azure-osx - include: # This builder create and push the Docker images for all architectures - stage: build @@ -62,23 +56,6 @@ jobs: - go run build/ci.go install -dlgo -arch arm64 -cc aarch64-linux-gnu-gcc - go run build/ci.go archive -arch arm64 -type tar -signer LINUX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds - # This builder does the OSX Azure uploads - - stage: build - if: type = push - os: osx - osx_image: xcode14.2 - go: 1.23.1 # See https://github.com/ethereum/go-ethereum/pull/30478 - env: - - azure-osx - git: - submodules: false # avoid cloning ethereum/tests - script: - - ln -sf /Users/travis/gopath/bin/go1.23.1 /usr/local/bin/go # Work around travis go-setup bug - - go run build/ci.go install -dlgo - - go run build/ci.go archive -type tar -signer OSX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds - - go run build/ci.go install -dlgo -arch arm64 - - go run build/ci.go archive -arch arm64 -type tar -signer OSX_SIGNING_KEY -signify SIGNIFY_KEY -upload gethstore/builds - # These builders run the tests - stage: build if: type = push From f0cdc40cebd3fcb26d21ced0f1093efd7a2c949f Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 31 Mar 2025 18:29:33 +0200 Subject: [PATCH 35/42] version: begin v1.15.8 release cycle reloaded --- version/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/version/version.go b/version/version.go index 2098af17b5..72fd903a1f 100644 --- a/version/version.go +++ b/version/version.go @@ -17,8 +17,8 @@ package version const ( - Major = 1 // Major version component of the current release - Minor = 15 // Minor version component of the current release - Patch = 7 // Patch version component of the current release - Meta = "stable" // Version metadata to append to the version string + Major = 1 // Major version component of the current release + Minor = 15 // Minor version component of the current release + Patch = 8 // Patch version component of the current release + Meta = "unstable" // Version metadata to append to the version string ) From bc36f2de83a5ba8e805af6ce4ea9da3ab7cb1df9 Mon Sep 17 00:00:00 2001 From: Sina M <1591639+s1na@users.noreply.github.com> Date: Tue, 1 Apr 2025 13:42:01 +0200 Subject: [PATCH 36/42] eth, eth/filters: implement API error code for pruned blocks (#31361) Implements #31275 --------- Co-authored-by: Jared Wasinger Co-authored-by: Felix Lange --- core/blockchain_reader.go | 5 ++ eth/api_backend.go | 46 ++++++++++++++++--- eth/ethconfig/historymode.go | 6 +++ eth/filters/api.go | 5 ++ eth/filters/filter.go | 20 ++++++-- eth/filters/filter_system.go | 10 ++++ eth/filters/filter_system_test.go | 4 ++ internal/ethapi/api.go | 58 +++++++++++++----------- internal/ethapi/api_test.go | 7 +++ internal/ethapi/backend.go | 1 + internal/ethapi/transaction_args_test.go | 2 + rpc/types.go | 2 +- 12 files changed, 128 insertions(+), 38 deletions(-) diff --git a/core/blockchain_reader.go b/core/blockchain_reader.go index 025b912ceb..415a0f5442 100644 --- a/core/blockchain_reader.go +++ b/core/blockchain_reader.go @@ -86,6 +86,11 @@ func (bc *BlockChain) GetHeaderByNumber(number uint64) *types.Header { return bc.hc.GetHeaderByNumber(number) } +// GetBlockNumber retrieves the block number associated with a block hash. +func (bc *BlockChain) GetBlockNumber(hash common.Hash) *uint64 { + return bc.hc.GetBlockNumber(hash) +} + // GetHeadersFrom returns a contiguous segment of headers, in rlp-form, going // backwards from the given number. func (bc *BlockChain) GetHeadersFrom(number, count uint64) []rlp.RawValue { diff --git a/eth/api_backend.go b/eth/api_backend.go index c8c5ca707f..944c357e78 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -34,6 +34,7 @@ import ( "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/eth/tracers" "github.com/ethereum/go-ethereum/ethdb" @@ -91,7 +92,13 @@ func (b *EthAPIBackend) HeaderByNumber(ctx context.Context, number rpc.BlockNumb } return block, nil } - return b.eth.blockchain.GetHeaderByNumber(uint64(number)), nil + var bn uint64 + if number == rpc.EarliestBlockNumber { + bn = b.eth.blockchain.HistoryPruningCutoff() + } else { + bn = uint64(number) + } + return b.eth.blockchain.GetHeaderByNumber(bn), nil } func (b *EthAPIBackend) HeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Header, error) { @@ -143,11 +150,27 @@ func (b *EthAPIBackend) BlockByNumber(ctx context.Context, number rpc.BlockNumbe } return b.eth.blockchain.GetBlock(header.Hash(), header.Number.Uint64()), nil } - return b.eth.blockchain.GetBlockByNumber(uint64(number)), nil + bn := uint64(number) // the resolved number + if number == rpc.EarliestBlockNumber { + bn = b.eth.blockchain.HistoryPruningCutoff() + } + block := b.eth.blockchain.GetBlockByNumber(bn) + if block == nil && bn < b.eth.blockchain.HistoryPruningCutoff() { + return nil, ðconfig.PrunedHistoryError{} + } + return block, nil } func (b *EthAPIBackend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error) { - return b.eth.blockchain.GetBlockByHash(hash), nil + number := b.eth.blockchain.GetBlockNumber(hash) + if number == nil { + return nil, nil + } + block := b.eth.blockchain.GetBlock(hash, *number) + if block == nil && *number < b.eth.blockchain.HistoryPruningCutoff() { + return nil, ðconfig.PrunedHistoryError{} + } + return block, nil } // GetBody returns body of a block. It does not resolve special block numbers. @@ -155,10 +178,14 @@ func (b *EthAPIBackend) GetBody(ctx context.Context, hash common.Hash, number rp if number < 0 || hash == (common.Hash{}) { return nil, errors.New("invalid arguments; expect hash and no special block numbers") } - if body := b.eth.blockchain.GetBody(hash); body != nil { - return body, nil + body := b.eth.blockchain.GetBody(hash) + if body == nil { + if uint64(number) < b.eth.blockchain.HistoryPruningCutoff() { + return nil, ðconfig.PrunedHistoryError{} + } + return nil, errors.New("block body not found") } - return nil, errors.New("block body not found") + return body, nil } func (b *EthAPIBackend) BlockByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Block, error) { @@ -175,6 +202,9 @@ func (b *EthAPIBackend) BlockByNumberOrHash(ctx context.Context, blockNrOrHash r } block := b.eth.blockchain.GetBlock(hash, header.Number.Uint64()) if block == nil { + if header.Number.Uint64() < b.eth.blockchain.HistoryPruningCutoff() { + return nil, ðconfig.PrunedHistoryError{} + } return nil, errors.New("header found, but block body is missing") } return block, nil @@ -234,6 +264,10 @@ func (b *EthAPIBackend) StateAndHeaderByNumberOrHash(ctx context.Context, blockN return nil, nil, errors.New("invalid arguments; neither block nor hash specified") } +func (b *EthAPIBackend) HistoryPruningCutoff() uint64 { + return b.eth.blockchain.HistoryPruningCutoff() +} + func (b *EthAPIBackend) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) { return b.eth.blockchain.GetReceiptsByHash(hash), nil } diff --git a/eth/ethconfig/historymode.go b/eth/ethconfig/historymode.go index c3661004e8..a595d72feb 100644 --- a/eth/ethconfig/historymode.go +++ b/eth/ethconfig/historymode.go @@ -90,3 +90,9 @@ var HistoryPrunePoints = map[common.Hash]*HistoryPrunePoint{ BlockHash: common.HexToHash("0x229f6b18ca1552f1d5146deceb5387333f40dc6275aebee3f2c5c4ece07d02db"), }, } + +// PrunedHistoryError is returned when the requested history is pruned. +type PrunedHistoryError struct{} + +func (e *PrunedHistoryError) Error() string { return "pruned history unavailable" } +func (e *PrunedHistoryError) ErrorCode() int { return 4444 } diff --git a/eth/filters/api.go b/eth/filters/api.go index e3057a2af2..6593cbef27 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/rpc" ) @@ -354,9 +355,13 @@ func (api *FilterAPI) GetLogs(ctx context.Context, crit FilterCriteria) ([]*type if crit.ToBlock != nil { end = crit.ToBlock.Int64() } + // Block numbers below 0 are special cases. if begin > 0 && end > 0 && begin > end { return nil, errInvalidBlockRange } + if begin > 0 && begin < int64(api.events.backend.HistoryPruningCutoff()) { + return nil, ðconfig.PrunedHistoryError{} + } // Construct the range filter filter = api.sys.NewRangeFilter(begin, end, crit.Addresses, crit.Topics) } diff --git a/eth/filters/filter.go b/eth/filters/filter.go index b743c25994..e44b37d047 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/filtermaps" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" ) @@ -86,6 +87,9 @@ func (f *Filter) Logs(ctx context.Context) ([]*types.Log, error) { if header == nil { return nil, errors.New("unknown block") } + if header.Number.Uint64() < f.sys.backend.HistoryPruningCutoff() { + return nil, ðconfig.PrunedHistoryError{} + } return f.blockLogs(ctx, header) } @@ -114,11 +118,19 @@ func (f *Filter) Logs(ctx context.Context) ([]*types.Log, error) { return 0, errors.New("safe header not found") } return hdr.Number.Uint64(), nil + case rpc.EarliestBlockNumber.Int64(): + earliest := f.sys.backend.HistoryPruningCutoff() + hdr, _ := f.sys.backend.HeaderByNumber(ctx, rpc.BlockNumber(earliest)) + if hdr == nil { + return 0, errors.New("earliest header not found") + } + return hdr.Number.Uint64(), nil + default: + if number < 0 { + return 0, errors.New("negative block number") + } + return uint64(number), nil } - if number < 0 { - return 0, errors.New("negative block number") - } - return uint64(number), nil } // range query need to resolve the special begin/end block number diff --git a/eth/filters/filter_system.go b/eth/filters/filter_system.go index 7531a1ecfc..aca3c03ad6 100644 --- a/eth/filters/filter_system.go +++ b/eth/filters/filter_system.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/filtermaps" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" @@ -64,6 +65,7 @@ type Backend interface { CurrentHeader() *types.Header ChainConfig() *params.ChainConfig + HistoryPruningCutoff() uint64 SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription @@ -304,6 +306,14 @@ func (es *EventSystem) SubscribeLogs(crit ethereum.FilterQuery, logs chan []*typ return nil, errPendingLogsUnsupported } + if from == rpc.EarliestBlockNumber { + from = rpc.BlockNumber(es.backend.HistoryPruningCutoff()) + } + // Queries beyond the pruning cutoff are not supported. + if uint64(from) < es.backend.HistoryPruningCutoff() { + return nil, ðconfig.PrunedHistoryError{} + } + // only interested in new mined logs if from == rpc.LatestBlockNumber && to == rpc.LatestBlockNumber { return es.subscribeLogs(crit, logs), nil diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index c35d823f5a..3bb019d105 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -181,6 +181,10 @@ func (b *testBackend) setPending(block *types.Block, receipts types.Receipts) { b.pendingReceipts = receipts } +func (b *testBackend) HistoryPruningCutoff() uint64 { + return 0 +} + func newTestFilterSystem(db ethdb.Database, cfg Config) (*testBackend, *FilterSystem) { backend := &testBackend{db: db} sys := NewFilterSystem(backend, cfg) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 36a5df8087..3b699748b8 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -549,21 +549,23 @@ func (api *BlockChainAPI) GetUncleByBlockHashAndIndex(ctx context.Context, block } // GetUncleCountByBlockNumber returns number of uncles in the block for the given block number -func (api *BlockChainAPI) GetUncleCountByBlockNumber(ctx context.Context, blockNr rpc.BlockNumber) *hexutil.Uint { - if block, _ := api.b.BlockByNumber(ctx, blockNr); block != nil { +func (api *BlockChainAPI) GetUncleCountByBlockNumber(ctx context.Context, blockNr rpc.BlockNumber) (*hexutil.Uint, error) { + block, err := api.b.BlockByNumber(ctx, blockNr) + if block != nil { n := hexutil.Uint(len(block.Uncles())) - return &n + return &n, nil } - return nil + return nil, err } // GetUncleCountByBlockHash returns number of uncles in the block for the given block hash -func (api *BlockChainAPI) GetUncleCountByBlockHash(ctx context.Context, blockHash common.Hash) *hexutil.Uint { - if block, _ := api.b.BlockByHash(ctx, blockHash); block != nil { +func (api *BlockChainAPI) GetUncleCountByBlockHash(ctx context.Context, blockHash common.Hash) (*hexutil.Uint, error) { + block, err := api.b.BlockByHash(ctx, blockHash) + if block != nil { n := hexutil.Uint(len(block.Uncles())) - return &n + return &n, nil } - return nil + return nil, err } // GetCode returns the code stored at the given address in the state for the given block number. @@ -596,9 +598,7 @@ func (api *BlockChainAPI) GetStorageAt(ctx context.Context, address common.Addre func (api *BlockChainAPI) GetBlockReceipts(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) ([]map[string]interface{}, error) { block, err := api.b.BlockByNumberOrHash(ctx, blockNrOrHash) if block == nil || err != nil { - // When the block doesn't exist, the RPC method should return JSON null - // as per specification. - return nil, nil + return nil, err } receipts, err := api.b.GetReceipts(ctx, block.Hash()) if err != nil { @@ -1258,37 +1258,41 @@ func NewTransactionAPI(b Backend, nonceLock *AddrLocker) *TransactionAPI { } // GetBlockTransactionCountByNumber returns the number of transactions in the block with the given block number. -func (api *TransactionAPI) GetBlockTransactionCountByNumber(ctx context.Context, blockNr rpc.BlockNumber) *hexutil.Uint { - if block, _ := api.b.BlockByNumber(ctx, blockNr); block != nil { +func (api *TransactionAPI) GetBlockTransactionCountByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*hexutil.Uint, error) { + block, err := api.b.BlockByNumber(ctx, blockNr) + if block != nil { n := hexutil.Uint(len(block.Transactions())) - return &n + return &n, nil } - return nil + return nil, err } // GetBlockTransactionCountByHash returns the number of transactions in the block with the given hash. -func (api *TransactionAPI) GetBlockTransactionCountByHash(ctx context.Context, blockHash common.Hash) *hexutil.Uint { - if block, _ := api.b.BlockByHash(ctx, blockHash); block != nil { +func (api *TransactionAPI) GetBlockTransactionCountByHash(ctx context.Context, blockHash common.Hash) (*hexutil.Uint, error) { + block, err := api.b.BlockByHash(ctx, blockHash) + if block != nil { n := hexutil.Uint(len(block.Transactions())) - return &n + return &n, nil } - return nil + return nil, err } // GetTransactionByBlockNumberAndIndex returns the transaction for the given block number and index. -func (api *TransactionAPI) GetTransactionByBlockNumberAndIndex(ctx context.Context, blockNr rpc.BlockNumber, index hexutil.Uint) *RPCTransaction { - if block, _ := api.b.BlockByNumber(ctx, blockNr); block != nil { - return newRPCTransactionFromBlockIndex(block, uint64(index), api.b.ChainConfig()) +func (api *TransactionAPI) GetTransactionByBlockNumberAndIndex(ctx context.Context, blockNr rpc.BlockNumber, index hexutil.Uint) (*RPCTransaction, error) { + block, err := api.b.BlockByNumber(ctx, blockNr) + if block != nil { + return newRPCTransactionFromBlockIndex(block, uint64(index), api.b.ChainConfig()), nil } - return nil + return nil, err } // GetTransactionByBlockHashAndIndex returns the transaction for the given block hash and index. -func (api *TransactionAPI) GetTransactionByBlockHashAndIndex(ctx context.Context, blockHash common.Hash, index hexutil.Uint) *RPCTransaction { - if block, _ := api.b.BlockByHash(ctx, blockHash); block != nil { - return newRPCTransactionFromBlockIndex(block, uint64(index), api.b.ChainConfig()) +func (api *TransactionAPI) GetTransactionByBlockHashAndIndex(ctx context.Context, blockHash common.Hash, index hexutil.Uint) (*RPCTransaction, error) { + block, err := api.b.BlockByHash(ctx, blockHash) + if block != nil { + return newRPCTransactionFromBlockIndex(block, uint64(index), api.b.ChainConfig()), nil } - return nil + return nil, err } // GetRawTransactionByBlockNumberAndIndex returns the bytes of the transaction for the given block number and index. diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go index 1ed1a8c8d8..37210aa78b 100644 --- a/internal/ethapi/api_test.go +++ b/internal/ethapi/api_test.go @@ -520,8 +520,12 @@ func (b testBackend) BlockByNumber(ctx context.Context, number rpc.BlockNumber) if number == rpc.PendingBlockNumber { return b.pending, nil } + if number == rpc.EarliestBlockNumber { + number = 0 + } return b.chain.GetBlockByNumber(uint64(number)), nil } + func (b testBackend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error) { return b.chain.GetBlockByHash(hash), nil } @@ -618,6 +622,9 @@ func (b testBackend) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscripti func (b testBackend) NewMatcherBackend() filtermaps.MatcherBackend { panic("implement me") } + +func (b testBackend) HistoryPruningCutoff() uint64 { return b.chain.HistoryPruningCutoff() } + func TestEstimateGas(t *testing.T) { t.Parallel() // Initialize test accounts diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index 9e2ea2c876..c4bf2e0591 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -85,6 +85,7 @@ type Backend interface { ChainConfig() *params.ChainConfig Engine() consensus.Engine + HistoryPruningCutoff() uint64 // This is copied from filters.Backend // eth/filters needs to be initialized from this backend type, so methods needed by diff --git a/internal/ethapi/transaction_args_test.go b/internal/ethapi/transaction_args_test.go index a5fd9bb0d4..b4d11a3033 100644 --- a/internal/ethapi/transaction_args_test.go +++ b/internal/ethapi/transaction_args_test.go @@ -402,3 +402,5 @@ func (b *backendMock) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) func (b *backendMock) Engine() consensus.Engine { return nil } func (b *backendMock) NewMatcherBackend() filtermaps.MatcherBackend { return nil } + +func (b *backendMock) HistoryPruningCutoff() uint64 { return 0 } diff --git a/rpc/types.go b/rpc/types.go index 2e53174b87..85f15344e8 100644 --- a/rpc/types.go +++ b/rpc/types.go @@ -63,11 +63,11 @@ type jsonWriter interface { type BlockNumber int64 const ( + EarliestBlockNumber = BlockNumber(-5) SafeBlockNumber = BlockNumber(-4) FinalizedBlockNumber = BlockNumber(-3) LatestBlockNumber = BlockNumber(-2) PendingBlockNumber = BlockNumber(-1) - EarliestBlockNumber = BlockNumber(0) ) // UnmarshalJSON parses the given JSON fragment into a BlockNumber. It supports: From 1bd70ba57aa943340e7a685ee3d89db072ac6ef6 Mon Sep 17 00:00:00 2001 From: John <33443230+gazzua@users.noreply.github.com> Date: Tue, 1 Apr 2025 21:07:47 +0900 Subject: [PATCH 37/42] p2p/nat: improve AddMapping code (#31486) It introduces a new variable to store the external port returned by the addAnyPortMapping function and ensures that the correct external port is returned even in case of an error. --------- Co-authored-by: Felix Lange --- p2p/nat/natupnp.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/p2p/nat/natupnp.go b/p2p/nat/natupnp.go index f1bb955892..1014dc69d2 100644 --- a/p2p/nat/natupnp.go +++ b/p2p/nat/natupnp.go @@ -82,7 +82,7 @@ func (n *upnp) ExternalIP() (addr net.IP, err error) { func (n *upnp) AddMapping(protocol string, extport, intport int, desc string, lifetime time.Duration) (uint16, error) { ip, err := n.internalAddress() if err != nil { - return 0, nil // TODO: Shouldn't we return the error? + return 0, err } protocol = strings.ToUpper(protocol) lifetimeS := uint32(lifetime / time.Second) @@ -94,14 +94,15 @@ func (n *upnp) AddMapping(protocol string, extport, intport int, desc string, li if err == nil { return uint16(extport), nil } - - return uint16(extport), n.withRateLimit(func() error { + // Try addAnyPortMapping if mapping specified port didn't work. + err = n.withRateLimit(func() error { p, err := n.addAnyPortMapping(protocol, extport, intport, ip, desc, lifetimeS) if err == nil { extport = int(p) } return err }) + return uint16(extport), err } func (n *upnp) addAnyPortMapping(protocol string, extport, intport int, ip net.IP, desc string, lifetimeS uint32) (uint16, error) { From 4add312c8a8332b76e5263066a475e962637c9ac Mon Sep 17 00:00:00 2001 From: Delweng Date: Tue, 1 Apr 2025 20:10:22 +0800 Subject: [PATCH 38/42] cmd: apply snapshot cache flag in the MakeChain (#31534) --- cmd/utils/flags.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index ae58c2d053..fb2892d2c1 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -2189,6 +2189,8 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readonly bool) (*core.BlockCh } if !ctx.Bool(SnapshotFlag.Name) { cache.SnapshotLimit = 0 // Disabled + } else if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheSnapshotFlag.Name) { + cache.SnapshotLimit = ctx.Int(CacheFlag.Name) * ctx.Int(CacheSnapshotFlag.Name) / 100 } // If we're in readonly, do not bother generating snapshot data. if readonly { From 7e3170fb5ce4e03348d29d1a153c2591680058a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Tue, 1 Apr 2025 14:29:20 +0200 Subject: [PATCH 39/42] core/filtermaps: add metrics (#31511) This PR adds metrics related to map rendering and pattern matching to the `core/filtermaps` package. --- core/filtermaps/filtermaps.go | 21 +++++++++++++++++++++ core/filtermaps/map_renderer.go | 22 +++++++++++++++++++++- core/filtermaps/matcher.go | 12 ++++++++++++ 3 files changed, 54 insertions(+), 1 deletion(-) diff --git a/core/filtermaps/filtermaps.go b/core/filtermaps/filtermaps.go index 5722f17daa..18b1c7dc79 100644 --- a/core/filtermaps/filtermaps.go +++ b/core/filtermaps/filtermaps.go @@ -30,6 +30,23 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" +) + +var ( + mapCountGauge = metrics.NewRegisteredGauge("filtermaps/maps/count", nil) // actual number of rendered maps + mapLogValueMeter = metrics.NewRegisteredMeter("filtermaps/maps/logvalues", nil) // number of log values processed + mapBlockMeter = metrics.NewRegisteredMeter("filtermaps/maps/blocks", nil) // number of block delimiters processed + mapRenderTimer = metrics.NewRegisteredTimer("filtermaps/maps/rendertime", nil) // time elapsed while rendering a single map + mapWriteTimer = metrics.NewRegisteredTimer("filtermaps/maps/writetime", nil) // time elapsed while writing a batch of finished maps to db + matchRequestTimer = metrics.NewRegisteredTimer("filtermaps/match/requesttime", nil) // processing time a matching request in a single epoch + matchEpochTimer = metrics.NewRegisteredTimer("filtermaps/match/epochtime", nil) // total processing time a matching request + matchBaseRowAccessMeter = metrics.NewRegisteredMeter("filtermaps/match/baserowaccess", nil) // number of accessed rows on layer 0 + matchBaseRowSizeMeter = metrics.NewRegisteredMeter("filtermaps/match/baserowsize", nil) // size of accessed rows on layer 0 + matchExtRowAccessMeter = metrics.NewRegisteredMeter("filtermaps/match/extrowaccess", nil) // number of accessed rows on higher layers + matchExtRowSizeMeter = metrics.NewRegisteredMeter("filtermaps/match/extrowsize", nil) // size of accessed rows on higher layers + matchLogLookup = metrics.NewRegisteredMeter("filtermaps/match/loglookup", nil) // number of log lookups based on potential matches + matchAllMeter = metrics.NewRegisteredMeter("filtermaps/match/matchall", nil) // number of requests returned with ErrMatchAll ) const ( @@ -429,8 +446,12 @@ func (f *FilterMaps) setRange(batch ethdb.KeyValueWriter, newView *ChainView, ne TailPartialEpoch: newRange.tailPartialEpoch, } rawdb.WriteFilterMapsRange(batch, rs) + if !isTempRange { + mapCountGauge.Update(int64(newRange.maps.Count() + newRange.tailPartialEpoch)) + } } else { rawdb.DeleteFilterMapsRange(batch) + mapCountGauge.Update(0) } } diff --git a/core/filtermaps/map_renderer.go b/core/filtermaps/map_renderer.go index 1eaaa9bb1a..28f943abb3 100644 --- a/core/filtermaps/map_renderer.go +++ b/core/filtermaps/map_renderer.go @@ -21,6 +21,7 @@ import ( "fmt" "math" "sort" + "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/lru" @@ -301,6 +302,11 @@ func (r *mapRenderer) run(stopCb func() bool, writeCb func()) (bool, error) { // renderCurrentMap renders a single map. func (r *mapRenderer) renderCurrentMap(stopCb func() bool) (bool, error) { + var ( + totalTime time.Duration + logValuesProcessed, blocksProcessed int64 + ) + start := time.Now() if !r.iterator.updateChainView(r.f.targetView) { return false, errChainUpdate } @@ -316,9 +322,11 @@ func (r *mapRenderer) renderCurrentMap(stopCb func() bool) (bool, error) { for r.iterator.lvIndex < uint64(r.currentMap.mapIndex+1)<= valuesPerCallback { + totalTime += time.Since(start) if stopCb() { return false, nil } + start = time.Now() if !r.iterator.updateChainView(r.f.targetView) { return false, errChainUpdate } @@ -343,8 +351,10 @@ func (r *mapRenderer) renderCurrentMap(stopCb func() bool) (bool, error) { return false, fmt.Errorf("failed to advance log iterator at %d while rendering map %d: %v", r.iterator.lvIndex, r.currentMap.mapIndex, err) } if !r.iterator.skipToBoundary { + logValuesProcessed++ r.currentMap.lastBlock = r.iterator.blockNumber if r.iterator.blockStart { + blocksProcessed++ r.currentMap.blockLvPtrs = append(r.currentMap.blockLvPtrs, r.iterator.lvIndex) } if !r.f.testDisableSnapshots && r.renderBefore >= r.f.indexedRange.maps.AfterLast() && @@ -358,12 +368,18 @@ func (r *mapRenderer) renderCurrentMap(stopCb func() bool) (bool, error) { r.currentMap.headDelimiter = r.iterator.lvIndex } r.currentMap.lastBlockId = r.f.targetView.getBlockId(r.currentMap.lastBlock) + totalTime += time.Since(start) + mapRenderTimer.Update(totalTime) + mapLogValueMeter.Mark(logValuesProcessed) + mapBlockMeter.Mark(blocksProcessed) return true, nil } // writeFinishedMaps writes rendered maps to the database and updates // filterMapsRange and indexedView accordingly. func (r *mapRenderer) writeFinishedMaps(pauseCb func() bool) error { + var totalTime time.Duration + start := time.Now() if len(r.finishedMaps) == 0 { return nil } @@ -379,7 +395,7 @@ func (r *mapRenderer) writeFinishedMaps(pauseCb func() bool) error { if err != nil { return fmt.Errorf("failed to get updated rendered range: %v", err) } - renderedView := r.f.targetView // stopCb callback might still change targetView while writing finished maps + renderedView := r.f.targetView // pauseCb callback might still change targetView while writing finished maps batch := r.f.db.NewBatch() var writeCnt int @@ -393,7 +409,9 @@ func (r *mapRenderer) writeFinishedMaps(pauseCb func() bool) error { // do not exit while in partially written state but do allow processing // events and pausing while block processing is in progress r.f.indexLock.Unlock() + totalTime += time.Since(start) pauseCb() + start = time.Now() r.f.indexLock.Lock() batch = r.f.db.NewBatch() } @@ -477,6 +495,8 @@ func (r *mapRenderer) writeFinishedMaps(pauseCb func() bool) error { if err := batch.Write(); err != nil { log.Crit("Error writing log index update batch", "error", err) } + totalTime += time.Since(start) + mapWriteTimer.Update(totalTime) return nil } diff --git a/core/filtermaps/matcher.go b/core/filtermaps/matcher.go index 6c05672cbc..5738bf166a 100644 --- a/core/filtermaps/matcher.go +++ b/core/filtermaps/matcher.go @@ -125,6 +125,7 @@ func GetPotentialMatches(ctx context.Context, backend MatcherBackend, firstBlock start := time.Now() res, err := m.process() + matchRequestTimer.Update(time.Since(start)) if doRuntimeStats { log.Info("Log search finished", "elapsed", time.Since(start)) @@ -202,6 +203,7 @@ func (m *matcherEnv) process() ([]*types.Log, error) { logs = append(logs, tasks[waitEpoch].logs...) if err := tasks[waitEpoch].err; err != nil { if err == ErrMatchAll { + matchAllMeter.Mark(1) return logs, err } return logs, fmt.Errorf("failed to process log index epoch %d: %v", waitEpoch, err) @@ -220,6 +222,7 @@ func (m *matcherEnv) process() ([]*types.Log, error) { // processEpoch returns the potentially matching logs from the given epoch. func (m *matcherEnv) processEpoch(epochIndex uint32) ([]*types.Log, error) { + start := time.Now() var logs []*types.Log // create a list of map indices to process fm, lm := epochIndex< Date: Tue, 1 Apr 2025 22:13:37 +0800 Subject: [PATCH 40/42] accounts/abi/abigen: fix a flaky bind test case `NewSingleStructArgument` (#31501) found the failed testcase here https://ci.appveyor.com/project/ethereum/go-ethereum/builds/51767091/job/rbjke432c05pufja add a timeout to wait the tx to be mined. --------- Signed-off-by: jsvisa Co-authored-by: Jared Wasinger --- accounts/abi/abigen/bind_test.go | 55 ++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 21 deletions(-) diff --git a/accounts/abi/abigen/bind_test.go b/accounts/abi/abigen/bind_test.go index 3871560912..195064fb7a 100644 --- a/accounts/abi/abigen/bind_test.go +++ b/accounts/abi/abigen/bind_test.go @@ -541,7 +541,7 @@ var bindTests = []struct { struct A { bytes32 B; } - + function F() public view returns (A[] memory a, uint256[] memory c, bool[] memory d) { A[] memory a = new A[](2); a[0].B = bytes32(uint256(1234) << 96); @@ -549,7 +549,7 @@ var bindTests = []struct { bool[] memory d; return (a, c, d); } - + function G() public view returns (A[] memory a) { A[] memory a = new A[](2); a[0].B = bytes32(uint256(1234) << 96); @@ -571,10 +571,10 @@ var bindTests = []struct { // Generate a new random account and a funded simulator key, _ := crypto.GenerateKey() auth, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - + sim := backends.NewSimulatedBackend(types.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000000000)}}, 10000000) defer sim.Close() - + // Deploy a structs method invoker contract and execute its default method _, _, structs, err := DeployStructs(auth, sim) if err != nil { @@ -1701,13 +1701,13 @@ var bindTests = []struct { `NewFallbacks`, ` pragma solidity >=0.6.0 <0.7.0; - + contract NewFallbacks { event Fallback(bytes data); fallback() external { emit Fallback(msg.data); } - + event Received(address addr, uint value); receive() external payable { emit Received(msg.sender, msg.value); @@ -1719,7 +1719,7 @@ var bindTests = []struct { ` "bytes" "math/big" - + "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" "github.com/ethereum/go-ethereum/core/types" @@ -1728,22 +1728,22 @@ var bindTests = []struct { ` key, _ := crypto.GenerateKey() addr := crypto.PubkeyToAddress(key.PublicKey) - + sim := backends.NewSimulatedBackend(types.GenesisAlloc{addr: {Balance: big.NewInt(10000000000000000)}}, 1000000) defer sim.Close() - + opts, _ := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) _, _, c, err := DeployNewFallbacks(opts, sim) if err != nil { t.Fatalf("Failed to deploy contract: %v", err) } sim.Commit() - + // Test receive function opts.Value = big.NewInt(100) c.Receive(opts) sim.Commit() - + var gotEvent bool iter, _ := c.FilterReceived(nil) defer iter.Close() @@ -1760,14 +1760,14 @@ var bindTests = []struct { if !gotEvent { t.Fatal("Expect to receive event emitted by receive") } - + // Test fallback function gotEvent = false opts.Value = nil calldata := []byte{0x01, 0x02, 0x03} c.Fallback(opts, calldata) sim.Commit() - + iter2, _ := c.FilterFallback(nil) defer iter2.Close() for iter2.Next() { @@ -1806,7 +1806,9 @@ var bindTests = []struct { []string{"608060405234801561001057600080fd5b50610113806100206000396000f3fe6080604052348015600f57600080fd5b506004361060285760003560e01c806324ec1d3f14602d575b600080fd5b60336035565b005b7fb4b2ff75e30cb4317eaae16dd8a187dd89978df17565104caa6c2797caae27d460405180604001604052806001815260200160028152506040516078919060ba565b60405180910390a1565b6040820160008201516096600085018260ad565b50602082015160a7602085018260ad565b50505050565b60b48160d3565b82525050565b600060408201905060cd60008301846082565b92915050565b600081905091905056fea26469706673582212208823628796125bf9941ce4eda18da1be3cf2931b231708ab848e1bd7151c0c9a64736f6c63430008070033"}, []string{`[{"anonymous":false,"inputs":[{"components":[{"internalType":"uint256","name":"a","type":"uint256"},{"internalType":"uint256","name":"b","type":"uint256"}],"indexed":false,"internalType":"struct Test.MyStruct","name":"s","type":"tuple"}],"name":"StructEvent","type":"event"},{"inputs":[],"name":"TestEvent","outputs":[],"stateMutability":"nonpayable","type":"function"}]`}, ` + "context" "math/big" + "time" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" @@ -1828,12 +1830,23 @@ var bindTests = []struct { } sim.Commit() - _, err = d.TestEvent(user) + tx, err := d.TestEvent(user) if err != nil { t.Fatalf("Failed to call contract %v", err) } sim.Commit() + // Wait for the transaction to be mined + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + receipt, err := bind.WaitMined(ctx, sim, tx) + if err != nil { + t.Fatalf("Failed to wait for tx to be mined: %v", err) + } + if receipt.Status != types.ReceiptStatusSuccessful { + t.Fatal("Transaction failed") + } + it, err := d.FilterStructEvent(nil) if err != nil { t.Fatalf("Failed to filter contract event %v", err) @@ -1862,7 +1875,7 @@ var bindTests = []struct { `NewErrors`, ` pragma solidity >0.8.4; - + contract NewErrors { error MyError(uint256); error MyError1(uint256); @@ -1878,7 +1891,7 @@ var bindTests = []struct { ` "context" "math/big" - + "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" "github.com/ethereum/go-ethereum/core/types" @@ -1892,7 +1905,7 @@ var bindTests = []struct { sim = backends.NewSimulatedBackend(types.GenesisAlloc{user.From: {Balance: big.NewInt(1000000000000000000)}}, ethconfig.Defaults.Miner.GasCeil) ) defer sim.Close() - + _, tx, contract, err := DeployNewErrors(user, sim) if err != nil { t.Fatal(err) @@ -1917,12 +1930,12 @@ var bindTests = []struct { name: `ConstructorWithStructParam`, contract: ` pragma solidity >=0.8.0 <0.9.0; - + contract ConstructorWithStructParam { struct StructType { uint256 field; } - + constructor(StructType memory st) {} } `, @@ -1951,7 +1964,7 @@ var bindTests = []struct { t.Fatalf("DeployConstructorWithStructParam() got err %v; want nil err", err) } sim.Commit() - + if _, err = bind.WaitDeployed(context.Background(), sim, tx); err != nil { t.Logf("Deployment tx: %+v", tx) t.Errorf("bind.WaitDeployed(nil, %T, ) got err %v; want nil err", sim, err) @@ -2000,7 +2013,7 @@ var bindTests = []struct { t.Fatalf("DeployNameConflict() got err %v; want nil err", err) } sim.Commit() - + if _, err = bind.WaitDeployed(context.Background(), sim, tx); err != nil { t.Logf("Deployment tx: %+v", tx) t.Errorf("bind.WaitDeployed(nil, %T, ) got err %v; want nil err", sim, err) From a9e6c8daae7aa2691e227d7a79323f306f529ffd Mon Sep 17 00:00:00 2001 From: Ng Wei Han <47109095+weiihann@users.noreply.github.com> Date: Wed, 2 Apr 2025 15:06:54 +0800 Subject: [PATCH 41/42] triedb/pathdb: improve perf by separating nodes map (#31306) This PR refactors the `nodeSet` structure in the path database to use separate maps for account and storage trie nodes, resulting in performance improvements. The change maintains the same API while optimizing the internal data structure. --- triedb/pathdb/nodes.go | 195 ++++++++++++++++++++++++++--------------- 1 file changed, 126 insertions(+), 69 deletions(-) diff --git a/triedb/pathdb/nodes.go b/triedb/pathdb/nodes.go index c56e38066b..f90bd0f01c 100644 --- a/triedb/pathdb/nodes.go +++ b/triedb/pathdb/nodes.go @@ -36,8 +36,9 @@ import ( // transition, typically corresponding to a block execution. It can also represent // the combined trie node set from several aggregated state transitions. type nodeSet struct { - size uint64 // aggregated size of the trie node - nodes map[common.Hash]map[string]*trienode.Node // node set, mapped by owner and path + size uint64 // aggregated size of the trie node + accountNodes map[string]*trienode.Node // account trie nodes, mapped by path + storageNodes map[common.Hash]map[string]*trienode.Node // storage trie nodes, mapped by owner and path } // newNodeSet constructs the set with the provided dirty trie nodes. @@ -46,7 +47,17 @@ func newNodeSet(nodes map[common.Hash]map[string]*trienode.Node) *nodeSet { if nodes == nil { nodes = make(map[common.Hash]map[string]*trienode.Node) } - s := &nodeSet{nodes: nodes} + s := &nodeSet{ + accountNodes: make(map[string]*trienode.Node), + storageNodes: make(map[common.Hash]map[string]*trienode.Node), + } + for owner, subset := range nodes { + if owner == (common.Hash{}) { + s.accountNodes = subset + } else { + s.storageNodes[owner] = subset + } + } s.computeSize() return s } @@ -54,13 +65,12 @@ func newNodeSet(nodes map[common.Hash]map[string]*trienode.Node) *nodeSet { // computeSize calculates the database size of the held trie nodes. func (s *nodeSet) computeSize() { var size uint64 - for owner, subset := range s.nodes { - var prefix int - if owner != (common.Hash{}) { - prefix = common.HashLength // owner (32 bytes) for storage trie nodes - } + for path, n := range s.accountNodes { + size += uint64(len(n.Blob) + len(path)) + } + for _, subset := range s.storageNodes { for path, n := range subset { - size += uint64(prefix + len(n.Blob) + len(path)) + size += uint64(common.HashLength + len(n.Blob) + len(path)) } } s.size = size @@ -79,15 +89,18 @@ func (s *nodeSet) updateSize(delta int64) { // node retrieves the trie node with node path and its trie identifier. func (s *nodeSet) node(owner common.Hash, path []byte) (*trienode.Node, bool) { - subset, ok := s.nodes[owner] + // Account trie node + if owner == (common.Hash{}) { + n, ok := s.accountNodes[string(path)] + return n, ok + } + // Storage trie node + subset, ok := s.storageNodes[owner] if !ok { return nil, false } n, ok := subset[string(path)] - if !ok { - return nil, false - } - return n, true + return n, ok } // merge integrates the provided dirty nodes into the set. The provided nodeset @@ -97,15 +110,24 @@ func (s *nodeSet) merge(set *nodeSet) { delta int64 // size difference resulting from node merging overwrite counter // counter of nodes being overwritten ) - for owner, subset := range set.nodes { - var prefix int - if owner != (common.Hash{}) { - prefix = common.HashLength + + // Merge account nodes + for path, n := range set.accountNodes { + if orig, exist := s.accountNodes[path]; !exist { + delta += int64(len(n.Blob) + len(path)) + } else { + delta += int64(len(n.Blob) - len(orig.Blob)) + overwrite.add(len(orig.Blob) + len(path)) } - current, exist := s.nodes[owner] + s.accountNodes[path] = n + } + + // Merge storage nodes + for owner, subset := range set.storageNodes { + current, exist := s.storageNodes[owner] if !exist { for path, n := range subset { - delta += int64(prefix + len(n.Blob) + len(path)) + delta += int64(common.HashLength + len(n.Blob) + len(path)) } // Perform a shallow copy of the map for the subset instead of claiming it // directly from the provided nodeset to avoid potential concurrent map @@ -113,19 +135,19 @@ func (s *nodeSet) merge(set *nodeSet) { // accessible even after merging. Therefore, ownership of the nodes map // should still belong to the original layer, and any modifications to it // should be prevented. - s.nodes[owner] = maps.Clone(subset) + s.storageNodes[owner] = maps.Clone(subset) continue } for path, n := range subset { if orig, exist := current[path]; !exist { - delta += int64(prefix + len(n.Blob) + len(path)) + delta += int64(common.HashLength + len(n.Blob) + len(path)) } else { delta += int64(len(n.Blob) - len(orig.Blob)) - overwrite.add(prefix + len(orig.Blob) + len(path)) + overwrite.add(common.HashLength + len(orig.Blob) + len(path)) } current[path] = n } - s.nodes[owner] = current + s.storageNodes[owner] = current } overwrite.report(gcTrieNodeMeter, gcTrieNodeBytesMeter) s.updateSize(delta) @@ -136,34 +158,38 @@ func (s *nodeSet) merge(set *nodeSet) { func (s *nodeSet) revertTo(db ethdb.KeyValueReader, nodes map[common.Hash]map[string]*trienode.Node) { var delta int64 for owner, subset := range nodes { - current, ok := s.nodes[owner] - if !ok { - panic(fmt.Sprintf("non-existent subset (%x)", owner)) - } - for path, n := range subset { - orig, ok := current[path] - if !ok { - // There is a special case in merkle tree that one child is removed - // from a fullNode which only has two children, and then a new child - // with different position is immediately inserted into the fullNode. - // In this case, the clean child of the fullNode will also be marked - // as dirty because of node collapse and expansion. In case of database - // rollback, don't panic if this "clean" node occurs which is not - // present in buffer. - var blob []byte - if owner == (common.Hash{}) { - blob = rawdb.ReadAccountTrieNode(db, []byte(path)) - } else { - blob = rawdb.ReadStorageTrieNode(db, owner, []byte(path)) + if owner == (common.Hash{}) { + // Account trie nodes + for path, n := range subset { + orig, ok := s.accountNodes[path] + if !ok { + blob := rawdb.ReadAccountTrieNode(db, []byte(path)) + if bytes.Equal(blob, n.Blob) { + continue + } + panic(fmt.Sprintf("non-existent account node (%v) blob: %v", path, crypto.Keccak256Hash(n.Blob).Hex())) } - // Ignore the clean node in the case described above. - if bytes.Equal(blob, n.Blob) { - continue - } - panic(fmt.Sprintf("non-existent node (%x %v) blob: %v", owner, path, crypto.Keccak256Hash(n.Blob).Hex())) + s.accountNodes[path] = n + delta += int64(len(n.Blob)) - int64(len(orig.Blob)) + } + } else { + // Storage trie nodes + current, ok := s.storageNodes[owner] + if !ok { + panic(fmt.Sprintf("non-existent subset (%x)", owner)) + } + for path, n := range subset { + orig, ok := current[path] + if !ok { + blob := rawdb.ReadStorageTrieNode(db, owner, []byte(path)) + if bytes.Equal(blob, n.Blob) { + continue + } + panic(fmt.Sprintf("non-existent storage node (%x %v) blob: %v", owner, path, crypto.Keccak256Hash(n.Blob).Hex())) + } + current[path] = n + delta += int64(len(n.Blob)) - int64(len(orig.Blob)) } - current[path] = n - delta += int64(len(n.Blob)) - int64(len(orig.Blob)) } } s.updateSize(delta) @@ -184,8 +210,21 @@ type journalNodes struct { // encode serializes the content of trie nodes into the provided writer. func (s *nodeSet) encode(w io.Writer) error { - nodes := make([]journalNodes, 0, len(s.nodes)) - for owner, subset := range s.nodes { + nodes := make([]journalNodes, 0, len(s.storageNodes)+1) + + // Encode account nodes + if len(s.accountNodes) > 0 { + entry := journalNodes{Owner: common.Hash{}} + for path, node := range s.accountNodes { + entry.Nodes = append(entry.Nodes, journalNode{ + Path: []byte(path), + Blob: node.Blob, + }) + } + nodes = append(nodes, entry) + } + // Encode storage nodes + for owner, subset := range s.storageNodes { entry := journalNodes{Owner: owner} for path, node := range subset { entry.Nodes = append(entry.Nodes, journalNode{ @@ -204,43 +243,61 @@ func (s *nodeSet) decode(r *rlp.Stream) error { if err := r.Decode(&encoded); err != nil { return fmt.Errorf("load nodes: %v", err) } - nodes := make(map[common.Hash]map[string]*trienode.Node) + s.accountNodes = make(map[string]*trienode.Node) + s.storageNodes = make(map[common.Hash]map[string]*trienode.Node) + for _, entry := range encoded { - subset := make(map[string]*trienode.Node) - for _, n := range entry.Nodes { - if len(n.Blob) > 0 { - subset[string(n.Path)] = trienode.New(crypto.Keccak256Hash(n.Blob), n.Blob) - } else { - subset[string(n.Path)] = trienode.NewDeleted() + if entry.Owner == (common.Hash{}) { + // Account nodes + for _, n := range entry.Nodes { + if len(n.Blob) > 0 { + s.accountNodes[string(n.Path)] = trienode.New(crypto.Keccak256Hash(n.Blob), n.Blob) + } else { + s.accountNodes[string(n.Path)] = trienode.NewDeleted() + } } + } else { + // Storage nodes + subset := make(map[string]*trienode.Node) + for _, n := range entry.Nodes { + if len(n.Blob) > 0 { + subset[string(n.Path)] = trienode.New(crypto.Keccak256Hash(n.Blob), n.Blob) + } else { + subset[string(n.Path)] = trienode.NewDeleted() + } + } + s.storageNodes[entry.Owner] = subset } - nodes[entry.Owner] = subset } - s.nodes = nodes s.computeSize() return nil } // write flushes nodes into the provided database batch as a whole. func (s *nodeSet) write(batch ethdb.Batch, clean *fastcache.Cache) int { - return writeNodes(batch, s.nodes, clean) + nodes := make(map[common.Hash]map[string]*trienode.Node) + if len(s.accountNodes) > 0 { + nodes[common.Hash{}] = s.accountNodes + } + for owner, subset := range s.storageNodes { + nodes[owner] = subset + } + return writeNodes(batch, nodes, clean) } // reset clears all cached trie node data. func (s *nodeSet) reset() { - s.nodes = make(map[common.Hash]map[string]*trienode.Node) + s.accountNodes = make(map[string]*trienode.Node) + s.storageNodes = make(map[common.Hash]map[string]*trienode.Node) s.size = 0 } // dbsize returns the approximate size of db write. func (s *nodeSet) dbsize() int { var m int - for owner, nodes := range s.nodes { - if owner == (common.Hash{}) { - m += len(nodes) * len(rawdb.TrieNodeAccountPrefix) // database key prefix - } else { - m += len(nodes) * (len(rawdb.TrieNodeStoragePrefix)) // database key prefix - } + m += len(s.accountNodes) * len(rawdb.TrieNodeAccountPrefix) // database key prefix + for _, nodes := range s.storageNodes { + m += len(nodes) * (len(rawdb.TrieNodeStoragePrefix)) // database key prefix } return m + int(s.size) } From ee30681a8d4d176a3561db20e9c8867dafe97441 Mon Sep 17 00:00:00 2001 From: minh-bq Date: Wed, 2 Apr 2025 14:47:56 +0700 Subject: [PATCH 42/42] core/txpool: add GetMetadata to transaction pool (#31433) This is an alternative to #31309 With eth/68, transaction announcement must have transaction type and size. So in announceTransactions, we need to query the transaction from transaction pool with its hash. This creates overhead in case of blob transaction which needs to load data from billy and RLP decode. This commit creates a lightweight lookup from transaction hash to transaction size and a function GetMetadata to query transaction type and transaction size given the transaction hash. --------- Co-authored-by: Gary Rong --- core/txpool/blobpool/blobpool.go | 81 ++++++++++++++++---------- core/txpool/blobpool/blobpool_test.go | 12 +++- core/txpool/blobpool/evictheap_test.go | 8 +-- core/txpool/blobpool/lookup.go | 35 ++++++++--- core/txpool/legacypool/legacypool.go | 13 +++++ core/txpool/subpool.go | 10 ++++ core/txpool/txpool.go | 11 ++++ eth/handler.go | 4 ++ eth/handler_test.go | 16 +++++ eth/protocols/eth/broadcast.go | 6 +- eth/protocols/eth/handler.go | 5 ++ 11 files changed, 156 insertions(+), 45 deletions(-) diff --git a/core/txpool/blobpool/blobpool.go b/core/txpool/blobpool/blobpool.go index 59a5645040..5a20c3ce5a 100644 --- a/core/txpool/blobpool/blobpool.go +++ b/core/txpool/blobpool/blobpool.go @@ -87,8 +87,9 @@ type blobTxMeta struct { hash common.Hash // Transaction hash to maintain the lookup table vhashes []common.Hash // Blob versioned hashes to maintain the lookup table - id uint64 // Storage ID in the pool's persistent store - size uint32 // Byte size in the pool's persistent store + id uint64 // Storage ID in the pool's persistent store + storageSize uint32 // Byte size in the pool's persistent store + size uint64 // RLP-encoded size of transaction including the attached blob nonce uint64 // Needed to prioritize inclusion order within an account costCap *uint256.Int // Needed to validate cumulative balance sufficiency @@ -108,19 +109,20 @@ type blobTxMeta struct { // newBlobTxMeta retrieves the indexed metadata fields from a blob transaction // and assembles a helper struct to track in memory. -func newBlobTxMeta(id uint64, size uint32, tx *types.Transaction) *blobTxMeta { +func newBlobTxMeta(id uint64, size uint64, storageSize uint32, tx *types.Transaction) *blobTxMeta { meta := &blobTxMeta{ - hash: tx.Hash(), - vhashes: tx.BlobHashes(), - id: id, - size: size, - nonce: tx.Nonce(), - costCap: uint256.MustFromBig(tx.Cost()), - execTipCap: uint256.MustFromBig(tx.GasTipCap()), - execFeeCap: uint256.MustFromBig(tx.GasFeeCap()), - blobFeeCap: uint256.MustFromBig(tx.BlobGasFeeCap()), - execGas: tx.Gas(), - blobGas: tx.BlobGas(), + hash: tx.Hash(), + vhashes: tx.BlobHashes(), + id: id, + storageSize: storageSize, + size: size, + nonce: tx.Nonce(), + costCap: uint256.MustFromBig(tx.Cost()), + execTipCap: uint256.MustFromBig(tx.GasTipCap()), + execFeeCap: uint256.MustFromBig(tx.GasFeeCap()), + blobFeeCap: uint256.MustFromBig(tx.BlobGasFeeCap()), + execGas: tx.Gas(), + blobGas: tx.BlobGas(), } meta.basefeeJumps = dynamicFeeJumps(meta.execFeeCap) meta.blobfeeJumps = dynamicFeeJumps(meta.blobFeeCap) @@ -480,7 +482,7 @@ func (p *BlobPool) parseTransaction(id uint64, size uint32, blob []byte) error { return errors.New("missing blob sidecar") } - meta := newBlobTxMeta(id, size, tx) + meta := newBlobTxMeta(id, tx.Size(), size, tx) if p.lookup.exists(meta.hash) { // This path is only possible after a crash, where deleted items are not // removed via the normal shutdown-startup procedure and thus may get @@ -507,7 +509,7 @@ func (p *BlobPool) parseTransaction(id uint64, size uint32, blob []byte) error { p.spent[sender] = new(uint256.Int).Add(p.spent[sender], meta.costCap) p.lookup.track(meta) - p.stored += uint64(meta.size) + p.stored += uint64(meta.storageSize) return nil } @@ -539,7 +541,7 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 ids = append(ids, txs[i].id) nonces = append(nonces, txs[i].nonce) - p.stored -= uint64(txs[i].size) + p.stored -= uint64(txs[i].storageSize) p.lookup.untrack(txs[i]) // Included transactions blobs need to be moved to the limbo @@ -580,7 +582,7 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 nonces = append(nonces, txs[0].nonce) p.spent[addr] = new(uint256.Int).Sub(p.spent[addr], txs[0].costCap) - p.stored -= uint64(txs[0].size) + p.stored -= uint64(txs[0].storageSize) p.lookup.untrack(txs[0]) // Included transactions blobs need to be moved to the limbo @@ -636,7 +638,7 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 dropRepeatedMeter.Mark(1) p.spent[addr] = new(uint256.Int).Sub(p.spent[addr], txs[i].costCap) - p.stored -= uint64(txs[i].size) + p.stored -= uint64(txs[i].storageSize) p.lookup.untrack(txs[i]) if err := p.store.Delete(id); err != nil { @@ -658,7 +660,7 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 nonces = append(nonces, txs[j].nonce) p.spent[addr] = new(uint256.Int).Sub(p.spent[addr], txs[j].costCap) - p.stored -= uint64(txs[j].size) + p.stored -= uint64(txs[j].storageSize) p.lookup.untrack(txs[j]) } txs = txs[:i] @@ -696,7 +698,7 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 nonces = append(nonces, last.nonce) p.spent[addr] = new(uint256.Int).Sub(p.spent[addr], last.costCap) - p.stored -= uint64(last.size) + p.stored -= uint64(last.storageSize) p.lookup.untrack(last) } if len(txs) == 0 { @@ -736,7 +738,7 @@ func (p *BlobPool) recheck(addr common.Address, inclusions map[common.Hash]uint6 nonces = append(nonces, last.nonce) p.spent[addr] = new(uint256.Int).Sub(p.spent[addr], last.costCap) - p.stored -= uint64(last.size) + p.stored -= uint64(last.storageSize) p.lookup.untrack(last) } p.index[addr] = txs @@ -1002,7 +1004,7 @@ func (p *BlobPool) reinject(addr common.Address, txhash common.Hash) error { } // Update the indices and metrics - meta := newBlobTxMeta(id, p.store.Size(id), tx) + meta := newBlobTxMeta(id, tx.Size(), p.store.Size(id), tx) if _, ok := p.index[addr]; !ok { if err := p.reserve(addr, true); err != nil { log.Warn("Failed to reserve account for blob pool", "tx", tx.Hash(), "from", addr, "err", err) @@ -1016,7 +1018,7 @@ func (p *BlobPool) reinject(addr common.Address, txhash common.Hash) error { p.spent[addr] = new(uint256.Int).Add(p.spent[addr], meta.costCap) } p.lookup.track(meta) - p.stored += uint64(meta.size) + p.stored += uint64(meta.storageSize) return nil } @@ -1041,7 +1043,7 @@ func (p *BlobPool) SetGasTip(tip *big.Int) { nonces = []uint64{tx.nonce} ) p.spent[addr] = new(uint256.Int).Sub(p.spent[addr], txs[i].costCap) - p.stored -= uint64(tx.size) + p.stored -= uint64(tx.storageSize) p.lookup.untrack(tx) txs[i] = nil @@ -1051,7 +1053,7 @@ func (p *BlobPool) SetGasTip(tip *big.Int) { nonces = append(nonces, tx.nonce) p.spent[addr] = new(uint256.Int).Sub(p.spent[addr], tx.costCap) - p.stored -= uint64(tx.size) + p.stored -= uint64(tx.storageSize) p.lookup.untrack(tx) txs[i+1+j] = nil } @@ -1236,6 +1238,25 @@ func (p *BlobPool) GetRLP(hash common.Hash) []byte { return p.getRLP(hash) } +// GetMetadata returns the transaction type and transaction size with the +// given transaction hash. +// +// The size refers the length of the 'rlp encoding' of a blob transaction +// including the attached blobs. +func (p *BlobPool) GetMetadata(hash common.Hash) *txpool.TxMetadata { + p.lock.RLock() + defer p.lock.RUnlock() + + size, ok := p.lookup.sizeOfTx(hash) + if !ok { + return nil + } + return &txpool.TxMetadata{ + Type: types.BlobTxType, + Size: size, + } +} + // GetBlobs returns a number of blobs are proofs for the given versioned hashes. // This is a utility method for the engine API, enabling consensus clients to // retrieve blobs from the pools directly instead of the network. @@ -1375,7 +1396,7 @@ func (p *BlobPool) add(tx *types.Transaction) (err error) { if err != nil { return err } - meta := newBlobTxMeta(id, p.store.Size(id), tx) + meta := newBlobTxMeta(id, tx.Size(), p.store.Size(id), tx) var ( next = p.state.GetNonce(from) @@ -1403,7 +1424,7 @@ func (p *BlobPool) add(tx *types.Transaction) (err error) { p.lookup.untrack(prev) p.lookup.track(meta) - p.stored += uint64(meta.size) - uint64(prev.size) + p.stored += uint64(meta.storageSize) - uint64(prev.storageSize) } else { // Transaction extends previously scheduled ones p.index[from] = append(p.index[from], meta) @@ -1413,7 +1434,7 @@ func (p *BlobPool) add(tx *types.Transaction) (err error) { } p.spent[from] = new(uint256.Int).Add(p.spent[from], meta.costCap) p.lookup.track(meta) - p.stored += uint64(meta.size) + p.stored += uint64(meta.storageSize) } // Recompute the rolling eviction fields. In case of a replacement, this will // recompute all subsequent fields. In case of an append, this will only do @@ -1500,7 +1521,7 @@ func (p *BlobPool) drop() { p.index[from] = txs p.spent[from] = new(uint256.Int).Sub(p.spent[from], drop.costCap) } - p.stored -= uint64(drop.size) + p.stored -= uint64(drop.storageSize) p.lookup.untrack(drop) // Remove the transaction from the pool's eviction heap: diff --git a/core/txpool/blobpool/blobpool_test.go b/core/txpool/blobpool/blobpool_test.go index d9137cb679..b7c6cfa51e 100644 --- a/core/txpool/blobpool/blobpool_test.go +++ b/core/txpool/blobpool/blobpool_test.go @@ -376,7 +376,7 @@ func verifyPoolInternals(t *testing.T, pool *BlobPool) { var stored uint64 for _, txs := range pool.index { for _, tx := range txs { - stored += uint64(tx.size) + stored += uint64(tx.storageSize) } } if pool.stored != stored { @@ -1553,6 +1553,16 @@ func TestAdd(t *testing.T) { if err := pool.add(signed); !errors.Is(err, add.err) { t.Errorf("test %d, tx %d: adding transaction error mismatch: have %v, want %v", i, j, err, add.err) } + if add.err == nil { + size, exist := pool.lookup.sizeOfTx(signed.Hash()) + if !exist { + t.Errorf("test %d, tx %d: failed to lookup transaction's size", i, j) + } + if size != signed.Size() { + t.Errorf("test %d, tx %d: transaction's size mismatches: have %v, want %v", + i, j, size, signed.Size()) + } + } verifyPoolInternals(t, pool) } verifyPoolInternals(t, pool) diff --git a/core/txpool/blobpool/evictheap_test.go b/core/txpool/blobpool/evictheap_test.go index e392932401..de4076e298 100644 --- a/core/txpool/blobpool/evictheap_test.go +++ b/core/txpool/blobpool/evictheap_test.go @@ -146,7 +146,7 @@ func TestPriceHeapSorting(t *testing.T) { ) index[addr] = []*blobTxMeta{{ id: uint64(j), - size: 128 * 1024, + storageSize: 128 * 1024, nonce: 0, execTipCap: execTip, execFeeCap: execFee, @@ -205,7 +205,7 @@ func benchmarkPriceHeapReinit(b *testing.B, datacap uint64) { ) index[addr] = []*blobTxMeta{{ id: uint64(i), - size: 128 * 1024, + storageSize: 128 * 1024, nonce: 0, execTipCap: execTip, execFeeCap: execFee, @@ -281,7 +281,7 @@ func benchmarkPriceHeapOverflow(b *testing.B, datacap uint64) { ) index[addr] = []*blobTxMeta{{ id: uint64(i), - size: 128 * 1024, + storageSize: 128 * 1024, nonce: 0, execTipCap: execTip, execFeeCap: execFee, @@ -312,7 +312,7 @@ func benchmarkPriceHeapOverflow(b *testing.B, datacap uint64) { ) metas[i] = &blobTxMeta{ id: uint64(int(blobs) + i), - size: 128 * 1024, + storageSize: 128 * 1024, nonce: 0, execTipCap: execTip, execFeeCap: execFee, diff --git a/core/txpool/blobpool/lookup.go b/core/txpool/blobpool/lookup.go index b5cf4d3799..7607cd487a 100644 --- a/core/txpool/blobpool/lookup.go +++ b/core/txpool/blobpool/lookup.go @@ -20,18 +20,24 @@ import ( "github.com/ethereum/go-ethereum/common" ) +type txMetadata struct { + id uint64 // the billy id of transction + size uint64 // the RLP encoded size of transaction (blobs are included) +} + // lookup maps blob versioned hashes to transaction hashes that include them, -// and transaction hashes to billy entries that include them. +// transaction hashes to billy entries that include them, transaction hashes +// to the transaction size type lookup struct { blobIndex map[common.Hash]map[common.Hash]struct{} - txIndex map[common.Hash]uint64 + txIndex map[common.Hash]*txMetadata } // newLookup creates a new index for tracking blob to tx; and tx to billy mappings. func newLookup() *lookup { return &lookup{ blobIndex: make(map[common.Hash]map[common.Hash]struct{}), - txIndex: make(map[common.Hash]uint64), + txIndex: make(map[common.Hash]*txMetadata), } } @@ -43,8 +49,11 @@ func (l *lookup) exists(txhash common.Hash) bool { // storeidOfTx returns the datastore storage item id of a transaction. func (l *lookup) storeidOfTx(txhash common.Hash) (uint64, bool) { - id, ok := l.txIndex[txhash] - return id, ok + meta, ok := l.txIndex[txhash] + if !ok { + return 0, false + } + return meta.id, true } // storeidOfBlob returns the datastore storage item id of a blob. @@ -61,6 +70,15 @@ func (l *lookup) storeidOfBlob(vhash common.Hash) (uint64, bool) { return 0, false // Weird, don't choke } +// sizeOfTx returns the RLP-encoded size of transaction +func (l *lookup) sizeOfTx(txhash common.Hash) (uint64, bool) { + meta, ok := l.txIndex[txhash] + if !ok { + return 0, false + } + return meta.size, true +} + // track inserts a new set of mappings from blob versioned hashes to transaction // hashes; and from transaction hashes to datastore storage item ids. func (l *lookup) track(tx *blobTxMeta) { @@ -71,8 +89,11 @@ func (l *lookup) track(tx *blobTxMeta) { } l.blobIndex[vhash][tx.hash] = struct{}{} // may be double mapped if a tx contains the same blob twice } - // Map the transaction hash to the datastore id - l.txIndex[tx.hash] = tx.id + // Map the transaction hash to the datastore id and RLP-encoded transaction size + l.txIndex[tx.hash] = &txMetadata{ + id: tx.id, + size: tx.size, + } } // untrack removes a set of mappings from blob versioned hashes to transaction diff --git a/core/txpool/legacypool/legacypool.go b/core/txpool/legacypool/legacypool.go index dafd185836..9066f3e16b 100644 --- a/core/txpool/legacypool/legacypool.go +++ b/core/txpool/legacypool/legacypool.go @@ -1035,6 +1035,19 @@ func (pool *LegacyPool) GetRLP(hash common.Hash) []byte { return encoded } +// GetMetadata returns the transaction type and transaction size with the +// given transaction hash. +func (pool *LegacyPool) GetMetadata(hash common.Hash) *txpool.TxMetadata { + tx := pool.all.Get(hash) + if tx == nil { + return nil + } + return &txpool.TxMetadata{ + Type: tx.Type(), + Size: tx.Size(), + } +} + // GetBlobs is not supported by the legacy transaction pool, it is just here to // implement the txpool.SubPool interface. func (pool *LegacyPool) GetBlobs(vhashes []common.Hash) ([]*kzg4844.Blob, []*kzg4844.Proof) { diff --git a/core/txpool/subpool.go b/core/txpool/subpool.go index 1392cfb274..f5cb852d8f 100644 --- a/core/txpool/subpool.go +++ b/core/txpool/subpool.go @@ -86,6 +86,12 @@ type PendingFilter struct { OnlyBlobTxs bool // Return only blob transactions (block blob-space filling) } +// TxMetadata denotes the metadata of a transaction. +type TxMetadata struct { + Type uint8 // The type of the transaction + Size uint64 // The length of the 'rlp encoding' of a transaction +} + // SubPool represents a specialized transaction pool that lives on its own (e.g. // blob pool). Since independent of how many specialized pools we have, they do // need to be updated in lockstep and assemble into one coherent view for block @@ -127,6 +133,10 @@ type SubPool interface { // GetRLP returns a RLP-encoded transaction if it is contained in the pool. GetRLP(hash common.Hash) []byte + // GetMetadata returns the transaction type and transaction size with the + // given transaction hash. + GetMetadata(hash common.Hash) *TxMetadata + // GetBlobs returns a number of blobs are proofs for the given versioned hashes. // This is a utility method for the engine API, enabling consensus clients to // retrieve blobs from the pools directly instead of the network. diff --git a/core/txpool/txpool.go b/core/txpool/txpool.go index 649c5d1a78..083aac92c6 100644 --- a/core/txpool/txpool.go +++ b/core/txpool/txpool.go @@ -354,6 +354,17 @@ func (p *TxPool) GetRLP(hash common.Hash) []byte { return nil } +// GetMetadata returns the transaction type and transaction size with the given +// hash. +func (p *TxPool) GetMetadata(hash common.Hash) *TxMetadata { + for _, subpool := range p.subpools { + if meta := subpool.GetMetadata(hash); meta != nil { + return meta + } + } + return nil +} + // GetBlobs returns a number of blobs are proofs for the given versioned hashes. // This is a utility method for the engine API, enabling consensus clients to // retrieve blobs from the pools directly instead of the network. diff --git a/eth/handler.go b/eth/handler.go index 7179c9980b..b2ad6effdb 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -71,6 +71,10 @@ type txPool interface { // with given tx hash. GetRLP(hash common.Hash) []byte + // GetMetadata returns the transaction type and transaction size with the + // given transaction hash. + GetMetadata(hash common.Hash) *txpool.TxMetadata + // Add should add the given transactions to the pool. Add(txs []*types.Transaction, sync bool) []error diff --git a/eth/handler_test.go b/eth/handler_test.go index 0c6b9854e6..fb3103f241 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -93,6 +93,22 @@ func (p *testTxPool) GetRLP(hash common.Hash) []byte { return nil } +// GetMetadata returns the transaction type and transaction size with the given +// hash. +func (p *testTxPool) GetMetadata(hash common.Hash) *txpool.TxMetadata { + p.lock.Lock() + defer p.lock.Unlock() + + tx := p.pool[hash] + if tx != nil { + return &txpool.TxMetadata{ + Type: tx.Type(), + Size: tx.Size(), + } + } + return nil +} + // Add appends a batch of transactions to the pool, and notifies any // listeners if the addition channel is non nil func (p *testTxPool) Add(txs []*types.Transaction, sync bool) []error { diff --git a/eth/protocols/eth/broadcast.go b/eth/protocols/eth/broadcast.go index f0ed1d6bc9..21cea0d4ef 100644 --- a/eth/protocols/eth/broadcast.go +++ b/eth/protocols/eth/broadcast.go @@ -116,10 +116,10 @@ func (p *Peer) announceTransactions() { size common.StorageSize ) for count = 0; count < len(queue) && size < maxTxPacketSize; count++ { - if tx := p.txpool.Get(queue[count]); tx != nil { + if meta := p.txpool.GetMetadata(queue[count]); meta != nil { pending = append(pending, queue[count]) - pendingTypes = append(pendingTypes, tx.Type()) - pendingSizes = append(pendingSizes, uint32(tx.Size())) + pendingTypes = append(pendingTypes, meta.Type) + pendingSizes = append(pendingSizes, uint32(meta.Size)) size += common.HashLength } } diff --git a/eth/protocols/eth/handler.go b/eth/protocols/eth/handler.go index eca6777bd6..f2a3cb0292 100644 --- a/eth/protocols/eth/handler.go +++ b/eth/protocols/eth/handler.go @@ -22,6 +22,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/txpool" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/p2p" @@ -90,6 +91,10 @@ type TxPool interface { // GetRLP retrieves the RLP-encoded transaction from the local txpool with // the given hash. GetRLP(hash common.Hash) []byte + + // GetMetadata returns the transaction type and transaction size with the + // given transaction hash. + GetMetadata(hash common.Hash) *txpool.TxMetadata } // MakeProtocols constructs the P2P protocol definitions for `eth`.