From f14ec73a8866c6fa9d2436b3c7b23f6c43c39dd7 Mon Sep 17 00:00:00 2001 From: Zsolt Felfoldi Date: Thu, 20 Mar 2025 12:00:42 +0100 Subject: [PATCH] core/rawdb: implement filtermaps and bloombits database removal in core/rawdb --- core/filtermaps/filtermaps.go | 29 ++++++++++------------------- core/rawdb/accessors_indexes.go | 22 ++++++++++++++++++++++ core/rawdb/database.go | 2 +- core/rawdb/schema.go | 16 ++++++++-------- 4 files changed, 41 insertions(+), 28 deletions(-) diff --git a/core/filtermaps/filtermaps.go b/core/filtermaps/filtermaps.go index 1ed0ce88bd..5f7296914e 100644 --- a/core/filtermaps/filtermaps.go +++ b/core/filtermaps/filtermaps.go @@ -17,7 +17,6 @@ package filtermaps import ( - "bytes" "errors" "fmt" "os" @@ -293,7 +292,7 @@ func (f *FilterMaps) reset() bool { // deleting the range first ensures that resetDb will be called again at next // startup and any leftover data will be removed even if it cannot finish now. rawdb.DeleteFilterMapsRange(f.db) - return f.removeDbWithPrefix([]byte(rawdb.FilterMapsPrefix), "Resetting log index database") + return f.safeDeleteRange(rawdb.DeleteFilterMapsDb, "Resetting log index database") } // init initializes an empty log index according to the current targetView. @@ -338,28 +337,20 @@ func (f *FilterMaps) init() error { // removeBloomBits removes old bloom bits data from the database. func (f *FilterMaps) removeBloomBits() { - f.removeDbWithPrefix(rawdb.BloomBitsPrefix, "Removing old bloom bits database") - f.removeDbWithPrefix(rawdb.BloomBitsIndexPrefix, "Removing old bloom bits chain index") + f.safeDeleteRange(rawdb.DeleteBloomBitsDb, "Removing old bloom bits database") f.closeWg.Done() } -// removeDbWithPrefix removes data with the given prefix from the database and -// returns true if everything was successfully removed. -func (f *FilterMaps) removeDbWithPrefix(prefix []byte, action string) bool { - it := f.db.NewIterator(prefix, nil) - hasData := it.Next() - it.Release() - if !hasData { - return true - } - - end := bytes.Clone(prefix) - end[len(end)-1]++ +// safeDeleteRange calls the specified database range deleter function +// repeatedly as long as it returns leveldb.ErrTooManyKeys. +// This wrapper is necessary because of the leveldb fallback implementation +// of DeleteRange. +func (f *FilterMaps) safeDeleteRange(removeFn func(ethdb.KeyValueRangeDeleter) error, action string) bool { start := time.Now() var retry bool for { - err := f.db.DeleteRange(prefix, end) - if err == nil { + err := removeFn(f.db) + if retry && err == nil { log.Info(action+" finished", "elapsed", time.Since(start)) return true } @@ -373,7 +364,7 @@ func (f *FilterMaps) removeDbWithPrefix(prefix []byte, action string) bool { default: } if !retry { - log.Info(action + " in progress...") + log.Info(action + " in progress...", "elapsed", time.Since(start)) retry = true } } diff --git a/core/rawdb/accessors_indexes.go b/core/rawdb/accessors_indexes.go index e6b4316b45..1d5897cef1 100644 --- a/core/rawdb/accessors_indexes.go +++ b/core/rawdb/accessors_indexes.go @@ -17,6 +17,7 @@ package rawdb import ( + "bytes" "encoding/binary" "errors" "math/big" @@ -449,3 +450,24 @@ func DeleteFilterMapsRange(db ethdb.KeyValueWriter) { log.Crit("Failed to delete filter maps range", "err", err) } } + +// deletePrefixRange deletes everything with the given prefix from the database. +func deletePrefixRange(db ethdb.KeyValueRangeDeleter, prefix []byte) error { + end := bytes.Clone(prefix) + end[len(end)-1]++ + return db.DeleteRange(prefix, end) +} + +// DeleteFilterMapsDb removes the entire filter maps database +func DeleteFilterMapsDb(db ethdb.KeyValueRangeDeleter) error { + return deletePrefixRange(db, []byte(filterMapsPrefix)) +} + +// DeleteFilterMapsDb removes the old bloombits database and the associated +// chain indexer database. +func DeleteBloomBitsDb(db ethdb.KeyValueRangeDeleter) error { + if err := deletePrefixRange(db, bloomBitsPrefix); err != nil { + return err + } + return deletePrefixRange(db, bloomBitsIndexPrefix) +} diff --git a/core/rawdb/database.go b/core/rawdb/database.go index bf00bffeae..4c87e66cfd 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -436,7 +436,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { metadata.Add(size) case bytes.HasPrefix(key, genesisPrefix) && len(key) == (len(genesisPrefix)+common.HashLength): metadata.Add(size) - case bytes.HasPrefix(key, []byte(FilterMapsPrefix)): + case bytes.HasPrefix(key, []byte(filterMapsPrefix)): filterMaps.Add(size) case bytes.HasPrefix(key, skeletonHeaderPrefix) && len(key) == (len(skeletonHeaderPrefix)+8): beaconHeaders.Add(size) diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 756047dec6..c21a96bd24 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -106,7 +106,7 @@ var ( blockReceiptsPrefix = []byte("r") // blockReceiptsPrefix + num (uint64 big endian) + hash -> block receipts txLookupPrefix = []byte("l") // txLookupPrefix + hash -> transaction/receipt lookup metadata - BloomBitsPrefix = []byte("B") // BloomBitsPrefix + bit (uint16 big endian) + section (uint64 big endian) + hash -> bloom bits + bloomBitsPrefix = []byte("B") // bloomBitsPrefix + bit (uint16 big endian) + section (uint64 big endian) + hash -> bloom bits SnapshotAccountPrefix = []byte("a") // SnapshotAccountPrefix + account hash -> account trie value SnapshotStoragePrefix = []byte("o") // SnapshotStoragePrefix + account hash + storage hash -> storage trie value CodePrefix = []byte("c") // CodePrefix + code hash -> account code @@ -128,8 +128,8 @@ var ( configPrefix = []byte("ethereum-config-") // config prefix for the db genesisPrefix = []byte("ethereum-genesis-") // genesis state prefix for the db - // BloomBitsIndexPrefix is the data table of a chain indexer to track its progress - BloomBitsIndexPrefix = []byte("iB") + // bloomBitsIndexPrefix is the data table of a chain indexer to track its progress + bloomBitsIndexPrefix = []byte("iB") ChtPrefix = []byte("chtRootV2-") // ChtPrefix + chtNum (uint64 big endian) -> trie root hash ChtTablePrefix = []byte("cht-") @@ -145,11 +145,11 @@ var ( FixedCommitteeRootKey = []byte("fixedRoot-") // bigEndian64(syncPeriod) -> committee root hash SyncCommitteeKey = []byte("committee-") // bigEndian64(syncPeriod) -> serialized committee - FilterMapsPrefix = "fm-" - filterMapsRangeKey = []byte(FilterMapsPrefix + "R") - filterMapRowPrefix = []byte(FilterMapsPrefix + "r") // filterMapRowPrefix + mapRowIndex (uint64 big endian) -> filter row - filterMapLastBlockPrefix = []byte(FilterMapsPrefix + "b") // filterMapLastBlockPrefix + mapIndex (uint32 big endian) -> block number (uint64 big endian) - filterMapBlockLVPrefix = []byte(FilterMapsPrefix + "p") // filterMapBlockLVPrefix + num (uint64 big endian) -> log value pointer (uint64 big endian) + filterMapsPrefix = "fm-" + filterMapsRangeKey = []byte(filterMapsPrefix + "R") + filterMapRowPrefix = []byte(filterMapsPrefix + "r") // filterMapRowPrefix + mapRowIndex (uint64 big endian) -> filter row + filterMapLastBlockPrefix = []byte(filterMapsPrefix + "b") // filterMapLastBlockPrefix + mapIndex (uint32 big endian) -> block number (uint64 big endian) + filterMapBlockLVPrefix = []byte(filterMapsPrefix + "p") // filterMapBlockLVPrefix + num (uint64 big endian) -> log value pointer (uint64 big endian) preimageCounter = metrics.NewRegisteredCounter("db/preimage/total", nil) preimageHitsCounter = metrics.NewRegisteredCounter("db/preimage/hits", nil)