diff --git a/core/rawdb/database.go b/core/rawdb/database.go index edbe067b33..5a6a8a3773 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -373,28 +373,23 @@ type stat struct { count uint64 } -// Add size to the stat and increase the counter by 1 -func (s *stat) Add(size common.StorageSize) { +func (s *stat) empty() bool { + return atomic.LoadUint64(&s.count) == 0 +} + +func (s *stat) add(size common.StorageSize) { atomic.AddUint64(&s.size, uint64(size)) atomic.AddUint64(&s.count, 1) } -func (s *stat) Size() string { +func (s *stat) sizeString() string { return common.StorageSize(atomic.LoadUint64(&s.size)).String() } -func (s *stat) Count() string { +func (s *stat) countString() string { return counter(atomic.LoadUint64(&s.count)).String() } -func (s *stat) GetSize() common.StorageSize { - return common.StorageSize(atomic.LoadUint64(&s.size)) -} - -func (s *stat) GetCount() counter { - return counter(atomic.LoadUint64(&s.count)) -} - // InspectDatabase traverses the entire database and checks the size // of all different categories of data. func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { @@ -444,22 +439,19 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { unaccountedMu sync.Mutex ) - processRange := func(ctx context.Context, rangePrefix []byte) error { - var start []byte - - if len(keyStart) > 0 && len(rangePrefix) > 0 { - r0, s0 := rangePrefix[0], keyStart[0] - if r0 < s0 { - // Skip ranges that are entirely before keyStart + inspectRange := func(ctx context.Context, r byte) error { + var s []byte + if len(keyStart) > 0 { + switch { + case r < keyStart[0]: return nil - } else if r0 == s0 { - // Only apply keyStart to the range that contains it, - // skip the first byte, as it's already in rangePrefix - start = keyStart[1:] + case r == keyStart[0]: + s = keyStart[1:] + default: + // entire key range is included for inspection } } - - it := db.NewIterator(slices.Concat(keyPrefix, rangePrefix), start) + it := db.NewIterator(append(keyPrefix, r), s) defer it.Release() for it.Next() { @@ -472,86 +464,86 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { switch { case bytes.HasPrefix(key, headerPrefix) && len(key) == (len(headerPrefix)+8+common.HashLength): - headers.Add(size) + headers.add(size) case bytes.HasPrefix(key, blockBodyPrefix) && len(key) == (len(blockBodyPrefix)+8+common.HashLength): - bodies.Add(size) + bodies.add(size) case bytes.HasPrefix(key, blockReceiptsPrefix) && len(key) == (len(blockReceiptsPrefix)+8+common.HashLength): - receipts.Add(size) + receipts.add(size) case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerTDSuffix): - tds.Add(size) + tds.add(size) case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerHashSuffix): - numHashPairings.Add(size) + numHashPairings.add(size) case bytes.HasPrefix(key, headerNumberPrefix) && len(key) == (len(headerNumberPrefix)+common.HashLength): - hashNumPairings.Add(size) + hashNumPairings.add(size) case IsLegacyTrieNode(key, it.Value()): - legacyTries.Add(size) + legacyTries.add(size) case bytes.HasPrefix(key, stateIDPrefix) && len(key) == len(stateIDPrefix)+common.HashLength: - stateLookups.Add(size) + stateLookups.add(size) case IsAccountTrieNode(key): - accountTries.Add(size) + accountTries.add(size) case IsStorageTrieNode(key): - storageTries.Add(size) + storageTries.add(size) case bytes.HasPrefix(key, CodePrefix) && len(key) == len(CodePrefix)+common.HashLength: - codes.Add(size) + codes.add(size) case bytes.HasPrefix(key, txLookupPrefix) && len(key) == (len(txLookupPrefix)+common.HashLength): - txLookups.Add(size) + txLookups.add(size) case bytes.HasPrefix(key, SnapshotAccountPrefix) && len(key) == (len(SnapshotAccountPrefix)+common.HashLength): - accountSnaps.Add(size) + accountSnaps.add(size) case bytes.HasPrefix(key, SnapshotStoragePrefix) && len(key) == (len(SnapshotStoragePrefix)+2*common.HashLength): - storageSnaps.Add(size) + storageSnaps.add(size) case bytes.HasPrefix(key, PreimagePrefix) && len(key) == (len(PreimagePrefix)+common.HashLength): - preimages.Add(size) + preimages.add(size) case bytes.HasPrefix(key, configPrefix) && len(key) == (len(configPrefix)+common.HashLength): - metadata.Add(size) + metadata.add(size) case bytes.HasPrefix(key, genesisPrefix) && len(key) == (len(genesisPrefix)+common.HashLength): - metadata.Add(size) + metadata.add(size) case bytes.HasPrefix(key, skeletonHeaderPrefix) && len(key) == (len(skeletonHeaderPrefix)+8): - beaconHeaders.Add(size) + beaconHeaders.add(size) case bytes.HasPrefix(key, CliqueSnapshotPrefix) && len(key) == 7+common.HashLength: - cliqueSnaps.Add(size) + cliqueSnaps.add(size) // new log index case bytes.HasPrefix(key, filterMapRowPrefix) && len(key) <= len(filterMapRowPrefix)+9: - filterMapRows.Add(size) + filterMapRows.add(size) case bytes.HasPrefix(key, filterMapLastBlockPrefix) && len(key) == len(filterMapLastBlockPrefix)+4: - filterMapLastBlock.Add(size) + filterMapLastBlock.add(size) case bytes.HasPrefix(key, filterMapBlockLVPrefix) && len(key) == len(filterMapBlockLVPrefix)+8: - filterMapBlockLV.Add(size) + filterMapBlockLV.add(size) // old log index (deprecated) case bytes.HasPrefix(key, bloomBitsPrefix) && len(key) == (len(bloomBitsPrefix)+10+common.HashLength): - bloomBits.Add(size) + bloomBits.add(size) case bytes.HasPrefix(key, bloomBitsMetaPrefix) && len(key) < len(bloomBitsMetaPrefix)+8: - bloomBits.Add(size) + bloomBits.add(size) // Path-based historic state indexes case bytes.HasPrefix(key, StateHistoryIndexPrefix) && len(key) >= len(StateHistoryIndexPrefix)+common.HashLength: - stateIndex.Add(size) + stateIndex.add(size) // Verkle trie data is detected, determine the sub-category case bytes.HasPrefix(key, VerklePrefix): remain := key[len(VerklePrefix):] switch { case IsAccountTrieNode(remain): - verkleTries.Add(size) + verkleTries.add(size) case bytes.HasPrefix(remain, stateIDPrefix) && len(remain) == len(stateIDPrefix)+common.HashLength: - verkleStateLookups.Add(size) + verkleStateLookups.add(size) case bytes.Equal(remain, persistentStateIDKey): - metadata.Add(size) + metadata.add(size) case bytes.Equal(remain, trieJournalKey): - metadata.Add(size) + metadata.add(size) case bytes.Equal(remain, snapSyncStatusFlagKey): - metadata.Add(size) + metadata.add(size) default: - unaccounted.Add(size) + unaccounted.add(size) } // Metadata keys case slices.ContainsFunc(knownMetadataKeys, func(x []byte) bool { return bytes.Equal(x, key) }): - metadata.Add(size) + metadata.add(size) default: - unaccounted.Add(size) + unaccounted.add(size) if len(key) >= 2 { prefix := [2]byte(key[:2]) unaccountedMu.Lock() @@ -576,10 +568,10 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { eg, ctx = errgroup.WithContext(context.Background()) workers = runtime.NumCPU() ) + ctx.Done() eg.SetLimit(workers) - log.Info("Starting parallel database inspection", "workers", workers) - + // Progress reporter done := make(chan struct{}) go func() { ticker := time.NewTicker(8 * time.Second) @@ -588,7 +580,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { for { select { case <-ticker.C: - log.Info("Inspecting database progress", "count", count.Load(), "size", common.StorageSize(total.Load()), "elapsed", common.PrettyDuration(time.Since(start))) + log.Info("Inspecting database", "count", count.Load(), "size", common.StorageSize(total.Load()), "elapsed", common.PrettyDuration(time.Since(start))) case <-done: return } @@ -597,8 +589,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { // Inspect key-value database in parallel. for i := 0; i < 256; i++ { - rangePrefix := []byte{byte(i)} - eg.Go(func() error { return processRange(ctx, rangePrefix) }) + eg.Go(func() error { return inspectRange(ctx, byte(i)) }) } if err := eg.Wait(); err != nil { @@ -609,31 +600,31 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { // Display the database statistic of key-value store. stats := [][]string{ - {"Key-Value store", "Headers", headers.Size(), headers.Count()}, - {"Key-Value store", "Bodies", bodies.Size(), bodies.Count()}, - {"Key-Value store", "Receipt lists", receipts.Size(), receipts.Count()}, - {"Key-Value store", "Difficulties (deprecated)", tds.Size(), tds.Count()}, - {"Key-Value store", "Block number->hash", numHashPairings.Size(), numHashPairings.Count()}, - {"Key-Value store", "Block hash->number", hashNumPairings.Size(), hashNumPairings.Count()}, - {"Key-Value store", "Transaction index", txLookups.Size(), txLookups.Count()}, - {"Key-Value store", "Log index filter-map rows", filterMapRows.Size(), filterMapRows.Count()}, - {"Key-Value store", "Log index last-block-of-map", filterMapLastBlock.Size(), filterMapLastBlock.Count()}, - {"Key-Value store", "Log index block-lv", filterMapBlockLV.Size(), filterMapBlockLV.Count()}, - {"Key-Value store", "Log bloombits (deprecated)", bloomBits.Size(), bloomBits.Count()}, - {"Key-Value store", "Contract codes", codes.Size(), codes.Count()}, - {"Key-Value store", "Hash trie nodes", legacyTries.Size(), legacyTries.Count()}, - {"Key-Value store", "Path trie state lookups", stateLookups.Size(), stateLookups.Count()}, - {"Key-Value store", "Path trie account nodes", accountTries.Size(), accountTries.Count()}, - {"Key-Value store", "Path trie storage nodes", storageTries.Size(), storageTries.Count()}, - {"Key-Value store", "Path state history indexes", stateIndex.Size(), stateIndex.Count()}, - {"Key-Value store", "Verkle trie nodes", verkleTries.Size(), verkleTries.Count()}, - {"Key-Value store", "Verkle trie state lookups", verkleStateLookups.Size(), verkleStateLookups.Count()}, - {"Key-Value store", "Trie preimages", preimages.Size(), preimages.Count()}, - {"Key-Value store", "Account snapshot", accountSnaps.Size(), accountSnaps.Count()}, - {"Key-Value store", "Storage snapshot", storageSnaps.Size(), storageSnaps.Count()}, - {"Key-Value store", "Beacon sync headers", beaconHeaders.Size(), beaconHeaders.Count()}, - {"Key-Value store", "Clique snapshots", cliqueSnaps.Size(), cliqueSnaps.Count()}, - {"Key-Value store", "Singleton metadata", metadata.Size(), metadata.Count()}, + {"Key-Value store", "Headers", headers.sizeString(), headers.countString()}, + {"Key-Value store", "Bodies", bodies.sizeString(), bodies.countString()}, + {"Key-Value store", "Receipt lists", receipts.sizeString(), receipts.countString()}, + {"Key-Value store", "Difficulties (deprecated)", tds.sizeString(), tds.countString()}, + {"Key-Value store", "Block number->hash", numHashPairings.sizeString(), numHashPairings.countString()}, + {"Key-Value store", "Block hash->number", hashNumPairings.sizeString(), hashNumPairings.countString()}, + {"Key-Value store", "Transaction index", txLookups.sizeString(), txLookups.countString()}, + {"Key-Value store", "Log index filter-map rows", filterMapRows.sizeString(), filterMapRows.countString()}, + {"Key-Value store", "Log index last-block-of-map", filterMapLastBlock.sizeString(), filterMapLastBlock.countString()}, + {"Key-Value store", "Log index block-lv", filterMapBlockLV.sizeString(), filterMapBlockLV.countString()}, + {"Key-Value store", "Log bloombits (deprecated)", bloomBits.sizeString(), bloomBits.countString()}, + {"Key-Value store", "Contract codes", codes.sizeString(), codes.countString()}, + {"Key-Value store", "Hash trie nodes", legacyTries.sizeString(), legacyTries.countString()}, + {"Key-Value store", "Path trie state lookups", stateLookups.sizeString(), stateLookups.countString()}, + {"Key-Value store", "Path trie account nodes", accountTries.sizeString(), accountTries.countString()}, + {"Key-Value store", "Path trie storage nodes", storageTries.sizeString(), storageTries.countString()}, + {"Key-Value store", "Path state history indexes", stateIndex.sizeString(), stateIndex.countString()}, + {"Key-Value store", "Verkle trie nodes", verkleTries.sizeString(), verkleTries.countString()}, + {"Key-Value store", "Verkle trie state lookups", verkleStateLookups.sizeString(), verkleStateLookups.countString()}, + {"Key-Value store", "Trie preimages", preimages.sizeString(), preimages.countString()}, + {"Key-Value store", "Account snapshot", accountSnaps.sizeString(), accountSnaps.countString()}, + {"Key-Value store", "Storage snapshot", storageSnaps.sizeString(), storageSnaps.countString()}, + {"Key-Value store", "Beacon sync headers", beaconHeaders.sizeString(), beaconHeaders.countString()}, + {"Key-Value store", "Clique snapshots", cliqueSnaps.sizeString(), cliqueSnaps.countString()}, + {"Key-Value store", "Singleton metadata", metadata.sizeString(), metadata.countString()}, } // Inspect all registered append-only file store then. @@ -659,13 +650,12 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { table.AppendBulk(stats) table.Render() - if unaccounted.GetSize() > 0 { - log.Error("Database contains unaccounted data", "size", unaccounted.GetSize(), "count", unaccounted.GetCount()) + if !unaccounted.empty() { + log.Error("Database contains unaccounted data", "size", unaccounted.sizeString(), "count", unaccounted.countString()) for _, e := range slices.SortedFunc(maps.Values(unaccountedKeys), bytes.Compare) { log.Error(fmt.Sprintf(" example key: %x", e)) } } - log.Info("Database inspection completed", "count", count.Load(), "size", common.StorageSize(total.Load()), "elapsed", common.PrettyDuration(time.Since(start))) return nil }