forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-07-24 05:36:46 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

core/rawdb: polish

Browse source
This commit is contained in:
Gary Rong 2025-09-01 10:36:46 +08:00
parent d778ea25e4
commit 9c92f0ec37
1 changed files with 81 additions and 91 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

172
core/rawdb/database.go
View file

@ -373,28 +373,23 @@ type stat struct {
count uint64 count uint64
} }
// Add size to the stat and increase the counter by 1 func (s *stat) empty() bool {
func (s *stat) Add(size common.StorageSize) { return atomic.LoadUint64(&s.count) == 0
}
func (s *stat) add(size common.StorageSize) {
atomic.AddUint64(&s.size, uint64(size)) atomic.AddUint64(&s.size, uint64(size))
atomic.AddUint64(&s.count, 1) atomic.AddUint64(&s.count, 1)
} }
func (s *stat) Size() string { func (s *stat) sizeString() string {
return common.StorageSize(atomic.LoadUint64(&s.size)).String() return common.StorageSize(atomic.LoadUint64(&s.size)).String()
} }
func (s *stat) Count() string { func (s *stat) countString() string {
return counter(atomic.LoadUint64(&s.count)).String() return counter(atomic.LoadUint64(&s.count)).String()
} }
func (s *stat) GetSize() common.StorageSize {
return common.StorageSize(atomic.LoadUint64(&s.size))
}
func (s *stat) GetCount() counter {
return counter(atomic.LoadUint64(&s.count))
}
// InspectDatabase traverses the entire database and checks the size // InspectDatabase traverses the entire database and checks the size
// of all different categories of data. // of all different categories of data.
func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
@ -444,22 +439,19 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
unaccountedMu sync.Mutex unaccountedMu sync.Mutex
) )
processRange := func(ctx context.Context, rangePrefix []byte) error { inspectRange := func(ctx context.Context, r byte) error {
var start []byte var s []byte
if len(keyStart) > 0 {
if len(keyStart) > 0 && len(rangePrefix) > 0 { switch {
r0, s0 := rangePrefix[0], keyStart[0] case r < keyStart[0]:
if r0 < s0 {
// Skip ranges that are entirely before keyStart
return nil return nil
} else if r0 == s0 { case r == keyStart[0]:
// Only apply keyStart to the range that contains it, s = keyStart[1:]
// skip the first byte, as it's already in rangePrefix default:
start = keyStart[1:] // entire key range is included for inspection
} }
} }
it := db.NewIterator(append(keyPrefix, r), s)
it := db.NewIterator(slices.Concat(keyPrefix, rangePrefix), start)
defer it.Release() defer it.Release()
for it.Next() { for it.Next() {
@ -472,86 +464,86 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
switch { switch {
case bytes.HasPrefix(key, headerPrefix) && len(key) == (len(headerPrefix)+8+common.HashLength): case bytes.HasPrefix(key, headerPrefix) && len(key) == (len(headerPrefix)+8+common.HashLength):
headers.Add(size) headers.add(size)
case bytes.HasPrefix(key, blockBodyPrefix) && len(key) == (len(blockBodyPrefix)+8+common.HashLength): case bytes.HasPrefix(key, blockBodyPrefix) && len(key) == (len(blockBodyPrefix)+8+common.HashLength):
bodies.Add(size) bodies.add(size)
case bytes.HasPrefix(key, blockReceiptsPrefix) && len(key) == (len(blockReceiptsPrefix)+8+common.HashLength): case bytes.HasPrefix(key, blockReceiptsPrefix) && len(key) == (len(blockReceiptsPrefix)+8+common.HashLength):
receipts.Add(size) receipts.add(size)
case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerTDSuffix): case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerTDSuffix):
tds.Add(size) tds.add(size)
case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerHashSuffix): case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerHashSuffix):
numHashPairings.Add(size) numHashPairings.add(size)
case bytes.HasPrefix(key, headerNumberPrefix) && len(key) == (len(headerNumberPrefix)+common.HashLength): case bytes.HasPrefix(key, headerNumberPrefix) && len(key) == (len(headerNumberPrefix)+common.HashLength):
hashNumPairings.Add(size) hashNumPairings.add(size)
case IsLegacyTrieNode(key, it.Value()): case IsLegacyTrieNode(key, it.Value()):
legacyTries.Add(size) legacyTries.add(size)
case bytes.HasPrefix(key, stateIDPrefix) && len(key) == len(stateIDPrefix)+common.HashLength: case bytes.HasPrefix(key, stateIDPrefix) && len(key) == len(stateIDPrefix)+common.HashLength:
stateLookups.Add(size) stateLookups.add(size)
case IsAccountTrieNode(key): case IsAccountTrieNode(key):
accountTries.Add(size) accountTries.add(size)
case IsStorageTrieNode(key): case IsStorageTrieNode(key):
storageTries.Add(size) storageTries.add(size)
case bytes.HasPrefix(key, CodePrefix) && len(key) == len(CodePrefix)+common.HashLength: case bytes.HasPrefix(key, CodePrefix) && len(key) == len(CodePrefix)+common.HashLength:
codes.Add(size) codes.add(size)
case bytes.HasPrefix(key, txLookupPrefix) && len(key) == (len(txLookupPrefix)+common.HashLength): case bytes.HasPrefix(key, txLookupPrefix) && len(key) == (len(txLookupPrefix)+common.HashLength):
txLookups.Add(size) txLookups.add(size)
case bytes.HasPrefix(key, SnapshotAccountPrefix) && len(key) == (len(SnapshotAccountPrefix)+common.HashLength): case bytes.HasPrefix(key, SnapshotAccountPrefix) && len(key) == (len(SnapshotAccountPrefix)+common.HashLength):
accountSnaps.Add(size) accountSnaps.add(size)
case bytes.HasPrefix(key, SnapshotStoragePrefix) && len(key) == (len(SnapshotStoragePrefix)+2*common.HashLength): case bytes.HasPrefix(key, SnapshotStoragePrefix) && len(key) == (len(SnapshotStoragePrefix)+2*common.HashLength):
storageSnaps.Add(size) storageSnaps.add(size)
case bytes.HasPrefix(key, PreimagePrefix) && len(key) == (len(PreimagePrefix)+common.HashLength): case bytes.HasPrefix(key, PreimagePrefix) && len(key) == (len(PreimagePrefix)+common.HashLength):
preimages.Add(size) preimages.add(size)
case bytes.HasPrefix(key, configPrefix) && len(key) == (len(configPrefix)+common.HashLength): case bytes.HasPrefix(key, configPrefix) && len(key) == (len(configPrefix)+common.HashLength):
metadata.Add(size) metadata.add(size)
case bytes.HasPrefix(key, genesisPrefix) && len(key) == (len(genesisPrefix)+common.HashLength): case bytes.HasPrefix(key, genesisPrefix) && len(key) == (len(genesisPrefix)+common.HashLength):
metadata.Add(size) metadata.add(size)
case bytes.HasPrefix(key, skeletonHeaderPrefix) && len(key) == (len(skeletonHeaderPrefix)+8): case bytes.HasPrefix(key, skeletonHeaderPrefix) && len(key) == (len(skeletonHeaderPrefix)+8):
beaconHeaders.Add(size) beaconHeaders.add(size)
case bytes.HasPrefix(key, CliqueSnapshotPrefix) && len(key) == 7+common.HashLength: case bytes.HasPrefix(key, CliqueSnapshotPrefix) && len(key) == 7+common.HashLength:
cliqueSnaps.Add(size) cliqueSnaps.add(size)
// new log index // new log index
case bytes.HasPrefix(key, filterMapRowPrefix) && len(key) <= len(filterMapRowPrefix)+9: case bytes.HasPrefix(key, filterMapRowPrefix) && len(key) <= len(filterMapRowPrefix)+9:
filterMapRows.Add(size) filterMapRows.add(size)
case bytes.HasPrefix(key, filterMapLastBlockPrefix) && len(key) == len(filterMapLastBlockPrefix)+4: case bytes.HasPrefix(key, filterMapLastBlockPrefix) && len(key) == len(filterMapLastBlockPrefix)+4:
filterMapLastBlock.Add(size) filterMapLastBlock.add(size)
case bytes.HasPrefix(key, filterMapBlockLVPrefix) && len(key) == len(filterMapBlockLVPrefix)+8: case bytes.HasPrefix(key, filterMapBlockLVPrefix) && len(key) == len(filterMapBlockLVPrefix)+8:
filterMapBlockLV.Add(size) filterMapBlockLV.add(size)
// old log index (deprecated) // old log index (deprecated)
case bytes.HasPrefix(key, bloomBitsPrefix) && len(key) == (len(bloomBitsPrefix)+10+common.HashLength): case bytes.HasPrefix(key, bloomBitsPrefix) && len(key) == (len(bloomBitsPrefix)+10+common.HashLength):
bloomBits.Add(size) bloomBits.add(size)
case bytes.HasPrefix(key, bloomBitsMetaPrefix) && len(key) < len(bloomBitsMetaPrefix)+8: case bytes.HasPrefix(key, bloomBitsMetaPrefix) && len(key) < len(bloomBitsMetaPrefix)+8:
bloomBits.Add(size) bloomBits.add(size)
// Path-based historic state indexes // Path-based historic state indexes
case bytes.HasPrefix(key, StateHistoryIndexPrefix) && len(key) >= len(StateHistoryIndexPrefix)+common.HashLength: case bytes.HasPrefix(key, StateHistoryIndexPrefix) && len(key) >= len(StateHistoryIndexPrefix)+common.HashLength:
stateIndex.Add(size) stateIndex.add(size)
// Verkle trie data is detected, determine the sub-category // Verkle trie data is detected, determine the sub-category
case bytes.HasPrefix(key, VerklePrefix): case bytes.HasPrefix(key, VerklePrefix):
remain := key[len(VerklePrefix):] remain := key[len(VerklePrefix):]
switch { switch {
case IsAccountTrieNode(remain): case IsAccountTrieNode(remain):
verkleTries.Add(size) verkleTries.add(size)
case bytes.HasPrefix(remain, stateIDPrefix) && len(remain) == len(stateIDPrefix)+common.HashLength: case bytes.HasPrefix(remain, stateIDPrefix) && len(remain) == len(stateIDPrefix)+common.HashLength:
verkleStateLookups.Add(size) verkleStateLookups.add(size)
case bytes.Equal(remain, persistentStateIDKey): case bytes.Equal(remain, persistentStateIDKey):
metadata.Add(size) metadata.add(size)
case bytes.Equal(remain, trieJournalKey): case bytes.Equal(remain, trieJournalKey):
metadata.Add(size) metadata.add(size)
case bytes.Equal(remain, snapSyncStatusFlagKey): case bytes.Equal(remain, snapSyncStatusFlagKey):
metadata.Add(size) metadata.add(size)
default: default:
unaccounted.Add(size) unaccounted.add(size)
} }
// Metadata keys // Metadata keys
case slices.ContainsFunc(knownMetadataKeys, func(x []byte) bool { return bytes.Equal(x, key) }): case slices.ContainsFunc(knownMetadataKeys, func(x []byte) bool { return bytes.Equal(x, key) }):
metadata.Add(size) metadata.add(size)
default: default:
unaccounted.Add(size) unaccounted.add(size)
if len(key) >= 2 { if len(key) >= 2 {
prefix := [2]byte(key[:2]) prefix := [2]byte(key[:2])
unaccountedMu.Lock() unaccountedMu.Lock()
@ -576,10 +568,10 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
eg, ctx = errgroup.WithContext(context.Background()) eg, ctx = errgroup.WithContext(context.Background())
workers = runtime.NumCPU() workers = runtime.NumCPU()
) )
ctx.Done()
eg.SetLimit(workers) eg.SetLimit(workers)
log.Info("Starting parallel database inspection", "workers", workers) // Progress reporter
done := make(chan struct{}) done := make(chan struct{})
go func() { go func() {
ticker := time.NewTicker(8 * time.Second) ticker := time.NewTicker(8 * time.Second)
@ -588,7 +580,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
for { for {
select { select {
case <-ticker.C: case <-ticker.C:
log.Info("Inspecting database progress", "count", count.Load(), "size", common.StorageSize(total.Load()), "elapsed", common.PrettyDuration(time.Since(start))) log.Info("Inspecting database", "count", count.Load(), "size", common.StorageSize(total.Load()), "elapsed", common.PrettyDuration(time.Since(start)))
case <-done: case <-done:
return return
} }
@ -597,8 +589,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
// Inspect key-value database in parallel. // Inspect key-value database in parallel.
for i := 0; i < 256; i++ { for i := 0; i < 256; i++ {
rangePrefix := []byte{byte(i)} eg.Go(func() error { return inspectRange(ctx, byte(i)) })
eg.Go(func() error { return processRange(ctx, rangePrefix) })
} }
if err := eg.Wait(); err != nil { if err := eg.Wait(); err != nil {
@ -609,31 +600,31 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
// Display the database statistic of key-value store. // Display the database statistic of key-value store.
stats := [][]string{ stats := [][]string{
{"Key-Value store", "Headers", headers.Size(), headers.Count()}, {"Key-Value store", "Headers", headers.sizeString(), headers.countString()},
{"Key-Value store", "Bodies", bodies.Size(), bodies.Count()}, {"Key-Value store", "Bodies", bodies.sizeString(), bodies.countString()},
{"Key-Value store", "Receipt lists", receipts.Size(), receipts.Count()}, {"Key-Value store", "Receipt lists", receipts.sizeString(), receipts.countString()},
{"Key-Value store", "Difficulties (deprecated)", tds.Size(), tds.Count()}, {"Key-Value store", "Difficulties (deprecated)", tds.sizeString(), tds.countString()},
{"Key-Value store", "Block number->hash", numHashPairings.Size(), numHashPairings.Count()}, {"Key-Value store", "Block number->hash", numHashPairings.sizeString(), numHashPairings.countString()},
{"Key-Value store", "Block hash->number", hashNumPairings.Size(), hashNumPairings.Count()}, {"Key-Value store", "Block hash->number", hashNumPairings.sizeString(), hashNumPairings.countString()},
{"Key-Value store", "Transaction index", txLookups.Size(), txLookups.Count()}, {"Key-Value store", "Transaction index", txLookups.sizeString(), txLookups.countString()},
{"Key-Value store", "Log index filter-map rows", filterMapRows.Size(), filterMapRows.Count()}, {"Key-Value store", "Log index filter-map rows", filterMapRows.sizeString(), filterMapRows.countString()},
{"Key-Value store", "Log index last-block-of-map", filterMapLastBlock.Size(), filterMapLastBlock.Count()}, {"Key-Value store", "Log index last-block-of-map", filterMapLastBlock.sizeString(), filterMapLastBlock.countString()},
{"Key-Value store", "Log index block-lv", filterMapBlockLV.Size(), filterMapBlockLV.Count()}, {"Key-Value store", "Log index block-lv", filterMapBlockLV.sizeString(), filterMapBlockLV.countString()},
{"Key-Value store", "Log bloombits (deprecated)", bloomBits.Size(), bloomBits.Count()}, {"Key-Value store", "Log bloombits (deprecated)", bloomBits.sizeString(), bloomBits.countString()},
{"Key-Value store", "Contract codes", codes.Size(), codes.Count()}, {"Key-Value store", "Contract codes", codes.sizeString(), codes.countString()},
{"Key-Value store", "Hash trie nodes", legacyTries.Size(), legacyTries.Count()}, {"Key-Value store", "Hash trie nodes", legacyTries.sizeString(), legacyTries.countString()},
{"Key-Value store", "Path trie state lookups", stateLookups.Size(), stateLookups.Count()}, {"Key-Value store", "Path trie state lookups", stateLookups.sizeString(), stateLookups.countString()},
{"Key-Value store", "Path trie account nodes", accountTries.Size(), accountTries.Count()}, {"Key-Value store", "Path trie account nodes", accountTries.sizeString(), accountTries.countString()},
{"Key-Value store", "Path trie storage nodes", storageTries.Size(), storageTries.Count()}, {"Key-Value store", "Path trie storage nodes", storageTries.sizeString(), storageTries.countString()},
{"Key-Value store", "Path state history indexes", stateIndex.Size(), stateIndex.Count()}, {"Key-Value store", "Path state history indexes", stateIndex.sizeString(), stateIndex.countString()},
{"Key-Value store", "Verkle trie nodes", verkleTries.Size(), verkleTries.Count()}, {"Key-Value store", "Verkle trie nodes", verkleTries.sizeString(), verkleTries.countString()},
{"Key-Value store", "Verkle trie state lookups", verkleStateLookups.Size(), verkleStateLookups.Count()}, {"Key-Value store", "Verkle trie state lookups", verkleStateLookups.sizeString(), verkleStateLookups.countString()},
{"Key-Value store", "Trie preimages", preimages.Size(), preimages.Count()}, {"Key-Value store", "Trie preimages", preimages.sizeString(), preimages.countString()},
{"Key-Value store", "Account snapshot", accountSnaps.Size(), accountSnaps.Count()}, {"Key-Value store", "Account snapshot", accountSnaps.sizeString(), accountSnaps.countString()},
{"Key-Value store", "Storage snapshot", storageSnaps.Size(), storageSnaps.Count()}, {"Key-Value store", "Storage snapshot", storageSnaps.sizeString(), storageSnaps.countString()},
{"Key-Value store", "Beacon sync headers", beaconHeaders.Size(), beaconHeaders.Count()}, {"Key-Value store", "Beacon sync headers", beaconHeaders.sizeString(), beaconHeaders.countString()},
{"Key-Value store", "Clique snapshots", cliqueSnaps.Size(), cliqueSnaps.Count()}, {"Key-Value store", "Clique snapshots", cliqueSnaps.sizeString(), cliqueSnaps.countString()},
{"Key-Value store", "Singleton metadata", metadata.Size(), metadata.Count()}, {"Key-Value store", "Singleton metadata", metadata.sizeString(), metadata.countString()},
} }
// Inspect all registered append-only file store then. // Inspect all registered append-only file store then.
@ -659,13 +650,12 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
table.AppendBulk(stats) table.AppendBulk(stats)
table.Render() table.Render()
if unaccounted.GetSize() > 0 { if !unaccounted.empty() {
log.Error("Database contains unaccounted data", "size", unaccounted.GetSize(), "count", unaccounted.GetCount()) log.Error("Database contains unaccounted data", "size", unaccounted.sizeString(), "count", unaccounted.countString())
for _, e := range slices.SortedFunc(maps.Values(unaccountedKeys), bytes.Compare) { for _, e := range slices.SortedFunc(maps.Values(unaccountedKeys), bytes.Compare) {
log.Error(fmt.Sprintf(" example key: %x", e)) log.Error(fmt.Sprintf(" example key: %x", e))
} }
} }
log.Info("Database inspection completed", "count", count.Load(), "size", common.StorageSize(total.Load()), "elapsed", common.PrettyDuration(time.Since(start)))
return nil return nil
} }