mirror of
https://github.com/ethereum/go-ethereum.git
synced 2026-07-24 05:36:46 +00:00
core/rawdb: polish
This commit is contained in:
Gary Rong
parent
d778ea25e4
commit
9c92f0ec37
1 changed files with 81 additions and 91 deletions
|
|
@ -373,28 +373,23 @@ type stat struct {
|
|||
count uint64
|
||||
}
|
||||
|
||||
// Add size to the stat and increase the counter by 1
|
||||
func (s *stat) Add(size common.StorageSize) {
|
||||
func (s *stat) empty() bool {
|
||||
return atomic.LoadUint64(&s.count) == 0
|
||||
}
|
||||
|
||||
func (s *stat) add(size common.StorageSize) {
|
||||
atomic.AddUint64(&s.size, uint64(size))
|
||||
atomic.AddUint64(&s.count, 1)
|
||||
}
|
||||
|
||||
func (s *stat) Size() string {
|
||||
func (s *stat) sizeString() string {
|
||||
return common.StorageSize(atomic.LoadUint64(&s.size)).String()
|
||||
}
|
||||
|
||||
func (s *stat) Count() string {
|
||||
func (s *stat) countString() string {
|
||||
return counter(atomic.LoadUint64(&s.count)).String()
|
||||
}
|
||||
|
||||
func (s *stat) GetSize() common.StorageSize {
|
||||
return common.StorageSize(atomic.LoadUint64(&s.size))
|
||||
}
|
||||
|
||||
func (s *stat) GetCount() counter {
|
||||
return counter(atomic.LoadUint64(&s.count))
|
||||
}
|
||||
|
||||
// InspectDatabase traverses the entire database and checks the size
|
||||
// of all different categories of data.
|
||||
func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
|
||||
|
|
@ -444,22 +439,19 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
|
|||
unaccountedMu sync.Mutex
|
||||
)
|
||||
|
||||
processRange := func(ctx context.Context, rangePrefix []byte) error {
|
||||
var start []byte
|
||||
|
||||
if len(keyStart) > 0 && len(rangePrefix) > 0 {
|
||||
r0, s0 := rangePrefix[0], keyStart[0]
|
||||
if r0 < s0 {
|
||||
// Skip ranges that are entirely before keyStart
|
||||
inspectRange := func(ctx context.Context, r byte) error {
|
||||
var s []byte
|
||||
if len(keyStart) > 0 {
|
||||
switch {
|
||||
case r < keyStart[0]:
|
||||
return nil
|
||||
} else if r0 == s0 {
|
||||
// Only apply keyStart to the range that contains it,
|
||||
// skip the first byte, as it's already in rangePrefix
|
||||
start = keyStart[1:]
|
||||
case r == keyStart[0]:
|
||||
s = keyStart[1:]
|
||||
default:
|
||||
// entire key range is included for inspection
|
||||
}
|
||||
}
|
||||
|
||||
it := db.NewIterator(slices.Concat(keyPrefix, rangePrefix), start)
|
||||
it := db.NewIterator(append(keyPrefix, r), s)
|
||||
defer it.Release()
|
||||
|
||||
for it.Next() {
|
||||
|
|
@ -472,86 +464,86 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
|
|||
|
||||
switch {
|
||||
case bytes.HasPrefix(key, headerPrefix) && len(key) == (len(headerPrefix)+8+common.HashLength):
|
||||
headers.Add(size)
|
||||
headers.add(size)
|
||||
case bytes.HasPrefix(key, blockBodyPrefix) && len(key) == (len(blockBodyPrefix)+8+common.HashLength):
|
||||
bodies.Add(size)
|
||||
bodies.add(size)
|
||||
case bytes.HasPrefix(key, blockReceiptsPrefix) && len(key) == (len(blockReceiptsPrefix)+8+common.HashLength):
|
||||
receipts.Add(size)
|
||||
receipts.add(size)
|
||||
case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerTDSuffix):
|
||||
tds.Add(size)
|
||||
tds.add(size)
|
||||
case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerHashSuffix):
|
||||
numHashPairings.Add(size)
|
||||
numHashPairings.add(size)
|
||||
case bytes.HasPrefix(key, headerNumberPrefix) && len(key) == (len(headerNumberPrefix)+common.HashLength):
|
||||
hashNumPairings.Add(size)
|
||||
hashNumPairings.add(size)
|
||||
case IsLegacyTrieNode(key, it.Value()):
|
||||
legacyTries.Add(size)
|
||||
legacyTries.add(size)
|
||||
case bytes.HasPrefix(key, stateIDPrefix) && len(key) == len(stateIDPrefix)+common.HashLength:
|
||||
stateLookups.Add(size)
|
||||
stateLookups.add(size)
|
||||
case IsAccountTrieNode(key):
|
||||
accountTries.Add(size)
|
||||
accountTries.add(size)
|
||||
case IsStorageTrieNode(key):
|
||||
storageTries.Add(size)
|
||||
storageTries.add(size)
|
||||
case bytes.HasPrefix(key, CodePrefix) && len(key) == len(CodePrefix)+common.HashLength:
|
||||
codes.Add(size)
|
||||
codes.add(size)
|
||||
case bytes.HasPrefix(key, txLookupPrefix) && len(key) == (len(txLookupPrefix)+common.HashLength):
|
||||
txLookups.Add(size)
|
||||
txLookups.add(size)
|
||||
case bytes.HasPrefix(key, SnapshotAccountPrefix) && len(key) == (len(SnapshotAccountPrefix)+common.HashLength):
|
||||
accountSnaps.Add(size)
|
||||
accountSnaps.add(size)
|
||||
case bytes.HasPrefix(key, SnapshotStoragePrefix) && len(key) == (len(SnapshotStoragePrefix)+2*common.HashLength):
|
||||
storageSnaps.Add(size)
|
||||
storageSnaps.add(size)
|
||||
case bytes.HasPrefix(key, PreimagePrefix) && len(key) == (len(PreimagePrefix)+common.HashLength):
|
||||
preimages.Add(size)
|
||||
preimages.add(size)
|
||||
case bytes.HasPrefix(key, configPrefix) && len(key) == (len(configPrefix)+common.HashLength):
|
||||
metadata.Add(size)
|
||||
metadata.add(size)
|
||||
case bytes.HasPrefix(key, genesisPrefix) && len(key) == (len(genesisPrefix)+common.HashLength):
|
||||
metadata.Add(size)
|
||||
metadata.add(size)
|
||||
case bytes.HasPrefix(key, skeletonHeaderPrefix) && len(key) == (len(skeletonHeaderPrefix)+8):
|
||||
beaconHeaders.Add(size)
|
||||
beaconHeaders.add(size)
|
||||
case bytes.HasPrefix(key, CliqueSnapshotPrefix) && len(key) == 7+common.HashLength:
|
||||
cliqueSnaps.Add(size)
|
||||
cliqueSnaps.add(size)
|
||||
|
||||
// new log index
|
||||
case bytes.HasPrefix(key, filterMapRowPrefix) && len(key) <= len(filterMapRowPrefix)+9:
|
||||
filterMapRows.Add(size)
|
||||
filterMapRows.add(size)
|
||||
case bytes.HasPrefix(key, filterMapLastBlockPrefix) && len(key) == len(filterMapLastBlockPrefix)+4:
|
||||
filterMapLastBlock.Add(size)
|
||||
filterMapLastBlock.add(size)
|
||||
case bytes.HasPrefix(key, filterMapBlockLVPrefix) && len(key) == len(filterMapBlockLVPrefix)+8:
|
||||
filterMapBlockLV.Add(size)
|
||||
filterMapBlockLV.add(size)
|
||||
|
||||
// old log index (deprecated)
|
||||
case bytes.HasPrefix(key, bloomBitsPrefix) && len(key) == (len(bloomBitsPrefix)+10+common.HashLength):
|
||||
bloomBits.Add(size)
|
||||
bloomBits.add(size)
|
||||
case bytes.HasPrefix(key, bloomBitsMetaPrefix) && len(key) < len(bloomBitsMetaPrefix)+8:
|
||||
bloomBits.Add(size)
|
||||
bloomBits.add(size)
|
||||
|
||||
// Path-based historic state indexes
|
||||
case bytes.HasPrefix(key, StateHistoryIndexPrefix) && len(key) >= len(StateHistoryIndexPrefix)+common.HashLength:
|
||||
stateIndex.Add(size)
|
||||
stateIndex.add(size)
|
||||
|
||||
// Verkle trie data is detected, determine the sub-category
|
||||
case bytes.HasPrefix(key, VerklePrefix):
|
||||
remain := key[len(VerklePrefix):]
|
||||
switch {
|
||||
case IsAccountTrieNode(remain):
|
||||
verkleTries.Add(size)
|
||||
verkleTries.add(size)
|
||||
case bytes.HasPrefix(remain, stateIDPrefix) && len(remain) == len(stateIDPrefix)+common.HashLength:
|
||||
verkleStateLookups.Add(size)
|
||||
verkleStateLookups.add(size)
|
||||
case bytes.Equal(remain, persistentStateIDKey):
|
||||
metadata.Add(size)
|
||||
metadata.add(size)
|
||||
case bytes.Equal(remain, trieJournalKey):
|
||||
metadata.Add(size)
|
||||
metadata.add(size)
|
||||
case bytes.Equal(remain, snapSyncStatusFlagKey):
|
||||
metadata.Add(size)
|
||||
metadata.add(size)
|
||||
default:
|
||||
unaccounted.Add(size)
|
||||
unaccounted.add(size)
|
||||
}
|
||||
|
||||
// Metadata keys
|
||||
case slices.ContainsFunc(knownMetadataKeys, func(x []byte) bool { return bytes.Equal(x, key) }):
|
||||
metadata.Add(size)
|
||||
metadata.add(size)
|
||||
|
||||
default:
|
||||
unaccounted.Add(size)
|
||||
unaccounted.add(size)
|
||||
if len(key) >= 2 {
|
||||
prefix := [2]byte(key[:2])
|
||||
unaccountedMu.Lock()
|
||||
|
|
@ -576,10 +568,10 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
|
|||
eg, ctx = errgroup.WithContext(context.Background())
|
||||
workers = runtime.NumCPU()
|
||||
)
|
||||
ctx.Done()
|
||||
eg.SetLimit(workers)
|
||||
|
||||
log.Info("Starting parallel database inspection", "workers", workers)
|
||||
|
||||
// Progress reporter
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
ticker := time.NewTicker(8 * time.Second)
|
||||
|
|
@ -588,7 +580,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
|
|||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
log.Info("Inspecting database progress", "count", count.Load(), "size", common.StorageSize(total.Load()), "elapsed", common.PrettyDuration(time.Since(start)))
|
||||
log.Info("Inspecting database", "count", count.Load(), "size", common.StorageSize(total.Load()), "elapsed", common.PrettyDuration(time.Since(start)))
|
||||
case <-done:
|
||||
return
|
||||
}
|
||||
|
|
@ -597,8 +589,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
|
|||
|
||||
// Inspect key-value database in parallel.
|
||||
for i := 0; i < 256; i++ {
|
||||
rangePrefix := []byte{byte(i)}
|
||||
eg.Go(func() error { return processRange(ctx, rangePrefix) })
|
||||
eg.Go(func() error { return inspectRange(ctx, byte(i)) })
|
||||
}
|
||||
|
||||
if err := eg.Wait(); err != nil {
|
||||
|
|
@ -609,31 +600,31 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
|
|||
|
||||
// Display the database statistic of key-value store.
|
||||
stats := [][]string{
|
||||
{"Key-Value store", "Headers", headers.Size(), headers.Count()},
|
||||
{"Key-Value store", "Bodies", bodies.Size(), bodies.Count()},
|
||||
{"Key-Value store", "Receipt lists", receipts.Size(), receipts.Count()},
|
||||
{"Key-Value store", "Difficulties (deprecated)", tds.Size(), tds.Count()},
|
||||
{"Key-Value store", "Block number->hash", numHashPairings.Size(), numHashPairings.Count()},
|
||||
{"Key-Value store", "Block hash->number", hashNumPairings.Size(), hashNumPairings.Count()},
|
||||
{"Key-Value store", "Transaction index", txLookups.Size(), txLookups.Count()},
|
||||
{"Key-Value store", "Log index filter-map rows", filterMapRows.Size(), filterMapRows.Count()},
|
||||
{"Key-Value store", "Log index last-block-of-map", filterMapLastBlock.Size(), filterMapLastBlock.Count()},
|
||||
{"Key-Value store", "Log index block-lv", filterMapBlockLV.Size(), filterMapBlockLV.Count()},
|
||||
{"Key-Value store", "Log bloombits (deprecated)", bloomBits.Size(), bloomBits.Count()},
|
||||
{"Key-Value store", "Contract codes", codes.Size(), codes.Count()},
|
||||
{"Key-Value store", "Hash trie nodes", legacyTries.Size(), legacyTries.Count()},
|
||||
{"Key-Value store", "Path trie state lookups", stateLookups.Size(), stateLookups.Count()},
|
||||
{"Key-Value store", "Path trie account nodes", accountTries.Size(), accountTries.Count()},
|
||||
{"Key-Value store", "Path trie storage nodes", storageTries.Size(), storageTries.Count()},
|
||||
{"Key-Value store", "Path state history indexes", stateIndex.Size(), stateIndex.Count()},
|
||||
{"Key-Value store", "Verkle trie nodes", verkleTries.Size(), verkleTries.Count()},
|
||||
{"Key-Value store", "Verkle trie state lookups", verkleStateLookups.Size(), verkleStateLookups.Count()},
|
||||
{"Key-Value store", "Trie preimages", preimages.Size(), preimages.Count()},
|
||||
{"Key-Value store", "Account snapshot", accountSnaps.Size(), accountSnaps.Count()},
|
||||
{"Key-Value store", "Storage snapshot", storageSnaps.Size(), storageSnaps.Count()},
|
||||
{"Key-Value store", "Beacon sync headers", beaconHeaders.Size(), beaconHeaders.Count()},
|
||||
{"Key-Value store", "Clique snapshots", cliqueSnaps.Size(), cliqueSnaps.Count()},
|
||||
{"Key-Value store", "Singleton metadata", metadata.Size(), metadata.Count()},
|
||||
{"Key-Value store", "Headers", headers.sizeString(), headers.countString()},
|
||||
{"Key-Value store", "Bodies", bodies.sizeString(), bodies.countString()},
|
||||
{"Key-Value store", "Receipt lists", receipts.sizeString(), receipts.countString()},
|
||||
{"Key-Value store", "Difficulties (deprecated)", tds.sizeString(), tds.countString()},
|
||||
{"Key-Value store", "Block number->hash", numHashPairings.sizeString(), numHashPairings.countString()},
|
||||
{"Key-Value store", "Block hash->number", hashNumPairings.sizeString(), hashNumPairings.countString()},
|
||||
{"Key-Value store", "Transaction index", txLookups.sizeString(), txLookups.countString()},
|
||||
{"Key-Value store", "Log index filter-map rows", filterMapRows.sizeString(), filterMapRows.countString()},
|
||||
{"Key-Value store", "Log index last-block-of-map", filterMapLastBlock.sizeString(), filterMapLastBlock.countString()},
|
||||
{"Key-Value store", "Log index block-lv", filterMapBlockLV.sizeString(), filterMapBlockLV.countString()},
|
||||
{"Key-Value store", "Log bloombits (deprecated)", bloomBits.sizeString(), bloomBits.countString()},
|
||||
{"Key-Value store", "Contract codes", codes.sizeString(), codes.countString()},
|
||||
{"Key-Value store", "Hash trie nodes", legacyTries.sizeString(), legacyTries.countString()},
|
||||
{"Key-Value store", "Path trie state lookups", stateLookups.sizeString(), stateLookups.countString()},
|
||||
{"Key-Value store", "Path trie account nodes", accountTries.sizeString(), accountTries.countString()},
|
||||
{"Key-Value store", "Path trie storage nodes", storageTries.sizeString(), storageTries.countString()},
|
||||
{"Key-Value store", "Path state history indexes", stateIndex.sizeString(), stateIndex.countString()},
|
||||
{"Key-Value store", "Verkle trie nodes", verkleTries.sizeString(), verkleTries.countString()},
|
||||
{"Key-Value store", "Verkle trie state lookups", verkleStateLookups.sizeString(), verkleStateLookups.countString()},
|
||||
{"Key-Value store", "Trie preimages", preimages.sizeString(), preimages.countString()},
|
||||
{"Key-Value store", "Account snapshot", accountSnaps.sizeString(), accountSnaps.countString()},
|
||||
{"Key-Value store", "Storage snapshot", storageSnaps.sizeString(), storageSnaps.countString()},
|
||||
{"Key-Value store", "Beacon sync headers", beaconHeaders.sizeString(), beaconHeaders.countString()},
|
||||
{"Key-Value store", "Clique snapshots", cliqueSnaps.sizeString(), cliqueSnaps.countString()},
|
||||
{"Key-Value store", "Singleton metadata", metadata.sizeString(), metadata.countString()},
|
||||
}
|
||||
|
||||
// Inspect all registered append-only file store then.
|
||||
|
|
@ -659,13 +650,12 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
|
|||
table.AppendBulk(stats)
|
||||
table.Render()
|
||||
|
||||
if unaccounted.GetSize() > 0 {
|
||||
log.Error("Database contains unaccounted data", "size", unaccounted.GetSize(), "count", unaccounted.GetCount())
|
||||
if !unaccounted.empty() {
|
||||
log.Error("Database contains unaccounted data", "size", unaccounted.sizeString(), "count", unaccounted.countString())
|
||||
for _, e := range slices.SortedFunc(maps.Values(unaccountedKeys), bytes.Compare) {
|
||||
log.Error(fmt.Sprintf(" example key: %x", e))
|
||||
}
|
||||
}
|
||||
log.Info("Database inspection completed", "count", count.Load(), "size", common.StorageSize(total.Load()), "elapsed", common.PrettyDuration(time.Since(start)))
|
||||
return nil
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue