Merge pull request #876 from gzliudan/dbcmd_inspect

cmd/XDC, core/rawdb: add db command inspect

cmd/XDC/dbcmd.go

@@ -19,12 +19,14 @@ package main
 import (
 	"fmt"
 	"os"
+	"path/filepath"
 	"time"
 
 	"github.com/XinFinOrg/XDPoSChain/cmd/utils"
 	"github.com/XinFinOrg/XDPoSChain/common"
 	"github.com/XinFinOrg/XDPoSChain/common/hexutil"
 	"github.com/XinFinOrg/XDPoSChain/console"
+	"github.com/XinFinOrg/XDPoSChain/core/rawdb"
 	"github.com/XinFinOrg/XDPoSChain/ethdb"
 	"github.com/XinFinOrg/XDPoSChain/log"
 	"github.com/urfave/cli/v2"
@@ -38,8 +40,6 @@ var (
 		ArgsUsage: " ",
 		Flags: []cli.Flag{
 			utils.DataDirFlag,
-			utils.XDCXDataDirFlag,
-			utils.LightModeFlag,
 		},
 		Description: `
 Remove blockchain and state databases`,
@@ -49,6 +49,7 @@ Remove blockchain and state databases`,
 		Usage:     "Low level database operations",
 		ArgsUsage: "",
 		Subcommands: []*cli.Command{
+			dbInspectCmd,
 			dbStatCmd,
 			dbCompactCmd,
 			dbGetCmd,
@@ -56,24 +57,61 @@ Remove blockchain and state databases`,
 			dbPutCmd,
 		},
 	}
+	dbInspectCmd = &cli.Command{
+		Action:    inspect,
+		Name:      "inspect",
+		ArgsUsage: "<prefix> <start>",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.SyncModeFlag,
+			utils.MainnetFlag,
+			utils.TestnetFlag,
+			utils.DevnetFlag,
+		},
+		Usage:       "Inspect the storage size for each type of data in the database",
+		Description: `This commands iterates the entire database. If the optional 'prefix' and 'start' arguments are provided, then the iteration is limited to the given subset of data.`,
+	}
 	dbStatCmd = &cli.Command{
 		Action: dbStats,
 		Name:   "stats",
 		Usage:  "Print leveldb statistics",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.SyncModeFlag,
+			utils.MainnetFlag,
+			utils.TestnetFlag,
+			utils.DevnetFlag,
+		},
 	}
 	dbCompactCmd = &cli.Command{
 		Action: dbCompact,
 		Name:   "compact",
 		Usage:  "Compact leveldb database. WARNING: May take a very long time",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.SyncModeFlag,
+			utils.MainnetFlag,
+			utils.TestnetFlag,
+			utils.DevnetFlag,
+			utils.CacheFlag,
+			utils.CacheDatabaseFlag,
+		},
 		Description: `This command performs a database compaction.
 WARNING: This operation may take a very long time to finish, and may cause database
 corruption if it is aborted during execution'!`,
 	}
 	dbGetCmd = &cli.Command{
-		Action:      dbGet,
+		Action:    dbGet,
-		Name:        "get",
+		Name:      "get",
-		Usage:       "Show the value of a database key",
+		Usage:     "Show the value of a database key",
-		ArgsUsage:   "<hex-encoded key>",
+		ArgsUsage: "<hex-encoded key>",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.SyncModeFlag,
+			utils.MainnetFlag,
+			utils.TestnetFlag,
+			utils.DevnetFlag,
+		},
 		Description: "This command looks up the specified database key from the database.",
 	}
 	dbDeleteCmd = &cli.Command{
@@ -81,6 +119,13 @@ corruption if it is aborted during execution'!`,
 		Name:      "delete",
 		Usage:     "Delete a database key (WARNING: may corrupt your database)",
 		ArgsUsage: "<hex-encoded key>",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.SyncModeFlag,
+			utils.MainnetFlag,
+			utils.TestnetFlag,
+			utils.DevnetFlag,
+		},
 		Description: `This command deletes the specified database key from the database.
 WARNING: This is a low-level operation which may cause database corruption!`,
 	}
@@ -89,6 +134,13 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 		Name:      "put",
 		Usage:     "Set the value of a database key (WARNING: may corrupt your database)",
 		ArgsUsage: "<hex-encoded key> <hex-encoded value>",
+		Flags: []cli.Flag{
+			utils.DataDirFlag,
+			utils.SyncModeFlag,
+			utils.MainnetFlag,
+			utils.TestnetFlag,
+			utils.DevnetFlag,
+		},
 		Description: `This command sets a given database key to the given value.
 WARNING: This is a low-level operation which may cause database corruption!`,
 	}
@@ -97,35 +149,80 @@ WARNING: This is a low-level operation which may cause database corruption!`,
 func removeDB(ctx *cli.Context) error {
 	stack, _ := makeConfigNode(ctx)
 	for _, name := range []string{"chaindata", "lightchaindata"} {
-		// Ensure the database exists in the first place
-		logger := log.New("database", name)
-
 		dbdir := stack.ResolvePath(name)
-		if !common.FileExist(dbdir) {
+		if common.FileExist(dbdir) {
-			logger.Info("Database doesn't exist, skipping", "path", dbdir)
+			confirmAndRemoveDB(dbdir, name)
-			continue
+		} else {
+			log.Info("Database doesn't exist, skipping", "path", dbdir)
 		}
-		confirmAndRemoveDB(dbdir, name)
 	}
 	return nil
 }
 
+// removeFolder deletes all files (not folders) inside the directory 'dir' (but
+// not files in subfolders).
+func removeFolder(dir string) {
+	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		// If we're at the top level folder, recurse into
+		if path == dir {
+			return nil
+		}
+		// Delete all the files, but not subfolders
+		if !info.IsDir() {
+			os.Remove(path)
+			return nil
+		}
+		return filepath.SkipDir
+	})
+}
+
 // confirmAndRemoveDB prompts the user for a last confirmation and removes the
 // folder if accepted.
-func confirmAndRemoveDB(database string, kind string) {
+func confirmAndRemoveDB(path string, kind string) {
-	confirm, err := console.Stdin.PromptConfirm(fmt.Sprintf("Remove %s (%s)?", kind, database))
+	confirm, err := console.Stdin.PromptConfirm(fmt.Sprintf("Remove %s (%s)?", kind, path))
 	switch {
 	case err != nil:
 		utils.Fatalf("%v", err)
 	case !confirm:
-		log.Warn("Database deletion aborted", "path", database)
+		log.Warn("Database deletion aborted", "path", path)
 	default:
 		start := time.Now()
-		os.RemoveAll(database)
+		removeFolder(path)
-		log.Info("Database successfully deleted", "path", database, "elapsed", common.PrettyDuration(time.Since(start)))
+		log.Info("Database successfully deleted", "kind", kind, "path", path, "elapsed", common.PrettyDuration(time.Since(start)))
 	}
 }
 
+func inspect(ctx *cli.Context) error {
+	var (
+		prefix []byte
+		start  []byte
+	)
+	if ctx.NArg() > 2 {
+		return fmt.Errorf("max 2 arguments: %v", ctx.Command.ArgsUsage)
+	}
+	if ctx.NArg() >= 1 {
+		if d, err := hexutil.Decode(ctx.Args().Get(0)); err != nil {
+			return fmt.Errorf("failed to hex-decode 'prefix': %v", err)
+		} else {
+			prefix = d
+		}
+	}
+	if ctx.NArg() >= 2 {
+		if d, err := hexutil.Decode(ctx.Args().Get(1)); err != nil {
+			return fmt.Errorf("failed to hex-decode 'start': %v", err)
+		} else {
+			start = d
+		}
+	}
+	stack, _ := makeConfigNode(ctx)
+	defer stack.Close()
+
+	db := utils.MakeChainDatabase(ctx, stack, true)
+	defer db.Close()
+
+	return rawdb.InspectDatabase(db, prefix, start)
+}
+
 func showLeveldbStats(db ethdb.Stater) {
 	if stats, err := db.Stat("leveldb.stats"); err != nil {
 		log.Warn("Failed to read database stats", "error", err)

cmd/utils/cmd.go

@@ -303,6 +303,8 @@ func ExportPreimages(db ethdb.Database, fn string) error {
 	}
 	// Iterate over the preimages and export them
 	it := db.NewIterator([]byte("secure-key-"), nil)
+	defer it.Release()
+
 	for it.Next() {
 		if err := rlp.Encode(writer, it.Value()); err != nil {
 			return err

core/rawdb/accessors_chain.go

@@ -148,7 +148,7 @@ func WriteHeadFastBlockHash(db ethdb.KeyValueWriter, hash common.Hash) {
 // ReadFastTrieProgress retrieves the number of tries nodes fast synced to allow
 // reportinc correct numbers across restarts.
 func ReadFastTrieProgress(db ethdb.KeyValueReader) uint64 {
-	data, _ := db.Get(trieSyncKey)
+	data, _ := db.Get(fastTrieProgressKey)
 	if len(data) == 0 {
 		return 0
 	}
@@ -158,7 +158,7 @@ func ReadFastTrieProgress(db ethdb.KeyValueReader) uint64 {
 // WriteFastTrieProgress stores the fast sync trie process counter to support
 // retrieving it across restarts.
 func WriteFastTrieProgress(db ethdb.KeyValueWriter, count uint64) error {
-	if err := db.Put(trieSyncKey, new(big.Int).SetUint64(count).Bytes()); err != nil {
+	if err := db.Put(fastTrieProgressKey, new(big.Int).SetUint64(count).Bytes()); err != nil {
 		log.Crit("Failed to store fast sync trie progress", "err", err)
 	}
 	return nil

core/rawdb/database.go

@@ -17,11 +17,17 @@
 package rawdb
 
 import (
+	"bytes"
 	"fmt"
+	"os"
+	"time"
 
+	"github.com/XinFinOrg/XDPoSChain/common"
 	"github.com/XinFinOrg/XDPoSChain/ethdb"
 	"github.com/XinFinOrg/XDPoSChain/ethdb/leveldb"
 	"github.com/XinFinOrg/XDPoSChain/ethdb/memorydb"
+	"github.com/XinFinOrg/XDPoSChain/log"
+	"github.com/olekukonko/tablewriter"
 )
 
 // freezerdb is a database wrapper that enabled freezer data retrievals.
@@ -116,3 +122,138 @@ func NewLevelDBDatabase(file string, cache int, handles int, namespace string, r
 	}
 	return NewDatabase(db), nil
 }
+
+// InspectDatabase traverses the entire database and checks the size
+// of all different categories of data.
+func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
+	it := db.NewIterator(keyPrefix, keyStart)
+	defer it.Release()
+
+	var (
+		count  int64
+		start  = time.Now()
+		logged = time.Now()
+
+		// Key-value store statistics
+		total           common.StorageSize
+		headerSize      common.StorageSize
+		bodySize        common.StorageSize
+		receiptSize     common.StorageSize
+		tdSize          common.StorageSize
+		numHashPairing  common.StorageSize
+		hashNumPairing  common.StorageSize
+		trieSize        common.StorageSize
+		txlookupSize    common.StorageSize
+		preimageSize    common.StorageSize
+		bloomBitsSize   common.StorageSize
+		cliqueSnapsSize common.StorageSize
+
+		// Ancient store statistics
+		ancientHeaders  common.StorageSize
+		ancientBodies   common.StorageSize
+		ancientReceipts common.StorageSize
+		ancientHashes   common.StorageSize
+		ancientTds      common.StorageSize
+
+		// Les statistic
+		chtTrieNodes   common.StorageSize
+		bloomTrieNodes common.StorageSize
+
+		// Meta- and unaccounted data
+		metadata    common.StorageSize
+		unaccounted common.StorageSize
+	)
+	// Inspect key-value database first.
+	for it.Next() {
+		var (
+			key  = it.Key()
+			size = common.StorageSize(len(key) + len(it.Value()))
+		)
+		total += size
+		switch {
+		case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerTDSuffix):
+			tdSize += size
+		case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerHashSuffix):
+			numHashPairing += size
+		case bytes.HasPrefix(key, headerPrefix) && len(key) == (len(headerPrefix)+8+common.HashLength):
+			headerSize += size
+		case bytes.HasPrefix(key, headerNumberPrefix) && len(key) == (len(headerNumberPrefix)+common.HashLength):
+			hashNumPairing += size
+		case bytes.HasPrefix(key, blockBodyPrefix) && len(key) == (len(blockBodyPrefix)+8+common.HashLength):
+			bodySize += size
+		case bytes.HasPrefix(key, blockReceiptsPrefix) && len(key) == (len(blockReceiptsPrefix)+8+common.HashLength):
+			receiptSize += size
+		case bytes.HasPrefix(key, txLookupPrefix) && len(key) == (len(txLookupPrefix)+common.HashLength):
+			txlookupSize += size
+		case bytes.HasPrefix(key, preimagePrefix) && len(key) == (len(preimagePrefix)+common.HashLength):
+			preimageSize += size
+		case bytes.HasPrefix(key, bloomBitsPrefix) && len(key) == (len(bloomBitsPrefix)+10+common.HashLength):
+			bloomBitsSize += size
+		case bytes.HasPrefix(key, []byte("clique-")) && len(key) == 7+common.HashLength:
+			cliqueSnapsSize += size
+		case bytes.HasPrefix(key, []byte("cht-")) && len(key) == 4+common.HashLength:
+			chtTrieNodes += size
+		case bytes.HasPrefix(key, []byte("blt-")) && len(key) == 4+common.HashLength:
+			bloomTrieNodes += size
+		case len(key) == common.HashLength:
+			trieSize += size
+		default:
+			var accounted bool
+			for _, meta := range [][]byte{databaseVersionKey, headHeaderKey, headBlockKey, headFastBlockKey, fastTrieProgressKey} {
+				if bytes.Equal(key, meta) {
+					metadata += size
+					accounted = true
+					break
+				}
+			}
+			if !accounted {
+				unaccounted += size
+			}
+		}
+		count += 1
+		if count%1000 == 0 && time.Since(logged) > 8*time.Second {
+			log.Info("Inspecting database", "count", count, "elapsed", common.PrettyDuration(time.Since(start)))
+			logged = time.Now()
+		}
+	}
+	// Inspect append-only file store then.
+	ancients := []*common.StorageSize{&ancientHeaders, &ancientBodies, &ancientReceipts, &ancientHashes, &ancientTds}
+	for i, category := range []string{freezerHeaderTable, freezerBodiesTable, freezerReceiptTable, freezerHashTable, freezerDifficultyTable} {
+		if size, err := db.AncientSize(category); err == nil {
+			*ancients[i] += common.StorageSize(size)
+			total += common.StorageSize(size)
+		}
+	}
+	// Display the database statistic.
+	stats := [][]string{
+		{"Key-Value store", "Headers", headerSize.String()},
+		{"Key-Value store", "Bodies", bodySize.String()},
+		{"Key-Value store", "Receipts", receiptSize.String()},
+		{"Key-Value store", "Difficulties", tdSize.String()},
+		{"Key-Value store", "Block number->hash", numHashPairing.String()},
+		{"Key-Value store", "Block hash->number", hashNumPairing.String()},
+		{"Key-Value store", "Transaction index", txlookupSize.String()},
+		{"Key-Value store", "Bloombit index", bloomBitsSize.String()},
+		{"Key-Value store", "Trie nodes", trieSize.String()},
+		{"Key-Value store", "Trie preimages", preimageSize.String()},
+		{"Key-Value store", "Clique snapshots", cliqueSnapsSize.String()},
+		{"Key-Value store", "Singleton metadata", metadata.String()},
+		{"Ancient store", "Headers", ancientHeaders.String()},
+		{"Ancient store", "Bodies", ancientBodies.String()},
+		{"Ancient store", "Receipts", ancientReceipts.String()},
+		{"Ancient store", "Difficulties", ancientTds.String()},
+		{"Ancient store", "Block number->hash", ancientHashes.String()},
+		{"Light client", "CHT trie nodes", chtTrieNodes.String()},
+		{"Light client", "Bloom trie nodes", bloomTrieNodes.String()},
+	}
+	table := tablewriter.NewWriter(os.Stdout)
+	table.SetHeader([]string{"Database", "Category", "Size"})
+	table.SetFooter([]string{"", "Total", total.String()})
+	table.AppendBulk(stats)
+	table.Render()
+
+	if unaccounted > 0 {
+		log.Error("Database contains unaccounted data", "size", unaccounted)
+	}
+	return nil
+}

core/rawdb/schema.go

@@ -38,7 +38,8 @@ var (
 	// headFastBlockKey tracks the latest known incomplete block's hash during fast sync.
 	headFastBlockKey = []byte("LastFast")
 
-	trieSyncKey = []byte("TrieSync")
+	// fastTrieProgressKey tracks the number of trie entries imported during fast sync.
+	fastTrieProgressKey = []byte("TrieSync")
 
 	// Data item prefixes (use single byte to avoid mixing data types, avoid `i`, used for indexes).
 	headerPrefix       = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header
@@ -80,6 +81,9 @@ const (
 
 	// freezerReceiptTable indicates the name of the freezer receipts table.
 	freezerReceiptTable = "receipts"
+
+	// freezerDifficultyTable indicates the name of the freezer total difficulty table.
+	freezerDifficultyTable = "diffs"
 )
 
 // LegacyTxLookupEntry is the legacy TxLookupEntry definition with some unnecessary