From b4f7b076a5cc938038a2b306efeb10470dcb247a Mon Sep 17 00:00:00 2001 From: lightclient Date: Tue, 17 Feb 2026 19:23:34 +0000 Subject: [PATCH] cmd/geth,trie: stream trie inspect to disk then summarize after --- cmd/geth/dbcmd.go | 67 +++-- trie/inspect.go | 592 ++++++++++++++++++++++++++++++++++++------- trie/inspect_test.go | 15 +- 3 files changed, 562 insertions(+), 112 deletions(-) diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index 01000f5b99..8212959876 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -55,6 +55,19 @@ var ( Name: "remove.chain", Usage: "If set, selects the state data for removal", } + inspectTrieTopFlag = &cli.IntFlag{ + Name: "top", + Usage: "Print the top N results per ranking category", + Value: 10, + } + inspectTrieDumpPathFlag = &cli.StringFlag{ + Name: "dump-path", + Usage: "Path for the trie statistics dump file", + } + inspectTrieSummarizeFlag = &cli.StringFlag{ + Name: "summarize", + Usage: "Summarize an existing trie dump file (skip trie traversal)", + } removedbCommand = &cli.Command{ Action: removeDB, @@ -99,8 +112,14 @@ Remove blockchain and state databases`, Action: inspectTrie, Name: "inspect-trie", ArgsUsage: "", - Flags: slices.Concat([]cli.Flag{utils.ExcludeStorageFlag, utils.TopFlag, utils.OutputFileFlag}, utils.NetworkFlags, utils.DatabaseFlags), - Usage: "Print detailed trie information about the structure of account trie and storage tries.", + Flags: slices.Concat([]cli.Flag{ + utils.ExcludeStorageFlag, + inspectTrieTopFlag, + utils.OutputFileFlag, + inspectTrieDumpPathFlag, + inspectTrieSummarizeFlag, + }, utils.NetworkFlags, utils.DatabaseFlags), + Usage: "Print detailed trie information about the structure of account trie and storage tries.", Description: `This commands iterates the entrie trie-backed state. If the 'blocknum' is not specified, the latest block number will be used by default.`, } @@ -398,9 +417,28 @@ func checkStateContent(ctx *cli.Context) error { } func inspectTrie(ctx *cli.Context) error { + topN := ctx.Int(inspectTrieTopFlag.Name) + if topN <= 0 { + return fmt.Errorf("invalid --%s value %d (must be > 0)", inspectTrieTopFlag.Name, topN) + } + config := &trie.InspectConfig{ + NoStorage: ctx.Bool(utils.ExcludeStorageFlag.Name), + TopN: topN, + Path: ctx.String(utils.OutputFileFlag.Name), + } + + if summarizePath := ctx.String(inspectTrieSummarizeFlag.Name); summarizePath != "" { + if ctx.NArg() > 0 { + return fmt.Errorf("block number argument is not supported with --%s", inspectTrieSummarizeFlag.Name) + } + config.DumpPath = summarizePath + log.Info("Summarizing trie dump", "path", summarizePath, "top", topN) + return trie.Summarize(summarizePath, config) + } if ctx.NArg() > 1 { return fmt.Errorf("excessive number of arguments: %v", ctx.Command.ArgsUsage) } + stack, _ := makeConfigNode(ctx) db := utils.MakeChainDatabase(ctx, stack, false) defer stack.Close() @@ -413,8 +451,8 @@ func inspectTrie(ctx *cli.Context) error { ) switch { case ctx.NArg() == 0 || ctx.Args().Get(0) == "latest": - hash := rawdb.ReadHeadHeaderHash(db) - n, ok := rawdb.ReadHeaderNumber(db, hash) + head := rawdb.ReadHeadHeaderHash(db) + n, ok := rawdb.ReadHeaderNumber(db, head) if !ok { return fmt.Errorf("could not load head block hash") } @@ -430,7 +468,6 @@ func inspectTrie(ctx *cli.Context) error { } } - // Load head block number based on canonical hash, if applicable. if number != math.MaxUint64 { hash = rawdb.ReadCanonicalHash(db, number) if hash == (common.Hash{}) { @@ -439,24 +476,20 @@ func inspectTrie(ctx *cli.Context) error { blockHeader := rawdb.ReadHeader(db, hash, number) trieRoot = blockHeader.Root } - if (trieRoot == common.Hash{}) { + if trieRoot == (common.Hash{}) { log.Error("Empty root hash") } + config.DumpPath = ctx.String(inspectTrieDumpPathFlag.Name) + if config.DumpPath == "" { + config.DumpPath = stack.ResolvePath("trie-dump.bin") + } + triedb := utils.MakeTrieDatabase(ctx, stack, db, false, true, false) defer triedb.Close() - log.Info("Inspecting trie", "root", trieRoot, "block", number) - config := &trie.InspectConfig{ - NoStorage: ctx.Bool(utils.ExcludeStorageFlag.Name), - TopN: ctx.Int(utils.TopFlag.Name), - Path: ctx.String(utils.OutputFileFlag.Name), - } - err := trie.Inspect(triedb, trieRoot, config) - if err != nil { - return err - } - return nil + log.Info("Inspecting trie", "root", trieRoot, "block", number, "dump", config.DumpPath, "top", topN) + return trie.Inspect(triedb, trieRoot, config) } func showDBStats(db ethdb.KeyValueStater) { diff --git a/trie/inspect.go b/trie/inspect.go index 1ddb0e6d1f..be8948dbfe 100644 --- a/trie/inspect.go +++ b/trie/inspect.go @@ -17,10 +17,16 @@ package trie import ( + "bufio" "bytes" + "container/heap" + "encoding/binary" "encoding/json" + "errors" "fmt" + "io" "os" + "slices" "sort" "strings" "sync" @@ -34,58 +40,162 @@ import ( "golang.org/x/sync/semaphore" ) +const ( + inspectDumpRecordSize = 32 + trieStatLevels*3*4 + inspectDefaultTopN = 10 + inspectParallelism = int64(16) +) + // inspector is used by the inner inspect function to coordinate across threads. type inspector struct { triedb database.NodeDatabase root common.Hash - config *InspectConfig - stats map[common.Hash]*LevelStats - m sync.Mutex // protects stats + config *InspectConfig + accountStat *LevelStats sem *semaphore.Weighted wg sync.WaitGroup + + // Pass 1: dump file writer. + dumpMu sync.Mutex + dumpBuf *bufio.Writer + dumpFile *os.File + + errMu sync.Mutex + err error } -// InspectConfig is a set of options to control inspection and format the -// output. TopN will print the deepest min(len(results), N) storage tries. -// If path is set, a JSON version of the data will be written to a file at this path. +// InspectConfig is a set of options to control inspection and format the output. +// TopN determines the maximum number of entries retained for each top-list. +// Path controls optional JSON output. DumpPath controls the pass-1 dump location. type InspectConfig struct { NoStorage bool TopN int Path string + DumpPath string } // Inspect walks the trie with the given root and records the number and type of -// nodes at each depth. It works by recursively calling the inner inspect -// function on each child node. +// nodes at each depth. Storage trie stats are streamed to disk in fixed-size +// records, then summarized in a second pass. func Inspect(triedb database.NodeDatabase, root common.Hash, config *InspectConfig) error { trie, err := New(TrieID(root), triedb) if err != nil { return fmt.Errorf("fail to open trie %s: %w", root, err) } + config = normalizeInspectConfig(config) + + dumpFile, err := os.OpenFile(config.DumpPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + if err != nil { + return fmt.Errorf("failed to create trie dump %s: %w", config.DumpPath, err) + } + in := inspector{ + triedb: triedb, + root: root, + config: config, + accountStat: NewLevelStats(), + sem: semaphore.NewWeighted(inspectParallelism), + dumpBuf: bufio.NewWriterSize(dumpFile, 1<<20), + dumpFile: dumpFile, + } + + in.inspect(trie, trie.root, 0, []byte{}, in.accountStat) + in.wg.Wait() + + // Persist account trie stats as the sentinel record. + in.writeDumpRecord(common.Hash{}, in.accountStat) + if err := in.closeDump(); err != nil { + in.setError(err) + } + if err := in.getError(); err != nil { + return err + } + return Summarize(config.DumpPath, config) +} + +func normalizeInspectConfig(config *InspectConfig) *InspectConfig { if config == nil { config = &InspectConfig{} } - in := inspector{ - triedb: triedb, - root: root, - config: config, - stats: make(map[common.Hash]*LevelStats), - sem: semaphore.NewWeighted(int64(128)), + if config.TopN <= 0 { + config.TopN = inspectDefaultTopN } - in.stats[root] = NewLevelStats() + if config.DumpPath == "" { + config.DumpPath = "trie-dump.bin" + } + return config +} - in.inspect(trie, trie.root, 0, []byte{}, in.stats[root]) - in.wg.Wait() - if len(config.Path) > 0 { - if err := in.writeJSON(); err != nil { - log.Crit("Error during json encodeing", "error", err) +func (in *inspector) closeDump() error { + var ret error + if in.dumpBuf != nil { + if err := in.dumpBuf.Flush(); err != nil { + ret = errors.Join(ret, fmt.Errorf("failed to flush trie dump %s: %w", in.config.DumpPath, err)) } - } else { - in.displayResult() } - return nil + if in.dumpFile != nil { + if err := in.dumpFile.Close(); err != nil { + ret = errors.Join(ret, fmt.Errorf("failed to close trie dump %s: %w", in.config.DumpPath, err)) + } + } + return ret +} + +func (in *inspector) setError(err error) { + if err == nil { + return + } + in.errMu.Lock() + defer in.errMu.Unlock() + in.err = errors.Join(in.err, err) +} + +func (in *inspector) getError() error { + in.errMu.Lock() + defer in.errMu.Unlock() + return in.err +} + +func (in *inspector) hasError() bool { + return in.getError() != nil +} + +func (in *inspector) spawn(fn func()) bool { + if !in.sem.TryAcquire(1) { + return false + } + in.wg.Add(1) + go func() { + defer in.sem.Release(1) + defer in.wg.Done() + fn() + }() + return true +} + +func (in *inspector) writeDumpRecord(owner common.Hash, s *LevelStats) { + if in.hasError() { + return + } + var buf [inspectDumpRecordSize]byte + copy(buf[:32], owner[:]) + + off := 32 + for i := 0; i < trieStatLevels; i++ { + binary.LittleEndian.PutUint32(buf[off:], uint32(s.level[i].short.Load())) + off += 4 + binary.LittleEndian.PutUint32(buf[off:], uint32(s.level[i].full.Load())) + off += 4 + binary.LittleEndian.PutUint32(buf[off:], uint32(s.level[i].value.Load())) + off += 4 + } + in.dumpMu.Lock() + _, err := in.dumpBuf.Write(buf[:]) + in.dumpMu.Unlock() + if err != nil { + in.setError(fmt.Errorf("failed writing trie dump record: %w", err)) + } } // inspect is called recursively down the trie. At each level it records the @@ -102,22 +212,21 @@ func (in *inspector) inspect(trie *Trie, n node, height uint32, path []byte, sta // - value: if account, begin inspecting storage trie. switch n := (n).(type) { case *shortNode: - in.inspect(trie, n.Val, height+1, append(path, n.Key...), stat) + nextPath := slices.Concat(path, n.Key) + in.inspect(trie, n.Val, height+1, nextPath, stat) case *fullNode: for idx, child := range n.Children { if child == nil { continue } - childPath := append(path, byte(idx)) - if in.sem.TryAcquire(1) { - in.wg.Add(1) - go func() { - in.inspect(trie, child, height+1, childPath, stat) - in.wg.Done() - }() - } else { - in.inspect(trie, child, height+1, childPath, stat) + childPath := slices.Concat(path, []byte{byte(idx)}) + childNode := child + if in.spawn(func() { + in.inspect(trie, childNode, height+1, childPath, stat) + }) { + continue } + in.inspect(trie, childNode, height+1, childPath, stat) } case hashNode: resolved, err := trie.resolveWithoutTrack(n, path) @@ -143,7 +252,6 @@ func (in *inspector) inspect(trie *Trie, n node, height uint32, path []byte, sta break } - // Start inspecting storage trie. if !in.config.NoStorage { owner := common.BytesToHash(hexToCompact(path)) storage, err := New(StorageTrieID(in.root, owner, account.Root), in.triedb) @@ -151,86 +259,383 @@ func (in *inspector) inspect(trie *Trie, n node, height uint32, path []byte, sta log.Error("Failed to open account storage trie", "node", n, "error", err, "height", height, "path", common.Bytes2Hex(path)) break } - stat := NewLevelStats() - - in.m.Lock() - in.stats[owner] = stat - in.m.Unlock() - - in.wg.Add(1) - go func() { - in.inspect(storage, storage.root, 0, []byte{}, stat) - in.wg.Done() - }() + storageStat := NewLevelStats() + run := func() { + in.inspect(storage, storage.root, 0, []byte{}, storageStat) + in.writeDumpRecord(owner, storageStat) + } + if in.spawn(run) { + break + } + run() } default: panic(fmt.Sprintf("%T: invalid node: %v", n, n)) } - // Record stats for current height + // Record stats for current height. stat.add(n, height) } -// Display results prints out the inspect results. -func (in *inspector) displayResult() { - fmt.Println("Results for trie", in.root) - in.stats[in.root].display("Accounts trie") - fmt.Println("===") - fmt.Println() - - if !in.config.NoStorage { - // Sort stats by max node depth. - keys, stats := sortedLevelStats(in.stats).sort() - - fmt.Println("Results for top storage tries") - for i := range keys[0:min(in.config.TopN, len(keys))] { - fmt.Printf("%d: %s\n", i+1, keys[i]) - stats[i].display("storage trie") - } +// Summarize performs pass 2 over a trie dump and reports account stats, +// aggregate storage statistics, and top-N rankings. +func Summarize(dumpPath string, config *InspectConfig) error { + config = normalizeInspectConfig(config) + if dumpPath == "" { + dumpPath = config.DumpPath } -} - -func (in *inspector) writeJSON() error { - file, err := os.OpenFile(in.config.Path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0777) + if dumpPath == "" { + return errors.New("missing dump path") + } + file, err := os.Open(dumpPath) if err != nil { - return err + return fmt.Errorf("failed to open trie dump %s: %w", dumpPath, err) } - enc := json.NewEncoder(file) + defer file.Close() - accountTrie := newJsonStat(in.stats[in.root], "account trie") - if err := enc.Encode(accountTrie); err != nil { - return err - } - if !in.config.NoStorage { - // Sort stats by max node depth. - keys, stats := sortedLevelStats(in.stats).sort() - for i := range keys[0:min(in.config.TopN, len(keys))] { - storageTrie := newJsonStat(stats[i], fmt.Sprintf("%x", keys[i])) - if err := enc.Encode(storageTrie); err != nil { - return err - } + if info, err := file.Stat(); err == nil { + if info.Size()%inspectDumpRecordSize != 0 { + return fmt.Errorf("invalid trie dump size %d (not a multiple of %d)", info.Size(), inspectDumpRecordSize) } } + + depthTop := newStorageTopN(config.TopN, compareStorageByDepth) + totalTop := newStorageTopN(config.TopN, compareStorageByTotal) + valueTop := newStorageTopN(config.TopN, compareStorageByValue) + + summary := &inspectSummary{} + reader := bufio.NewReaderSize(file, 1<<20) + var buf [inspectDumpRecordSize]byte + + for { + _, err := io.ReadFull(reader, buf[:]) + if errors.Is(err, io.EOF) { + break + } + if errors.Is(err, io.ErrUnexpectedEOF) { + return fmt.Errorf("truncated trie dump %s", dumpPath) + } + if err != nil { + return fmt.Errorf("failed reading trie dump %s: %w", dumpPath, err) + } + + record := decodeDumpRecord(buf[:]) + snapshot := newStorageSnapshot(record.Owner, record.Levels) + if record.Owner == (common.Hash{}) { + summary.Account = snapshot + continue + } + summary.StorageCount++ + summary.DepthHistogram[snapshot.MaxDepth]++ + summary.StorageTotals.Short += snapshot.Summary.Short + summary.StorageTotals.Full += snapshot.Summary.Full + summary.StorageTotals.Value += snapshot.Summary.Value + + depthTop.TryInsert(snapshot) + totalTop.TryInsert(snapshot) + valueTop.TryInsert(snapshot) + } + if summary.Account == nil { + return fmt.Errorf("dump file %s does not contain the account trie sentinel record", dumpPath) + } + summary.TopByDepth = depthTop.Sorted() + summary.TopByTotalNodes = totalTop.Sorted() + summary.TopByValueNodes = valueTop.Sorted() + + if config.Path != "" { + return summary.writeJSON(config.Path) + } + summary.display() return nil } -// sortedLevelStats implements sort(). -type sortedLevelStats map[common.Hash]*LevelStats +type dumpRecord struct { + Owner common.Hash + Levels [trieStatLevels]jsonLevel +} -// sort returns the keys and stats in descending order of maximum trie node depth. -func (s sortedLevelStats) sort() ([]common.Hash, []*LevelStats) { +func decodeDumpRecord(raw []byte) dumpRecord { var ( - keys = make([]common.Hash, 0, len(s)) - stats = make([]*LevelStats, 0, len(s)) + record dumpRecord + off = 32 ) - for k := range s { - keys = append(keys, k) + copy(record.Owner[:], raw[:32]) + for i := 0; i < trieStatLevels; i++ { + record.Levels[i] = jsonLevel{ + Short: uint64(binary.LittleEndian.Uint32(raw[off:])), + Full: uint64(binary.LittleEndian.Uint32(raw[off+4:])), + Value: uint64(binary.LittleEndian.Uint32(raw[off+8:])), + } + off += 12 } - sort.Slice(keys, func(i, j int) bool { return s[keys[i]].maxDepth() > s[keys[j]].maxDepth() }) - for _, k := range keys { - stats = append(stats, s[k]) + return record +} + +type storageSnapshot struct { + Owner common.Hash + Levels [trieStatLevels]jsonLevel + Summary jsonLevel + MaxDepth int + TotalNodes uint64 +} + +func newStorageSnapshot(owner common.Hash, levels [trieStatLevels]jsonLevel) *storageSnapshot { + snapshot := &storageSnapshot{Owner: owner, Levels: levels} + for i := 0; i < trieStatLevels; i++ { + level := levels[i] + if level.Short != 0 || level.Full != 0 || level.Value != 0 { + snapshot.MaxDepth = i + } + snapshot.Summary.Short += level.Short + snapshot.Summary.Full += level.Full + snapshot.Summary.Value += level.Value } - return keys, stats + snapshot.TotalNodes = snapshot.Summary.Short + snapshot.Summary.Full + snapshot.Summary.Value + return snapshot +} + +func (s *storageSnapshot) toLevelStats() *LevelStats { + stats := NewLevelStats() + for i := 0; i < trieStatLevels; i++ { + stats.level[i].short.Store(s.Levels[i].Short) + stats.level[i].full.Store(s.Levels[i].Full) + stats.level[i].value.Store(s.Levels[i].Value) + } + return stats +} + +type storageCompare func(a, b *storageSnapshot) int + +type topStorage struct { + limit int + cmp storageCompare + heap storageHeap +} + +type storageHeap struct { + items []*storageSnapshot + cmp storageCompare +} + +func (h storageHeap) Len() int { return len(h.items) } + +func (h storageHeap) Less(i, j int) bool { + // Keep the weakest entry at the root (min-heap semantics). + return h.cmp(h.items[i], h.items[j]) < 0 +} + +func (h storageHeap) Swap(i, j int) { h.items[i], h.items[j] = h.items[j], h.items[i] } + +func (h *storageHeap) Push(x any) { + h.items = append(h.items, x.(*storageSnapshot)) +} + +func (h *storageHeap) Pop() any { + item := h.items[len(h.items)-1] + h.items = h.items[:len(h.items)-1] + return item +} + +func newStorageTopN(limit int, cmp storageCompare) *topStorage { + h := storageHeap{cmp: cmp} + heap.Init(&h) + return &topStorage{limit: limit, cmp: cmp, heap: h} +} + +func (t *topStorage) TryInsert(item *storageSnapshot) { + if t.limit <= 0 { + return + } + if t.heap.Len() < t.limit { + heap.Push(&t.heap, item) + return + } + if t.cmp(item, t.heap.items[0]) <= 0 { + return + } + heap.Pop(&t.heap) + heap.Push(&t.heap, item) +} + +func (t *topStorage) Sorted() []*storageSnapshot { + out := append([]*storageSnapshot(nil), t.heap.items...) + sort.Slice(out, func(i, j int) bool { return t.cmp(out[i], out[j]) > 0 }) + return out +} + +func compareStorageByDepth(a, b *storageSnapshot) int { + if cmp := compareInt(a.MaxDepth, b.MaxDepth); cmp != 0 { + return cmp + } + if cmp := compareUint64(a.TotalNodes, b.TotalNodes); cmp != 0 { + return cmp + } + if cmp := compareUint64(a.Summary.Value, b.Summary.Value); cmp != 0 { + return cmp + } + return bytes.Compare(a.Owner[:], b.Owner[:]) +} + +func compareStorageByTotal(a, b *storageSnapshot) int { + if cmp := compareUint64(a.TotalNodes, b.TotalNodes); cmp != 0 { + return cmp + } + if cmp := compareInt(a.MaxDepth, b.MaxDepth); cmp != 0 { + return cmp + } + if cmp := compareUint64(a.Summary.Value, b.Summary.Value); cmp != 0 { + return cmp + } + return bytes.Compare(a.Owner[:], b.Owner[:]) +} + +func compareStorageByValue(a, b *storageSnapshot) int { + if cmp := compareUint64(a.Summary.Value, b.Summary.Value); cmp != 0 { + return cmp + } + if cmp := compareInt(a.MaxDepth, b.MaxDepth); cmp != 0 { + return cmp + } + if cmp := compareUint64(a.TotalNodes, b.TotalNodes); cmp != 0 { + return cmp + } + return bytes.Compare(a.Owner[:], b.Owner[:]) +} + +func compareInt(a, b int) int { + switch { + case a > b: + return 1 + case a < b: + return -1 + default: + return 0 + } +} + +func compareUint64(a, b uint64) int { + switch { + case a > b: + return 1 + case a < b: + return -1 + default: + return 0 + } +} + +type inspectSummary struct { + Account *storageSnapshot + StorageCount uint64 + StorageTotals jsonLevel + DepthHistogram [trieStatLevels]uint64 + TopByDepth []*storageSnapshot + TopByTotalNodes []*storageSnapshot + TopByValueNodes []*storageSnapshot +} + +func (s *inspectSummary) display() { + s.Account.toLevelStats().display("Accounts trie") + fmt.Println("Storage trie aggregate summary") + fmt.Printf("Total storage tries: %d\n", s.StorageCount) + totalNodes := s.StorageTotals.Short + s.StorageTotals.Full + s.StorageTotals.Value + fmt.Printf("Total nodes: %d\n", totalNodes) + fmt.Printf(" Short nodes: %d\n", s.StorageTotals.Short) + fmt.Printf(" Full nodes: %d\n", s.StorageTotals.Full) + fmt.Printf(" Value nodes: %d\n", s.StorageTotals.Value) + + b := new(strings.Builder) + table := tablewriter.NewWriter(b) + table.SetHeader([]string{"Max Depth", "Storage Tries"}) + for i, count := range s.DepthHistogram { + table.AppendBulk([][]string{{fmt.Sprint(i), fmt.Sprint(count)}}) + } + table.Render() + fmt.Print(b.String()) + fmt.Println() + + s.displayTop("Top storage tries by max depth", s.TopByDepth) + s.displayTop("Top storage tries by total node count", s.TopByTotalNodes) + s.displayTop("Top storage tries by value (slot) count", s.TopByValueNodes) +} + +func (s *inspectSummary) displayTop(title string, list []*storageSnapshot) { + fmt.Println(title) + if len(list) == 0 { + fmt.Println("No storage tries found") + fmt.Println() + return + } + for i, item := range list { + fmt.Printf("%d: %s\n", i+1, item.Owner) + item.toLevelStats().display("storage trie") + } +} + +func (s *inspectSummary) writeJSON(path string) error { + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0o644) + if err != nil { + return err + } + defer file.Close() + + enc := json.NewEncoder(file) + enc.SetIndent("", " ") + return enc.Encode(s.toJSON()) +} + +func (s *inspectSummary) toJSON() *jsonSummary { + jsonSummary := &jsonSummary{ + AccountTrie: newJsonStat(s.Account.toLevelStats(), "account trie"), + StorageSummary: jsonStorageSummary{ + TotalStorageTries: s.StorageCount, + Totals: s.StorageTotals, + DepthHistogram: s.DepthHistogram, + }, + TopByDepth: snapshotsToJSON(s.TopByDepth), + TopByTotalNodes: snapshotsToJSON(s.TopByTotalNodes), + TopByValueNodes: snapshotsToJSON(s.TopByValueNodes), + } + return jsonSummary +} + +type jsonSummary struct { + AccountTrie *jsonStat + StorageSummary jsonStorageSummary + TopByDepth []jsonStorageStat + TopByTotalNodes []jsonStorageStat + TopByValueNodes []jsonStorageStat +} + +type jsonStorageSummary struct { + TotalStorageTries uint64 + Totals jsonLevel + DepthHistogram [trieStatLevels]uint64 +} + +type jsonStorageStat struct { + Owner string + MaxDepth int + TotalNodes uint64 + ValueNodes uint64 + Levels []jsonLevel + Summary jsonLevel +} + +func snapshotsToJSON(list []*storageSnapshot) []jsonStorageStat { + out := make([]jsonStorageStat, 0, len(list)) + for _, item := range list { + stat := newJsonStat(item.toLevelStats(), item.Owner.Hex()) + out = append(out, jsonStorageStat{ + Owner: item.Owner.Hex(), + MaxDepth: item.MaxDepth, + TotalNodes: item.TotalNodes, + ValueNodes: item.Summary.Value, + Levels: stat.Levels, + Summary: stat.Summary, + }) + } + return out } // display will print a table displaying the trie's node statistics. @@ -247,7 +652,7 @@ func (s *LevelStats) display(title string) { stat := &stat{} for i := range s.level { if s.level[i].empty() { - break + continue } short, full, value := s.level[i].load() table.AppendBulk([][]string{{"-", fmt.Sprint(i), fmt.Sprint(short), fmt.Sprint(full), fmt.Sprint(value)}}) @@ -276,7 +681,6 @@ type jsonStat struct { func newJsonStat(s *LevelStats, name string) *jsonStat { ret := jsonStat{Name: name, Summary: jsonLevel{}} for i := 0; i < len(s.level); i++ { - // only count non-empty levels if s.level[i].empty() { continue } diff --git a/trie/inspect_test.go b/trie/inspect_test.go index eb9e16722a..c3145cb964 100644 --- a/trie/inspect_test.go +++ b/trie/inspect_test.go @@ -18,6 +18,7 @@ package trie import ( "math/rand" + "path/filepath" "testing" "github.com/ethereum/go-ethereum/core/rawdb" @@ -46,9 +47,21 @@ func TestInspect(t *testing.T) { db.Update(root, types.EmptyRootHash, trienode.NewWithNodeSet(nodes)) db.Commit(root) - if err := Inspect(db, root, &InspectConfig{TopN: 1}); err != nil { + tempDir := t.TempDir() + dumpPath := filepath.Join(tempDir, "trie-dump.bin") + if err := Inspect(db, root, &InspectConfig{ + TopN: 1, + DumpPath: dumpPath, + Path: filepath.Join(tempDir, "trie-summary.json"), + }); err != nil { t.Fatalf("inspect failed: %v", err) } + if err := Summarize(dumpPath, &InspectConfig{ + TopN: 1, + Path: filepath.Join(tempDir, "trie-summary-reanalysis.json"), + }); err != nil { + t.Fatalf("summarize failed: %v", err) + } } func makeAccountsWithStorage(db *testDb, size int, storage bool) (addresses [][20]byte, accounts [][]byte) {