feat: rawdb.InspectDatabaseOption (#111)

## Why this should be merged

Allows for `ava-labs/coreth` equivalent modifications of
`rawdb.InspectDatabase()` through external logic injection.

## How this works

Variadic options to:

1. Record a database statistic;
2. Mark a database statistic as metadata;
3. Filter statistics for printing.

## How this was tested

Testable example acting as a unit test.

---------

Signed-off-by: Arran Schlosberg <519948+ARR4N@users.noreply.github.com>
Co-authored-by: Quentin McGaw <quentin.mcgaw@avalabs.org>
This commit is contained in:
Arran Schlosberg 2025-02-07 17:32:51 +00:00 committed by GitHub
parent d210cc4fce
commit c74b645360
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 291 additions and 2 deletions

View file

@ -31,6 +31,7 @@ import (
"github.com/ava-labs/libevm/ethdb/leveldb"
"github.com/ava-labs/libevm/ethdb/memorydb"
"github.com/ava-labs/libevm/ethdb/pebble"
"github.com/ava-labs/libevm/libevm/options"
"github.com/ava-labs/libevm/log"
"github.com/olekukonko/tablewriter"
)
@ -451,7 +452,8 @@ func (s *stat) Count() string {
// InspectDatabase traverses the entire database and checks the size
// of all different categories of data.
func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte, opts ...InspectDatabaseOption) error {
libevmConfig := options.As[inspectDatabaseConfig](opts...)
it := db.NewIterator(keyPrefix, keyStart)
defer it.Release()
@ -549,6 +551,9 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
bytes.HasPrefix(key, BloomTrieIndexPrefix) ||
bytes.HasPrefix(key, BloomTriePrefix): // Bloomtrie sub
bloomTrieNodes.Add(size)
case libevmConfig.recordStat(key, size):
case libevmConfig.isMetadata(key):
metadata.Add(size)
default:
var accounted bool
for _, meta := range [][]byte{
@ -617,7 +622,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
table := tablewriter.NewWriter(os.Stdout)
table.SetHeader([]string{"Database", "Category", "Size", "Items"})
table.SetFooter([]string{"", "Total", total.String(), " "})
table.AppendBulk(stats)
table.AppendBulk(libevmConfig.transformStats(stats))
table.Render()
if unaccounted.size > 0 {

View file

@ -0,0 +1,90 @@
// Copyright 2025 the libevm authors.
//
// The libevm additions to go-ethereum are free software: you can redistribute
// them and/or modify them under the terms of the GNU Lesser General Public License
// as published by the Free Software Foundation, either version 3 of the License,
// or (at your option) any later version.
//
// The libevm additions are distributed in the hope that they will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser
// General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see
// <http://www.gnu.org/licenses/>.
package rawdb
import (
"github.com/ava-labs/libevm/common"
"github.com/ava-labs/libevm/libevm/options"
)
// An InspectDatabaseOption configures the behaviour of [InspectDatabase]. For
// each type of option, only one instance can be used in the same call to
// InspectDatabase().
type InspectDatabaseOption = options.Option[inspectDatabaseConfig]
type inspectDatabaseConfig struct {
statRecorder func([]byte, common.StorageSize) bool
isMeta func([]byte) bool
statsTransformer func([][]string) [][]string
}
func (c inspectDatabaseConfig) recordStat(key []byte, size common.StorageSize) bool {
if r := c.statRecorder; r != nil {
return r(key, size)
}
return false
}
func (c inspectDatabaseConfig) isMetadata(key []byte) bool {
if m := c.isMeta; m != nil {
return m(key)
}
return false
}
func (c inspectDatabaseConfig) transformStats(stats [][]string) [][]string {
if f := c.statsTransformer; f != nil {
return f(stats)
}
return stats
}
func newInspectOpt(fn func(*inspectDatabaseConfig)) InspectDatabaseOption {
return options.Func[inspectDatabaseConfig](fn)
}
// WithDatabaseStatRecorder returns an option that results in `rec` being called
// for every `key` not otherwise matched by the [InspectDatabase] iterator loop.
// The returned boolean signals whether the recorder matches the key, thus
// stopping further matches.
func WithDatabaseStatRecorder(rec func(key []byte, size common.StorageSize) bool) InspectDatabaseOption {
return newInspectOpt(func(c *inspectDatabaseConfig) {
c.statRecorder = rec
})
}
// A DatabaseStat stores total size and counts for a parameter measured by
// [InspectDatabase]. It is exported for use with [WithDatabaseStatRecorder].
type DatabaseStat = stat
// WithDatabaseMetadataKeys returns an option that results in the `key` size
// being counted with the metadata statistic i.f.f. the function returns true.
func WithDatabaseMetadataKeys(isMetadata func(key []byte) bool) InspectDatabaseOption {
return newInspectOpt(func(c *inspectDatabaseConfig) {
c.isMeta = isMetadata
})
}
// WithDatabaseStatsTransformer returns an option that causes all statistics rows to
// be passed to the provided function, with its return value being printed
// instead of the original values.
// Each row contains 4 columns: database, category, size and count.
func WithDatabaseStatsTransformer(transform func(rows [][]string) [][]string) InspectDatabaseOption {
return newInspectOpt(func(c *inspectDatabaseConfig) {
c.statsTransformer = transform
})
}

View file

@ -0,0 +1,194 @@
// Copyright 2025 the libevm authors.
//
// The libevm additions to go-ethereum are free software: you can redistribute
// them and/or modify them under the terms of the GNU Lesser General Public License
// as published by the Free Software Foundation, either version 3 of the License,
// or (at your option) any later version.
//
// The libevm additions are distributed in the hope that they will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser
// General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see
// <http://www.gnu.org/licenses/>.
package rawdb_test
import (
"bytes"
"fmt"
"sort"
"github.com/ava-labs/libevm/common"
// To ensure that all methods are available to importing packages, this test
// is defined in package `rawdb_test` instead of `rawdb`.
"github.com/ava-labs/libevm/core/rawdb"
"github.com/ava-labs/libevm/ethdb"
)
// ExampleDatabaseStat demonstrates the method signatures of DatabaseStat, which
// exposes an otherwise unexported type that won't have its methods documented.
func ExampleDatabaseStat() {
var stat rawdb.DatabaseStat
stat.Add(common.StorageSize(1)) // only to demonstrate param type
stat.Add(2)
fmt.Println("Sum:", stat.Size()) // sum of all values passed to Add()
fmt.Println("Count:", stat.Count()) // number of calls to Add()
// Output:
// Sum: 3.00 B
// Count: 2
}
func ExampleInspectDatabase() {
db := &stubDatabase{
iterator: &stubIterator{
kvs: []keyValue{
// Bloom bits total = 5 + 1 = 6
{key: []byte("iBxxx"), value: []byte("m")},
// Optional stat record total = 5 + 7 = 12
{key: []byte("mykey"), value: []byte("myvalue")},
// metadata total = 13 + 7 = 20
{key: []byte("mymetadatakey"), value: []byte("myvalue")},
},
},
}
keyPrefix := []byte(nil)
keyStart := []byte(nil)
var (
myStat rawdb.DatabaseStat
)
options := []rawdb.InspectDatabaseOption{
rawdb.WithDatabaseStatRecorder(func(key []byte, size common.StorageSize) bool {
if bytes.Equal(key, []byte("mykey")) {
myStat.Add(size)
return true
}
return false
}),
rawdb.WithDatabaseMetadataKeys(func(key []byte) bool {
return bytes.Equal(key, []byte("mymetadatakey"))
}),
rawdb.WithDatabaseStatsTransformer(func(rows [][]string) [][]string {
sort.Slice(rows, func(i, j int) bool {
ri, rj := rows[i], rows[j]
if ri[0] != rj[0] {
return ri[0] < rj[0]
}
return ri[1] < rj[1]
})
return append(
rows,
[]string{"My database", "My category", myStat.Size(), myStat.Count()},
)
}),
}
err := rawdb.InspectDatabase(db, keyPrefix, keyStart, options...)
if err != nil {
fmt.Println(err)
}
// Output:
// +-----------------------+-------------------------+---------+-------+
// | DATABASE | CATEGORY | SIZE | ITEMS |
// +-----------------------+-------------------------+---------+-------+
// | Ancient store (Chain) | Bodies | 0.00 B | 0 |
// | Ancient store (Chain) | Diffs | 0.00 B | 0 |
// | Ancient store (Chain) | Hashes | 0.00 B | 0 |
// | Ancient store (Chain) | Headers | 0.00 B | 0 |
// | Ancient store (Chain) | Receipts | 0.00 B | 0 |
// | Key-Value store | Account snapshot | 0.00 B | 0 |
// | Key-Value store | Beacon sync headers | 0.00 B | 0 |
// | Key-Value store | Block hash->number | 0.00 B | 0 |
// | Key-Value store | Block number->hash | 0.00 B | 0 |
// | Key-Value store | Bloombit index | 6.00 B | 1 |
// | Key-Value store | Bodies | 0.00 B | 0 |
// | Key-Value store | Clique snapshots | 0.00 B | 0 |
// | Key-Value store | Contract codes | 0.00 B | 0 |
// | Key-Value store | Difficulties | 0.00 B | 0 |
// | Key-Value store | Hash trie nodes | 0.00 B | 0 |
// | Key-Value store | Headers | 0.00 B | 0 |
// | Key-Value store | Path trie account nodes | 0.00 B | 0 |
// | Key-Value store | Path trie state lookups | 0.00 B | 0 |
// | Key-Value store | Path trie storage nodes | 0.00 B | 0 |
// | Key-Value store | Receipt lists | 0.00 B | 0 |
// | Key-Value store | Singleton metadata | 20.00 B | 1 |
// | Key-Value store | Storage snapshot | 0.00 B | 0 |
// | Key-Value store | Transaction index | 0.00 B | 0 |
// | Key-Value store | Trie preimages | 0.00 B | 0 |
// | Light client | Bloom trie nodes | 0.00 B | 0 |
// | Light client | CHT trie nodes | 0.00 B | 0 |
// | My database | My category | 12.00 B | 1 |
// +-----------------------+-------------------------+---------+-------+
// | TOTAL | 38.00 B | |
// +-----------------------+-------------------------+---------+-------+
}
type stubDatabase struct {
ethdb.Database
iterator ethdb.Iterator
}
func (s *stubDatabase) NewIterator(keyPrefix, keyStart []byte) ethdb.Iterator {
return s.iterator
}
// AncientSize is used in [InspectDatabase] to determine the ancient sizes.
func (s *stubDatabase) AncientSize(kind string) (uint64, error) {
return 0, nil
}
func (s *stubDatabase) Ancients() (uint64, error) {
return 0, nil
}
func (s *stubDatabase) Tail() (uint64, error) {
return 0, nil
}
func (s *stubDatabase) Get(key []byte) ([]byte, error) {
return nil, nil
}
func (s *stubDatabase) ReadAncients(fn func(ethdb.AncientReaderOp) error) error {
return nil
}
type stubIterator struct {
ethdb.Iterator
i int // see [stubIterator.pos]
kvs []keyValue
}
type keyValue struct {
key []byte
value []byte
}
// pos returns the true iterator position, which is otherwise off by one because
// Next() is called _before_ usage.
func (s *stubIterator) pos() int {
return s.i - 1
}
func (s *stubIterator) Next() bool {
s.i++
return s.pos() < len(s.kvs)
}
func (s *stubIterator) Release() {}
func (s *stubIterator) Key() []byte {
return s.kvs[s.pos()].key
}
func (s *stubIterator) Value() []byte {
return s.kvs[s.pos()].value
}