From ebc7dc231ad8d9d43e4f7eec0a14f7a9de3ff879 Mon Sep 17 00:00:00 2001 From: rroblak Date: Tue, 12 Sep 2023 03:09:26 -0700 Subject: [PATCH] Add CLI flags to config LevelDB table/total sizes (#981) * Add CLI flags to config LevelDB table/total sizes I wired up CLI flags to allow configuring LevelDB table and total sizes: - `--leveldb.compaction.table.size`, LevelDB SSTable file size factor in MiB (default: 2) - `--leveldb.compaction.table.multiplier`, multiplier on LevelDB SSTable file size (default: 1) - `--leveldb.compaction.total.size`, total size factor in MiB of LevelDB levels (default: 10) - `--leveldb.compaction.total.multiplier`, multiplier on LevelDB total level size (default: 10) N.B. that the default values for these configs are exactly the same as before this changset and so Bor behavior should not change unless these flags are deliberately overridden. Bor/Geth inherited the default values from [the `goleveldb` defaults](https://github.com/syndtr/goleveldb/blob/126854af5e6d8295ef8e8bee3040dd8380ae72e8/leveldb/opt/options.go). We (Alchemy) found it necessary to override these configs as follows to keep Bor archive nodes tracking the canonical chain: - `--leveldb.compaction.table.size=4` - `--leveldb.compaction.total.size=20` These overrides double the size of LevelDB SSTable files (2 MiB -> 4 MiB) and also the total amount of data in each level (100 MiB -> 200 MiB, 1,000 MiB -> 2,000 MiB, etc.). The idea is to have LevelDB read and write data in larger chunks while keeping the proportional frequency of compaction operations the same as in the original defaults defined by Dean and Ghemawat. Without these overrides we found that our archive nodes would tend to fall into a "LevelDB compaction loop of death" where the incoming stream of blockchain data could not be flowed into LevelDB's structure quickly enough, resulting in the node blocking writes for long periods of time while LevelDB's single-threaded compaction organized the data. Over time the nodes would fall farther and farther behind the canonical chain head, metaphorically dying a slow node's death. These configs can be changed on existing node databases (resyncing is not necessary). LevelDB appears to work correctly with SSTable files of different sizes. Note that the database does not undergo any sort of migration when changing these configs. Only newly-written files (due to new data or compaction) are affected by these configs. * Update docs * Adjust line spacing for linter * Replace map with `ExtraDBConfig` * Rename `LevelDbConfig` to `ExtraDBConfig` * Regenerate docs --- cmd/geth/chaincmd.go | 4 ++-- cmd/geth/dao_test.go | 2 +- cmd/utils/flags.go | 40 +++++++++++++++++++++++++++++++++-- core/bench_test.go | 8 +++---- core/rawdb/database.go | 26 +++++++++++++++++++---- docs/cli/server.md | 10 +++++++++ eth/backend.go | 12 ++++++++++- eth/ethconfig/config.go | 6 ++++++ eth/filters/bench_test.go | 6 +++--- eth/filters/filter_test.go | 4 ++-- ethdb/leveldb/leveldb.go | 34 +++++++++++++++++++++++++++-- internal/cli/server/config.go | 25 ++++++++++++++++++++++ internal/cli/server/flags.go | 30 ++++++++++++++++++++++++++ internal/cli/snapshot.go | 3 ++- les/client.go | 15 +++++++++++-- les/server.go | 3 ++- node/node.go | 18 +++++++++------- node/node_test.go | 7 +++--- 18 files changed, 217 insertions(+), 36 deletions(-) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index afb18d8346..cce5d874b1 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -195,7 +195,7 @@ func initGenesis(ctx *cli.Context) error { defer stack.Close() for _, name := range []string{"chaindata", "lightchaindata"} { - chaindb, err := stack.OpenDatabaseWithFreezer(name, 0, 0, ctx.String(utils.AncientFlag.Name), "", false) + chaindb, err := stack.OpenDatabaseWithFreezer(name, 0, 0, ctx.String(utils.AncientFlag.Name), "", false, rawdb.ExtraDBConfig{}) if err != nil { utils.Fatalf("Failed to open database: %v", err) } @@ -229,7 +229,7 @@ func dumpGenesis(ctx *cli.Context) error { // dump whatever already exists in the datadir stack, _ := makeConfigNode(ctx) for _, name := range []string{"chaindata", "lightchaindata"} { - db, err := stack.OpenDatabase(name, 0, 0, "", true) + db, err := stack.OpenDatabase(name, 0, 0, "", true, rawdb.ExtraDBConfig{}) if err != nil { if !os.IsNotExist(err) { diff --git a/cmd/geth/dao_test.go b/cmd/geth/dao_test.go index 2013789e32..3e4fc32daf 100644 --- a/cmd/geth/dao_test.go +++ b/cmd/geth/dao_test.go @@ -123,7 +123,7 @@ func testDAOForkBlockNewChain(t *testing.T, test int, genesis string, expectBloc // Retrieve the DAO config flag from the database path := filepath.Join(datadir, "geth", "chaindata") - db, err := rawdb.NewLevelDBDatabase(path, 0, 0, "", false) + db, err := rawdb.NewLevelDBDatabase(path, 0, 0, "", false, rawdb.ExtraDBConfig{}) if err != nil { t.Fatalf("test %d: failed to open test database: %v", test, err) } diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 0313149341..78e3d8ffa2 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -461,6 +461,31 @@ var ( Value: 50, Category: flags.PerfCategory, } + + LevelDbCompactionTableSizeFlag = &cli.Uint64Flag{ + Name: "leveldb.compaction.table.size", + Usage: "LevelDB SSTable/file size in mebibytes", + Category: flags.PerfCategory, + } + + LevelDbCompactionTableSizeMultiplierFlag = &cli.Float64Flag{ + Name: "leveldb.compaction.table.size.multiplier", + Usage: "Multiplier on LevelDB SSTable/file size. Size for a level is determined by: `leveldb.compaction.table.size * (leveldb.compaction.table.size.multiplier ^ Level)`", + Category: flags.PerfCategory, + } + + LevelDbCompactionTotalSizeFlag = &cli.Uint64Flag{ + Name: "leveldb.compaction.total.size", + Usage: "Total size in mebibytes of SSTables in a given LevelDB level. Size for a level is determined by: `leveldb.compaction.total.size * (leveldb.compaction.total.size.multiplier ^ Level)`", + Category: flags.PerfCategory, + } + + LevelDbCompactionTotalSizeMultiplierFlag = &cli.Float64Flag{ + Name: "leveldb.compaction.total.size.multiplier", + Usage: "Multiplier on level size on LevelDB levels. Size for a level is determined by: `leveldb.compaction.total.size * (leveldb.compaction.total.size.multiplier ^ Level)`", + Category: flags.PerfCategory, + } + CacheTrieFlag = &cli.IntFlag{ Name: "cache.trie", Usage: "Percentage of cache memory allowance to use for trie caching (default = 15% full mode, 30% archive mode)", @@ -2291,6 +2316,8 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly bool) ethdb. err error chainDb ethdb.Database + + dbOptions = resolveExtraDBConfig(ctx) ) switch { @@ -2304,9 +2331,9 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly bool) ethdb. chainDb = remotedb.New(client) case ctx.String(SyncModeFlag.Name) == "light": - chainDb, err = stack.OpenDatabase("lightchaindata", cache, handles, "", readonly) + chainDb, err = stack.OpenDatabase("lightchaindata", cache, handles, "", readonly, dbOptions) default: - chainDb, err = stack.OpenDatabaseWithFreezer("chaindata", cache, handles, ctx.String(AncientFlag.Name), "", readonly) + chainDb, err = stack.OpenDatabaseWithFreezer("chaindata", cache, handles, ctx.String(AncientFlag.Name), "", readonly, dbOptions) } if err != nil { @@ -2316,6 +2343,15 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly bool) ethdb. return chainDb } +func resolveExtraDBConfig(ctx *cli.Context) rawdb.ExtraDBConfig { + return rawdb.ExtraDBConfig{ + LevelDBCompactionTableSize: ctx.Uint64(LevelDbCompactionTableSizeFlag.Name), + LevelDBCompactionTableSizeMultiplier: ctx.Float64(LevelDbCompactionTableSizeMultiplierFlag.Name), + LevelDBCompactionTotalSize: ctx.Uint64(LevelDbCompactionTotalSizeFlag.Name), + LevelDBCompactionTotalSizeMultiplier: ctx.Float64(LevelDbCompactionTotalSizeMultiplierFlag.Name), + } +} + func IsNetworkPreset(ctx *cli.Context) bool { for _, flag := range NetworkFlags { bFlag, _ := flag.(*cli.BoolFlag) diff --git a/core/bench_test.go b/core/bench_test.go index 1f9e0f91e6..dd7257f380 100644 --- a/core/bench_test.go +++ b/core/bench_test.go @@ -193,7 +193,7 @@ func benchInsertChain(b *testing.B, disk bool, gen func(int, *BlockGen)) { } else { dir := b.TempDir() - db, err = rawdb.NewLevelDBDatabase(dir, 128, 128, "", false) + db, err = rawdb.NewLevelDBDatabase(dir, 128, 128, "", false, rawdb.ExtraDBConfig{}) if err != nil { b.Fatalf("cannot create temporary database: %v", err) } @@ -296,7 +296,7 @@ func makeChainForBench(db ethdb.Database, full bool, count uint64) { func benchWriteChain(b *testing.B, full bool, count uint64) { for i := 0; i < b.N; i++ { dir := b.TempDir() - db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "", false) + db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "", false, rawdb.ExtraDBConfig{}) if err != nil { b.Fatalf("error opening database at %v: %v", dir, err) @@ -310,7 +310,7 @@ func benchWriteChain(b *testing.B, full bool, count uint64) { func benchReadChain(b *testing.B, full bool, count uint64) { dir := b.TempDir() - db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "", false) + db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "", false, rawdb.ExtraDBConfig{}) if err != nil { b.Fatalf("error opening database at %v: %v", dir, err) } @@ -325,7 +325,7 @@ func benchReadChain(b *testing.B, full bool, count uint64) { b.ResetTimer() for i := 0; i < b.N; i++ { - db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "", false) + db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "", false, rawdb.ExtraDBConfig{}) if err != nil { b.Fatalf("error opening database at %v: %v", dir, err) } diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 5cc13d0632..baf1d2cf79 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -321,8 +321,8 @@ func NewMemoryDatabaseWithCap(size int) ethdb.Database { // NewLevelDBDatabase creates a persistent key-value database without a freezer // moving immutable chain segments into cold storage. -func NewLevelDBDatabase(file string, cache int, handles int, namespace string, readonly bool) (ethdb.Database, error) { - db, err := leveldb.New(file, cache, handles, namespace, readonly) +func NewLevelDBDatabase(file string, cache int, handles int, namespace string, readonly bool, extraDBConfig ExtraDBConfig) (ethdb.Database, error) { + db, err := leveldb.New(file, cache, handles, namespace, readonly, resolveLevelDBConfig(extraDBConfig)) if err != nil { return nil, err } @@ -332,6 +332,15 @@ func NewLevelDBDatabase(file string, cache int, handles int, namespace string, r return NewDatabase(db), nil } +func resolveLevelDBConfig(config ExtraDBConfig) leveldb.LevelDBConfig { + return leveldb.LevelDBConfig{ + CompactionTableSize: config.LevelDBCompactionTableSize, + CompactionTableSizeMultiplier: config.LevelDBCompactionTableSizeMultiplier, + CompactionTotalSize: config.LevelDBCompactionTotalSize, + CompactionTotalSizeMultiplier: config.LevelDBCompactionTotalSizeMultiplier, + } +} + const ( dbPebble = "pebble" dbLeveldb = "leveldb" @@ -366,6 +375,14 @@ type OpenOptions struct { Cache int // the capacity(in megabytes) of the data caching Handles int // number of files to be open simultaneously ReadOnly bool + ExtraDBConfig ExtraDBConfig +} + +type ExtraDBConfig struct { + LevelDBCompactionTableSize uint64 // LevelDB SSTable/file size in mebibytes + LevelDBCompactionTableSizeMultiplier float64 // Multiplier on LevelDB SSTable/file size + LevelDBCompactionTotalSize uint64 // Total size in mebibytes of SSTables in a given LevelDB level + LevelDBCompactionTotalSizeMultiplier float64 // Multiplier on level size on LevelDB levels } // openKeyValueDatabase opens a disk-based key-value database, e.g. leveldb or pebble. @@ -393,9 +410,10 @@ func openKeyValueDatabase(o OpenOptions) (ethdb.Database, error) { return nil, fmt.Errorf("unknown db.engine %v", o.Type) } - log.Info("Using leveldb as the backing database") // Use leveldb, either as default (no explicit choice), or pre-existing, or chosen explicitly - return NewLevelDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly) + log.Info("Using leveldb as the backing database") + + return NewLevelDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly, o.ExtraDBConfig) } // Open opens both a disk-based key-value database such as leveldb or pebble, but also diff --git a/docs/cli/server.md b/docs/cli/server.md index 213d491fff..865b54e3f0 100644 --- a/docs/cli/server.md +++ b/docs/cli/server.md @@ -126,6 +126,16 @@ The ```bor server``` command runs the Bor client. - ```fdlimit```: Raise the open file descriptor resource limit (default = system fd limit) (default: 0) +### ExtraDB Options + +- ```leveldb.compaction.table.size```: LevelDB SSTable/file size in mebibytes (default: 2) + +- ```leveldb.compaction.table.size.multiplier```: Multiplier on LevelDB SSTable/file size. Size for a level is determined by: `leveldb.compaction.table.size * (leveldb.compaction.table.size.multiplier ^ Level)` (default: 1) + +- ```leveldb.compaction.total.size```: Total size in mebibytes of SSTables in a given LevelDB level. Size for a level is determined by: `leveldb.compaction.total.size * (leveldb.compaction.total.size.multiplier ^ Level)` (default: 10) + +- ```leveldb.compaction.total.size.multiplier```: Multiplier on level size on LevelDB levels. Size for a level is determined by: `leveldb.compaction.total.size * (leveldb.compaction.total.size.multiplier ^ Level)` (default: 10) + ### JsonRPC Options - ```rpc.gascap```: Sets a cap on gas that can be used in eth_call/estimateGas (0=infinite) (default: 50000000) diff --git a/eth/backend.go b/eth/backend.go index 93004cc196..061edb0c06 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -141,7 +141,8 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { log.Info("Allocated trie memory caches", "clean", common.StorageSize(config.TrieCleanCache)*1024*1024, "dirty", common.StorageSize(config.TrieDirtyCache)*1024*1024) // Assemble the Ethereum object - chainDb, err := stack.OpenDatabaseWithFreezer("chaindata", config.DatabaseCache, config.DatabaseHandles, config.DatabaseFreezer, "ethereum/db/chaindata/", false) + extraDBConfig := resolveExtraDBConfig(config) + chainDb, err := stack.OpenDatabaseWithFreezer("chaindata", config.DatabaseCache, config.DatabaseHandles, config.DatabaseFreezer, "ethereum/db/chaindata/", false, extraDBConfig) if err != nil { return nil, err } @@ -332,6 +333,15 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { return ethereum, nil } +func resolveExtraDBConfig(config *ethconfig.Config) rawdb.ExtraDBConfig { + return rawdb.ExtraDBConfig{ + LevelDBCompactionTableSize: config.LevelDbCompactionTableSize, + LevelDBCompactionTableSizeMultiplier: config.LevelDbCompactionTableSizeMultiplier, + LevelDBCompactionTotalSize: config.LevelDbCompactionTotalSize, + LevelDBCompactionTotalSizeMultiplier: config.LevelDbCompactionTotalSizeMultiplier, + } +} + func makeExtraData(extra []byte) []byte { if len(extra) == 0 { // create default extradata diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 77b127a3a8..7a037aec31 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -169,6 +169,12 @@ type Config struct { DatabaseCache int DatabaseFreezer string + // Database - LevelDB options + LevelDbCompactionTableSize uint64 + LevelDbCompactionTableSizeMultiplier float64 + LevelDbCompactionTotalSize uint64 + LevelDbCompactionTotalSizeMultiplier float64 + TrieCleanCache int TrieCleanCacheJournal string `toml:",omitempty"` // Disk journal directory for trie cache to survive node restarts TrieCleanCacheRejournal time.Duration `toml:",omitempty"` // Time interval to regenerate the journal for clean cache diff --git a/eth/filters/bench_test.go b/eth/filters/bench_test.go index 89a45bc2d8..3e52a6c11b 100644 --- a/eth/filters/bench_test.go +++ b/eth/filters/bench_test.go @@ -68,7 +68,7 @@ func benchmarkBloomBits(b *testing.B, sectionSize uint64) { b.Log("Running bloombits benchmark section size:", sectionSize) - db, err := rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024, "", false) + db, err := rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024, "", false, rawdb.ExtraDBConfig{}) if err != nil { b.Fatalf("error opening database at %v: %v", benchDataDir, err) } @@ -145,7 +145,7 @@ func benchmarkBloomBits(b *testing.B, sectionSize uint64) { for i := 0; i < benchFilterCnt; i++ { if i%20 == 0 { db.Close() - db, _ = rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024, "", false) + db, _ = rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024, "", false, rawdb.ExtraDBConfig{}) backend = &testBackend{db: db, sections: cnt} sys = NewFilterSystem(backend, Config{}) } @@ -187,7 +187,7 @@ func BenchmarkNoBloomBits(b *testing.B) { b.Log("Running benchmark without bloombits") - db, err := rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024, "", false) + db, err := rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024, "", false, rawdb.ExtraDBConfig{}) if err != nil { b.Fatalf("error opening database at %v: %v", benchDataDir, err) } diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go index 4bea788f25..431db44f75 100644 --- a/eth/filters/filter_test.go +++ b/eth/filters/filter_test.go @@ -43,7 +43,7 @@ func makeReceipt(addr common.Address) *types.Receipt { func BenchmarkFilters(b *testing.B) { var ( - db, _ = rawdb.NewLevelDBDatabase(b.TempDir(), 0, 0, "", false) + db, _ = rawdb.NewLevelDBDatabase(b.TempDir(), 0, 0, "", false, rawdb.ExtraDBConfig{}) _, sys = newTestFilterSystem(b, db, Config{}) key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr1 = crypto.PubkeyToAddress(key1.PublicKey) @@ -106,7 +106,7 @@ func BenchmarkFilters(b *testing.B) { func TestFilters(t *testing.T) { var ( - db, _ = rawdb.NewLevelDBDatabase(t.TempDir(), 0, 0, "", false) + db, _ = rawdb.NewLevelDBDatabase(t.TempDir(), 0, 0, "", false, rawdb.ExtraDBConfig{}) _, sys = newTestFilterSystem(t, db, Config{}) key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr = crypto.PubkeyToAddress(key1.PublicKey) diff --git a/ethdb/leveldb/leveldb.go b/ethdb/leveldb/leveldb.go index 6c81f15f75..617cfd2763 100644 --- a/ethdb/leveldb/leveldb.go +++ b/ethdb/leveldb/leveldb.go @@ -83,9 +83,16 @@ type Database struct { log log.Logger // Contextual logger tracking the database path } +type LevelDBConfig struct { + CompactionTableSize uint64 // LevelDB SSTable/file size in mebibytes + CompactionTableSizeMultiplier float64 // Multiplier on LevelDB SSTable/file size + CompactionTotalSize uint64 // Total size in mebibytes of SSTables in a given LevelDB level + CompactionTotalSizeMultiplier float64 // Multiplier on level size on LevelDB levels +} + // New returns a wrapped LevelDB object. The namespace is the prefix that the // metrics reporting should use for surfacing internal stats. -func New(file string, cache int, handles int, namespace string, readonly bool) (*Database, error) { +func New(file string, cache int, handles int, namespace string, readonly bool, config LevelDBConfig) (*Database, error) { return NewCustom(file, namespace, func(options *opt.Options) { // Ensure we have some minimal caching and file guarantees if cache < minCache { @@ -100,6 +107,22 @@ func New(file string, cache int, handles int, namespace string, readonly bool) ( options.BlockCacheCapacity = cache / 2 * opt.MiB options.WriteBuffer = cache / 4 * opt.MiB // Two of these are used internally + if config.CompactionTableSize != 0 { + options.CompactionTableSize = int(config.CompactionTableSize * opt.MiB) + } + + if config.CompactionTableSizeMultiplier != 0 { + options.CompactionTableSizeMultiplier = config.CompactionTableSizeMultiplier + } + + if config.CompactionTotalSize != 0 { + options.CompactionTotalSize = int(config.CompactionTotalSize * opt.MiB) + } + + if config.CompactionTotalSizeMultiplier != 0 { + options.CompactionTotalSizeMultiplier = config.CompactionTotalSizeMultiplier + } + if readonly { options.ReadOnly = true } @@ -114,7 +137,14 @@ func NewCustom(file string, namespace string, customize func(options *opt.Option logger := log.New("database", file) usedCache := options.GetBlockCacheCapacity() + options.GetWriteBuffer()*2 - logCtx := []interface{}{"cache", common.StorageSize(usedCache), "handles", options.GetOpenFilesCacheCapacity()} + logCtx := []interface{}{ + "cache", common.StorageSize(usedCache), + "handles", options.GetOpenFilesCacheCapacity(), + "compactionTableSize", options.CompactionTableSize, + "compactionTableSizeMultiplier", options.CompactionTableSizeMultiplier, + "compactionTotalSize", options.CompactionTotalSize, + "compactionTotalSizeMultiplier", options.CompactionTotalSizeMultiplier} + if options.ReadOnly { logCtx = append(logCtx, "readonly", "true") } diff --git a/internal/cli/server/config.go b/internal/cli/server/config.go index c5c7608ca8..0a9c7dd704 100644 --- a/internal/cli/server/config.go +++ b/internal/cli/server/config.go @@ -121,6 +121,8 @@ type Config struct { // Cache has the cache related settings Cache *CacheConfig `hcl:"cache,block" toml:"cache,block"` + ExtraDB *ExtraDBConfig `hcl:"leveldb,block" toml:"leveldb,block"` + // Account has the validator account related settings Accounts *AccountsConfig `hcl:"accounts,block" toml:"accounts,block"` @@ -551,6 +553,13 @@ type CacheConfig struct { FDLimit int `hcl:"fdlimit,optional" toml:"fdlimit,optional"` } +type ExtraDBConfig struct { + LevelDbCompactionTableSize uint64 `hcl:"compactiontablesize,optional" toml:"compactiontablesize,optional"` + LevelDbCompactionTableSizeMultiplier float64 `hcl:"compactiontablesizemultiplier,optional" toml:"compactiontablesizemultiplier,optional"` + LevelDbCompactionTotalSize uint64 `hcl:"compactiontotalsize,optional" toml:"compactiontotalsize,optional"` + LevelDbCompactionTotalSizeMultiplier float64 `hcl:"compactiontotalsizemultiplier,optional" toml:"compactiontotalsizemultiplier,optional"` +} + type AccountsConfig struct { // Unlock is the list of addresses to unlock in the node Unlock []string `hcl:"unlock,optional" toml:"unlock,optional"` @@ -742,6 +751,14 @@ func DefaultConfig() *Config { TrieTimeout: 60 * time.Minute, FDLimit: 0, }, + ExtraDB: &ExtraDBConfig{ + // These are LevelDB defaults, specifying here for clarity in code and in logging. + // See: https://github.com/syndtr/goleveldb/blob/126854af5e6d8295ef8e8bee3040dd8380ae72e8/leveldb/opt/options.go + LevelDbCompactionTableSize: 2, // MiB + LevelDbCompactionTableSizeMultiplier: 1, + LevelDbCompactionTotalSize: 10, // MiB + LevelDbCompactionTotalSizeMultiplier: 10, + }, Accounts: &AccountsConfig{ Unlock: []string{}, PasswordFile: "", @@ -1101,6 +1118,14 @@ func (c *Config) buildEth(stack *node.Node, accountManager *accounts.Manager) (* n.TriesInMemory = c.Cache.TriesInMemory } + // LevelDB + { + n.LevelDbCompactionTableSize = c.ExtraDB.LevelDbCompactionTableSize + n.LevelDbCompactionTableSizeMultiplier = c.ExtraDB.LevelDbCompactionTableSizeMultiplier + n.LevelDbCompactionTotalSize = c.ExtraDB.LevelDbCompactionTotalSize + n.LevelDbCompactionTotalSizeMultiplier = c.ExtraDB.LevelDbCompactionTotalSizeMultiplier + } + n.RPCGasCap = c.JsonRPC.GasCap if n.RPCGasCap != 0 { log.Info("Set global gas cap", "cap", n.RPCGasCap) diff --git a/internal/cli/server/flags.go b/internal/cli/server/flags.go index 34d3ace94f..1edacc9ac0 100644 --- a/internal/cli/server/flags.go +++ b/internal/cli/server/flags.go @@ -448,6 +448,36 @@ func (c *Command) Flags() *flagset.Flagset { Group: "Cache", }) + // LevelDB options + f.Uint64Flag(&flagset.Uint64Flag{ + Name: "leveldb.compaction.table.size", + Usage: "LevelDB SSTable/file size in mebibytes", + Value: &c.cliConfig.ExtraDB.LevelDbCompactionTableSize, + Default: c.cliConfig.ExtraDB.LevelDbCompactionTableSize, + Group: "ExtraDB", + }) + f.Float64Flag(&flagset.Float64Flag{ + Name: "leveldb.compaction.table.size.multiplier", + Usage: "Multiplier on LevelDB SSTable/file size. Size for a level is determined by: `leveldb.compaction.table.size * (leveldb.compaction.table.size.multiplier ^ Level)`", + Value: &c.cliConfig.ExtraDB.LevelDbCompactionTableSizeMultiplier, + Default: c.cliConfig.ExtraDB.LevelDbCompactionTableSizeMultiplier, + Group: "ExtraDB", + }) + f.Uint64Flag(&flagset.Uint64Flag{ + Name: "leveldb.compaction.total.size", + Usage: "Total size in mebibytes of SSTables in a given LevelDB level. Size for a level is determined by: `leveldb.compaction.total.size * (leveldb.compaction.total.size.multiplier ^ Level)`", + Value: &c.cliConfig.ExtraDB.LevelDbCompactionTotalSize, + Default: c.cliConfig.ExtraDB.LevelDbCompactionTotalSize, + Group: "ExtraDB", + }) + f.Float64Flag(&flagset.Float64Flag{ + Name: "leveldb.compaction.total.size.multiplier", + Usage: "Multiplier on level size on LevelDB levels. Size for a level is determined by: `leveldb.compaction.total.size * (leveldb.compaction.total.size.multiplier ^ Level)`", + Value: &c.cliConfig.ExtraDB.LevelDbCompactionTotalSizeMultiplier, + Default: c.cliConfig.ExtraDB.LevelDbCompactionTotalSizeMultiplier, + Group: "ExtraDB", + }) + // rpc options f.Uint64Flag(&flagset.Uint64Flag{ Name: "rpc.gascap", diff --git a/internal/cli/snapshot.go b/internal/cli/snapshot.go index 7257cde4b7..60034103d3 100644 --- a/internal/cli/snapshot.go +++ b/internal/cli/snapshot.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state/pruner" "github.com/ethereum/go-ethereum/internal/cli/flagset" "github.com/ethereum/go-ethereum/internal/cli/server" @@ -161,7 +162,7 @@ func (c *PruneStateCommand) Run(args []string) int { return 1 } - chaindb, err := node.OpenDatabaseWithFreezer(chaindataPath, int(c.cache), dbHandles, c.datadirAncient, "", false) + chaindb, err := node.OpenDatabaseWithFreezer(chaindataPath, int(c.cache), dbHandles, c.datadirAncient, "", false, rawdb.ExtraDBConfig{}) if err != nil { c.UI.Error(err.Error()) diff --git a/les/client.go b/les/client.go index 9c31b2dd9e..72a74db33e 100644 --- a/les/client.go +++ b/les/client.go @@ -86,12 +86,14 @@ type LightEthereum struct { // New creates an instance of the light client. func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { - chainDb, err := stack.OpenDatabase("lightchaindata", config.DatabaseCache, config.DatabaseHandles, "eth/db/chaindata/", false) + extraDBConfig := resolveExtraDBConfig(config) + + chainDb, err := stack.OpenDatabase("lightchaindata", config.DatabaseCache, config.DatabaseHandles, "eth/db/chaindata/", false, extraDBConfig) if err != nil { return nil, err } - lesDb, err := stack.OpenDatabase("les.client", 0, 0, "eth/db/lesclient/", false) + lesDb, err := stack.OpenDatabase("les.client", 0, 0, "eth/db/lesclient/", false, extraDBConfig) if err != nil { return nil, err } @@ -227,6 +229,15 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { return leth, nil } +func resolveExtraDBConfig(config *ethconfig.Config) rawdb.ExtraDBConfig { + return rawdb.ExtraDBConfig{ + LevelDBCompactionTableSize: config.LevelDbCompactionTableSize, + LevelDBCompactionTableSizeMultiplier: config.LevelDbCompactionTableSizeMultiplier, + LevelDBCompactionTotalSize: config.LevelDbCompactionTotalSize, + LevelDBCompactionTotalSizeMultiplier: config.LevelDbCompactionTotalSizeMultiplier, + } +} + // VfluxRequest sends a batch of requests to the given node through discv5 UDP TalkRequest and returns the responses func (s *LightEthereum) VfluxRequest(n *enode.Node, reqs vflux.Requests) vflux.Replies { if !s.udpEnabled { diff --git a/les/server.go b/les/server.go index 713f84ea55..4a5751222b 100644 --- a/les/server.go +++ b/les/server.go @@ -78,7 +78,8 @@ type LesServer struct { } func NewLesServer(node *node.Node, e ethBackend, config *ethconfig.Config) (*LesServer, error) { - lesDb, err := node.OpenDatabase("les.server", 0, 0, "eth/db/lesserver/", false) + dbOptions := resolveExtraDBConfig(config) + lesDb, err := node.OpenDatabase("les.server", 0, 0, "eth/db/lesserver/", false, dbOptions) if err != nil { return nil, err } diff --git a/node/node.go b/node/node.go index cc7215fa6e..428a5a4de9 100644 --- a/node/node.go +++ b/node/node.go @@ -771,7 +771,7 @@ func (n *Node) EventMux() *event.TypeMux { // OpenDatabase opens an existing database with the given name (or creates one if no // previous can be found) from within the node's instance directory. If the node is // ephemeral, a memory database is returned. -func (n *Node) OpenDatabase(name string, cache, handles int, namespace string, readonly bool) (ethdb.Database, error) { +func (n *Node) OpenDatabase(name string, cache, handles int, namespace string, readonly bool, extraDBConfig rawdb.ExtraDBConfig) (ethdb.Database, error) { n.lock.Lock() defer n.lock.Unlock() @@ -787,12 +787,13 @@ func (n *Node) OpenDatabase(name string, cache, handles int, namespace string, r db = rawdb.NewMemoryDatabase() } else { db, err = rawdb.Open(rawdb.OpenOptions{ - Type: n.config.DBEngine, - Directory: n.ResolvePath(name), - Namespace: namespace, - Cache: cache, - Handles: handles, - ReadOnly: readonly, + Type: n.config.DBEngine, + Directory: n.ResolvePath(name), + Namespace: namespace, + Cache: cache, + Handles: handles, + ReadOnly: readonly, + ExtraDBConfig: extraDBConfig, }) } @@ -808,7 +809,7 @@ func (n *Node) OpenDatabase(name string, cache, handles int, namespace string, r // also attaching a chain freezer to it that moves ancient chain data from the // database to immutable append-only files. If the node is an ephemeral one, a // memory database is returned. -func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, ancient string, namespace string, readonly bool) (ethdb.Database, error) { +func (n *Node) OpenDatabaseWithFreezer(name string, cache int, handles int, ancient string, namespace string, readonly bool, extraDBConfig rawdb.ExtraDBConfig) (ethdb.Database, error) { n.lock.Lock() defer n.lock.Unlock() @@ -831,6 +832,7 @@ func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, ancient Cache: cache, Handles: handles, ReadOnly: readonly, + ExtraDBConfig: extraDBConfig, }) } diff --git a/node/node_test.go b/node/node_test.go index dc291f5d91..f53b003684 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -26,6 +26,7 @@ import ( "strings" "testing" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/p2p" @@ -157,7 +158,7 @@ func TestNodeCloseClosesDB(t *testing.T) { stack, _ := New(testNodeConfig()) defer stack.Close() - db, err := stack.OpenDatabase("mydb", 0, 0, "", false) + db, err := stack.OpenDatabase("mydb", 0, 0, "", false, rawdb.ExtraDBConfig{}) if err != nil { t.Fatal("can't open DB:", err) } @@ -184,7 +185,7 @@ func TestNodeOpenDatabaseFromLifecycleStart(t *testing.T) { stack.RegisterLifecycle(&InstrumentedService{ startHook: func() { - db, err = stack.OpenDatabase("mydb", 0, 0, "", false) + db, err = stack.OpenDatabase("mydb", 0, 0, "", false, rawdb.ExtraDBConfig{}) if err != nil { t.Fatal("can't open DB:", err) } @@ -205,7 +206,7 @@ func TestNodeOpenDatabaseFromLifecycleStop(t *testing.T) { stack.RegisterLifecycle(&InstrumentedService{ stopHook: func() { - db, err := stack.OpenDatabase("mydb", 0, 0, "", false) + db, err := stack.OpenDatabase("mydb", 0, 0, "", false, rawdb.ExtraDBConfig{}) if err != nil { t.Fatal("can't open DB:", err) }