diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index afb18d8346..cce5d874b1 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -195,7 +195,7 @@ func initGenesis(ctx *cli.Context) error { defer stack.Close() for _, name := range []string{"chaindata", "lightchaindata"} { - chaindb, err := stack.OpenDatabaseWithFreezer(name, 0, 0, ctx.String(utils.AncientFlag.Name), "", false) + chaindb, err := stack.OpenDatabaseWithFreezer(name, 0, 0, ctx.String(utils.AncientFlag.Name), "", false, rawdb.ExtraDBConfig{}) if err != nil { utils.Fatalf("Failed to open database: %v", err) } @@ -229,7 +229,7 @@ func dumpGenesis(ctx *cli.Context) error { // dump whatever already exists in the datadir stack, _ := makeConfigNode(ctx) for _, name := range []string{"chaindata", "lightchaindata"} { - db, err := stack.OpenDatabase(name, 0, 0, "", true) + db, err := stack.OpenDatabase(name, 0, 0, "", true, rawdb.ExtraDBConfig{}) if err != nil { if !os.IsNotExist(err) { diff --git a/cmd/geth/dao_test.go b/cmd/geth/dao_test.go index 2013789e32..3e4fc32daf 100644 --- a/cmd/geth/dao_test.go +++ b/cmd/geth/dao_test.go @@ -123,7 +123,7 @@ func testDAOForkBlockNewChain(t *testing.T, test int, genesis string, expectBloc // Retrieve the DAO config flag from the database path := filepath.Join(datadir, "geth", "chaindata") - db, err := rawdb.NewLevelDBDatabase(path, 0, 0, "", false) + db, err := rawdb.NewLevelDBDatabase(path, 0, 0, "", false, rawdb.ExtraDBConfig{}) if err != nil { t.Fatalf("test %d: failed to open test database: %v", test, err) } diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 0313149341..78e3d8ffa2 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -461,6 +461,31 @@ var ( Value: 50, Category: flags.PerfCategory, } + + LevelDbCompactionTableSizeFlag = &cli.Uint64Flag{ + Name: "leveldb.compaction.table.size", + Usage: "LevelDB SSTable/file size in mebibytes", + Category: flags.PerfCategory, + } + + LevelDbCompactionTableSizeMultiplierFlag = &cli.Float64Flag{ + Name: "leveldb.compaction.table.size.multiplier", + Usage: "Multiplier on LevelDB SSTable/file size. Size for a level is determined by: `leveldb.compaction.table.size * (leveldb.compaction.table.size.multiplier ^ Level)`", + Category: flags.PerfCategory, + } + + LevelDbCompactionTotalSizeFlag = &cli.Uint64Flag{ + Name: "leveldb.compaction.total.size", + Usage: "Total size in mebibytes of SSTables in a given LevelDB level. Size for a level is determined by: `leveldb.compaction.total.size * (leveldb.compaction.total.size.multiplier ^ Level)`", + Category: flags.PerfCategory, + } + + LevelDbCompactionTotalSizeMultiplierFlag = &cli.Float64Flag{ + Name: "leveldb.compaction.total.size.multiplier", + Usage: "Multiplier on level size on LevelDB levels. Size for a level is determined by: `leveldb.compaction.total.size * (leveldb.compaction.total.size.multiplier ^ Level)`", + Category: flags.PerfCategory, + } + CacheTrieFlag = &cli.IntFlag{ Name: "cache.trie", Usage: "Percentage of cache memory allowance to use for trie caching (default = 15% full mode, 30% archive mode)", @@ -2291,6 +2316,8 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly bool) ethdb. err error chainDb ethdb.Database + + dbOptions = resolveExtraDBConfig(ctx) ) switch { @@ -2304,9 +2331,9 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly bool) ethdb. chainDb = remotedb.New(client) case ctx.String(SyncModeFlag.Name) == "light": - chainDb, err = stack.OpenDatabase("lightchaindata", cache, handles, "", readonly) + chainDb, err = stack.OpenDatabase("lightchaindata", cache, handles, "", readonly, dbOptions) default: - chainDb, err = stack.OpenDatabaseWithFreezer("chaindata", cache, handles, ctx.String(AncientFlag.Name), "", readonly) + chainDb, err = stack.OpenDatabaseWithFreezer("chaindata", cache, handles, ctx.String(AncientFlag.Name), "", readonly, dbOptions) } if err != nil { @@ -2316,6 +2343,15 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly bool) ethdb. return chainDb } +func resolveExtraDBConfig(ctx *cli.Context) rawdb.ExtraDBConfig { + return rawdb.ExtraDBConfig{ + LevelDBCompactionTableSize: ctx.Uint64(LevelDbCompactionTableSizeFlag.Name), + LevelDBCompactionTableSizeMultiplier: ctx.Float64(LevelDbCompactionTableSizeMultiplierFlag.Name), + LevelDBCompactionTotalSize: ctx.Uint64(LevelDbCompactionTotalSizeFlag.Name), + LevelDBCompactionTotalSizeMultiplier: ctx.Float64(LevelDbCompactionTotalSizeMultiplierFlag.Name), + } +} + func IsNetworkPreset(ctx *cli.Context) bool { for _, flag := range NetworkFlags { bFlag, _ := flag.(*cli.BoolFlag) diff --git a/core/bench_test.go b/core/bench_test.go index 1f9e0f91e6..dd7257f380 100644 --- a/core/bench_test.go +++ b/core/bench_test.go @@ -193,7 +193,7 @@ func benchInsertChain(b *testing.B, disk bool, gen func(int, *BlockGen)) { } else { dir := b.TempDir() - db, err = rawdb.NewLevelDBDatabase(dir, 128, 128, "", false) + db, err = rawdb.NewLevelDBDatabase(dir, 128, 128, "", false, rawdb.ExtraDBConfig{}) if err != nil { b.Fatalf("cannot create temporary database: %v", err) } @@ -296,7 +296,7 @@ func makeChainForBench(db ethdb.Database, full bool, count uint64) { func benchWriteChain(b *testing.B, full bool, count uint64) { for i := 0; i < b.N; i++ { dir := b.TempDir() - db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "", false) + db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "", false, rawdb.ExtraDBConfig{}) if err != nil { b.Fatalf("error opening database at %v: %v", dir, err) @@ -310,7 +310,7 @@ func benchWriteChain(b *testing.B, full bool, count uint64) { func benchReadChain(b *testing.B, full bool, count uint64) { dir := b.TempDir() - db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "", false) + db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "", false, rawdb.ExtraDBConfig{}) if err != nil { b.Fatalf("error opening database at %v: %v", dir, err) } @@ -325,7 +325,7 @@ func benchReadChain(b *testing.B, full bool, count uint64) { b.ResetTimer() for i := 0; i < b.N; i++ { - db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "", false) + db, err := rawdb.NewLevelDBDatabase(dir, 128, 1024, "", false, rawdb.ExtraDBConfig{}) if err != nil { b.Fatalf("error opening database at %v: %v", dir, err) } diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 5cc13d0632..baf1d2cf79 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -321,8 +321,8 @@ func NewMemoryDatabaseWithCap(size int) ethdb.Database { // NewLevelDBDatabase creates a persistent key-value database without a freezer // moving immutable chain segments into cold storage. -func NewLevelDBDatabase(file string, cache int, handles int, namespace string, readonly bool) (ethdb.Database, error) { - db, err := leveldb.New(file, cache, handles, namespace, readonly) +func NewLevelDBDatabase(file string, cache int, handles int, namespace string, readonly bool, extraDBConfig ExtraDBConfig) (ethdb.Database, error) { + db, err := leveldb.New(file, cache, handles, namespace, readonly, resolveLevelDBConfig(extraDBConfig)) if err != nil { return nil, err } @@ -332,6 +332,15 @@ func NewLevelDBDatabase(file string, cache int, handles int, namespace string, r return NewDatabase(db), nil } +func resolveLevelDBConfig(config ExtraDBConfig) leveldb.LevelDBConfig { + return leveldb.LevelDBConfig{ + CompactionTableSize: config.LevelDBCompactionTableSize, + CompactionTableSizeMultiplier: config.LevelDBCompactionTableSizeMultiplier, + CompactionTotalSize: config.LevelDBCompactionTotalSize, + CompactionTotalSizeMultiplier: config.LevelDBCompactionTotalSizeMultiplier, + } +} + const ( dbPebble = "pebble" dbLeveldb = "leveldb" @@ -366,6 +375,14 @@ type OpenOptions struct { Cache int // the capacity(in megabytes) of the data caching Handles int // number of files to be open simultaneously ReadOnly bool + ExtraDBConfig ExtraDBConfig +} + +type ExtraDBConfig struct { + LevelDBCompactionTableSize uint64 // LevelDB SSTable/file size in mebibytes + LevelDBCompactionTableSizeMultiplier float64 // Multiplier on LevelDB SSTable/file size + LevelDBCompactionTotalSize uint64 // Total size in mebibytes of SSTables in a given LevelDB level + LevelDBCompactionTotalSizeMultiplier float64 // Multiplier on level size on LevelDB levels } // openKeyValueDatabase opens a disk-based key-value database, e.g. leveldb or pebble. @@ -393,9 +410,10 @@ func openKeyValueDatabase(o OpenOptions) (ethdb.Database, error) { return nil, fmt.Errorf("unknown db.engine %v", o.Type) } - log.Info("Using leveldb as the backing database") // Use leveldb, either as default (no explicit choice), or pre-existing, or chosen explicitly - return NewLevelDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly) + log.Info("Using leveldb as the backing database") + + return NewLevelDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly, o.ExtraDBConfig) } // Open opens both a disk-based key-value database such as leveldb or pebble, but also diff --git a/docs/cli/server.md b/docs/cli/server.md index 213d491fff..865b54e3f0 100644 --- a/docs/cli/server.md +++ b/docs/cli/server.md @@ -126,6 +126,16 @@ The ```bor server``` command runs the Bor client. - ```fdlimit```: Raise the open file descriptor resource limit (default = system fd limit) (default: 0) +### ExtraDB Options + +- ```leveldb.compaction.table.size```: LevelDB SSTable/file size in mebibytes (default: 2) + +- ```leveldb.compaction.table.size.multiplier```: Multiplier on LevelDB SSTable/file size. Size for a level is determined by: `leveldb.compaction.table.size * (leveldb.compaction.table.size.multiplier ^ Level)` (default: 1) + +- ```leveldb.compaction.total.size```: Total size in mebibytes of SSTables in a given LevelDB level. Size for a level is determined by: `leveldb.compaction.total.size * (leveldb.compaction.total.size.multiplier ^ Level)` (default: 10) + +- ```leveldb.compaction.total.size.multiplier```: Multiplier on level size on LevelDB levels. Size for a level is determined by: `leveldb.compaction.total.size * (leveldb.compaction.total.size.multiplier ^ Level)` (default: 10) + ### JsonRPC Options - ```rpc.gascap```: Sets a cap on gas that can be used in eth_call/estimateGas (0=infinite) (default: 50000000) diff --git a/eth/backend.go b/eth/backend.go index 93004cc196..061edb0c06 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -141,7 +141,8 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { log.Info("Allocated trie memory caches", "clean", common.StorageSize(config.TrieCleanCache)*1024*1024, "dirty", common.StorageSize(config.TrieDirtyCache)*1024*1024) // Assemble the Ethereum object - chainDb, err := stack.OpenDatabaseWithFreezer("chaindata", config.DatabaseCache, config.DatabaseHandles, config.DatabaseFreezer, "ethereum/db/chaindata/", false) + extraDBConfig := resolveExtraDBConfig(config) + chainDb, err := stack.OpenDatabaseWithFreezer("chaindata", config.DatabaseCache, config.DatabaseHandles, config.DatabaseFreezer, "ethereum/db/chaindata/", false, extraDBConfig) if err != nil { return nil, err } @@ -332,6 +333,15 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { return ethereum, nil } +func resolveExtraDBConfig(config *ethconfig.Config) rawdb.ExtraDBConfig { + return rawdb.ExtraDBConfig{ + LevelDBCompactionTableSize: config.LevelDbCompactionTableSize, + LevelDBCompactionTableSizeMultiplier: config.LevelDbCompactionTableSizeMultiplier, + LevelDBCompactionTotalSize: config.LevelDbCompactionTotalSize, + LevelDBCompactionTotalSizeMultiplier: config.LevelDbCompactionTotalSizeMultiplier, + } +} + func makeExtraData(extra []byte) []byte { if len(extra) == 0 { // create default extradata diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 77b127a3a8..7a037aec31 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -169,6 +169,12 @@ type Config struct { DatabaseCache int DatabaseFreezer string + // Database - LevelDB options + LevelDbCompactionTableSize uint64 + LevelDbCompactionTableSizeMultiplier float64 + LevelDbCompactionTotalSize uint64 + LevelDbCompactionTotalSizeMultiplier float64 + TrieCleanCache int TrieCleanCacheJournal string `toml:",omitempty"` // Disk journal directory for trie cache to survive node restarts TrieCleanCacheRejournal time.Duration `toml:",omitempty"` // Time interval to regenerate the journal for clean cache diff --git a/eth/filters/bench_test.go b/eth/filters/bench_test.go index 89a45bc2d8..3e52a6c11b 100644 --- a/eth/filters/bench_test.go +++ b/eth/filters/bench_test.go @@ -68,7 +68,7 @@ func benchmarkBloomBits(b *testing.B, sectionSize uint64) { b.Log("Running bloombits benchmark section size:", sectionSize) - db, err := rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024, "", false) + db, err := rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024, "", false, rawdb.ExtraDBConfig{}) if err != nil { b.Fatalf("error opening database at %v: %v", benchDataDir, err) } @@ -145,7 +145,7 @@ func benchmarkBloomBits(b *testing.B, sectionSize uint64) { for i := 0; i < benchFilterCnt; i++ { if i%20 == 0 { db.Close() - db, _ = rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024, "", false) + db, _ = rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024, "", false, rawdb.ExtraDBConfig{}) backend = &testBackend{db: db, sections: cnt} sys = NewFilterSystem(backend, Config{}) } @@ -187,7 +187,7 @@ func BenchmarkNoBloomBits(b *testing.B) { b.Log("Running benchmark without bloombits") - db, err := rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024, "", false) + db, err := rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024, "", false, rawdb.ExtraDBConfig{}) if err != nil { b.Fatalf("error opening database at %v: %v", benchDataDir, err) } diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go index 4bea788f25..431db44f75 100644 --- a/eth/filters/filter_test.go +++ b/eth/filters/filter_test.go @@ -43,7 +43,7 @@ func makeReceipt(addr common.Address) *types.Receipt { func BenchmarkFilters(b *testing.B) { var ( - db, _ = rawdb.NewLevelDBDatabase(b.TempDir(), 0, 0, "", false) + db, _ = rawdb.NewLevelDBDatabase(b.TempDir(), 0, 0, "", false, rawdb.ExtraDBConfig{}) _, sys = newTestFilterSystem(b, db, Config{}) key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr1 = crypto.PubkeyToAddress(key1.PublicKey) @@ -106,7 +106,7 @@ func BenchmarkFilters(b *testing.B) { func TestFilters(t *testing.T) { var ( - db, _ = rawdb.NewLevelDBDatabase(t.TempDir(), 0, 0, "", false) + db, _ = rawdb.NewLevelDBDatabase(t.TempDir(), 0, 0, "", false, rawdb.ExtraDBConfig{}) _, sys = newTestFilterSystem(t, db, Config{}) key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr = crypto.PubkeyToAddress(key1.PublicKey) diff --git a/ethdb/leveldb/leveldb.go b/ethdb/leveldb/leveldb.go index 6c81f15f75..617cfd2763 100644 --- a/ethdb/leveldb/leveldb.go +++ b/ethdb/leveldb/leveldb.go @@ -83,9 +83,16 @@ type Database struct { log log.Logger // Contextual logger tracking the database path } +type LevelDBConfig struct { + CompactionTableSize uint64 // LevelDB SSTable/file size in mebibytes + CompactionTableSizeMultiplier float64 // Multiplier on LevelDB SSTable/file size + CompactionTotalSize uint64 // Total size in mebibytes of SSTables in a given LevelDB level + CompactionTotalSizeMultiplier float64 // Multiplier on level size on LevelDB levels +} + // New returns a wrapped LevelDB object. The namespace is the prefix that the // metrics reporting should use for surfacing internal stats. -func New(file string, cache int, handles int, namespace string, readonly bool) (*Database, error) { +func New(file string, cache int, handles int, namespace string, readonly bool, config LevelDBConfig) (*Database, error) { return NewCustom(file, namespace, func(options *opt.Options) { // Ensure we have some minimal caching and file guarantees if cache < minCache { @@ -100,6 +107,22 @@ func New(file string, cache int, handles int, namespace string, readonly bool) ( options.BlockCacheCapacity = cache / 2 * opt.MiB options.WriteBuffer = cache / 4 * opt.MiB // Two of these are used internally + if config.CompactionTableSize != 0 { + options.CompactionTableSize = int(config.CompactionTableSize * opt.MiB) + } + + if config.CompactionTableSizeMultiplier != 0 { + options.CompactionTableSizeMultiplier = config.CompactionTableSizeMultiplier + } + + if config.CompactionTotalSize != 0 { + options.CompactionTotalSize = int(config.CompactionTotalSize * opt.MiB) + } + + if config.CompactionTotalSizeMultiplier != 0 { + options.CompactionTotalSizeMultiplier = config.CompactionTotalSizeMultiplier + } + if readonly { options.ReadOnly = true } @@ -114,7 +137,14 @@ func NewCustom(file string, namespace string, customize func(options *opt.Option logger := log.New("database", file) usedCache := options.GetBlockCacheCapacity() + options.GetWriteBuffer()*2 - logCtx := []interface{}{"cache", common.StorageSize(usedCache), "handles", options.GetOpenFilesCacheCapacity()} + logCtx := []interface{}{ + "cache", common.StorageSize(usedCache), + "handles", options.GetOpenFilesCacheCapacity(), + "compactionTableSize", options.CompactionTableSize, + "compactionTableSizeMultiplier", options.CompactionTableSizeMultiplier, + "compactionTotalSize", options.CompactionTotalSize, + "compactionTotalSizeMultiplier", options.CompactionTotalSizeMultiplier} + if options.ReadOnly { logCtx = append(logCtx, "readonly", "true") } diff --git a/internal/cli/server/config.go b/internal/cli/server/config.go index c5c7608ca8..0a9c7dd704 100644 --- a/internal/cli/server/config.go +++ b/internal/cli/server/config.go @@ -121,6 +121,8 @@ type Config struct { // Cache has the cache related settings Cache *CacheConfig `hcl:"cache,block" toml:"cache,block"` + ExtraDB *ExtraDBConfig `hcl:"leveldb,block" toml:"leveldb,block"` + // Account has the validator account related settings Accounts *AccountsConfig `hcl:"accounts,block" toml:"accounts,block"` @@ -551,6 +553,13 @@ type CacheConfig struct { FDLimit int `hcl:"fdlimit,optional" toml:"fdlimit,optional"` } +type ExtraDBConfig struct { + LevelDbCompactionTableSize uint64 `hcl:"compactiontablesize,optional" toml:"compactiontablesize,optional"` + LevelDbCompactionTableSizeMultiplier float64 `hcl:"compactiontablesizemultiplier,optional" toml:"compactiontablesizemultiplier,optional"` + LevelDbCompactionTotalSize uint64 `hcl:"compactiontotalsize,optional" toml:"compactiontotalsize,optional"` + LevelDbCompactionTotalSizeMultiplier float64 `hcl:"compactiontotalsizemultiplier,optional" toml:"compactiontotalsizemultiplier,optional"` +} + type AccountsConfig struct { // Unlock is the list of addresses to unlock in the node Unlock []string `hcl:"unlock,optional" toml:"unlock,optional"` @@ -742,6 +751,14 @@ func DefaultConfig() *Config { TrieTimeout: 60 * time.Minute, FDLimit: 0, }, + ExtraDB: &ExtraDBConfig{ + // These are LevelDB defaults, specifying here for clarity in code and in logging. + // See: https://github.com/syndtr/goleveldb/blob/126854af5e6d8295ef8e8bee3040dd8380ae72e8/leveldb/opt/options.go + LevelDbCompactionTableSize: 2, // MiB + LevelDbCompactionTableSizeMultiplier: 1, + LevelDbCompactionTotalSize: 10, // MiB + LevelDbCompactionTotalSizeMultiplier: 10, + }, Accounts: &AccountsConfig{ Unlock: []string{}, PasswordFile: "", @@ -1101,6 +1118,14 @@ func (c *Config) buildEth(stack *node.Node, accountManager *accounts.Manager) (* n.TriesInMemory = c.Cache.TriesInMemory } + // LevelDB + { + n.LevelDbCompactionTableSize = c.ExtraDB.LevelDbCompactionTableSize + n.LevelDbCompactionTableSizeMultiplier = c.ExtraDB.LevelDbCompactionTableSizeMultiplier + n.LevelDbCompactionTotalSize = c.ExtraDB.LevelDbCompactionTotalSize + n.LevelDbCompactionTotalSizeMultiplier = c.ExtraDB.LevelDbCompactionTotalSizeMultiplier + } + n.RPCGasCap = c.JsonRPC.GasCap if n.RPCGasCap != 0 { log.Info("Set global gas cap", "cap", n.RPCGasCap) diff --git a/internal/cli/server/flags.go b/internal/cli/server/flags.go index 34d3ace94f..1edacc9ac0 100644 --- a/internal/cli/server/flags.go +++ b/internal/cli/server/flags.go @@ -448,6 +448,36 @@ func (c *Command) Flags() *flagset.Flagset { Group: "Cache", }) + // LevelDB options + f.Uint64Flag(&flagset.Uint64Flag{ + Name: "leveldb.compaction.table.size", + Usage: "LevelDB SSTable/file size in mebibytes", + Value: &c.cliConfig.ExtraDB.LevelDbCompactionTableSize, + Default: c.cliConfig.ExtraDB.LevelDbCompactionTableSize, + Group: "ExtraDB", + }) + f.Float64Flag(&flagset.Float64Flag{ + Name: "leveldb.compaction.table.size.multiplier", + Usage: "Multiplier on LevelDB SSTable/file size. Size for a level is determined by: `leveldb.compaction.table.size * (leveldb.compaction.table.size.multiplier ^ Level)`", + Value: &c.cliConfig.ExtraDB.LevelDbCompactionTableSizeMultiplier, + Default: c.cliConfig.ExtraDB.LevelDbCompactionTableSizeMultiplier, + Group: "ExtraDB", + }) + f.Uint64Flag(&flagset.Uint64Flag{ + Name: "leveldb.compaction.total.size", + Usage: "Total size in mebibytes of SSTables in a given LevelDB level. Size for a level is determined by: `leveldb.compaction.total.size * (leveldb.compaction.total.size.multiplier ^ Level)`", + Value: &c.cliConfig.ExtraDB.LevelDbCompactionTotalSize, + Default: c.cliConfig.ExtraDB.LevelDbCompactionTotalSize, + Group: "ExtraDB", + }) + f.Float64Flag(&flagset.Float64Flag{ + Name: "leveldb.compaction.total.size.multiplier", + Usage: "Multiplier on level size on LevelDB levels. Size for a level is determined by: `leveldb.compaction.total.size * (leveldb.compaction.total.size.multiplier ^ Level)`", + Value: &c.cliConfig.ExtraDB.LevelDbCompactionTotalSizeMultiplier, + Default: c.cliConfig.ExtraDB.LevelDbCompactionTotalSizeMultiplier, + Group: "ExtraDB", + }) + // rpc options f.Uint64Flag(&flagset.Uint64Flag{ Name: "rpc.gascap", diff --git a/internal/cli/snapshot.go b/internal/cli/snapshot.go index 7257cde4b7..60034103d3 100644 --- a/internal/cli/snapshot.go +++ b/internal/cli/snapshot.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state/pruner" "github.com/ethereum/go-ethereum/internal/cli/flagset" "github.com/ethereum/go-ethereum/internal/cli/server" @@ -161,7 +162,7 @@ func (c *PruneStateCommand) Run(args []string) int { return 1 } - chaindb, err := node.OpenDatabaseWithFreezer(chaindataPath, int(c.cache), dbHandles, c.datadirAncient, "", false) + chaindb, err := node.OpenDatabaseWithFreezer(chaindataPath, int(c.cache), dbHandles, c.datadirAncient, "", false, rawdb.ExtraDBConfig{}) if err != nil { c.UI.Error(err.Error()) diff --git a/les/client.go b/les/client.go index 9c31b2dd9e..72a74db33e 100644 --- a/les/client.go +++ b/les/client.go @@ -86,12 +86,14 @@ type LightEthereum struct { // New creates an instance of the light client. func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { - chainDb, err := stack.OpenDatabase("lightchaindata", config.DatabaseCache, config.DatabaseHandles, "eth/db/chaindata/", false) + extraDBConfig := resolveExtraDBConfig(config) + + chainDb, err := stack.OpenDatabase("lightchaindata", config.DatabaseCache, config.DatabaseHandles, "eth/db/chaindata/", false, extraDBConfig) if err != nil { return nil, err } - lesDb, err := stack.OpenDatabase("les.client", 0, 0, "eth/db/lesclient/", false) + lesDb, err := stack.OpenDatabase("les.client", 0, 0, "eth/db/lesclient/", false, extraDBConfig) if err != nil { return nil, err } @@ -227,6 +229,15 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { return leth, nil } +func resolveExtraDBConfig(config *ethconfig.Config) rawdb.ExtraDBConfig { + return rawdb.ExtraDBConfig{ + LevelDBCompactionTableSize: config.LevelDbCompactionTableSize, + LevelDBCompactionTableSizeMultiplier: config.LevelDbCompactionTableSizeMultiplier, + LevelDBCompactionTotalSize: config.LevelDbCompactionTotalSize, + LevelDBCompactionTotalSizeMultiplier: config.LevelDbCompactionTotalSizeMultiplier, + } +} + // VfluxRequest sends a batch of requests to the given node through discv5 UDP TalkRequest and returns the responses func (s *LightEthereum) VfluxRequest(n *enode.Node, reqs vflux.Requests) vflux.Replies { if !s.udpEnabled { diff --git a/les/server.go b/les/server.go index 713f84ea55..4a5751222b 100644 --- a/les/server.go +++ b/les/server.go @@ -78,7 +78,8 @@ type LesServer struct { } func NewLesServer(node *node.Node, e ethBackend, config *ethconfig.Config) (*LesServer, error) { - lesDb, err := node.OpenDatabase("les.server", 0, 0, "eth/db/lesserver/", false) + dbOptions := resolveExtraDBConfig(config) + lesDb, err := node.OpenDatabase("les.server", 0, 0, "eth/db/lesserver/", false, dbOptions) if err != nil { return nil, err } diff --git a/node/node.go b/node/node.go index cc7215fa6e..428a5a4de9 100644 --- a/node/node.go +++ b/node/node.go @@ -771,7 +771,7 @@ func (n *Node) EventMux() *event.TypeMux { // OpenDatabase opens an existing database with the given name (or creates one if no // previous can be found) from within the node's instance directory. If the node is // ephemeral, a memory database is returned. -func (n *Node) OpenDatabase(name string, cache, handles int, namespace string, readonly bool) (ethdb.Database, error) { +func (n *Node) OpenDatabase(name string, cache, handles int, namespace string, readonly bool, extraDBConfig rawdb.ExtraDBConfig) (ethdb.Database, error) { n.lock.Lock() defer n.lock.Unlock() @@ -787,12 +787,13 @@ func (n *Node) OpenDatabase(name string, cache, handles int, namespace string, r db = rawdb.NewMemoryDatabase() } else { db, err = rawdb.Open(rawdb.OpenOptions{ - Type: n.config.DBEngine, - Directory: n.ResolvePath(name), - Namespace: namespace, - Cache: cache, - Handles: handles, - ReadOnly: readonly, + Type: n.config.DBEngine, + Directory: n.ResolvePath(name), + Namespace: namespace, + Cache: cache, + Handles: handles, + ReadOnly: readonly, + ExtraDBConfig: extraDBConfig, }) } @@ -808,7 +809,7 @@ func (n *Node) OpenDatabase(name string, cache, handles int, namespace string, r // also attaching a chain freezer to it that moves ancient chain data from the // database to immutable append-only files. If the node is an ephemeral one, a // memory database is returned. -func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, ancient string, namespace string, readonly bool) (ethdb.Database, error) { +func (n *Node) OpenDatabaseWithFreezer(name string, cache int, handles int, ancient string, namespace string, readonly bool, extraDBConfig rawdb.ExtraDBConfig) (ethdb.Database, error) { n.lock.Lock() defer n.lock.Unlock() @@ -831,6 +832,7 @@ func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, ancient Cache: cache, Handles: handles, ReadOnly: readonly, + ExtraDBConfig: extraDBConfig, }) } diff --git a/node/node_test.go b/node/node_test.go index dc291f5d91..f53b003684 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -26,6 +26,7 @@ import ( "strings" "testing" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/p2p" @@ -157,7 +158,7 @@ func TestNodeCloseClosesDB(t *testing.T) { stack, _ := New(testNodeConfig()) defer stack.Close() - db, err := stack.OpenDatabase("mydb", 0, 0, "", false) + db, err := stack.OpenDatabase("mydb", 0, 0, "", false, rawdb.ExtraDBConfig{}) if err != nil { t.Fatal("can't open DB:", err) } @@ -184,7 +185,7 @@ func TestNodeOpenDatabaseFromLifecycleStart(t *testing.T) { stack.RegisterLifecycle(&InstrumentedService{ startHook: func() { - db, err = stack.OpenDatabase("mydb", 0, 0, "", false) + db, err = stack.OpenDatabase("mydb", 0, 0, "", false, rawdb.ExtraDBConfig{}) if err != nil { t.Fatal("can't open DB:", err) } @@ -205,7 +206,7 @@ func TestNodeOpenDatabaseFromLifecycleStop(t *testing.T) { stack.RegisterLifecycle(&InstrumentedService{ stopHook: func() { - db, err := stack.OpenDatabase("mydb", 0, 0, "", false) + db, err := stack.OpenDatabase("mydb", 0, 0, "", false, rawdb.ExtraDBConfig{}) if err != nil { t.Fatal("can't open DB:", err) }