From dd9770c19ae25928ed47e0f43af29f7c47a2bcd5 Mon Sep 17 00:00:00 2001 From: Gary Rong Date: Tue, 22 Aug 2023 14:10:28 +0800 Subject: [PATCH] eth, trie/triedb/pathdb: reallocate excess memory to clean cache --- eth/backend.go | 11 ++++++++++- trie/triedb/pathdb/database.go | 16 ++++++++-------- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/eth/backend.go b/eth/backend.go index c6787870ca..48fc1e91d7 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -57,6 +57,7 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" + "github.com/ethereum/go-ethereum/trie/triedb/pathdb" ) // Config contains the configuration options of the ETH protocol. @@ -117,7 +118,9 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { log.Warn("Sanitizing invalid miner gas price", "provided", config.Miner.GasPrice, "updated", ethconfig.Defaults.Miner.GasPrice) config.Miner.GasPrice = new(big.Int).Set(ethconfig.Defaults.Miner.GasPrice) } - if config.NoPruning && config.TrieDirtyCache > 0 { + // Redistribute memory allocation from in-memory trie node garbage collection + // to other caches when an archive node is requested. + if config.StateScheme == rawdb.HashScheme && config.NoPruning && config.TrieDirtyCache > 0 { if config.SnapshotCache > 0 { config.TrieCleanCache += config.TrieDirtyCache * 3 / 5 config.SnapshotCache += config.TrieDirtyCache * 2 / 5 @@ -126,6 +129,12 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { } config.TrieDirtyCache = 0 } + // Optimize memory distribution by reallocating surplus allowance from the + // dirty cache to the clean cache. + if config.StateScheme == rawdb.PathScheme && config.TrieDirtyCache > pathdb.MaxBufferSize/1024/1024 { + config.TrieCleanCache += config.TrieDirtyCache - pathdb.MaxBufferSize/1024/1024 + config.TrieDirtyCache = pathdb.MaxBufferSize / 1024 / 1024 + } log.Info("Allocated trie memory caches", "clean", common.StorageSize(config.TrieCleanCache)*1024*1024, "dirty", common.StorageSize(config.TrieDirtyCache)*1024*1024) // Assemble the Ethereum object diff --git a/trie/triedb/pathdb/database.go b/trie/triedb/pathdb/database.go index dc64414e9b..390a94fd77 100644 --- a/trie/triedb/pathdb/database.go +++ b/trie/triedb/pathdb/database.go @@ -40,11 +40,11 @@ const ( // defaultCleanSize is the default memory allowance of clean cache. defaultCleanSize = 16 * 1024 * 1024 - // maxBufferSize is the maximum memory allowance of node buffer. + // MaxBufferSize is the maximum memory allowance of node buffer. // Too large nodebuffer will cause the system to pause for a long // time when write happens. Also, the largest batch that pebble can // support is 4GB, node will panic if batch size exceeds this limit. - maxBufferSize = 256 * 1024 * 1024 + MaxBufferSize = 256 * 1024 * 1024 // DefaultBufferSize is the default memory allowance of node buffer // that aggregates the writes from above until it's flushed into the @@ -96,9 +96,9 @@ type Config struct { // unreasonable or unworkable. func (c *Config) sanitize() *Config { conf := *c - if conf.DirtyCacheSize > maxBufferSize { - log.Warn("Sanitizing invalid node buffer size", "provided", common.StorageSize(conf.DirtyCacheSize), "updated", common.StorageSize(maxBufferSize)) - conf.DirtyCacheSize = maxBufferSize + if conf.DirtyCacheSize > MaxBufferSize { + log.Warn("Sanitizing invalid node buffer size", "provided", common.StorageSize(conf.DirtyCacheSize), "updated", common.StorageSize(MaxBufferSize)) + conf.DirtyCacheSize = MaxBufferSize } return &conf } @@ -439,9 +439,9 @@ func (db *Database) SetBufferSize(size int) error { db.lock.Lock() defer db.lock.Unlock() - if size > maxBufferSize { - log.Info("Capped node buffer size", "provided", common.StorageSize(size), "adjusted", common.StorageSize(maxBufferSize)) - size = maxBufferSize + if size > MaxBufferSize { + log.Info("Capped node buffer size", "provided", common.StorageSize(size), "adjusted", common.StorageSize(MaxBufferSize)) + size = MaxBufferSize } db.bufferSize = size return db.tree.bottom().setBufferSize(db.bufferSize)