forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-06-21 06:04:33 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

cmd, core, eth, trie: add trie read caching layer (#18087) (#946)

Browse source
This commit is contained in:
Daniel Liu 2025-04-16 17:27:43 +08:00 committed by GitHub
parent 5759893f2b
commit b9a6c8c32d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 68 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
cmd/XDC/main.go
View file

@ -95,7 +95,8 @@ var (
//utils.LightKDFFlag,
utils.CacheFlag,
utils.CacheDatabaseFlag,
//utils.CacheGCFlag,
utils.CacheTrieFlag,
utils.CacheGCFlag,
//utils.TrieCacheGenFlag,
utils.CacheLogSizeFlag,
utils.FDLimitFlag,

23
cmd/utils/flags.go
View file

@ -289,6 +289,12 @@ var (
Value: 50,
Category: flags.PerfCategory,
}
CacheTrieFlag = &cli.IntFlag{
Name: "cache.trie",
Usage: "Percentage of cache memory allowance to use for trie caching (default = 15% full mode, 30% archive mode)",
Value: 15,
Category: flags.PerfCategory,
}
CacheGCFlag = &cli.IntFlag{
Name: "cache-gc",
Aliases: []string{"cache.gc"},
@ -1424,8 +1430,11 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
}
cfg.NoPruning = ctx.String(GCModeFlag.Name) == "archive"
if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheTrieFlag.Name) {
cfg.TrieCleanCache = ctx.Int(CacheFlag.Name) * ctx.Int(CacheTrieFlag.Name) / 100
}
if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheGCFlag.Name) {
cfg.TrieCache = ctx.Int(CacheFlag.Name) * ctx.Int(CacheGCFlag.Name) / 100
cfg.TrieDirtyCache = ctx.Int(CacheFlag.Name) * ctx.Int(CacheGCFlag.Name) / 100
}
if ctx.IsSet(MinerThreadsFlag.Name) {
cfg.MinerThreads = ctx.Int(MinerThreadsFlag.Name)
@ -1673,12 +1682,16 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readonly bool) (chain *core.B
Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name)
}
cache := &core.CacheConfig{
Disabled: ctx.String(GCModeFlag.Name) == "archive",
TrieNodeLimit: ethconfig.Defaults.TrieCache,
TrieTimeLimit: ethconfig.Defaults.TrieTimeout,
Disabled: ctx.String(GCModeFlag.Name) == "archive",
TrieCleanLimit: ethconfig.Defaults.TrieCleanCache,
TrieDirtyLimit: ethconfig.Defaults.TrieDirtyCache,
TrieTimeLimit: ethconfig.Defaults.TrieTimeout,
}
if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheTrieFlag.Name) {
cache.TrieCleanLimit = ctx.Int(CacheFlag.Name) * ctx.Int(CacheTrieFlag.Name) / 100
}
if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheGCFlag.Name) {
cache.TrieNodeLimit = ctx.Int(CacheFlag.Name) * ctx.Int(CacheGCFlag.Name) / 100
cache.TrieCleanLimit = ctx.Int(CacheFlag.Name) * ctx.Int(CacheGCFlag.Name) / 100
}
vmcfg := vm.Config{EnablePreimageRecording: ctx.Bool(VMEnableDebugFlag.Name)}
chain, err = core.NewBlockChain(chainDb, cache, config, engine, vmcfg)

22
core/blockchain.go
View file

@ -127,10 +127,12 @@ const (
// CacheConfig contains the configuration values for the trie caching/pruning
// that's resident in a blockchain.
type CacheConfig struct {
Disabled bool // Whether to disable trie write caching (archive node)
TrieNodeLimit int // Memory limit (MB) at which to flush the current in-memory trie to disk
TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk
Disabled bool // Whether to disable trie write caching (archive node)
TrieCleanLimit int // Memory allowance (MB) to use for caching trie nodes in memory
TrieDirtyLimit int // Memory limit (MB) at which to start flushing dirty trie nodes to disk
TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk
}
type ResultProcessBlock struct {
logs []*types.Log
receipts []*types.Receipt
@ -226,8 +228,9 @@ type BlockChain struct {
func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *params.ChainConfig, engine consensus.Engine, vmConfig vm.Config) (*BlockChain, error) {
if cacheConfig == nil {
cacheConfig = &CacheConfig{
TrieNodeLimit: 256 * 1024 * 1024,
TrieTimeLimit: 5 * time.Minute,
TrieCleanLimit: 256,
TrieDirtyLimit: 256,
TrieTimeLimit: 5 * time.Minute,
}
}
@ -236,7 +239,7 @@ func NewBlockChain(db ethdb.Database, cacheConfig *CacheConfig, chainConfig *par
cacheConfig: cacheConfig,
db: db,
triegc: prque.New[int64, common.Hash](nil),
stateCache: state.NewDatabase(db),
stateCache: state.NewDatabaseWithCache(db, cacheConfig.TrieCleanLimit),
quit: make(chan struct{}),
chainmu: syncx.NewClosableMutex(),
bodyCache: lru.NewCache[common.Hash, *types.Body](bodyCacheLimit),
@ -589,6 +592,11 @@ func (bc *BlockChain) StateAt(root common.Hash) (*state.StateDB, error) {
return state.New(root, bc.stateCache)
}
// StateCache returns the caching database underpinning the blockchain instance.
func (bc *BlockChain) StateCache() state.Database {
return bc.stateCache
}
// OrderStateAt returns a new mutable state based on a particular point in time.
func (bc *BlockChain) OrderStateAt(block *types.Block) (*tradingstate.TradingStateDB, error) {
engine, ok := bc.Engine().(*XDPoS.XDPoS)
@ -1450,7 +1458,7 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.
//}
var (
nodes, imgs = triedb.Size()
limit = common.StorageSize(bc.cacheConfig.TrieNodeLimit) * 1024 * 1024
limit = common.StorageSize(bc.cacheConfig.TrieDirtyLimit) * 1024 * 1024
)
if nodes > limit || imgs > 4*1024*1024 {
triedb.Cap(limit - ethdb.IdealBatchSize)

10
core/state/database.go
View file

@ -102,15 +102,15 @@ type Trie interface {
}
// NewDatabase creates a backing store for state. The returned database is safe for
// concurrent use, but does not retain any recent trie nodes in memory. To keep some
// historical state in memory, use the NewDatabaseWithCache constructor.
// concurrent use and retains a few recent expanded trie nodes in memory. To keep
// more historical state in memory, use the NewDatabaseWithCache constructor.
func NewDatabase(db ethdb.Database) Database {
return NewDatabaseWithCache(db, 0)
}
// NewDatabaseWithCache creates a backing store for state. The returned database
// is safe for concurrent use and retains a lot of collapsed RLP trie nodes in a
// large memory cache.
// NewDatabase creates a backing store for state. The returned database is safe for
// concurrent use and retains both a few recent expanded trie nodes in memory, as
// well as a lot of collapsed RLP trie nodes in a large memory cache.
func NewDatabaseWithCache(db ethdb.Database, cache int) Database {
return &cachingDB{
db: trie.NewDatabaseWithCache(db, cache),

5
eth/api.go
View file

@ -462,12 +462,13 @@ func (api *PrivateDebugAPI) getModifiedAccounts(startBlock, endBlock *types.Bloc
if startBlock.Number().Uint64() >= endBlock.Number().Uint64() {
return nil, fmt.Errorf("start block height (%d) must be less than end block height (%d)", startBlock.Number().Uint64(), endBlock.Number().Uint64())
}
triedb := api.eth.BlockChain().StateCache().TrieDB()
oldTrie, err := trie.NewSecure(startBlock.Root(), trie.NewDatabase(api.eth.chainDb))
oldTrie, err := trie.NewSecure(startBlock.Root(), triedb)
if err != nil {
return nil, err
}
newTrie, err := trie.NewSecure(endBlock.Root(), trie.NewDatabase(api.eth.chainDb))
newTrie, err := trie.NewSecure(endBlock.Root(), triedb)
if err != nil {
return nil, err
}

4
eth/api_tracer.go
View file

@ -177,7 +177,7 @@ func (api *PrivateDebugAPI) traceChain(ctx context.Context, start, end *types.Bl
// Ensure we have a valid starting state before doing any work
origin := start.NumberU64()
database := state.NewDatabase(api.eth.ChainDb())
database := state.NewDatabaseWithCache(api.eth.ChainDb(), 16) // Chain tracing will probably start at genesis
if number := start.NumberU64(); number > 0 {
start = api.eth.blockchain.GetBlock(start.ParentHash(), start.NumberU64()-1)
@ -563,7 +563,7 @@ func (api *PrivateDebugAPI) computeStateDB(block *types.Block, reexec uint64) (*
}
// Otherwise try to reexec blocks until we find a state or reach our limit
origin := block.NumberU64()
database := state.NewDatabase(api.eth.ChainDb())
database := state.NewDatabaseWithCache(api.eth.ChainDb(), 16)
for i := uint64(0); i < reexec; i++ {
block = api.eth.blockchain.GetBlock(block.ParentHash(), block.NumberU64()-1)

7
eth/backend.go
View file

@ -175,7 +175,12 @@ func New(stack *node.Node, config *ethconfig.Config, XDCXServ *XDCx.XDCX, lendin
var (
vmConfig = vm.Config{EnablePreimageRecording: config.EnablePreimageRecording}
cacheConfig = &core.CacheConfig{Disabled: config.NoPruning, TrieNodeLimit: config.TrieCache, TrieTimeLimit: config.TrieTimeout}
cacheConfig = &core.CacheConfig{
Disabled: config.NoPruning,
TrieCleanLimit: config.TrieCleanCache,
TrieDirtyLimit: config.TrieDirtyCache,
TrieTimeLimit: config.TrieTimeout,
}
)
if eth.chainConfig.XDPoS != nil {
c := eth.engine.(*XDPoS.XDPoS)

6
eth/ethconfig/config.go
View file

@ -58,7 +58,8 @@ var Defaults = Config{
NetworkId: 0, // enable auto configuration of networkID == chainID
LightPeers: 100,
DatabaseCache: 768,
TrieCache: 256,
TrieCleanCache: 256,
TrieDirtyCache: 256,
TrieTimeout: 5 * time.Minute,
FilterLogCacheSize: 32,
GasPrice: big.NewInt(0.25 * params.Shannon),
@ -112,7 +113,8 @@ type Config struct {
SkipBcVersionCheck bool `toml:"-"`
DatabaseHandles int `toml:"-"`
DatabaseCache int
TrieCache int
TrieCleanCache int
TrieDirtyCache int
TrieTimeout time.Duration
// This is the number of blocks for which logs will be cached in the filter system.

16
eth/ethconfig/gen_config.go
View file

@ -29,7 +29,8 @@ func (c Config) MarshalTOML() (interface{}, error) {
SkipBcVersionCheck bool `toml:"-"`
DatabaseHandles int `toml:"-"`
DatabaseCache int
TrieCache int
TrieCleanCache int
TrieDirtyCache int
TrieTimeout time.Duration
FilterLogCacheSize int
Etherbase common.Address `toml:",omitempty"`
@ -53,7 +54,8 @@ func (c Config) MarshalTOML() (interface{}, error) {
enc.SkipBcVersionCheck = c.SkipBcVersionCheck
enc.DatabaseHandles = c.DatabaseHandles
enc.DatabaseCache = c.DatabaseCache
enc.TrieCache = c.TrieCache
enc.TrieCleanCache = c.TrieCleanCache
enc.TrieDirtyCache = c.TrieDirtyCache
enc.TrieTimeout = c.TrieTimeout
enc.FilterLogCacheSize = c.FilterLogCacheSize
enc.Etherbase = c.Etherbase
@ -81,7 +83,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
SkipBcVersionCheck *bool `toml:"-"`
DatabaseHandles *int `toml:"-"`
DatabaseCache *int
TrieCache *int
TrieCleanCache *int
TrieDirtyCache *int
TrieTimeout *time.Duration
FilterLogCacheSize *int
Etherbase *common.Address `toml:",omitempty"`
@ -126,8 +129,11 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
if dec.DatabaseCache != nil {
c.DatabaseCache = *dec.DatabaseCache
}
if dec.TrieCache != nil {
c.TrieCache = *dec.TrieCache
if dec.TrieCleanCache != nil {
c.TrieCleanCache = *dec.TrieCleanCache
}
if dec.TrieDirtyCache != nil {
c.TrieDirtyCache = *dec.TrieDirtyCache
}
if dec.TrieTimeout != nil {
c.TrieTimeout = *dec.TrieTimeout

2
tests/block_test_util.go
View file

@ -114,7 +114,7 @@ func (t *BlockTest) Run() error {
return fmt.Errorf("genesis block state root does not match test: computed=%x, test=%x", gblock.Root().Bytes(), t.json.Genesis.StateRoot)
}
chain, err := core.NewBlockChain(db, nil, config, ethash.NewShared(), vm.Config{})
chain, err := core.NewBlockChain(db, &core.CacheConfig{TrieCleanLimit: 0}, config, ethash.NewShared(), vm.Config{})
if err != nil {
return err
}

2
trie/database.go
View file

@ -336,7 +336,7 @@ func (db *Database) InsertPreimage(hash common.Hash, preimage []byte) {
db.preimagesSize += common.StorageSize(common.HashLength + len(preimage))
}
// Node retrieves a cached trie Node from memory, or returns nil if none can be
// node retrieves a cached trie Node from memory, or returns nil if none can be
// found in the memory Cache.
func (db *Database) node(hash common.Hash) node {
// Retrieve the Node from the clean Cache if available