forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-07-24 21:56:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

triedb/pathdb: change to the default datadir

Browse source
This commit is contained in:
Gary Rong 2025-06-27 14:03:37 +08:00
parent 3cc8e23b6a
commit 6bb3dfc8c7
10 changed files with 105 additions and 110 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
cmd/utils/flags.go
View file

@ -446,15 +446,6 @@ var (
Value: ethconfig.Defaults.BlobPool.PriceBump,
Category: flags.BlobPoolCategory,
}
// Trie database settings
TrieDBJournalFlag = &cli.BoolFlag{
Name: "triedb.journal",
Usage: "Enable persisting the trie database journal to disk (only used with pbss state scheme, default path: <datadir>/triedb.journal.rlp)",
Value: true,
Category: flags.TrieDatabaseCategory,
}
// Performance tuning settings
CacheFlag = &cli.IntFlag{
Name: "cache",
@ -992,7 +983,6 @@ var (
DataDirFlag,
AncientFlag,
EraFlag,
TrieDBJournalFlag,
RemoteDBFlag,
DBEngineFlag,
StateSchemeFlag,
@ -1654,9 +1644,6 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
if ctx.IsSet(StateSchemeFlag.Name) {
cfg.StateScheme = ctx.String(StateSchemeFlag.Name)
}
if !ctx.Bool(TrieDBJournalFlag.Name) {
cfg.TrieDBJournal = false
}
// Parse transaction history flag, if user is still using legacy config
// file with 'TxLookupLimit' configured, copy the value to 'TransactionHistory'.
if cfg.TransactionHistory == ethconfig.Defaults.TransactionHistory && cfg.TxLookupLimit != ethconfig.Defaults.TxLookupLimit {
@ -2216,9 +2203,6 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readonly bool) (*core.BlockCh
options.Preimages = true
log.Info("Enabling recording of key preimages since archive mode is used")
}
if !ctx.Bool(TrieDBJournalFlag.Name) {
options.TrieDBJournal = stack.ResolvePath("triedb.journal.rlp")
}
if !ctx.Bool(SnapshotFlag.Name) {
options.SnapshotLimit = 0 // Disabled
} else if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheSnapshotFlag.Name) {

4
core/blockchain.go
View file

@ -166,7 +166,7 @@ type BlockChainConfig struct {
TrieDirtyLimit int // Memory limit (MB) at which to start flushing dirty trie nodes to disk
TrieTimeLimit time.Duration // Time limit after which to flush the current in-memory trie to disk
TrieNoAsyncFlush bool // Whether the asynchronous buffer flushing is disallowed
TrieDBJournal string // Path to the journal used for persisting trie data across node restarts
TrieJournalDirectory string // Directory path to the journal used for persisting trie data across node restarts
Preimages bool // Whether to store preimage of trie key to the disk
StateHistory uint64 // Number of blocks from head whose state histories are reserved.
@ -247,7 +247,7 @@ func (cfg *BlockChainConfig) triedbConfig(isVerkle bool) *triedb.Config {
EnableStateIndexing: cfg.ArchiveMode,
TrieCleanSize: cfg.TrieCleanLimit * 1024 * 1024,
StateCleanSize: cfg.SnapshotLimit * 1024 * 1024,
JournalPath: cfg.TrieDBJournal,
JournalDirectory: cfg.TrieJournalDirectory,
// TODO(rjl493456442): The write buffer represents the memory limit used
// for flushing both trie data and state data to disk. The config name

11
eth/backend.go
View file

@ -236,14 +236,13 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
VmConfig: vm.Config{
EnablePreimageRecording: config.EnablePreimageRecording,
},
// Enables file journaling for the trie database. The journal files will be stored
// within the data directory. The corresponding paths will be either:
// - DATADIR/merkle.journal
// - DATADIR/verkle.journal
TrieJournalDirectory: stack.ResolvePath(""),
}
)
if config.TrieDBJournal {
options.TrieDBJournal = stack.ResolvePath("triedb.journal.rlp")
} else {
log.Warn("Trie database journal is persisted within the database")
}
if config.VMTrace != "" {
traceConfig := json.RawMessage("{}")
if config.VMTraceJsonConfig != "" {

15
eth/ethconfig/config.go
View file

@ -103,6 +103,12 @@ type Config struct {
LogHistory uint64 `toml:",omitempty"` // The maximum number of blocks from head where a log search index is maintained.
LogNoHistory bool `toml:",omitempty"` // No log search index is maintained.
LogExportCheckpoints string // export log index checkpoints to file
StateHistory uint64 `toml:",omitempty"` // The maximum number of blocks from head whose state histories are reserved.
// State scheme represents the scheme used to store ethereum states and trie
// nodes on top. It can be 'hash', 'path', or none which means use the scheme
// consistent with persistent state.
StateScheme string `toml:",omitempty"`
// RequiredBlocks is a set of block number -> hash mappings which must be in the
// canonical chain of all remote peers. Setting the option makes geth verify the
@ -116,20 +122,11 @@ type Config struct {
DatabaseFreezer string
DatabaseEra string
// Trie database options. TODO(rjl493456442) move all trie database options
// into a separate config structure.
TrieCleanCache int
TrieDirtyCache int
TrieTimeout time.Duration
SnapshotCache int
Preimages bool
StateHistory uint64 `toml:",omitempty"` // The maximum number of blocks from head whose state histories are reserved.
TrieDBJournal bool // Enable persisting the trie database journal to disk.
// State scheme represents the scheme used to store ethereum states and trie
// nodes on top. It can be 'hash', 'path', or none which means use the scheme
// consistent with persistent state.
StateScheme string `toml:",omitempty"`
// This is the number of blocks for which logs will be cached in the filter system.
FilterLogCacheSize int

30
eth/ethconfig/gen_config.go
View file

@ -30,6 +30,8 @@ func (c Config) MarshalTOML() (interface{}, error) {
LogHistory uint64 `toml:",omitempty"`
LogNoHistory bool `toml:",omitempty"`
LogExportCheckpoints string
StateHistory uint64 `toml:",omitempty"`
StateScheme string `toml:",omitempty"`
RequiredBlocks map[uint64]common.Hash `toml:"-"`
SkipBcVersionCheck bool `toml:"-"`
DatabaseHandles int `toml:"-"`
@ -41,9 +43,6 @@ func (c Config) MarshalTOML() (interface{}, error) {
TrieTimeout time.Duration
SnapshotCache int
Preimages bool
StateHistory uint64 `toml:",omitempty"`
TrieDBJournal bool
StateScheme string `toml:",omitempty"`
FilterLogCacheSize int
Miner miner.Config
TxPool legacypool.Config
@ -72,6 +71,8 @@ func (c Config) MarshalTOML() (interface{}, error) {
enc.LogHistory = c.LogHistory
enc.LogNoHistory = c.LogNoHistory
enc.LogExportCheckpoints = c.LogExportCheckpoints
enc.StateHistory = c.StateHistory
enc.StateScheme = c.StateScheme
enc.RequiredBlocks = c.RequiredBlocks
enc.SkipBcVersionCheck = c.SkipBcVersionCheck
enc.DatabaseHandles = c.DatabaseHandles
@ -83,9 +84,6 @@ func (c Config) MarshalTOML() (interface{}, error) {
enc.TrieTimeout = c.TrieTimeout
enc.SnapshotCache = c.SnapshotCache
enc.Preimages = c.Preimages
enc.StateHistory = c.StateHistory
enc.TrieDBJournal = c.TrieDBJournal
enc.StateScheme = c.StateScheme
enc.FilterLogCacheSize = c.FilterLogCacheSize
enc.Miner = c.Miner
enc.TxPool = c.TxPool
@ -118,6 +116,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
LogHistory *uint64 `toml:",omitempty"`
LogNoHistory *bool `toml:",omitempty"`
LogExportCheckpoints *string
StateHistory *uint64 `toml:",omitempty"`
StateScheme *string `toml:",omitempty"`
RequiredBlocks map[uint64]common.Hash `toml:"-"`
SkipBcVersionCheck *bool `toml:"-"`
DatabaseHandles *int `toml:"-"`
@ -129,9 +129,6 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
TrieTimeout *time.Duration
SnapshotCache *int
Preimages *bool
StateHistory *uint64 `toml:",omitempty"`
TrieDBJournal *bool
StateScheme *string `toml:",omitempty"`
FilterLogCacheSize *int
Miner *miner.Config
TxPool *legacypool.Config
@ -189,6 +186,12 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
if dec.LogExportCheckpoints != nil {
c.LogExportCheckpoints = *dec.LogExportCheckpoints
}
if dec.StateHistory != nil {
c.StateHistory = *dec.StateHistory
}
if dec.StateScheme != nil {
c.StateScheme = *dec.StateScheme
}
if dec.RequiredBlocks != nil {
c.RequiredBlocks = dec.RequiredBlocks
}
@ -222,15 +225,6 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error {
if dec.Preimages != nil {
c.Preimages = *dec.Preimages
}
if dec.StateHistory != nil {
c.StateHistory = *dec.StateHistory
}
if dec.TrieDBJournal != nil {
c.TrieDBJournal = *dec.TrieDBJournal
}
if dec.StateScheme != nil {
c.StateScheme = *dec.StateScheme
}
if dec.FilterLogCacheSize != nil {
c.FilterLogCacheSize = *dec.FilterLogCacheSize
}

1
internal/flags/categories.go
View file

@ -30,7 +30,6 @@ const (
APICategory = "API AND CONSOLE"
NetworkingCategory = "NETWORKING"
MinerCategory = "MINER"
TrieDatabaseCategory = "TRIE DATABASE"
GasPriceCategory = "GAS PRICE ORACLE"
VMCategory = "VIRTUAL MACHINE"
LoggingCategory = "LOGGING AND DEBUGGING"

21
triedb/pathdb/database.go
View file

@ -21,6 +21,7 @@ import (
"errors"
"fmt"
"io"
"path/filepath"
"sync"
"time"
@ -120,7 +121,7 @@ type Config struct {
StateCleanSize int // Maximum memory allowance (in bytes) for caching clean state data
WriteBufferSize int // Maximum memory allowance (in bytes) for write buffer
ReadOnly bool // Flag whether the database is opened in read only mode
JournalPath string // Absolute path of journal file (null means the journal data is persisted in key-value store)
JournalDirectory string // Absolute path of journal directory (null means the journal data is persisted in key-value store)
// Testing configurations
SnapshotNoBuild bool // Flag Whether the state generation is allowed
@ -157,8 +158,8 @@ func (c *Config) fields() []interface{} {
} else {
list = append(list, "history", fmt.Sprintf("last %d blocks", c.StateHistory))
}
if c.JournalPath != "" {
list = append(list, "journal", c.JournalPath)
if c.JournalDirectory != "" {
list = append(list, "journal-dir", c.JournalDirectory)
}
return list
}
@ -674,6 +675,20 @@ func (db *Database) modifyAllowed() error {
return nil
}
// journalPath returns the absolute path of journal for persisting state data.
func (db *Database) journalPath() string {
if db.config.JournalDirectory == "" {
return ""
}
var fname string
if db.isVerkle {
fname = fmt.Sprintf("verkle.journal")
} else {
fname = fmt.Sprintf("merkle.journal")
}
return filepath.Join(db.config.JournalDirectory, fname)
}
// AccountHistory inspects the account history within the specified range.
//
// Start: State ID of the first history object for the query. 0 implies the first

18
triedb/pathdb/database_test.go
View file

@ -125,7 +125,7 @@ type tester struct {
snapStorages map[common.Hash]map[common.Hash]map[common.Hash][]byte // Keyed by the hash of account address and the hash of storage key
}
func newTester(t *testing.T, historyLimit uint64, isVerkle bool, layers int, enableIndex bool, journal string) *tester {
func newTester(t *testing.T, historyLimit uint64, isVerkle bool, layers int, enableIndex bool, journalDir string) *tester {
var (
disk, _ = rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{Ancient: t.TempDir()})
db = New(disk, &Config{
@ -135,7 +135,7 @@ func newTester(t *testing.T, historyLimit uint64, isVerkle bool, layers int, ena
StateCleanSize: 256 * 1024,
WriteBufferSize: 256 * 1024,
NoAsyncFlush: true,
JournalPath: journal,
JournalDirectory: journalDir,
}, isVerkle)
obj = &tester{
@ -619,14 +619,14 @@ func TestJournal(t *testing.T) {
testJournal(t, filepath.Join(t.TempDir(), strconv.Itoa(rand.Intn(10000))))
}
func testJournal(t *testing.T, journal string) {
func testJournal(t *testing.T, journalDir string) {
// Redefine the diff layer depth allowance for faster testing.
maxDiffLayers = 4
defer func() {
maxDiffLayers = 128
}()
tester := newTester(t, 0, false, 12, false, journal)
tester := newTester(t, 0, false, 12, false, journalDir)
defer tester.release()
if err := tester.db.Journal(tester.lastHash()); err != nil {
@ -657,23 +657,23 @@ func TestCorruptedJournal(t *testing.T) {
rawdb.WriteTrieJournal(db, blob)
})
path := filepath.Join(t.TempDir(), strconv.Itoa(rand.Intn(10000)))
testCorruptedJournal(t, path, func(_ ethdb.Database) {
f, _ := os.OpenFile(path, os.O_WRONLY, 0644)
directory := filepath.Join(t.TempDir(), strconv.Itoa(rand.Intn(10000)))
testCorruptedJournal(t, directory, func(_ ethdb.Database) {
f, _ := os.OpenFile(filepath.Join(directory, "merkle.journal"), os.O_WRONLY, 0644)
f.WriteAt([]byte{0xa}, 0)
f.Sync()
f.Close()
})
}
func testCorruptedJournal(t *testing.T, journal string, modifyFn func(database ethdb.Database)) {
func testCorruptedJournal(t *testing.T, journalDir string, modifyFn func(database ethdb.Database)) {
// Redefine the diff layer depth allowance for faster testing.
maxDiffLayers = 4
defer func() {
maxDiffLayers = 128
}()
tester := newTester(t, 0, false, 12, false, journal)
tester := newTester(t, 0, false, 12, false, journalDir)
defer tester.release()
if err := tester.db.Journal(tester.lastHash()); err != nil {

2
triedb/pathdb/history_reader_test.go
View file

@ -126,7 +126,7 @@ func testHistoryReader(t *testing.T, historyLimit uint64) {
}()
//log.SetDefault(log.NewLogger(log.NewTerminalHandlerWithLevel(os.Stderr, log.LevelDebug, true)))
env := newTester(t, historyLimit, false, 64, true)
env := newTester(t, historyLimit, false, 64, true, "")
defer env.release()
waitIndexing(env.db)

49
triedb/pathdb/journal.go
View file

@ -22,7 +22,6 @@ import (
"fmt"
"io"
"os"
"path/filepath"
"time"
"github.com/ethereum/go-ethereum/common"
@ -33,6 +32,8 @@ import (
"github.com/ethereum/go-ethereum/rlp"
)
const tempJournalSuffix = ".tmp"
var (
errMissJournal = errors.New("journal not found")
errMissVersion = errors.New("version not found")
@ -54,12 +55,12 @@ const journalVersion uint64 = 3
// loadJournal tries to parse the layer journal from the disk.
func (db *Database) loadJournal(diskRoot common.Hash) (layer, error) {
var reader io.Reader
if db.config.JournalPath != "" && common.FileExist(db.config.JournalPath) {
if path := db.journalPath(); path != "" && common.FileExist(path) {
// If a journal file is specified, read it from there
log.Info("Load database journal from file", "path", db.config.JournalPath)
f, err := os.OpenFile(db.config.JournalPath, os.O_RDONLY, 0644)
log.Info("Load database journal from file", "path", path)
f, err := os.OpenFile(path, os.O_RDONLY, 0644)
if err != nil {
return nil, fmt.Errorf("failed to read journal file %s: %w", db.config.JournalPath, err)
return nil, fmt.Errorf("failed to read journal file %s: %w", path, err)
}
defer f.Close()
reader = f
@ -337,20 +338,24 @@ func (db *Database) Journal(root common.Hash) error {
var (
file *os.File
journal io.Writer
tmpName = db.config.JournalPath + ".tmp"
journalPath = db.journalPath()
)
if db.config.JournalPath != "" {
if journalPath != "" {
// Write into a temp file first
var err error
file, err = os.OpenFile(tmpName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
err := os.Mkdir(db.config.JournalDirectory, 0755)
if err != nil {
return fmt.Errorf("failed to open journal file %s: %w", tmpName, err)
return err
}
tmp := journalPath + tempJournalSuffix
file, err = os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
return fmt.Errorf("failed to open journal file %s: %w", tmp, err)
}
defer func() {
if file != nil {
file.Close()
os.Remove(tmpName) // Clean up temp file if we didn't successfully rename it
log.Warn("Removed leftover temporary journal file", "path", tmpName)
os.Remove(tmp) // Clean up temp file if we didn't successfully rename it
log.Warn("Removed leftover temporary journal file", "path", tmp)
}
}()
journal = file
@ -377,14 +382,17 @@ func (db *Database) Journal(root common.Hash) error {
}
// Store the journal into the database and return
var size int
if db.config.JournalPath == "" {
if file == nil {
data := journal.(*bytes.Buffer)
size = data.Len()
size := data.Len()
rawdb.WriteTrieJournal(db.diskdb, data.Bytes())
log.Info("Persisted dirty state to disk", "size", common.StorageSize(size), "elapsed", common.PrettyDuration(time.Since(start)))
} else {
stat, _ := journal.(*os.File).Stat()
size = int(stat.Size())
stat, err := file.Stat()
if err != nil {
return err
}
size := int(stat.Size())
// Close the temporary file and atomically rename it
if err := file.Sync(); err != nil {
@ -394,17 +402,16 @@ func (db *Database) Journal(root common.Hash) error {
return fmt.Errorf("failed to close the journal: %v", err)
}
// Replace the live journal with the newly generated one
if err := os.Rename(tmpName, db.config.JournalPath); err != nil {
if err := os.Rename(journalPath+tempJournalSuffix, journalPath); err != nil {
return fmt.Errorf("failed to rename the journal: %v", err)
}
if err := syncDir(filepath.Dir(db.config.JournalPath)); err != nil {
if err := syncDir(db.config.JournalDirectory); err != nil {
return fmt.Errorf("failed to fsync the dir: %v", err)
}
file = nil
log.Info("Persisted dirty state to file", "path", journalPath, "size", common.StorageSize(size), "elapsed", common.PrettyDuration(time.Since(start)))
}
// Set the db in read only mode to reject all following mutations
db.readOnly = true
log.Info("Persisted dirty state to disk", "size", common.StorageSize(size), "elapsed", common.PrettyDuration(time.Since(start)))
return nil
}