forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-07-25 06:06:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

add flag for eradb

Browse source
This commit is contained in:
Sina Mahmoodi 2025-04-14 19:40:11 +02:00 committed by lightclient
parent 7e720c961a
commit 930d969611
No known key found for this signature in database
GPG key ID: 657913021EF45A6A
17 changed files with 65 additions and 45 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
cmd/geth/chaincmd.go
View file

@ -277,7 +277,7 @@ func initGenesis(ctx *cli.Context) error {
overrides.OverrideVerkle = &v
}
chaindb, err := stack.OpenDatabaseWithFreezer("chaindata", 0, 0, ctx.String(utils.AncientFlag.Name), "", false)
chaindb, err := stack.OpenDatabaseWithFreezer("chaindata", 0, 0, ctx.String(utils.AncientFlag.Name), "", false, "")
if err != nil {
utils.Fatalf("Failed to open database: %v", err)
}

10
cmd/utils/flags.go
View file

@ -111,6 +111,10 @@ var (
Usage: "Root directory for ancient data (default = inside chaindata)",
Category: flags.EthCategory,
}
EraFlag = &flags.DirectoryFlag{
Name: "datadir.era",
Usage: "Root directory for era1 history (default = inside ancient/chain)",
}
MinFreeDiskSpaceFlag = &flags.DirectoryFlag{
Name: "datadir.minfreedisk",
Usage: "Minimum free disk space in MB, once reached triggers auto shut down (default = --cache.gc converted to MB, 0 = disabled)",
@ -977,6 +981,7 @@ var (
DatabaseFlags = []cli.Flag{
DataDirFlag,
AncientFlag,
EraFlag,
RemoteDBFlag,
DBEngineFlag,
StateSchemeFlag,
@ -1607,6 +1612,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
if ctx.IsSet(AncientFlag.Name) {
cfg.DatabaseFreezer = ctx.String(AncientFlag.Name)
}
if ctx.IsSet(EraFlag.Name) {
cfg.DatabaseEra = ctx.String(EraFlag.Name)
}
if gcmode := ctx.String(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" {
Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name)
@ -2072,7 +2080,7 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly bool) ethdb.
}
chainDb = remotedb.New(client)
default:
chainDb, err = stack.OpenDatabaseWithFreezer("chaindata", cache, handles, ctx.String(AncientFlag.Name), "eth/db/chaindata/", readonly)
chainDb, err = stack.OpenDatabaseWithFreezer("chaindata", cache, handles, ctx.String(AncientFlag.Name), "eth/db/chaindata/", readonly, ctx.String(EraFlag.Name))
}
if err != nil {
Fatalf("Could not open database: %v", err)

2
cmd/utils/history_test.go
View file

@ -158,7 +158,7 @@ func TestHistoryImportAndExport(t *testing.T) {
}
// Now import Era.
db2, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
db2, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false, "")
if err != nil {
panic(err)
}

8
core/blockchain_repair_test.go
View file

@ -1769,7 +1769,7 @@ func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme s
if err != nil {
t.Fatalf("Failed to create persistent key-value database: %v", err)
}
db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false, "")
if err != nil {
t.Fatalf("Failed to create persistent freezer database: %v", err)
}
@ -1854,7 +1854,7 @@ func testRepairWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme s
if err != nil {
t.Fatalf("Failed to reopen persistent key-value database: %v", err)
}
db, err = rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
db, err = rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false, "")
if err != nil {
t.Fatalf("Failed to reopen persistent freezer database: %v", err)
}
@ -1919,7 +1919,7 @@ func testIssue23496(t *testing.T, scheme string) {
if err != nil {
t.Fatalf("Failed to create persistent key-value database: %v", err)
}
db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false, "")
if err != nil {
t.Fatalf("Failed to create persistent freezer database: %v", err)
}
@ -1979,7 +1979,7 @@ func testIssue23496(t *testing.T, scheme string) {
if err != nil {
t.Fatalf("Failed to reopen persistent key-value database: %v", err)
}
db, err = rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
db, err = rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false, "")
if err != nil {
t.Fatalf("Failed to reopen persistent freezer database: %v", err)
}

2
core/blockchain_sethead_test.go
View file

@ -1973,7 +1973,7 @@ func testSetHeadWithScheme(t *testing.T, tt *rewindTest, snapshots bool, scheme
if err != nil {
t.Fatalf("Failed to create persistent key-value database: %v", err)
}
db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false, "")
if err != nil {
t.Fatalf("Failed to create persistent freezer database: %v", err)
}

4
core/blockchain_snapshot_test.go
View file

@ -70,7 +70,7 @@ func (basic *snapshotTestBasic) prepare(t *testing.T) (*BlockChain, []*types.Blo
if err != nil {
t.Fatalf("Failed to create persistent key-value database: %v", err)
}
db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false, "")
if err != nil {
t.Fatalf("Failed to create persistent freezer database: %v", err)
}
@ -265,7 +265,7 @@ func (snaptest *crashSnapshotTest) test(t *testing.T) {
if err != nil {
t.Fatalf("Failed to create persistent key-value database: %v", err)
}
newdb, err := rawdb.NewDatabaseWithFreezer(pdb, snaptest.ancient, "", false)
newdb, err := rawdb.NewDatabaseWithFreezer(pdb, snaptest.ancient, "", false, "")
if err != nil {
t.Fatalf("Failed to create persistent freezer database: %v", err)
}

22
core/blockchain_test.go
View file

@ -738,7 +738,7 @@ func testFastVsFullChains(t *testing.T, scheme string) {
t.Fatalf("failed to insert receipt %d: %v", n, err)
}
// Freezer style fast import the chain.
ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false, "")
if err != nil {
t.Fatalf("failed to create temp freezer db: %v", err)
}
@ -824,7 +824,7 @@ func testLightVsFastVsFullChainHeads(t *testing.T, scheme string) {
// makeDb creates a db instance for testing.
makeDb := func() ethdb.Database {
db, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
db, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false, "")
if err != nil {
t.Fatalf("failed to create temp freezer db: %v", err)
}
@ -1623,7 +1623,7 @@ func testLargeReorgTrieGC(t *testing.T, scheme string) {
competitor, _ := GenerateChain(genesis.Config, shared[len(shared)-1], engine, genDb, 2*state.TriesInMemory+1, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{3}) })
// Import the shared chain and the original canonical one
db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false, "")
defer db.Close()
chain, err := NewBlockChain(db, DefaultCacheConfigWithScheme(scheme), genesis, nil, engine, vm.Config{}, nil)
@ -1689,7 +1689,7 @@ func testBlockchainRecovery(t *testing.T, scheme string) {
_, blocks, receipts := GenerateChainWithGenesis(gspec, ethash.NewFaker(), int(height), nil)
// Import the chain as a ancient-first node and ensure all pointers are updated
ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false)
ancientDb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, "")
if err != nil {
t.Fatalf("failed to create temp freezer db: %v", err)
}
@ -1747,7 +1747,7 @@ func testLowDiffLongChain(t *testing.T, scheme string) {
})
// Import the canonical chain
diskdb, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
diskdb, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false, "")
defer diskdb.Close()
chain, err := NewBlockChain(diskdb, DefaultCacheConfigWithScheme(scheme), genesis, nil, engine, vm.Config{}, nil)
@ -1959,7 +1959,7 @@ func testInsertKnownChainData(t *testing.T, typ string, scheme string) {
b.OffsetTime(-9) // A higher difficulty
})
// Import the shared chain and the original canonical one
chaindb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
chaindb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false, "")
if err != nil {
t.Fatalf("failed to create temp freezer db: %v", err)
}
@ -2122,7 +2122,7 @@ func testInsertKnownChainDataWithMerging(t *testing.T, typ string, mergeHeight i
}
})
// Import the shared chain and the original canonical one
chaindb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
chaindb, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false, "")
if err != nil {
t.Fatalf("failed to create temp freezer db: %v", err)
}
@ -2496,7 +2496,7 @@ func testSideImportPrunedBlocks(t *testing.T, scheme string) {
if err != nil {
t.Fatalf("Failed to create persistent key-value database: %v", err)
}
db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false)
db, err := rawdb.NewDatabaseWithFreezer(pdb, ancient, "", false, "")
if err != nil {
t.Fatalf("Failed to create persistent freezer database: %v", err)
}
@ -3403,7 +3403,7 @@ func testSetCanonical(t *testing.T, scheme string) {
}
gen.AddTx(tx)
})
diskdb, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
diskdb, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false, "")
defer diskdb.Close()
chain, err := NewBlockChain(diskdb, DefaultCacheConfigWithScheme(scheme), gspec, nil, engine, vm.Config{}, nil)
@ -4199,7 +4199,7 @@ func testChainReorgSnapSync(t *testing.T, ancientLimit uint64) {
gen.SetCoinbase(common.Address{0: byte(0xb), 19: byte(i)})
})
db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false, "")
defer db.Close()
chain, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(rawdb.PathScheme), gspec, nil, beacon.New(ethash.NewFaker()), vm.Config{}, nil)
@ -4315,7 +4315,7 @@ func testInsertChainWithCutoff(t *testing.T, cutoff uint64, ancientLimit uint64,
config := DefaultCacheConfigWithScheme(rawdb.PathScheme)
config.ChainHistoryMode = history.KeepPostMerge
db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false, "")
defer db.Close()
chain, _ := NewBlockChain(db, DefaultCacheConfigWithScheme(rawdb.PathScheme), genesis, nil, beacon.New(ethash.NewFaker()), vm.Config{}, nil)
defer chain.Stop()

8
core/rawdb/accessors_chain_test.go
View file

@ -416,7 +416,7 @@ func checkReceiptsRLP(have, want types.Receipts) error {
func TestAncientStorage(t *testing.T) {
// Freezer style fast import the chain.
frdir := t.TempDir()
db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false)
db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false, "")
if err != nil {
t.Fatalf("failed to create database with ancient backend")
}
@ -469,7 +469,7 @@ func TestAncientStorage(t *testing.T) {
}
func TestWriteAncientHeaderChain(t *testing.T) {
db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), t.TempDir(), "", false)
db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), t.TempDir(), "", false, "")
if err != nil {
t.Fatalf("failed to create database with ancient backend")
}
@ -586,7 +586,7 @@ func TestHashesInRange(t *testing.T) {
func BenchmarkWriteAncientBlocks(b *testing.B) {
// Open freezer database.
frdir := b.TempDir()
db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false)
db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false, "")
if err != nil {
b.Fatalf("failed to create database with ancient backend")
}
@ -890,7 +890,7 @@ func TestHeadersRLPStorage(t *testing.T) {
// Have N headers in the freezer
frdir := t.TempDir()
db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false)
db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false, "")
if err != nil {
t.Fatalf("failed to create database with ancient backend")
}

16
core/rawdb/chain_freezer.go
View file

@ -19,7 +19,6 @@ package rawdb
import (
"errors"
"fmt"
"path"
"sync"
"time"
@ -58,7 +57,7 @@ type chainFreezer struct {
// state freezer (e.g. dev mode).
// - if non-empty directory is given, initializes the regular file-based
// state freezer.
func newChainFreezer(datadir string, namespace string, readonly bool) (*chainFreezer, error) {
func newChainFreezer(datadir string, namespace string, readonly bool, eraDir string) (*chainFreezer, error) {
var (
err error
freezer ethdb.AncientStore
@ -68,13 +67,12 @@ func newChainFreezer(datadir string, namespace string, readonly bool) (*chainFre
} else {
// Instantiate eradb outside of freezer to avoid
// creating an instance for the state freezer.
var (
edb *eradb.EraDatabase
eraDatadir = path.Join(datadir, "era")
)
edb, err = eradb.New(eraDatadir)
if err != nil {
return nil, err
var edb *eradb.EraDatabase
if eraDir != "" {
edb, err = eradb.New(eraDir)
if err != nil {
return nil, err
}
}
freezer, err = NewFreezer(datadir, namespace, readonly, freezerTableSize, chainFreezerTableConfigs, edb)
}

4
core/rawdb/database.go
View file

@ -190,7 +190,7 @@ func resolveChainFreezerDir(ancient string) string {
// value data store with a freezer moving immutable chain segments into cold
// storage. The passed ancient indicates the path of root ancient directory
// where the chain freezer can be opened.
func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace string, readonly bool) (ethdb.Database, error) {
func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace string, readonly bool, eraDir string) (ethdb.Database, error) {
// Create the idle freezer instance. If the given ancient directory is empty,
// in-memory chain freezer is used (e.g. dev mode); otherwise the regular
// file-based freezer is created.
@ -198,7 +198,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace st
if chainFreezerDir != "" {
chainFreezerDir = resolveChainFreezerDir(chainFreezerDir)
}
frdb, err := newChainFreezer(chainFreezerDir, namespace, readonly)
frdb, err := newChainFreezer(chainFreezerDir, namespace, readonly, eraDir)
if err != nil {
printChainMetadata(db)
return nil, err

6
core/txindexer_test.go
View file

@ -116,7 +116,7 @@ func TestTxIndexer(t *testing.T) {
},
}
for _, c := range cases {
db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false, "")
rawdb.WriteAncientBlocks(db, append([]*types.Block{gspec.ToBlock()}, blocks...), types.EncodeBlockReceiptLists(append([]types.Receipts{{}}, receipts...)))
// Index the initial blocks from ancient store
@ -235,7 +235,7 @@ func TestTxIndexerRepair(t *testing.T) {
},
}
for _, c := range cases {
db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false, "")
encReceipts := types.EncodeBlockReceiptLists(append([]types.Receipts{{}}, receipts...))
rawdb.WriteAncientBlocks(db, append([]*types.Block{gspec.ToBlock()}, blocks...), encReceipts)
@ -426,7 +426,7 @@ func TestTxIndexerReport(t *testing.T) {
},
}
for _, c := range cases {
db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false, "")
encReceipts := types.EncodeBlockReceiptLists(append([]types.Receipts{{}}, receipts...))
rawdb.WriteAncientBlocks(db, append([]*types.Block{gspec.ToBlock()}, blocks...), encReceipts)

2
eth/backend.go
View file

@ -128,7 +128,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
}
log.Info("Allocated trie memory caches", "clean", common.StorageSize(config.TrieCleanCache)*1024*1024, "dirty", common.StorageSize(config.TrieDirtyCache)*1024*1024)
chainDb, err := stack.OpenDatabaseWithFreezer("chaindata", config.DatabaseCache, config.DatabaseHandles, config.DatabaseFreezer, "eth/db/chaindata/", false)
chainDb, err := stack.OpenDatabaseWithFreezer("chaindata", config.DatabaseCache, config.DatabaseHandles, config.DatabaseFreezer, "eth/db/chaindata/", false, config.DatabaseEra)
if err != nil {
return nil, err
}

2
eth/downloader/downloader_test.go
View file

@ -56,7 +56,7 @@ func newTester(t *testing.T) *downloadTester {
// newTesterWithNotification creates a new downloader test mocker.
func newTesterWithNotification(t *testing.T, success func()) *downloadTester {
db, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
db, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false, "")
if err != nil {
panic(err)
}

1
eth/ethconfig/config.go
View file

@ -120,6 +120,7 @@ type Config struct {
DatabaseHandles int `toml:"-"`
DatabaseCache int
DatabaseFreezer string
DatabaseEra string
TrieCleanCache int
TrieDirtyCache int

3
node/database.go
View file

@ -32,6 +32,7 @@ type openOptions struct {
Type string // "leveldb" | "pebble"
Directory string // the datadir
AncientsDirectory string // the ancients-dir
EraDirectory string
Namespace string // the namespace for database relevant metrics
Cache int // the capacity(in megabytes) of the data caching
Handles int // number of files to be open simultaneously
@ -51,7 +52,7 @@ func openDatabase(o openOptions) (ethdb.Database, error) {
if len(o.AncientsDirectory) == 0 {
return kvdb, nil
}
frdb, err := rawdb.NewDatabaseWithFreezer(kvdb, o.AncientsDirectory, o.Namespace, o.ReadOnly)
frdb, err := rawdb.NewDatabaseWithFreezer(kvdb, o.AncientsDirectory, o.Namespace, o.ReadOnly, o.EraDirectory)
if err != nil {
kvdb.Close()
return nil, err

16
node/node.go
View file

@ -731,7 +731,7 @@ func (n *Node) OpenDatabase(name string, cache, handles int, namespace string, r
// also attaching a chain freezer to it that moves ancient chain data from the
// database to immutable append-only files. If the node is an ephemeral one, a
// memory database is returned.
func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, ancient string, namespace string, readonly bool) (ethdb.Database, error) {
func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, ancient string, namespace string, readonly bool, era string) (ethdb.Database, error) {
n.lock.Lock()
defer n.lock.Unlock()
if n.state == closedState {
@ -740,12 +740,13 @@ func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, ancient
var db ethdb.Database
var err error
if n.config.DataDir == "" {
db, err = rawdb.NewDatabaseWithFreezer(memorydb.New(), "", namespace, readonly)
db, err = rawdb.NewDatabaseWithFreezer(memorydb.New(), "", namespace, readonly, "")
} else {
db, err = openDatabase(openOptions{
Type: n.config.DBEngine,
Directory: n.ResolvePath(name),
AncientsDirectory: n.ResolveAncient(name, ancient),
EraDirectory: n.ResolveEra(ancient, era),
Namespace: namespace,
Cache: cache,
Handles: handles,
@ -774,6 +775,17 @@ func (n *Node) ResolveAncient(name string, ancient string) string {
return ancient
}
// ResolveEra returns the absolute path of the era directory.
func (n *Node) ResolveEra(ancient, era string) string {
switch {
case era == "":
era = filepath.Join(ancient, "era")
case !filepath.IsAbs(era):
era = n.ResolvePath(era)
}
return era
}
// closeTrackingDB wraps the Close method of a database. When the database is closed by the
// service, the wrapper removes it from the node's database map. This ensures that Node
// won't auto-close the database if it is closed by the service that opened it.

2
triedb/pathdb/database_test.go
View file

@ -123,7 +123,7 @@ type tester struct {
func newTester(t *testing.T, historyLimit uint64, isVerkle bool, layers int) *tester {
var (
disk, _ = rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false)
disk, _ = rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), t.TempDir(), "", false, "")
db = New(disk, &Config{
StateHistory: historyLimit,
TrieCleanSize: 256 * 1024,