mirror of
https://github.com/ethereum/go-ethereum.git
synced 2026-07-25 14:16:44 +00:00
core/rawdb: move erdb integration into chain freezer
This commit is contained in:
parent
ff11e10ac2
commit
d65c46526d
5 changed files with 68 additions and 50 deletions
|
|
@ -46,6 +46,9 @@ const (
|
||||||
type chainFreezer struct {
|
type chainFreezer struct {
|
||||||
ethdb.AncientStore // Ancient store for storing cold chain segment
|
ethdb.AncientStore // Ancient store for storing cold chain segment
|
||||||
|
|
||||||
|
// Optional Era database used as a backup for the pruned chain.
|
||||||
|
eradb *eradb.EraDatabase
|
||||||
|
|
||||||
quit chan struct{}
|
quit chan struct{}
|
||||||
wg sync.WaitGroup
|
wg sync.WaitGroup
|
||||||
trigger chan chan struct{} // Manual blocking freeze trigger, test determinism
|
trigger chan chan struct{} // Manual blocking freeze trigger, test determinism
|
||||||
|
|
@ -58,15 +61,17 @@ type chainFreezer struct {
|
||||||
// - if non-empty directory is given, initializes the regular file-based
|
// - if non-empty directory is given, initializes the regular file-based
|
||||||
// state freezer.
|
// state freezer.
|
||||||
func newChainFreezer(datadir string, namespace string, readonly bool, eraDir string) (*chainFreezer, error) {
|
func newChainFreezer(datadir string, namespace string, readonly bool, eraDir string) (*chainFreezer, error) {
|
||||||
var (
|
|
||||||
err error
|
|
||||||
freezer ethdb.AncientStore
|
|
||||||
)
|
|
||||||
if datadir == "" {
|
if datadir == "" {
|
||||||
freezer = NewMemoryFreezer(readonly, chainFreezerTableConfigs)
|
return &chainFreezer{
|
||||||
} else {
|
AncientStore: NewMemoryFreezer(readonly, chainFreezerTableConfigs),
|
||||||
// Instantiate eradb outside of freezer to avoid
|
quit: make(chan struct{}),
|
||||||
// creating an instance for the state freezer.
|
trigger: make(chan chan struct{}),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
freezer, err := NewFreezer(datadir, namespace, readonly, freezerTableSize, chainFreezerTableConfigs)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
var edb *eradb.EraDatabase
|
var edb *eradb.EraDatabase
|
||||||
if eraDir != "" {
|
if eraDir != "" {
|
||||||
edb, err = eradb.New(eraDir)
|
edb, err = eradb.New(eraDir)
|
||||||
|
|
@ -74,13 +79,9 @@ func newChainFreezer(datadir string, namespace string, readonly bool, eraDir str
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
freezer, err = NewFreezer(datadir, namespace, readonly, freezerTableSize, chainFreezerTableConfigs, edb)
|
|
||||||
}
|
|
||||||
if err != nil {
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
return &chainFreezer{
|
return &chainFreezer{
|
||||||
AncientStore: freezer,
|
AncientStore: freezer,
|
||||||
|
eradb: edb,
|
||||||
quit: make(chan struct{}),
|
quit: make(chan struct{}),
|
||||||
trigger: make(chan chan struct{}),
|
trigger: make(chan chan struct{}),
|
||||||
}, nil
|
}, nil
|
||||||
|
|
@ -94,7 +95,14 @@ func (f *chainFreezer) Close() error {
|
||||||
close(f.quit)
|
close(f.quit)
|
||||||
}
|
}
|
||||||
f.wg.Wait()
|
f.wg.Wait()
|
||||||
return f.AncientStore.Close()
|
|
||||||
|
var errs []error
|
||||||
|
if f.eradb != nil {
|
||||||
|
errs = append(errs, f.eradb.Close())
|
||||||
|
}
|
||||||
|
errs = append(errs, f.AncientStore.Close())
|
||||||
|
|
||||||
|
return errors.Join(errs...)
|
||||||
}
|
}
|
||||||
|
|
||||||
// readHeadNumber returns the number of chain head block. 0 is returned if the
|
// readHeadNumber returns the number of chain head block. 0 is returned if the
|
||||||
|
|
@ -344,3 +352,31 @@ func (f *chainFreezer) freezeRange(nfdb *nofreezedb, number, limit uint64) (hash
|
||||||
})
|
})
|
||||||
return hashes, err
|
return hashes, err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Ancient retrieves an ancient binary blob from the append-only immutable files.
|
||||||
|
func (f *chainFreezer) Ancient(kind string, number uint64) ([]byte, error) {
|
||||||
|
// Lookup the entry in the underlying ancient store, assuming that
|
||||||
|
// headers and hashes are always available.
|
||||||
|
if kind == ChainFreezerHeaderTable || kind == ChainFreezerHashTable {
|
||||||
|
return f.AncientStore.Ancient(kind, number)
|
||||||
|
}
|
||||||
|
tail, err := f.Tail()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
// Lookup the entry in the underlying ancient store if it's not pruned
|
||||||
|
if number >= tail {
|
||||||
|
return f.AncientStore.Ancient(kind, number)
|
||||||
|
}
|
||||||
|
// Lookup the entry in the optional era backend
|
||||||
|
if f.eradb == nil {
|
||||||
|
return nil, errOutOfBounds
|
||||||
|
}
|
||||||
|
switch kind {
|
||||||
|
case ChainFreezerBodiesTable:
|
||||||
|
return f.eradb.GetRawBody(number)
|
||||||
|
case ChainFreezerReceiptTable:
|
||||||
|
return f.eradb.GetRawReceipts(number)
|
||||||
|
}
|
||||||
|
return nil, errUnknownTable
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -54,7 +54,10 @@ func New(datadir string) (*EraDatabase, error) {
|
||||||
if err := os.MkdirAll(datadir, 0755); err != nil {
|
if err := os.MkdirAll(datadir, 0755); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
db := &EraDatabase{datadir: datadir, cache: lru.NewCache[uint64, *era.Era](openFileLimit)}
|
db := &EraDatabase{
|
||||||
|
datadir: datadir,
|
||||||
|
cache: lru.NewCache[uint64, *era.Era](openFileLimit),
|
||||||
|
}
|
||||||
closeEra := func(epoch uint64, e *era.Era) {
|
closeEra := func(epoch uint64, e *era.Era) {
|
||||||
if e == nil {
|
if e == nil {
|
||||||
log.Warn("Era1 cache contained nil value", "epoch", epoch)
|
log.Warn("Era1 cache contained nil value", "epoch", epoch)
|
||||||
|
|
@ -66,9 +69,11 @@ func New(datadir string) (*EraDatabase, error) {
|
||||||
}
|
}
|
||||||
// Take care to close era1 files when they are evicted from cache.
|
// Take care to close era1 files when they are evicted from cache.
|
||||||
db.cache.OnEvicted(closeEra)
|
db.cache.OnEvicted(closeEra)
|
||||||
|
|
||||||
// Concurrently calling GetRaw* methods can cause the same era1 file to be
|
// Concurrently calling GetRaw* methods can cause the same era1 file to be
|
||||||
// opened multiple times.
|
// opened multiple times.
|
||||||
db.cache.OnReplaced(closeEra)
|
db.cache.OnReplaced(closeEra)
|
||||||
|
|
||||||
log.Info("Opened erastore", "datadir", datadir)
|
log.Info("Opened erastore", "datadir", datadir)
|
||||||
return db, nil
|
return db, nil
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -25,7 +25,6 @@ import (
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
|
|
||||||
"github.com/ethereum/go-ethereum/core/rawdb/eradb"
|
|
||||||
"github.com/ethereum/go-ethereum/ethdb"
|
"github.com/ethereum/go-ethereum/ethdb"
|
||||||
"github.com/ethereum/go-ethereum/log"
|
"github.com/ethereum/go-ethereum/log"
|
||||||
"github.com/ethereum/go-ethereum/metrics"
|
"github.com/ethereum/go-ethereum/metrics"
|
||||||
|
|
@ -72,8 +71,6 @@ type Freezer struct {
|
||||||
tables map[string]*freezerTable // Data tables for storing everything
|
tables map[string]*freezerTable // Data tables for storing everything
|
||||||
instanceLock *flock.Flock // File-system lock to prevent double opens
|
instanceLock *flock.Flock // File-system lock to prevent double opens
|
||||||
closeOnce sync.Once
|
closeOnce sync.Once
|
||||||
|
|
||||||
eradb *eradb.EraDatabase
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewFreezer creates a freezer instance for maintaining immutable ordered
|
// NewFreezer creates a freezer instance for maintaining immutable ordered
|
||||||
|
|
@ -81,7 +78,7 @@ type Freezer struct {
|
||||||
//
|
//
|
||||||
// The 'tables' argument defines the data tables. If the value of a map
|
// The 'tables' argument defines the data tables. If the value of a map
|
||||||
// entry is true, snappy compression is disabled for the table.
|
// entry is true, snappy compression is disabled for the table.
|
||||||
func NewFreezer(datadir string, namespace string, readonly bool, maxTableSize uint32, tables map[string]freezerTableConfig, eradb *eradb.EraDatabase) (*Freezer, error) {
|
func NewFreezer(datadir string, namespace string, readonly bool, maxTableSize uint32, tables map[string]freezerTableConfig) (*Freezer, error) {
|
||||||
// Create the initial freezer object
|
// Create the initial freezer object
|
||||||
var (
|
var (
|
||||||
readMeter = metrics.NewRegisteredMeter(namespace+"ancient/read", nil)
|
readMeter = metrics.NewRegisteredMeter(namespace+"ancient/read", nil)
|
||||||
|
|
@ -121,7 +118,6 @@ func NewFreezer(datadir string, namespace string, readonly bool, maxTableSize ui
|
||||||
readonly: readonly,
|
readonly: readonly,
|
||||||
tables: make(map[string]*freezerTable),
|
tables: make(map[string]*freezerTable),
|
||||||
instanceLock: lock,
|
instanceLock: lock,
|
||||||
eradb: eradb,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create the tables.
|
// Create the tables.
|
||||||
|
|
@ -175,16 +171,8 @@ func (f *Freezer) Close() error {
|
||||||
if err := f.instanceLock.Unlock(); err != nil {
|
if err := f.instanceLock.Unlock(); err != nil {
|
||||||
errs = append(errs, err)
|
errs = append(errs, err)
|
||||||
}
|
}
|
||||||
if f.eradb != nil {
|
|
||||||
if err := f.eradb.Close(); err != nil {
|
|
||||||
errs = append(errs, err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
})
|
})
|
||||||
if errs != nil {
|
return errors.Join(errs...)
|
||||||
return fmt.Errorf("%v", errs)
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// AncientDatadir returns the path of the ancient store.
|
// AncientDatadir returns the path of the ancient store.
|
||||||
|
|
@ -204,17 +192,6 @@ func (f *Freezer) HasAncient(kind string, number uint64) (bool, error) {
|
||||||
// Ancient retrieves an ancient binary blob from the append-only immutable files.
|
// Ancient retrieves an ancient binary blob from the append-only immutable files.
|
||||||
func (f *Freezer) Ancient(kind string, number uint64) ([]byte, error) {
|
func (f *Freezer) Ancient(kind string, number uint64) ([]byte, error) {
|
||||||
if table := f.tables[kind]; table != nil {
|
if table := f.tables[kind]; table != nil {
|
||||||
if f.eradb != nil && table.config.prunable && number < f.tail.Load() {
|
|
||||||
// The requested item has been pruned. Attempt fetching from era1 file.
|
|
||||||
switch kind {
|
|
||||||
case ChainFreezerBodiesTable:
|
|
||||||
return f.eradb.GetRawBody(number)
|
|
||||||
case ChainFreezerReceiptTable:
|
|
||||||
return f.eradb.GetRawReceipts(number)
|
|
||||||
default:
|
|
||||||
return nil, errOutOfBounds
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return table.Retrieve(number)
|
return table.Retrieve(number)
|
||||||
}
|
}
|
||||||
return nil, errUnknownTable
|
return nil, errUnknownTable
|
||||||
|
|
|
||||||
|
|
@ -54,7 +54,7 @@ func newResettableFreezer(datadir string, namespace string, readonly bool, maxTa
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
opener := func() (*Freezer, error) {
|
opener := func() (*Freezer, error) {
|
||||||
return NewFreezer(datadir, namespace, readonly, maxTableSize, tables, nil)
|
return NewFreezer(datadir, namespace, readonly, maxTableSize, tables)
|
||||||
}
|
}
|
||||||
freezer, err := opener()
|
freezer, err := opener()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
|
||||||
|
|
@ -112,7 +112,7 @@ func TestFreezerModifyRollback(t *testing.T) {
|
||||||
|
|
||||||
// Reopen and check that the rolled-back data doesn't reappear.
|
// Reopen and check that the rolled-back data doesn't reappear.
|
||||||
tables := map[string]freezerTableConfig{"test": {noSnappy: true}}
|
tables := map[string]freezerTableConfig{"test": {noSnappy: true}}
|
||||||
f2, err := NewFreezer(dir, "", false, 2049, tables, nil)
|
f2, err := NewFreezer(dir, "", false, 2049, tables)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("can't reopen freezer after failed ModifyAncients: %v", err)
|
t.Fatalf("can't reopen freezer after failed ModifyAncients: %v", err)
|
||||||
}
|
}
|
||||||
|
|
@ -253,7 +253,7 @@ func TestFreezerReadonlyValidate(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
// Open non-readonly freezer and fill individual tables
|
// Open non-readonly freezer and fill individual tables
|
||||||
// with different amount of data.
|
// with different amount of data.
|
||||||
f, err := NewFreezer(dir, "", false, 2049, tables, nil)
|
f, err := NewFreezer(dir, "", false, 2049, tables)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal("can't open freezer", err)
|
t.Fatal("can't open freezer", err)
|
||||||
}
|
}
|
||||||
|
|
@ -276,7 +276,7 @@ func TestFreezerReadonlyValidate(t *testing.T) {
|
||||||
|
|
||||||
// Re-opening as readonly should fail when validating
|
// Re-opening as readonly should fail when validating
|
||||||
// table lengths.
|
// table lengths.
|
||||||
_, err = NewFreezer(dir, "", true, 2049, tables, nil)
|
_, err = NewFreezer(dir, "", true, 2049, tables)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("readonly freezer should fail with differing table lengths")
|
t.Fatal("readonly freezer should fail with differing table lengths")
|
||||||
}
|
}
|
||||||
|
|
@ -288,7 +288,7 @@ func TestFreezerConcurrentReadonly(t *testing.T) {
|
||||||
tables := map[string]freezerTableConfig{"a": {noSnappy: true}}
|
tables := map[string]freezerTableConfig{"a": {noSnappy: true}}
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
|
|
||||||
f, err := NewFreezer(dir, "", false, 2049, tables, nil)
|
f, err := NewFreezer(dir, "", false, 2049, tables)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal("can't open freezer", err)
|
t.Fatal("can't open freezer", err)
|
||||||
}
|
}
|
||||||
|
|
@ -314,7 +314,7 @@ func TestFreezerConcurrentReadonly(t *testing.T) {
|
||||||
go func(i int) {
|
go func(i int) {
|
||||||
defer wg.Done()
|
defer wg.Done()
|
||||||
|
|
||||||
f, err := NewFreezer(dir, "", true, 2049, tables, nil)
|
f, err := NewFreezer(dir, "", true, 2049, tables)
|
||||||
if err == nil {
|
if err == nil {
|
||||||
fs[i] = f
|
fs[i] = f
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -339,7 +339,7 @@ func newFreezerForTesting(t *testing.T, tables map[string]freezerTableConfig) (*
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
// note: using low max table size here to ensure the tests actually
|
// note: using low max table size here to ensure the tests actually
|
||||||
// switch between multiple files.
|
// switch between multiple files.
|
||||||
f, err := NewFreezer(dir, "", false, 2049, tables, nil)
|
f, err := NewFreezer(dir, "", false, 2049, tables)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatal("can't open freezer", err)
|
t.Fatal("can't open freezer", err)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue