From 0f06e3511534477cfb1f8aad23adf4eaa60debcc Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Mon, 17 Mar 2025 16:01:37 +0100 Subject: [PATCH] core/rawdb: allow for truncation in the freezer (#31362) Here we add the notion of prunable tables for the `TruncateTail` operation in the freezer. TruncateTail for the chain freezer now only truncates the body and receipts tables, leaving headers and hashes as-is. This change also requires changing the validation/repair at startup to allow for tables with different tail. For the header and hash tables, we now require them to start at number zero. --------- Co-authored-by: Felix Lange Co-authored-by: Gary Rong --- core/rawdb/ancient_scheme.go | 40 +++++++------ core/rawdb/ancient_utils.go | 12 ++-- core/rawdb/chain_freezer.go | 4 +- core/rawdb/freezer.go | 88 ++++++++++++++++++----------- core/rawdb/freezer_batch.go | 2 +- core/rawdb/freezer_memory.go | 28 +++++---- core/rawdb/freezer_memory_test.go | 14 +++-- core/rawdb/freezer_resettable.go | 2 +- core/rawdb/freezer_table.go | 50 ++++++++-------- core/rawdb/freezer_table_test.go | 94 +++++++++++++++---------------- core/rawdb/freezer_test.go | 28 +++++---- ethdb/database.go | 2 + 12 files changed, 208 insertions(+), 156 deletions(-) diff --git a/core/rawdb/ancient_scheme.go b/core/rawdb/ancient_scheme.go index 67bfa37ecc..1ffebed3e7 100644 --- a/core/rawdb/ancient_scheme.go +++ b/core/rawdb/ancient_scheme.go @@ -37,13 +37,21 @@ const ( ChainFreezerReceiptTable = "receipts" ) -// chainFreezerNoSnappy configures whether compression is disabled for the ancient-tables. -// Hashes and difficulties don't compress well. -var chainFreezerNoSnappy = map[string]bool{ - ChainFreezerHeaderTable: false, - ChainFreezerHashTable: true, - ChainFreezerBodiesTable: false, - ChainFreezerReceiptTable: false, +// chainFreezerTableConfigs configures the settings for tables in the chain freezer. +// Compression is disabled for hashes as they don't compress well. Additionally, +// tail truncation is disabled for the header and hash tables, as these are intended +// to be retained long-term. +var chainFreezerTableConfigs = map[string]freezerTableConfig{ + ChainFreezerHeaderTable: {noSnappy: false, prunable: false}, + ChainFreezerHashTable: {noSnappy: true, prunable: false}, + ChainFreezerBodiesTable: {noSnappy: false, prunable: true}, + ChainFreezerReceiptTable: {noSnappy: false, prunable: true}, +} + +// freezerTableConfig contains the settings for a freezer table. +type freezerTableConfig struct { + noSnappy bool // disables item compression + prunable bool // true for tables that can be pruned by TruncateTail } const ( @@ -58,13 +66,13 @@ const ( stateHistoryStorageData = "storage.data" ) -// stateFreezerNoSnappy configures whether compression is disabled for the state freezer. -var stateFreezerNoSnappy = map[string]bool{ - stateHistoryMeta: true, - stateHistoryAccountIndex: false, - stateHistoryStorageIndex: false, - stateHistoryAccountData: false, - stateHistoryStorageData: false, +// stateFreezerTableConfigs configures the settings for tables in the state freezer. +var stateFreezerTableConfigs = map[string]freezerTableConfig{ + stateHistoryMeta: {noSnappy: true, prunable: true}, + stateHistoryAccountIndex: {noSnappy: false, prunable: true}, + stateHistoryStorageIndex: {noSnappy: false, prunable: true}, + stateHistoryAccountData: {noSnappy: false, prunable: true}, + stateHistoryStorageData: {noSnappy: false, prunable: true}, } // The list of identifiers of ancient stores. @@ -85,7 +93,7 @@ var freezers = []string{ChainFreezerName, MerkleStateFreezerName, VerkleStateFre // state freezer. func NewStateFreezer(ancientDir string, verkle bool, readOnly bool) (ethdb.ResettableAncientStore, error) { if ancientDir == "" { - return NewMemoryFreezer(readOnly, stateFreezerNoSnappy), nil + return NewMemoryFreezer(readOnly, stateFreezerTableConfigs), nil } var name string if verkle { @@ -93,5 +101,5 @@ func NewStateFreezer(ancientDir string, verkle bool, readOnly bool) (ethdb.Reset } else { name = filepath.Join(ancientDir, MerkleStateFreezerName) } - return newResettableFreezer(name, "eth/db/state", readOnly, stateHistoryTableSize, stateFreezerNoSnappy) + return newResettableFreezer(name, "eth/db/state", readOnly, stateHistoryTableSize, stateFreezerTableConfigs) } diff --git a/core/rawdb/ancient_utils.go b/core/rawdb/ancient_utils.go index 6804d7a91a..f4909d86e7 100644 --- a/core/rawdb/ancient_utils.go +++ b/core/rawdb/ancient_utils.go @@ -51,7 +51,7 @@ func (info *freezerInfo) size() common.StorageSize { return total } -func inspect(name string, order map[string]bool, reader ethdb.AncientReader) (freezerInfo, error) { +func inspect(name string, order map[string]freezerTableConfig, reader ethdb.AncientReader) (freezerInfo, error) { info := freezerInfo{name: name} for t := range order { size, err := reader.AncientSize(t) @@ -82,7 +82,7 @@ func inspectFreezers(db ethdb.Database) ([]freezerInfo, error) { for _, freezer := range freezers { switch freezer { case ChainFreezerName: - info, err := inspect(ChainFreezerName, chainFreezerNoSnappy, db) + info, err := inspect(ChainFreezerName, chainFreezerTableConfigs, db) if err != nil { return nil, err } @@ -99,7 +99,7 @@ func inspectFreezers(db ethdb.Database) ([]freezerInfo, error) { } defer f.Close() - info, err := inspect(freezer, stateFreezerNoSnappy, f) + info, err := inspect(freezer, stateFreezerTableConfigs, f) if err != nil { return nil, err } @@ -119,13 +119,13 @@ func inspectFreezers(db ethdb.Database) ([]freezerInfo, error) { func InspectFreezerTable(ancient string, freezerName string, tableName string, start, end int64) error { var ( path string - tables map[string]bool + tables map[string]freezerTableConfig ) switch freezerName { case ChainFreezerName: - path, tables = resolveChainFreezerDir(ancient), chainFreezerNoSnappy + path, tables = resolveChainFreezerDir(ancient), chainFreezerTableConfigs case MerkleStateFreezerName, VerkleStateFreezerName: - path, tables = filepath.Join(ancient, freezerName), stateFreezerNoSnappy + path, tables = filepath.Join(ancient, freezerName), stateFreezerTableConfigs default: return fmt.Errorf("unknown freezer, supported ones: %v", freezers) } diff --git a/core/rawdb/chain_freezer.go b/core/rawdb/chain_freezer.go index 0627aca34c..f3c671f45a 100644 --- a/core/rawdb/chain_freezer.go +++ b/core/rawdb/chain_freezer.go @@ -62,9 +62,9 @@ func newChainFreezer(datadir string, namespace string, readonly bool) (*chainFre freezer ethdb.AncientStore ) if datadir == "" { - freezer = NewMemoryFreezer(readonly, chainFreezerNoSnappy) + freezer = NewMemoryFreezer(readonly, chainFreezerTableConfigs) } else { - freezer, err = NewFreezer(datadir, namespace, readonly, freezerTableSize, chainFreezerNoSnappy) + freezer, err = NewFreezer(datadir, namespace, readonly, freezerTableSize, chainFreezerTableConfigs) } if err != nil { return nil, err diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index c5a72eff7e..105d3af934 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -78,7 +78,7 @@ type Freezer struct { // // The 'tables' argument defines the data tables. If the value of a map // entry is true, snappy compression is disabled for the table. -func NewFreezer(datadir string, namespace string, readonly bool, maxTableSize uint32, tables map[string]bool) (*Freezer, error) { +func NewFreezer(datadir string, namespace string, readonly bool, maxTableSize uint32, tables map[string]freezerTableConfig) (*Freezer, error) { // Create the initial freezer object var ( readMeter = metrics.NewRegisteredMeter(namespace+"ancient/read", nil) @@ -121,8 +121,8 @@ func NewFreezer(datadir string, namespace string, readonly bool, maxTableSize ui } // Create the tables. - for name, disableSnappy := range tables { - table, err := newTable(datadir, name, readMeter, writeMeter, sizeGauge, maxTableSize, disableSnappy, readonly) + for name, config := range tables { + table, err := newTable(datadir, name, readMeter, writeMeter, sizeGauge, maxTableSize, config, readonly) if err != nil { for _, table := range freezer.tables { table.Close() @@ -301,7 +301,8 @@ func (f *Freezer) TruncateHead(items uint64) (uint64, error) { return oitems, nil } -// TruncateTail discards any recent data below the provided threshold number. +// TruncateTail discards all data below the specified threshold. Note that only +// 'prunable' tables will be truncated. func (f *Freezer) TruncateTail(tail uint64) (uint64, error) { if f.readonly { return 0, errReadOnly @@ -314,8 +315,10 @@ func (f *Freezer) TruncateTail(tail uint64) (uint64, error) { return old, nil } for _, table := range f.tables { - if err := table.truncateTail(tail); err != nil { - return 0, err + if table.config.prunable { + if err := table.truncateTail(tail); err != nil { + return 0, err + } } } f.tail.Store(tail) @@ -343,56 +346,77 @@ func (f *Freezer) validate() error { return nil } var ( - head uint64 - tail uint64 - name string + head uint64 + prunedTail *uint64 ) - // Hack to get boundary of any table - for kind, table := range f.tables { + // get any head value + for _, table := range f.tables { head = table.items.Load() - tail = table.itemHidden.Load() - name = kind break } - // Now check every table against those boundaries. for kind, table := range f.tables { + // all tables have to have the same head if head != table.items.Load() { - return fmt.Errorf("freezer tables %s and %s have differing head: %d != %d", kind, name, table.items.Load(), head) + return fmt.Errorf("freezer table %s has a differing head: %d != %d", kind, table.items.Load(), head) } - if tail != table.itemHidden.Load() { - return fmt.Errorf("freezer tables %s and %s have differing tail: %d != %d", kind, name, table.itemHidden.Load(), tail) + if !table.config.prunable { + // non-prunable tables have to start at 0 + if table.itemHidden.Load() != 0 { + return fmt.Errorf("non-prunable freezer table '%s' has a non-zero tail: %d", kind, table.itemHidden.Load()) + } + } else { + // prunable tables have to have the same length + if prunedTail == nil { + tmp := table.itemHidden.Load() + prunedTail = &tmp + } + if *prunedTail != table.itemHidden.Load() { + return fmt.Errorf("freezer table %s has differing tail: %d != %d", kind, table.itemHidden.Load(), *prunedTail) + } } } + + if prunedTail == nil { + tmp := uint64(0) + prunedTail = &tmp + } + f.frozen.Store(head) - f.tail.Store(tail) + f.tail.Store(*prunedTail) return nil } // repair truncates all data tables to the same length. func (f *Freezer) repair() error { var ( - head = uint64(math.MaxUint64) - tail = uint64(0) + head = uint64(math.MaxUint64) + prunedTail = uint64(0) ) + // get the minimal head and the maximum tail for _, table := range f.tables { - items := table.items.Load() - if head > items { - head = items - } - hidden := table.itemHidden.Load() - if hidden > tail { - tail = hidden - } + head = min(head, table.items.Load()) + prunedTail = max(prunedTail, table.itemHidden.Load()) } - for _, table := range f.tables { + // apply the pruning + for kind, table := range f.tables { + // all tables need to have the same head if err := table.truncateHead(head); err != nil { return err } - if err := table.truncateTail(tail); err != nil { - return err + if !table.config.prunable { + // non-prunable tables have to start at 0 + if table.itemHidden.Load() != 0 { + panic(fmt.Sprintf("non-prunable freezer table %s has non-zero tail: %v", kind, table.itemHidden.Load())) + } + } else { + // prunable tables have to have the same length + if err := table.truncateTail(prunedTail); err != nil { + return err + } } } + f.frozen.Store(head) - f.tail.Store(tail) + f.tail.Store(prunedTail) return nil } diff --git a/core/rawdb/freezer_batch.go b/core/rawdb/freezer_batch.go index 801d30f73f..99b63df4dc 100644 --- a/core/rawdb/freezer_batch.go +++ b/core/rawdb/freezer_batch.go @@ -96,7 +96,7 @@ type freezerTableBatch struct { // newBatch creates a new batch for the freezer table. func (t *freezerTable) newBatch() *freezerTableBatch { batch := &freezerTableBatch{t: t} - if !t.noCompression { + if !t.config.noSnappy { batch.sb = new(snappyBuffer) } batch.reset() diff --git a/core/rawdb/freezer_memory.go b/core/rawdb/freezer_memory.go index 2d3dbb07dd..4274546de5 100644 --- a/core/rawdb/freezer_memory.go +++ b/core/rawdb/freezer_memory.go @@ -30,17 +30,19 @@ import ( // memoryTable is used to store a list of sequential items in memory. type memoryTable struct { - name string // Table name items uint64 // Number of stored items in the table, including the deleted ones offset uint64 // Number of deleted items from the table data [][]byte // List of rlp-encoded items, sort in order size uint64 // Total memory size occupied by the table lock sync.RWMutex + + name string + config freezerTableConfig } // newMemoryTable initializes the memory table. -func newMemoryTable(name string) *memoryTable { - return &memoryTable{name: name} +func newMemoryTable(name string, config freezerTableConfig) *memoryTable { + return &memoryTable{name: name, config: config} } // has returns an indicator whether the specified data exists. @@ -218,10 +220,10 @@ type MemoryFreezer struct { } // NewMemoryFreezer initializes an in-memory freezer instance. -func NewMemoryFreezer(readonly bool, tableName map[string]bool) *MemoryFreezer { +func NewMemoryFreezer(readonly bool, tableName map[string]freezerTableConfig) *MemoryFreezer { tables := make(map[string]*memoryTable) - for name := range tableName { - tables[name] = newMemoryTable(name) + for name, cfg := range tableName { + tables[name] = newMemoryTable(name, cfg) } return &MemoryFreezer{ writeBatch: newMemoryBatch(), @@ -368,7 +370,9 @@ func (f *MemoryFreezer) TruncateHead(items uint64) (uint64, error) { return old, nil } -// TruncateTail discards any recent data below the provided threshold number. +// TruncateTail discards all data below the provided threshold number. +// Note this will only truncate 'prunable' tables. Block headers and canonical +// hashes cannot be truncated at this time. func (f *MemoryFreezer) TruncateTail(tail uint64) (uint64, error) { f.lock.Lock() defer f.lock.Unlock() @@ -381,8 +385,10 @@ func (f *MemoryFreezer) TruncateTail(tail uint64) (uint64, error) { return old, nil } for _, table := range f.tables { - if err := table.truncateTail(tail); err != nil { - return 0, err + if table.config.prunable { + if err := table.truncateTail(tail); err != nil { + return 0, err + } } } f.tail = tail @@ -412,8 +418,8 @@ func (f *MemoryFreezer) Reset() error { defer f.lock.Unlock() tables := make(map[string]*memoryTable) - for name := range f.tables { - tables[name] = newMemoryTable(name) + for name, table := range f.tables { + tables[name] = newMemoryTable(name, table.config) } f.tables = tables f.items, f.tail = 0, 0 diff --git a/core/rawdb/freezer_memory_test.go b/core/rawdb/freezer_memory_test.go index e71de0f629..4bd31d8027 100644 --- a/core/rawdb/freezer_memory_test.go +++ b/core/rawdb/freezer_memory_test.go @@ -25,16 +25,22 @@ import ( func TestMemoryFreezer(t *testing.T) { ancienttest.TestAncientSuite(t, func(kinds []string) ethdb.AncientStore { - tables := make(map[string]bool) + tables := make(map[string]freezerTableConfig) for _, kind := range kinds { - tables[kind] = true + tables[kind] = freezerTableConfig{ + noSnappy: true, + prunable: true, + } } return NewMemoryFreezer(false, tables) }) ancienttest.TestResettableAncientSuite(t, func(kinds []string) ethdb.ResettableAncientStore { - tables := make(map[string]bool) + tables := make(map[string]freezerTableConfig) for _, kind := range kinds { - tables[kind] = true + tables[kind] = freezerTableConfig{ + noSnappy: true, + prunable: true, + } } return NewMemoryFreezer(false, tables) }) diff --git a/core/rawdb/freezer_resettable.go b/core/rawdb/freezer_resettable.go index 7c77a06efc..2e64e6074c 100644 --- a/core/rawdb/freezer_resettable.go +++ b/core/rawdb/freezer_resettable.go @@ -49,7 +49,7 @@ type resettableFreezer struct { // // The reset function will delete directory atomically and re-create the // freezer from scratch. -func newResettableFreezer(datadir string, namespace string, readonly bool, maxTableSize uint32, tables map[string]bool) (*resettableFreezer, error) { +func newResettableFreezer(datadir string, namespace string, readonly bool, maxTableSize uint32, tables map[string]freezerTableConfig) (*resettableFreezer, error) { if err := cleanup(datadir); err != nil { return nil, err } diff --git a/core/rawdb/freezer_table.go b/core/rawdb/freezer_table.go index e553df7029..aec24b207e 100644 --- a/core/rawdb/freezer_table.go +++ b/core/rawdb/freezer_table.go @@ -100,11 +100,11 @@ type freezerTable struct { // should never be lower than itemOffset. itemHidden atomic.Uint64 - noCompression bool // if true, disables snappy compression. Note: does not work retroactively - readonly bool - maxFileSize uint32 // Max file size for data-files - name string - path string + config freezerTableConfig // if true, disables snappy compression. Note: does not work retroactively + readonly bool + maxFileSize uint32 // Max file size for data-files + name string + path string head *os.File // File descriptor for the data head of the table index *os.File // File descriptor for the indexEntry file of the table @@ -125,20 +125,20 @@ type freezerTable struct { } // newFreezerTable opens the given path as a freezer table. -func newFreezerTable(path, name string, disableSnappy, readonly bool) (*freezerTable, error) { - return newTable(path, name, metrics.NewInactiveMeter(), metrics.NewInactiveMeter(), metrics.NewGauge(), freezerTableSize, disableSnappy, readonly) +func newFreezerTable(path, name string, config freezerTableConfig, readonly bool) (*freezerTable, error) { + return newTable(path, name, metrics.NewInactiveMeter(), metrics.NewInactiveMeter(), metrics.NewGauge(), freezerTableSize, config, readonly) } // newTable opens a freezer table, creating the data and index files if they are // non-existent. Both files are truncated to the shortest common length to ensure // they don't go out of sync. -func newTable(path string, name string, readMeter, writeMeter *metrics.Meter, sizeGauge *metrics.Gauge, maxFilesize uint32, noCompression, readonly bool) (*freezerTable, error) { +func newTable(path string, name string, readMeter, writeMeter *metrics.Meter, sizeGauge *metrics.Gauge, maxFilesize uint32, config freezerTableConfig, readonly bool) (*freezerTable, error) { // Ensure the containing directory exists and open the indexEntry file if err := os.MkdirAll(path, 0755); err != nil { return nil, err } var idxName string - if noCompression { + if config.noSnappy { idxName = fmt.Sprintf("%s.ridx", name) // raw index file } else { idxName = fmt.Sprintf("%s.cidx", name) // compressed index file @@ -176,19 +176,19 @@ func newTable(path string, name string, readMeter, writeMeter *metrics.Meter, si } // Create the table and repair any past inconsistency tab := &freezerTable{ - index: index, - metadata: metadata, - lastSync: time.Now(), - files: make(map[uint32]*os.File), - readMeter: readMeter, - writeMeter: writeMeter, - sizeGauge: sizeGauge, - name: name, - path: path, - logger: log.New("database", path, "table", name), - noCompression: noCompression, - readonly: readonly, - maxFileSize: maxFilesize, + index: index, + metadata: metadata, + lastSync: time.Now(), + files: make(map[uint32]*os.File), + readMeter: readMeter, + writeMeter: writeMeter, + sizeGauge: sizeGauge, + name: name, + path: path, + logger: log.New("database", path, "table", name), + config: config, + readonly: readonly, + maxFileSize: maxFilesize, } if err := tab.repair(); err != nil { tab.Close() @@ -871,7 +871,7 @@ func (t *freezerTable) openFile(num uint32, opener func(string) (*os.File, error var exist bool if f, exist = t.files[num]; !exist { var name string - if t.noCompression { + if t.config.noSnappy { name = fmt.Sprintf("%s.%04d.rdat", t.name, num) } else { name = fmt.Sprintf("%s.%04d.cdat", t.name, num) @@ -987,13 +987,13 @@ func (t *freezerTable) RetrieveItems(start, count, maxBytes uint64) ([][]byte, e item := diskData[offset : offset+diskSize] offset += diskSize decompressedSize := diskSize - if !t.noCompression { + if !t.config.noSnappy { decompressedSize, _ = snappy.DecodedLen(item) } if i > 0 && maxBytes != 0 && uint64(outputSize+decompressedSize) > maxBytes { break } - if !t.noCompression { + if !t.config.noSnappy { data, err := snappy.Decode(nil, item) if err != nil { return nil, err diff --git a/core/rawdb/freezer_table_test.go b/core/rawdb/freezer_table_test.go index 9a72af6ccc..96edac7e4a 100644 --- a/core/rawdb/freezer_table_test.go +++ b/core/rawdb/freezer_table_test.go @@ -39,7 +39,7 @@ func TestFreezerBasics(t *testing.T) { // set cutoff at 50 bytes f, err := newTable(os.TempDir(), fmt.Sprintf("unittest-%d", rand.Uint64()), - metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, false) + metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, freezerTableConfig{noSnappy: true}, false) if err != nil { t.Fatal(err) } @@ -84,7 +84,7 @@ func TestFreezerBasicsClosing(t *testing.T) { f *freezerTable err error ) - f, err = newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false) + f, err = newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{noSnappy: true}, false) if err != nil { t.Fatal(err) } @@ -98,7 +98,7 @@ func TestFreezerBasicsClosing(t *testing.T) { require.NoError(t, batch.commit()) f.Close() - f, err = newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false) + f, err = newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{noSnappy: true}, false) if err != nil { t.Fatal(err) } @@ -115,7 +115,7 @@ func TestFreezerBasicsClosing(t *testing.T) { t.Fatalf("test %d, got \n%x != \n%x", y, got, exp) } f.Close() - f, err = newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false) + f, err = newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{noSnappy: true}, false) if err != nil { t.Fatal(err) } @@ -130,7 +130,7 @@ func TestFreezerRepairDanglingHead(t *testing.T) { // Fill table { - f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false) + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{noSnappy: true}, false) if err != nil { t.Fatal(err) } @@ -159,7 +159,7 @@ func TestFreezerRepairDanglingHead(t *testing.T) { // Now open it again { - f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false) + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{noSnappy: true}, false) if err != nil { t.Fatal(err) } @@ -182,7 +182,7 @@ func TestFreezerRepairDanglingHeadLarge(t *testing.T) { // Fill a table and close it { - f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false) + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{noSnappy: true}, false) if err != nil { t.Fatal(err) } @@ -208,7 +208,7 @@ func TestFreezerRepairDanglingHeadLarge(t *testing.T) { // Now open it again { - f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false) + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{noSnappy: true}, false) if err != nil { t.Fatal(err) } @@ -231,7 +231,7 @@ func TestFreezerRepairDanglingHeadLarge(t *testing.T) { // And if we open it, we should now be able to read all of them (new values) { - f, _ := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false) + f, _ := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{noSnappy: true}, false) for y := 1; y < 255; y++ { exp := getChunk(15, ^y) got, err := f.Retrieve(uint64(y)) @@ -253,7 +253,7 @@ func TestSnappyDetection(t *testing.T) { // Open with snappy { - f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false) + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{noSnappy: true}, false) if err != nil { t.Fatal(err) } @@ -264,7 +264,7 @@ func TestSnappyDetection(t *testing.T) { // Open with snappy { - f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false) + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{noSnappy: true}, false) if err != nil { t.Fatal(err) } @@ -277,7 +277,7 @@ func TestSnappyDetection(t *testing.T) { // Open without snappy { - f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, false, false) + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{noSnappy: false}, false) if err != nil { t.Fatal(err) } @@ -308,7 +308,7 @@ func TestFreezerRepairDanglingIndex(t *testing.T) { // Fill a table and close it { - f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false) + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{noSnappy: true}, false) if err != nil { t.Fatal(err) } @@ -344,7 +344,7 @@ func TestFreezerRepairDanglingIndex(t *testing.T) { // 45, 45, 15 // with 3+3+1 items { - f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false) + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{noSnappy: true}, false) if err != nil { t.Fatal(err) } @@ -365,7 +365,7 @@ func TestFreezerTruncate(t *testing.T) { // Fill table { - f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false) + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{noSnappy: true}, false) if err != nil { t.Fatal(err) } @@ -381,7 +381,7 @@ func TestFreezerTruncate(t *testing.T) { // Reopen, truncate { - f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false) + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{noSnappy: true}, false) if err != nil { t.Fatal(err) } @@ -406,7 +406,7 @@ func TestFreezerRepairFirstFile(t *testing.T) { // Fill table { - f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false) + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{noSnappy: true}, false) if err != nil { t.Fatal(err) } @@ -439,7 +439,7 @@ func TestFreezerRepairFirstFile(t *testing.T) { // Reopen { - f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false) + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{noSnappy: true}, false) if err != nil { t.Fatal(err) } @@ -474,7 +474,7 @@ func TestFreezerReadAndTruncate(t *testing.T) { // Fill table { - f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false) + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{noSnappy: true}, false) if err != nil { t.Fatal(err) } @@ -490,7 +490,7 @@ func TestFreezerReadAndTruncate(t *testing.T) { // Reopen and read all files { - f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false) + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{noSnappy: true}, false) if err != nil { t.Fatal(err) } @@ -521,7 +521,7 @@ func TestFreezerOffset(t *testing.T) { fname := fmt.Sprintf("offset-%d", rand.Uint64()) // Fill table - f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false) + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, freezerTableConfig{noSnappy: true}, false) if err != nil { t.Fatal(err) } @@ -545,7 +545,7 @@ func TestFreezerOffset(t *testing.T) { f.Close() // Now open again - f, err = newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false) + f, err = newTable(os.TempDir(), fname, rm, wm, sg, 40, freezerTableConfig{noSnappy: true}, false) if err != nil { t.Fatal(err) } @@ -597,7 +597,7 @@ func TestFreezerOffset(t *testing.T) { // Check that existing items have been moved to index 1M. { - f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false) + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, freezerTableConfig{noSnappy: true}, false) if err != nil { t.Fatal(err) } @@ -631,7 +631,7 @@ func TestTruncateTail(t *testing.T) { fname := fmt.Sprintf("truncate-tail-%d", rand.Uint64()) // Fill table - f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false) + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, freezerTableConfig{noSnappy: true}, false) if err != nil { t.Fatal(err) } @@ -682,7 +682,7 @@ func TestTruncateTail(t *testing.T) { // Reopen the table, the deletion information should be persisted as well f.Close() - f, err = newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false) + f, err = newTable(os.TempDir(), fname, rm, wm, sg, 40, freezerTableConfig{noSnappy: true}, false) if err != nil { t.Fatal(err) } @@ -716,7 +716,7 @@ func TestTruncateTail(t *testing.T) { // Reopen the table, the above testing should still pass f.Close() - f, err = newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false) + f, err = newTable(os.TempDir(), fname, rm, wm, sg, 40, freezerTableConfig{noSnappy: true}, false) if err != nil { t.Fatal(err) } @@ -772,7 +772,7 @@ func TestTruncateHead(t *testing.T) { fname := fmt.Sprintf("truncate-head-blow-tail-%d", rand.Uint64()) // Fill table - f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false) + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, freezerTableConfig{noSnappy: true}, false) if err != nil { t.Fatal(err) } @@ -883,7 +883,7 @@ func TestSequentialRead(t *testing.T) { rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge() fname := fmt.Sprintf("batchread-%d", rand.Uint64()) { // Fill table - f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false) + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{noSnappy: true}, false) if err != nil { t.Fatal(err) } @@ -893,7 +893,7 @@ func TestSequentialRead(t *testing.T) { f.Close() } { // Open it, iterate, verify iteration - f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false) + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{noSnappy: true}, false) if err != nil { t.Fatal(err) } @@ -914,7 +914,7 @@ func TestSequentialRead(t *testing.T) { } { // Open it, iterate, verify byte limit. The byte limit is less than item // size, so each lookup should only return one item - f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false) + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, freezerTableConfig{noSnappy: true}, false) if err != nil { t.Fatal(err) } @@ -943,7 +943,7 @@ func TestSequentialReadByteLimit(t *testing.T) { rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge() fname := fmt.Sprintf("batchread-2-%d", rand.Uint64()) { // Fill table - f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, true, false) + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, freezerTableConfig{noSnappy: true}, false) if err != nil { t.Fatal(err) } @@ -965,7 +965,7 @@ func TestSequentialReadByteLimit(t *testing.T) { {100, 109, 10}, } { { - f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, true, false) + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, freezerTableConfig{noSnappy: true}, false) if err != nil { t.Fatal(err) } @@ -993,7 +993,7 @@ func TestSequentialReadNoByteLimit(t *testing.T) { rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge() fname := fmt.Sprintf("batchread-3-%d", rand.Uint64()) { // Fill table - f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, true, false) + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, freezerTableConfig{noSnappy: true}, false) if err != nil { t.Fatal(err) } @@ -1011,7 +1011,7 @@ func TestSequentialReadNoByteLimit(t *testing.T) { {31, 30}, } { { - f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, true, false) + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, freezerTableConfig{noSnappy: true}, false) if err != nil { t.Fatal(err) } @@ -1038,7 +1038,7 @@ func TestFreezerReadonly(t *testing.T) { // Case 1: Check it fails on non-existent file. _, err := newTable(tmpdir, fmt.Sprintf("readonlytest-%d", rand.Uint64()), - metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, true) + metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, freezerTableConfig{noSnappy: true}, true) if err == nil { t.Fatal("readonly table instantiation should fail for non-existent table") } @@ -1053,7 +1053,7 @@ func TestFreezerReadonly(t *testing.T) { idxFile.Write(make([]byte, 17)) idxFile.Close() _, err = newTable(tmpdir, fname, - metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, true) + metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, freezerTableConfig{noSnappy: true}, true) if err == nil { t.Errorf("readonly table instantiation should fail for invalid index size") } @@ -1063,7 +1063,7 @@ func TestFreezerReadonly(t *testing.T) { // again in readonly triggers an error. fname = fmt.Sprintf("readonlytest-%d", rand.Uint64()) f, err := newTable(tmpdir, fname, - metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, false) + metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, freezerTableConfig{noSnappy: true}, false) if err != nil { t.Fatalf("failed to instantiate table: %v", err) } @@ -1076,7 +1076,7 @@ func TestFreezerReadonly(t *testing.T) { t.Fatal(err) } _, err = newTable(tmpdir, fname, - metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, true) + metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, freezerTableConfig{noSnappy: true}, true) if err == nil { t.Errorf("readonly table instantiation should fail for corrupt table file") } @@ -1085,7 +1085,7 @@ func TestFreezerReadonly(t *testing.T) { // Should be successful. fname = fmt.Sprintf("readonlytest-%d", rand.Uint64()) f, err = newTable(tmpdir, fname, - metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, false) + metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, freezerTableConfig{noSnappy: true}, false) if err != nil { t.Fatalf("failed to instantiate table: %v\n", err) } @@ -1094,7 +1094,7 @@ func TestFreezerReadonly(t *testing.T) { t.Fatal(err) } f, err = newTable(tmpdir, fname, - metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, true) + metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, freezerTableConfig{noSnappy: true}, true) if err != nil { t.Fatal(err) } @@ -1234,7 +1234,7 @@ func (randTest) Generate(r *rand.Rand, size int) reflect.Value { func runRandTest(rt randTest) bool { fname := fmt.Sprintf("randtest-%d", rand.Uint64()) - f, err := newTable(os.TempDir(), fname, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, false) + f, err := newTable(os.TempDir(), fname, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, freezerTableConfig{noSnappy: true}, false) if err != nil { panic("failed to initialize table") } @@ -1243,7 +1243,7 @@ func runRandTest(rt randTest) bool { switch step.op { case opReload: f.Close() - f, err = newTable(os.TempDir(), fname, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, false) + f, err = newTable(os.TempDir(), fname, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, freezerTableConfig{noSnappy: true}, false) if err != nil { rt[i].err = fmt.Errorf("failed to reload table %v", err) } @@ -1381,7 +1381,7 @@ func TestIndexValidation(t *testing.T) { } for _, c := range cases { fn := fmt.Sprintf("t-%d", rand.Uint64()) - f, err := newTable(os.TempDir(), fn, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 10*dataSize, true, false) + f, err := newTable(os.TempDir(), fn, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 10*dataSize, freezerTableConfig{noSnappy: true}, false) if err != nil { t.Fatal(err) } @@ -1392,7 +1392,7 @@ func TestIndexValidation(t *testing.T) { f.Close() // reopen the table, corruption should be truncated - f, err = newTable(os.TempDir(), fn, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 100, true, false) + f, err = newTable(os.TempDir(), fn, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 100, freezerTableConfig{noSnappy: true}, false) if err != nil { t.Fatal(err) } @@ -1422,7 +1422,7 @@ func TestFlushOffsetTracking(t *testing.T) { fileSize = 100 ) fn := fmt.Sprintf("t-%d", rand.Uint64()) - f, err := newTable(os.TempDir(), fn, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), fileSize, true, false) + f, err := newTable(os.TempDir(), fn, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), fileSize, freezerTableConfig{noSnappy: true}, false) if err != nil { t.Fatal(err) } @@ -1533,7 +1533,7 @@ func TestTailTruncationCrash(t *testing.T) { fileSize = 100 ) fn := fmt.Sprintf("t-%d", rand.Uint64()) - f, err := newTable(os.TempDir(), fn, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), fileSize, true, false) + f, err := newTable(os.TempDir(), fn, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), fileSize, freezerTableConfig{noSnappy: true}, false) if err != nil { t.Fatal(err) } @@ -1563,7 +1563,7 @@ func TestTailTruncationCrash(t *testing.T) { // the offset f.metadata.setFlushOffset(31*indexEntrySize, true) - f, err = newTable(os.TempDir(), fn, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), fileSize, true, false) + f, err = newTable(os.TempDir(), fn, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), fileSize, freezerTableConfig{noSnappy: true}, false) if err != nil { t.Fatal(err) } diff --git a/core/rawdb/freezer_test.go b/core/rawdb/freezer_test.go index 7d82ea305f..150734d3ac 100644 --- a/core/rawdb/freezer_test.go +++ b/core/rawdb/freezer_test.go @@ -31,7 +31,7 @@ import ( "github.com/stretchr/testify/require" ) -var freezerTestTableDef = map[string]bool{"test": true} +var freezerTestTableDef = map[string]freezerTableConfig{"test": {noSnappy: true}} func TestFreezerModify(t *testing.T) { t.Parallel() @@ -47,7 +47,7 @@ func TestFreezerModify(t *testing.T) { valuesRLP = append(valuesRLP, iv) } - tables := map[string]bool{"raw": true, "rlp": false} + tables := map[string]freezerTableConfig{"raw": {noSnappy: true}, "rlp": {noSnappy: false}} f, _ := newFreezerForTesting(t, tables) defer f.Close() @@ -111,7 +111,7 @@ func TestFreezerModifyRollback(t *testing.T) { f.Close() // Reopen and check that the rolled-back data doesn't reappear. - tables := map[string]bool{"test": true} + tables := map[string]freezerTableConfig{"test": {noSnappy: true}} f2, err := NewFreezer(dir, "", false, 2049, tables) if err != nil { t.Fatalf("can't reopen freezer after failed ModifyAncients: %v", err) @@ -249,7 +249,7 @@ func TestFreezerConcurrentModifyTruncate(t *testing.T) { } func TestFreezerReadonlyValidate(t *testing.T) { - tables := map[string]bool{"a": true, "b": true} + tables := map[string]freezerTableConfig{"a": {noSnappy: true}, "b": {noSnappy: true}} dir := t.TempDir() // Open non-readonly freezer and fill individual tables // with different amount of data. @@ -285,7 +285,7 @@ func TestFreezerReadonlyValidate(t *testing.T) { func TestFreezerConcurrentReadonly(t *testing.T) { t.Parallel() - tables := map[string]bool{"a": true} + tables := map[string]freezerTableConfig{"a": {noSnappy: true}} dir := t.TempDir() f, err := NewFreezer(dir, "", false, 2049, tables) @@ -333,7 +333,7 @@ func TestFreezerConcurrentReadonly(t *testing.T) { } } -func newFreezerForTesting(t *testing.T, tables map[string]bool) (*Freezer, string) { +func newFreezerForTesting(t *testing.T, tables map[string]freezerTableConfig) (*Freezer, string) { t.Helper() dir := t.TempDir() @@ -379,7 +379,7 @@ func checkAncientCount(t *testing.T, f *Freezer, kind string, n uint64) { func TestFreezerCloseSync(t *testing.T) { t.Parallel() - f, _ := newFreezerForTesting(t, map[string]bool{"a": true, "b": true}) + f, _ := newFreezerForTesting(t, map[string]freezerTableConfig{"a": {noSnappy: true}, "b": {noSnappy: true}}) defer f.Close() // Now, close and sync. This mimics the behaviour if the node is shut down, @@ -401,17 +401,23 @@ func TestFreezerCloseSync(t *testing.T) { func TestFreezerSuite(t *testing.T) { ancienttest.TestAncientSuite(t, func(kinds []string) ethdb.AncientStore { - tables := make(map[string]bool) + tables := make(map[string]freezerTableConfig) for _, kind := range kinds { - tables[kind] = true + tables[kind] = freezerTableConfig{ + noSnappy: true, + prunable: true, + } } f, _ := newFreezerForTesting(t, tables) return f }) ancienttest.TestResettableAncientSuite(t, func(kinds []string) ethdb.ResettableAncientStore { - tables := make(map[string]bool) + tables := make(map[string]freezerTableConfig) for _, kind := range kinds { - tables[kind] = true + tables[kind] = freezerTableConfig{ + noSnappy: true, + prunable: true, + } } f, _ := newResettableFreezer(t.TempDir(), "", false, 2048, tables) return f diff --git a/ethdb/database.go b/ethdb/database.go index 323f8f5d6f..b1577512f3 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -128,6 +128,8 @@ type AncientWriter interface { // is item_n(start from 0). The deleted items may not be removed from the ancient store // immediately, but only when the accumulated deleted data reach the threshold then // will be removed all together. + // + // Note that data marked as non-prunable will still be retained and remain accessible. TruncateTail(n uint64) (uint64, error) // Sync flushes all in-memory ancient store data to disk.