From 747f6a914e32f7813855ee94abb2b18bbc67356b Mon Sep 17 00:00:00 2001 From: Pratik Patil Date: Thu, 1 May 2025 13:53:54 +0530 Subject: [PATCH] upstream: fixed core/rawdb/* --- core/rawdb/chain_freezer.go | 2 +- core/rawdb/freezer.go | 2 +- core/rawdb/freezer_table.go | 127 ------------------------------- core/rawdb/freezer_table_test.go | 8 +- 4 files changed, 6 insertions(+), 133 deletions(-) diff --git a/core/rawdb/chain_freezer.go b/core/rawdb/chain_freezer.go index 4384d26dcd..d6c51e0efd 100644 --- a/core/rawdb/chain_freezer.go +++ b/core/rawdb/chain_freezer.go @@ -53,7 +53,7 @@ type chainFreezer struct { // NewChainFreezer is a small utility method around NewFreezer that sets the // default parameters for the chain storage. func NewChainFreezer(datadir string, namespace string, readonly bool, offset uint64) (*Freezer, error) { - return NewFreezer(datadir, namespace, readonly, offset, freezerTableSize, chainFreezerNoSnappy) + return NewFreezer(datadir, namespace, readonly, offset, freezerTableSize, chainFreezerTableConfigs) } // newChainFreezer initializes the freezer for ancient chain segment. diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index b023a1f998..bf68855795 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -493,7 +493,7 @@ func (f *Freezer) MigrateTable(kind string, convert convertLegacyFn) error { // Set up new dir for the migrated table, the content of which // we'll at the end move over to the ancients dir. migrationPath := filepath.Join(ancientsPath, "migration") - newTable, err := newFreezerTable(migrationPath, kind, table.noCompression, false) + newTable, err := newFreezerTable(migrationPath, kind, table.config, false) if err != nil { return err } diff --git a/core/rawdb/freezer_table.go b/core/rawdb/freezer_table.go index 47fded005a..9736108beb 100644 --- a/core/rawdb/freezer_table.go +++ b/core/rawdb/freezer_table.go @@ -411,108 +411,6 @@ func (t *freezerTable) repair() error { return nil } -// repairIndex validates the integrity of the index file. According to the design, -// the initial entry in the file denotes the earliest data file along with the -// count of deleted items. Following this, all subsequent entries in the file must -// be in order. This function identifies any corrupted entries and truncates items -// occurring after the corruption point. -// -// corruption can occur because of the power failure. In the Linux kernel, the -// file metadata update and data update are not necessarily performed at the -// same time. Typically, the metadata will be flushed/journalled ahead of the file -// data. Therefore, we make the pessimistic assumption that the file is first -// extended with invalid "garbage" data (normally zero bytes) and that afterwards -// the correct data replaces the garbage. As all the items in index file are -// supposed to be in-order, the leftover garbage must be truncated before the -// index data is utilized. -// -// It's important to note an exception that's unfortunately undetectable: when -// all index entries in the file are zero. Distinguishing whether they represent -// leftover garbage or if all items in the table have zero size is impossible. -// In such instances, the file will remain unchanged to prevent potential data -// loss or misinterpretation. -func (t *freezerTable) repairIndex() error { - // Retrieve the file sizes and prepare for validation - stat, err := t.index.Stat() - if err != nil { - return err - } - size := stat.Size() - - // Move the read cursor to the beginning of the file - _, err = t.index.Seek(0, io.SeekStart) - if err != nil { - return err - } - fr := bufio.NewReader(t.index) - - var ( - start = time.Now() - buff = make([]byte, indexEntrySize) - prev indexEntry - head indexEntry - - read = func() (indexEntry, error) { - n, err := io.ReadFull(fr, buff) - if err != nil { - return indexEntry{}, err - } - if n != indexEntrySize { - return indexEntry{}, fmt.Errorf("failed to read from index, n: %d", n) - } - var entry indexEntry - entry.unmarshalBinary(buff) - return entry, nil - } - truncate = func(offset int64) error { - if t.readonly { - return fmt.Errorf("index file is corrupted at %d, size: %d", offset, size) - } - if err := truncateFreezerFile(t.index, offset); err != nil { - return err - } - log.Warn("Truncated index file", "offset", offset, "truncated", size-offset) - return nil - } - ) - for offset := int64(0); offset < size; offset += indexEntrySize { - entry, err := read() - if err != nil { - return err - } - if offset == 0 { - head = entry - continue - } - // Ensure that the first non-head index refers to the earliest file, - // or the next file if the earliest file has no space to place the - // first item. - if offset == indexEntrySize { - if entry.filenum != head.filenum && entry.filenum != head.filenum+1 { - log.Error("Corrupted index item detected", "earliest", head.filenum, "filenumber", entry.filenum) - return truncate(offset) - } - prev = entry - continue - } - // ensure two consecutive index items are in order - if err := t.checkIndexItems(prev, entry); err != nil { - log.Error("Corrupted index item detected", "err", err) - return truncate(offset) - } - prev = entry - } - // Move the read cursor to the end of the file. While theoretically, the - // cursor should reach the end by reading all the items in the file, perform - // the seek operation anyway as a precaution. - _, err = t.index.Seek(0, io.SeekEnd) - if err != nil { - return err - } - log.Debug("Verified index file", "items", size/indexEntrySize, "elapsed", common.PrettyDuration(time.Since(start))) - return nil -} - // checkIndexItems validates the correctness of two consecutive index items based // on the following rules: // @@ -698,31 +596,6 @@ func (t *freezerTable) checkIndex(size int64) (int64, error) { return size, nil } -// checkIndexItems validates the correctness of two consecutive index items based -// on the following rules: -// -// - The file number of two consecutive index items must either be the same or -// increase monotonically. If the file number decreases or skips in a -// non-sequential manner, the index item is considered invalid. -// -// - For index items with the same file number, the data offset must be in -// non-decreasing order. Note: Two index items with the same file number -// and the same data offset are permitted if the entry size is zero. -// -// - The first index item in a new data file must not have a zero data offset. -func (t *freezerTable) checkIndexItems(a, b indexEntry) error { - if b.filenum != a.filenum && b.filenum != a.filenum+1 { - return fmt.Errorf("index items with inconsistent file number, prev: %d, next: %d", a.filenum, b.filenum) - } - if b.filenum == a.filenum && b.offset < a.offset { - return fmt.Errorf("index items with unordered offset, prev: %d, next: %d", a.offset, b.offset) - } - if b.filenum == a.filenum+1 && b.offset == 0 { - return fmt.Errorf("index items with zero offset, file number: %d", b.filenum) - } - return nil -} - // preopen opens all files that the freezer will need. This method should be called from an init-context, // since it assumes that it doesn't have to bother with locking // The rationale for doing preopen is to not have to do it from within Retrieve, thus not needing to ever diff --git a/core/rawdb/freezer_table_test.go b/core/rawdb/freezer_table_test.go index 72a1bb125c..3b49dd5b78 100644 --- a/core/rawdb/freezer_table_test.go +++ b/core/rawdb/freezer_table_test.go @@ -564,7 +564,7 @@ func TestFreezerOffset(t *testing.T) { } // Write 6 x 20 bytes, splitting out into three files - batch := f.newBatch() + batch := f.newBatch(0) require.NoError(t, batch.AppendRaw(0, getChunk(20, 0xFF))) require.NoError(t, batch.AppendRaw(1, getChunk(20, 0xEE))) @@ -589,7 +589,7 @@ func TestFreezerOffset(t *testing.T) { t.Log(f.dumpIndexString(0, 100)) // It should allow writing item 6. - batch = f.newBatch() + batch = f.newBatch(6) require.NoError(t, batch.AppendRaw(6, getChunk(20, 0x99))) require.NoError(t, batch.commit()) @@ -1540,7 +1540,7 @@ func TestFlushOffsetTracking(t *testing.T) { // Data files: // F1(10 items) -> F2(10 items) -> F3(10 items) -> F4(10 items, full) func(f *freezerTable) { - batch := f.newBatch() + batch := f.newBatch(0) for i := 0; i < 5; i++ { batch.AppendRaw(items+uint64(i), make([]byte, dataSize)) } @@ -1554,7 +1554,7 @@ func TestFlushOffsetTracking(t *testing.T) { // Data files: // F1(10 items) -> F2(10 items) -> F3(10 items) -> F4(10 items) -> F5(1 item) func(f *freezerTable) { - batch := f.newBatch() + batch := f.newBatch(0) batch.AppendRaw(items+5, make([]byte, dataSize)) batch.commit() },