upstream: fixed core/rawdb/*

core/rawdb/chain_freezer.go

@@ -53,7 +53,7 @@ type chainFreezer struct {
 // NewChainFreezer is a small utility method around NewFreezer that sets the
 // default parameters for the chain storage.
 func NewChainFreezer(datadir string, namespace string, readonly bool, offset uint64) (*Freezer, error) {
-	return NewFreezer(datadir, namespace, readonly, offset, freezerTableSize, chainFreezerNoSnappy)
+	return NewFreezer(datadir, namespace, readonly, offset, freezerTableSize, chainFreezerTableConfigs)
 }
 
 // newChainFreezer initializes the freezer for ancient chain segment.

core/rawdb/freezer.go

@@ -493,7 +493,7 @@ func (f *Freezer) MigrateTable(kind string, convert convertLegacyFn) error {
 	// Set up new dir for the migrated table, the content of which
 	// we'll at the end move over to the ancients dir.
 	migrationPath := filepath.Join(ancientsPath, "migration")
-	newTable, err := newFreezerTable(migrationPath, kind, table.noCompression, false)
+	newTable, err := newFreezerTable(migrationPath, kind, table.config, false)
 	if err != nil {
 		return err
 	}

core/rawdb/freezer_table.go

@@ -411,108 +411,6 @@ func (t *freezerTable) repair() error {
 	return nil
 }
 
-// repairIndex validates the integrity of the index file. According to the design,
-// the initial entry in the file denotes the earliest data file along with the
-// count of deleted items. Following this, all subsequent entries in the file must
-// be in order. This function identifies any corrupted entries and truncates items
-// occurring after the corruption point.
-//
-// corruption can occur because of the power failure. In the Linux kernel, the
-// file metadata update and data update are not necessarily performed at the
-// same time. Typically, the metadata will be flushed/journalled ahead of the file
-// data. Therefore, we make the pessimistic assumption that the file is first
-// extended with invalid "garbage" data (normally zero bytes) and that afterwards
-// the correct data replaces the garbage. As all the items in index file are
-// supposed to be in-order, the leftover garbage must be truncated before the
-// index data is utilized.
-//
-// It's important to note an exception that's unfortunately undetectable: when
-// all index entries in the file are zero. Distinguishing whether they represent
-// leftover garbage or if all items in the table have zero size is impossible.
-// In such instances, the file will remain unchanged to prevent potential data
-// loss or misinterpretation.
-func (t *freezerTable) repairIndex() error {
-	// Retrieve the file sizes and prepare for validation
-	stat, err := t.index.Stat()
-	if err != nil {
-		return err
-	}
-	size := stat.Size()
-
-	// Move the read cursor to the beginning of the file
-	_, err = t.index.Seek(0, io.SeekStart)
-	if err != nil {
-		return err
-	}
-	fr := bufio.NewReader(t.index)
-
-	var (
-		start = time.Now()
-		buff  = make([]byte, indexEntrySize)
-		prev  indexEntry
-		head  indexEntry
-
-		read = func() (indexEntry, error) {
-			n, err := io.ReadFull(fr, buff)
-			if err != nil {
-				return indexEntry{}, err
-			}
-			if n != indexEntrySize {
-				return indexEntry{}, fmt.Errorf("failed to read from index, n: %d", n)
-			}
-			var entry indexEntry
-			entry.unmarshalBinary(buff)
-			return entry, nil
-		}
-		truncate = func(offset int64) error {
-			if t.readonly {
-				return fmt.Errorf("index file is corrupted at %d, size: %d", offset, size)
-			}
-			if err := truncateFreezerFile(t.index, offset); err != nil {
-				return err
-			}
-			log.Warn("Truncated index file", "offset", offset, "truncated", size-offset)
-			return nil
-		}
-	)
-	for offset := int64(0); offset < size; offset += indexEntrySize {
-		entry, err := read()
-		if err != nil {
-			return err
-		}
-		if offset == 0 {
-			head = entry
-			continue
-		}
-		// Ensure that the first non-head index refers to the earliest file,
-		// or the next file if the earliest file has no space to place the
-		// first item.
-		if offset == indexEntrySize {
-			if entry.filenum != head.filenum && entry.filenum != head.filenum+1 {
-				log.Error("Corrupted index item detected", "earliest", head.filenum, "filenumber", entry.filenum)
-				return truncate(offset)
-			}
-			prev = entry
-			continue
-		}
-		// ensure two consecutive index items are in order
-		if err := t.checkIndexItems(prev, entry); err != nil {
-			log.Error("Corrupted index item detected", "err", err)
-			return truncate(offset)
-		}
-		prev = entry
-	}
-	// Move the read cursor to the end of the file. While theoretically, the
-	// cursor should reach the end by reading all the items in the file, perform
-	// the seek operation anyway as a precaution.
-	_, err = t.index.Seek(0, io.SeekEnd)
-	if err != nil {
-		return err
-	}
-	log.Debug("Verified index file", "items", size/indexEntrySize, "elapsed", common.PrettyDuration(time.Since(start)))
-	return nil
-}
-
 // checkIndexItems validates the correctness of two consecutive index items based
 // on the following rules:
 //
@@ -698,31 +596,6 @@ func (t *freezerTable) checkIndex(size int64) (int64, error) {
 	return size, nil
 }
 
-// checkIndexItems validates the correctness of two consecutive index items based
-// on the following rules:
-//
-//   - The file number of two consecutive index items must either be the same or
-//     increase monotonically. If the file number decreases or skips in a
-//     non-sequential manner, the index item is considered invalid.
-//
-//   - For index items with the same file number, the data offset must be in
-//     non-decreasing order. Note: Two index items with the same file number
-//     and the same data offset are permitted if the entry size is zero.
-//
-//   - The first index item in a new data file must not have a zero data offset.
-func (t *freezerTable) checkIndexItems(a, b indexEntry) error {
-	if b.filenum != a.filenum && b.filenum != a.filenum+1 {
-		return fmt.Errorf("index items with inconsistent file number, prev: %d, next: %d", a.filenum, b.filenum)
-	}
-	if b.filenum == a.filenum && b.offset < a.offset {
-		return fmt.Errorf("index items with unordered offset, prev: %d, next: %d", a.offset, b.offset)
-	}
-	if b.filenum == a.filenum+1 && b.offset == 0 {
-		return fmt.Errorf("index items with zero offset, file number: %d", b.filenum)
-	}
-	return nil
-}
-
 // preopen opens all files that the freezer will need. This method should be called from an init-context,
 // since it assumes that it doesn't have to bother with locking
 // The rationale for doing preopen is to not have to do it from within Retrieve, thus not needing to ever

core/rawdb/freezer_table_test.go

@@ -564,7 +564,7 @@ func TestFreezerOffset(t *testing.T) {
 	}
 
 	// Write 6 x 20 bytes, splitting out into three files
-	batch := f.newBatch()
+	batch := f.newBatch(0)
 	require.NoError(t, batch.AppendRaw(0, getChunk(20, 0xFF)))
 	require.NoError(t, batch.AppendRaw(1, getChunk(20, 0xEE)))
 
@@ -589,7 +589,7 @@ func TestFreezerOffset(t *testing.T) {
 	t.Log(f.dumpIndexString(0, 100))
 
 	// It should allow writing item 6.
-	batch = f.newBatch()
+	batch = f.newBatch(6)
 	require.NoError(t, batch.AppendRaw(6, getChunk(20, 0x99)))
 	require.NoError(t, batch.commit())
 
@@ -1540,7 +1540,7 @@ func TestFlushOffsetTracking(t *testing.T) {
 			// Data files:
 			//   F1(10 items) -> F2(10 items) -> F3(10 items) -> F4(10 items, full)
 			func(f *freezerTable) {
-				batch := f.newBatch()
+				batch := f.newBatch(0)
 				for i := 0; i < 5; i++ {
 					batch.AppendRaw(items+uint64(i), make([]byte, dataSize))
 				}
@@ -1554,7 +1554,7 @@ func TestFlushOffsetTracking(t *testing.T) {
 			// Data files:
 			//   F1(10 items) -> F2(10 items) -> F3(10 items) -> F4(10 items) -> F5(1 item)
 			func(f *freezerTable) {
-				batch := f.newBatch()
+				batch := f.newBatch(0)
 				batch.AppendRaw(items+5, make([]byte, dataSize))
 				batch.commit()
 			},