core/rawdb/freezer_table: add sizeHidden and sizeHiddenNoLock to freezerTable

core/rawdb/freezer: sizeHidden should be subtracted from Freezer.AncientSize

core/rawdb/freezer.go

@@ -226,7 +226,15 @@ func (f *Freezer) AncientSize(kind string) (uint64, error) {
 	defer f.writeLock.RUnlock()
 
 	if table := f.tables[kind]; table != nil {
-		return table.size()
+		size, err := table.size()
+		if err != nil {
+			return 0, err
+		}
+		hiddenSize, err := table.sizeHidden()
+		if err != nil {
+			return 0, err
+		}
+		return size - hiddenSize, nil
 	}
 	return 0, errUnknownTable
 }

core/rawdb/freezer_table.go

@@ -853,6 +853,39 @@ func (t *freezerTable) sizeNolock() (uint64, error) {
 	return total, nil
 }
 
+// sizeHidden returns the total data size of hidden items in the freezer table.
+func (t *freezerTable) sizeHidden() (uint64, error) {
+	t.lock.RLock()
+	defer t.lock.RUnlock()
+
+	return t.sizeHiddenNoLock()
+}
+
+// sizeHiddenNoLock returns the total data size of hidden items in the freezer table
+// without obtaining the mutex first.
+func (t *freezerTable) sizeHiddenNoLock() (uint64, error) {
+	hiddenItems := t.itemHidden.Load()
+	offsetItems := t.itemOffset.Load()
+	if hiddenItems <= offsetItems {
+		// Return 0 if there is no hidden items
+		return 0, nil
+	}
+
+	indices, err := t.getIndices(offsetItems, hiddenItems-offsetItems)
+	if err != nil {
+		return 0, err
+	}
+
+	var totalSize int = 0
+	for i, firstIndex := range indices[:len(indices)-1] {
+		secondIndex := indices[i+1]
+		offset1, offset2, _ := firstIndex.bounds(secondIndex)
+		size := int(offset2 - offset1)
+		totalSize += size
+	}
+	return uint64(totalSize), nil
+}
+
 // advanceHead should be called when the current head file would outgrow the file limits,
 // and a new file must be opened. The caller of this method must hold the write-lock
 // before calling this method.

core/rawdb/freezer_table_test.go

@@ -662,9 +662,10 @@ func TestTruncateTail(t *testing.T) {
 	t.Parallel()
 	rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge()
 	fname := fmt.Sprintf("truncate-tail-%d", rand.Uint64())
+	var maxFileSize uint32 = 50
 
 	// Fill table
-	f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false)
+	f, err := newTable(os.TempDir(), fname, rm, wm, sg, maxFileSize, true, false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -672,12 +673,12 @@ func TestTruncateTail(t *testing.T) {
 	// Write 7 x 20 bytes, splitting out into four files
 	batch := f.newBatch()
 	require.NoError(t, batch.AppendRaw(0, getChunk(20, 0xFF)))
-	require.NoError(t, batch.AppendRaw(1, getChunk(20, 0xEE)))
+	require.NoError(t, batch.AppendRaw(1, getChunk(21, 0xEE)))
-	require.NoError(t, batch.AppendRaw(2, getChunk(20, 0xdd)))
+	require.NoError(t, batch.AppendRaw(2, getChunk(22, 0xdd)))
-	require.NoError(t, batch.AppendRaw(3, getChunk(20, 0xcc)))
+	require.NoError(t, batch.AppendRaw(3, getChunk(23, 0xcc)))
-	require.NoError(t, batch.AppendRaw(4, getChunk(20, 0xbb)))
+	require.NoError(t, batch.AppendRaw(4, getChunk(24, 0xbb)))
-	require.NoError(t, batch.AppendRaw(5, getChunk(20, 0xaa)))
+	require.NoError(t, batch.AppendRaw(5, getChunk(25, 0xaa)))
-	require.NoError(t, batch.AppendRaw(6, getChunk(20, 0x11)))
+	require.NoError(t, batch.AppendRaw(6, getChunk(26, 0x11)))
 	require.NoError(t, batch.commit())
 
 	// nothing to do, all the items should still be there.
@@ -685,32 +686,34 @@ func TestTruncateTail(t *testing.T) {
 	fmt.Println(f.dumpIndexString(0, 1000))
 	checkRetrieve(t, f, map[uint64][]byte{
 		0: getChunk(20, 0xFF),
-		1: getChunk(20, 0xEE),
+		1: getChunk(21, 0xEE),
-		2: getChunk(20, 0xdd),
+		2: getChunk(22, 0xdd),
-		3: getChunk(20, 0xcc),
+		3: getChunk(23, 0xcc),
-		4: getChunk(20, 0xbb),
+		4: getChunk(24, 0xbb),
-		5: getChunk(20, 0xaa),
+		5: getChunk(25, 0xaa),
-		6: getChunk(20, 0x11),
+		6: getChunk(26, 0x11),
 	})
 
 	// truncate single element( item 0 ), deletion is only supported at file level
+	// hiddenSize should be 20 (size of item 0)
 	f.truncateTail(1)
 	fmt.Println(f.dumpIndexString(0, 1000))
 	checkRetrieveError(t, f, map[uint64]error{
 		0: errOutOfBounds,
 	})
 	checkRetrieve(t, f, map[uint64][]byte{
-		1: getChunk(20, 0xEE),
+		1: getChunk(21, 0xEE),
-		2: getChunk(20, 0xdd),
+		2: getChunk(22, 0xdd),
-		3: getChunk(20, 0xcc),
+		3: getChunk(23, 0xcc),
-		4: getChunk(20, 0xbb),
+		4: getChunk(24, 0xbb),
-		5: getChunk(20, 0xaa),
+		5: getChunk(25, 0xaa),
-		6: getChunk(20, 0x11),
+		6: getChunk(26, 0x11),
 	})
+	checkSizeHidden(t, f, 20)
 
 	// Reopen the table, the deletion information should be persisted as well
 	f.Close()
-	f, err = newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false)
+	f, err = newTable(os.TempDir(), fname, rm, wm, sg, maxFileSize, true, false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -718,31 +721,33 @@ func TestTruncateTail(t *testing.T) {
 		0: errOutOfBounds,
 	})
 	checkRetrieve(t, f, map[uint64][]byte{
-		1: getChunk(20, 0xEE),
+		1: getChunk(21, 0xEE),
-		2: getChunk(20, 0xdd),
+		2: getChunk(22, 0xdd),
-		3: getChunk(20, 0xcc),
+		3: getChunk(23, 0xcc),
-		4: getChunk(20, 0xbb),
+		4: getChunk(24, 0xbb),
-		5: getChunk(20, 0xaa),
+		5: getChunk(25, 0xaa),
-		6: getChunk(20, 0x11),
+		6: getChunk(26, 0x11),
 	})
+	checkSizeHidden(t, f, 20)
 
 	// truncate two elements( item 0, item 1 ), the file 0 should be deleted
+	// hiddenSize should be 0, as file 0 was deleted
 	f.truncateTail(2)
 	checkRetrieveError(t, f, map[uint64]error{
 		0: errOutOfBounds,
 		1: errOutOfBounds,
 	})
 	checkRetrieve(t, f, map[uint64][]byte{
-		2: getChunk(20, 0xdd),
+		2: getChunk(22, 0xdd),
-		3: getChunk(20, 0xcc),
+		3: getChunk(23, 0xcc),
-		4: getChunk(20, 0xbb),
+		4: getChunk(24, 0xbb),
-		5: getChunk(20, 0xaa),
+		5: getChunk(25, 0xaa),
-		6: getChunk(20, 0x11),
+		6: getChunk(26, 0x11),
 	})
 
 	// Reopen the table, the above testing should still pass
 	f.Close()
-	f, err = newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false)
+	f, err = newTable(os.TempDir(), fname, rm, wm, sg, maxFileSize, true, false)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -753,13 +758,50 @@ func TestTruncateTail(t *testing.T) {
 		1: errOutOfBounds,
 	})
 	checkRetrieve(t, f, map[uint64][]byte{
-		2: getChunk(20, 0xdd),
+		2: getChunk(22, 0xdd),
-		3: getChunk(20, 0xcc),
+		3: getChunk(23, 0xcc),
-		4: getChunk(20, 0xbb),
+		4: getChunk(24, 0xbb),
-		5: getChunk(20, 0xaa),
+		5: getChunk(25, 0xaa),
-		6: getChunk(20, 0x11),
+		6: getChunk(26, 0x11),
 	})
 
+	// truncate two elements( item 0, item 1, item 2 ), the file 0 should be deleted
+	// hiddenSize should be 22 (size of item 2), as item 0 and item 1 was deleted with file 0
+	f.truncateTail(3)
+	checkRetrieveError(t, f, map[uint64]error{
+		0: errOutOfBounds,
+		1: errOutOfBounds,
+		2: errOutOfBounds,
+	})
+	checkRetrieve(t, f, map[uint64][]byte{
+		3: getChunk(23, 0xcc),
+		4: getChunk(24, 0xbb),
+		5: getChunk(25, 0xaa),
+		6: getChunk(26, 0x11),
+	})
+	checkSizeHidden(t, f, 22)
+
+	// Reopen the table, the above testing should still pass
+	f.Close()
+	f, err = newTable(os.TempDir(), fname, rm, wm, sg, maxFileSize, true, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer f.Close()
+
+	checkRetrieveError(t, f, map[uint64]error{
+		0: errOutOfBounds,
+		1: errOutOfBounds,
+		2: errOutOfBounds,
+	})
+	checkRetrieve(t, f, map[uint64][]byte{
+		3: getChunk(23, 0xcc),
+		4: getChunk(24, 0xbb),
+		5: getChunk(25, 0xaa),
+		6: getChunk(26, 0x11),
+	})
+	checkSizeHidden(t, f, 22)
+
 	// truncate all, the entire freezer should be deleted
 	f.truncateTail(7)
 	checkRetrieveError(t, f, map[uint64]error{
@@ -823,6 +865,18 @@ func TestTruncateHead(t *testing.T) {
 	})
 }
 
+func checkSizeHidden(t *testing.T, f *freezerTable, size uint64) {
+	t.Helper()
+
+	sizeHidden, err := f.sizeHidden()
+	if err != nil {
+		t.Fatalf("can't get the size of hidden items: %v", err)
+	}
+	if sizeHidden != size {
+		t.Fatalf("the size of hidden items has wrong value %d (want %d)", sizeHidden, size)
+	}
+}
+
 func checkRetrieve(t *testing.T, f *freezerTable, items map[uint64][]byte) {
 	t.Helper()