forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-07-27 15:16:43 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

core/rawdb: rework logic

Browse source
This commit is contained in:
Marius van der Wijden 2025-03-14 16:54:26 +01:00
parent 2258c4b095
commit 365175f70c
10 changed files with 168 additions and 171 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
core/rawdb/ancient_scheme.go
View file

@ -39,11 +39,16 @@ const (
// chainFreezerNoSnappy configures whether compression is disabled for the ancient-tables.
// Hashes and difficulties don't compress well.
var chainFreezerNoSnappy = map[string]bool{
ChainFreezerHeaderTable: false,
ChainFreezerHashTable: true,
ChainFreezerBodiesTable: false,
ChainFreezerReceiptTable: false,
var chainFreezerNoSnappy = map[string]freezerTableConfig{
ChainFreezerHeaderTable: freezerTableConfig{snappy: false, prunable: false},
ChainFreezerHashTable: freezerTableConfig{snappy: true, prunable: false},
ChainFreezerBodiesTable: freezerTableConfig{snappy: false, prunable: true},
ChainFreezerReceiptTable: freezerTableConfig{snappy: false, prunable: true},
}
type freezerTableConfig struct {
snappy bool
prunable bool
}
const (
@ -59,12 +64,12 @@ const (
)
// stateFreezerNoSnappy configures whether compression is disabled for the state freezer.
var stateFreezerNoSnappy = map[string]bool{
stateHistoryMeta: true,
stateHistoryAccountIndex: false,
stateHistoryStorageIndex: false,
stateHistoryAccountData: false,
stateHistoryStorageData: false,
var stateFreezerNoSnappy = map[string]freezerTableConfig{
stateHistoryMeta: freezerTableConfig{snappy: true, prunable: false},
stateHistoryAccountIndex: freezerTableConfig{snappy: false, prunable: false},
stateHistoryStorageIndex: freezerTableConfig{snappy: false, prunable: false},
stateHistoryAccountData: freezerTableConfig{snappy: false, prunable: false},
stateHistoryStorageData: freezerTableConfig{snappy: false, prunable: false},
}
// The list of identifiers of ancient stores.

4
core/rawdb/ancient_utils.go
View file

@ -51,7 +51,7 @@ func (info *freezerInfo) size() common.StorageSize {
return total
}
func inspect(name string, order map[string]bool, reader ethdb.AncientReader) (freezerInfo, error) {
func inspect(name string, order map[string]freezerTableConfig, reader ethdb.AncientReader) (freezerInfo, error) {
info := freezerInfo{name: name}
for t := range order {
size, err := reader.AncientSize(t)
@ -119,7 +119,7 @@ func inspectFreezers(db ethdb.Database) ([]freezerInfo, error) {
func InspectFreezerTable(ancient string, freezerName string, tableName string, start, end int64) error {
var (
path string
tables map[string]bool
tables map[string]freezerTableConfig
)
switch freezerName {
case ChainFreezerName:

128
core/rawdb/freezer.go
View file

@ -78,7 +78,7 @@ type Freezer struct {
//
// The 'tables' argument defines the data tables. If the value of a map
// entry is true, snappy compression is disabled for the table.
func NewFreezer(datadir string, namespace string, readonly bool, maxTableSize uint32, tables map[string]bool) (*Freezer, error) {
func NewFreezer(datadir string, namespace string, readonly bool, maxTableSize uint32, tables map[string]freezerTableConfig) (*Freezer, error) {
// Create the initial freezer object
var (
readMeter = metrics.NewRegisteredMeter(namespace+"ancient/read", nil)
@ -121,8 +121,8 @@ func NewFreezer(datadir string, namespace string, readonly bool, maxTableSize ui
}
// Create the tables.
for name, disableSnappy := range tables {
table, err := newTable(datadir, name, readMeter, writeMeter, sizeGauge, maxTableSize, disableSnappy, readonly)
for name, config := range tables {
table, err := newTable(datadir, name, readMeter, writeMeter, sizeGauge, maxTableSize, config, readonly)
if err != nil {
for _, table := range freezer.tables {
table.Close()
@ -384,87 +384,79 @@ func (f *Freezer) validate() error {
return nil
}
check := func(tables map[string]*freezerTable) (uint64, uint64, error) {
var (
head uint64
tail uint64
name string
)
// Hack to get boundary of any table
for kind, table := range tables {
head = table.items.Load()
tail = table.itemHidden.Load()
name = kind
break
}
// Now check every table against those boundaries.
for kind, table := range tables {
if head != table.items.Load() {
return 0, 0, fmt.Errorf("freezer tables %s and %s have differing head: %d != %d", kind, name, table.items.Load(), head)
}
if tail != table.itemHidden.Load() {
return 0, 0, fmt.Errorf("freezer tables %s and %s have differing tail: %d != %d", kind, name, table.itemHidden.Load(), tail)
}
}
return head, tail, nil
var (
head uint64
prunedTail *uint64
)
// hack to get the head
for _, table := range f.tables {
head = table.items.Load()
break
}
headTables, otherTables := splitTables(f.tables)
// verify that all the non-header tables have the same size
head, tail, err := check(otherTables)
if err != nil {
return err
}
// verify that the header tables have the same size
if len(headTables) != 0 {
head, tail, err = check(headTables)
if err != nil {
return err
for kind, table := range f.tables {
// all tables have to have the same head
if head != table.items.Load() {
return fmt.Errorf("freezer table %s has a differing head: %d != %d", kind, table.items.Load(), head)
}
if !table.config.prunable {
// non-prunable tables have to start at 0
if table.itemHidden.Load() != 0 {
return fmt.Errorf("freezer table %s has a differing head: %d != %d", kind, table.items.Load(), 0)
}
} else {
// prunable tables have to have the same length
if prunedTail == nil {
tmp := table.itemHidden.Load()
prunedTail = &tmp
}
if *prunedTail != table.itemHidden.Load() {
return fmt.Errorf("freezer table %s has differing tail: %d != %d", kind, table.itemHidden.Load(), *prunedTail)
}
}
}
if prunedTail == nil {
tmp := uint64(0)
prunedTail = &tmp
}
f.frozen.Store(head)
f.tail.Store(tail)
f.tail.Store(*prunedTail)
return nil
}
// repair truncates all data tables to the same length.
func (f *Freezer) repair() error {
repair := func(tables map[string]*freezerTable) (uint64, uint64, error) {
var (
head = uint64(math.MaxUint64)
tail = uint64(0)
)
for _, table := range tables {
head = min(head, table.items.Load())
tail = max(tail, table.itemHidden.Load())
}
for _, table := range tables {
if err := table.truncateHead(head); err != nil {
return 0, 0, err
}
if err := table.truncateTail(tail); err != nil {
return 0, 0, err
}
}
return head, tail, nil
var (
head = uint64(math.MaxUint64)
prunedTail = uint64(0)
)
// get the minimal head and the maximum tail
for _, table := range f.tables {
head = min(head, table.items.Load())
prunedTail = max(prunedTail, table.itemHidden.Load())
}
headTables, otherTables := splitTables(f.tables)
// verify that all the non-header tables have the same size
head, tail, err := repair(otherTables)
if err != nil {
return err
}
// verify that the header tables have the same size
if len(headTables) != 0 {
head, tail, err = repair(headTables)
if err != nil {
// apply the pruning
for kind, table := range f.tables {
// all tables need to have the same head
if err := table.truncateHead(head); err != nil {
return err
}
if !table.config.prunable {
// non-prunable tables have to start at 0
if table.itemHidden.Load() != 0 {
panic(fmt.Sprintf("freezer table %s has non-zero tail: %v", kind, table.itemHidden.Load()))
}
} else {
// prunable tables have to have the same length
if err := table.truncateTail(prunedTail); err != nil {
return err
}
}
}
f.frozen.Store(head)
f.tail.Store(tail)
f.tail.Store(prunedTail)
return nil
}

2
core/rawdb/freezer_batch.go
View file

@ -96,7 +96,7 @@ type freezerTableBatch struct {
// newBatch creates a new batch for the freezer table.
func (t *freezerTable) newBatch() *freezerTableBatch {
batch := &freezerTableBatch{t: t}
if !t.noCompression {
if !t.config.snappy {
batch.sb = new(snappyBuffer)
}
batch.reset()

2
core/rawdb/freezer_memory.go
View file

@ -218,7 +218,7 @@ type MemoryFreezer struct {
}
// NewMemoryFreezer initializes an in-memory freezer instance.
func NewMemoryFreezer(readonly bool, tableName map[string]bool) *MemoryFreezer {
func NewMemoryFreezer(readonly bool, tableName map[string]freezerTableConfig) *MemoryFreezer {
tables := make(map[string]*memoryTable)
for name := range tableName {
tables[name] = newMemoryTable(name)

8
core/rawdb/freezer_memory_test.go
View file

@ -25,16 +25,16 @@ import (
func TestMemoryFreezer(t *testing.T) {
ancienttest.TestAncientSuite(t, func(kinds []string) ethdb.AncientStore {
tables := make(map[string]bool)
tables := make(map[string]freezerTableConfig)
for _, kind := range kinds {
tables[kind] = true
tables[kind] = freezerTableConfig{snappy: true}
}
return NewMemoryFreezer(false, tables)
})
ancienttest.TestResettableAncientSuite(t, func(kinds []string) ethdb.ResettableAncientStore {
tables := make(map[string]bool)
tables := make(map[string]freezerTableConfig)
for _, kind := range kinds {
tables[kind] = true
tables[kind] = freezerTableConfig{snappy: true}
}
return NewMemoryFreezer(false, tables)
})

2
core/rawdb/freezer_resettable.go
View file

@ -49,7 +49,7 @@ type resettableFreezer struct {
//
// The reset function will delete directory atomically and re-create the
// freezer from scratch.
func newResettableFreezer(datadir string, namespace string, readonly bool, maxTableSize uint32, tables map[string]bool) (*resettableFreezer, error) {
func newResettableFreezer(datadir string, namespace string, readonly bool, maxTableSize uint32, tables map[string]freezerTableConfig) (*resettableFreezer, error) {
if err := cleanup(datadir); err != nil {
return nil, err
}

50
core/rawdb/freezer_table.go
View file

@ -100,11 +100,11 @@ type freezerTable struct {
// should never be lower than itemOffset.
itemHidden atomic.Uint64
noCompression bool // if true, disables snappy compression. Note: does not work retroactively
readonly bool
maxFileSize uint32 // Max file size for data-files
name string
path string
config freezerTableConfig // if true, disables snappy compression. Note: does not work retroactively
readonly bool
maxFileSize uint32 // Max file size for data-files
name string
path string
head *os.File // File descriptor for the data head of the table
index *os.File // File descriptor for the indexEntry file of the table
@ -125,20 +125,20 @@ type freezerTable struct {
}
// newFreezerTable opens the given path as a freezer table.
func newFreezerTable(path, name string, disableSnappy, readonly bool) (*freezerTable, error) {
return newTable(path, name, metrics.NewInactiveMeter(), metrics.NewInactiveMeter(), metrics.NewGauge(), freezerTableSize, disableSnappy, readonly)
func newFreezerTable(path, name string, config freezerTableConfig, readonly bool) (*freezerTable, error) {
return newTable(path, name, metrics.NewInactiveMeter(), metrics.NewInactiveMeter(), metrics.NewGauge(), freezerTableSize, config, readonly)
}
// newTable opens a freezer table, creating the data and index files if they are
// non-existent. Both files are truncated to the shortest common length to ensure
// they don't go out of sync.
func newTable(path string, name string, readMeter, writeMeter *metrics.Meter, sizeGauge *metrics.Gauge, maxFilesize uint32, noCompression, readonly bool) (*freezerTable, error) {
func newTable(path string, name string, readMeter, writeMeter *metrics.Meter, sizeGauge *metrics.Gauge, maxFilesize uint32, config freezerTableConfig, readonly bool) (*freezerTable, error) {
// Ensure the containing directory exists and open the indexEntry file
if err := os.MkdirAll(path, 0755); err != nil {
return nil, err
}
var idxName string
if noCompression {
if config.snappy {
idxName = fmt.Sprintf("%s.ridx", name) // raw index file
} else {
idxName = fmt.Sprintf("%s.cidx", name) // compressed index file
@ -176,19 +176,19 @@ func newTable(path string, name string, readMeter, writeMeter *metrics.Meter, si
}
// Create the table and repair any past inconsistency
tab := &freezerTable{
index: index,
metadata: metadata,
lastSync: time.Now(),
files: make(map[uint32]*os.File),
readMeter: readMeter,
writeMeter: writeMeter,
sizeGauge: sizeGauge,
name: name,
path: path,
logger: log.New("database", path, "table", name),
noCompression: noCompression,
readonly: readonly,
maxFileSize: maxFilesize,
index: index,
metadata: metadata,
lastSync: time.Now(),
files: make(map[uint32]*os.File),
readMeter: readMeter,
writeMeter: writeMeter,
sizeGauge: sizeGauge,
name: name,
path: path,
logger: log.New("database", path, "table", name),
config: config,
readonly: readonly,
maxFileSize: maxFilesize,
}
if err := tab.repair(); err != nil {
tab.Close()
@ -871,7 +871,7 @@ func (t *freezerTable) openFile(num uint32, opener func(string) (*os.File, error
var exist bool
if f, exist = t.files[num]; !exist {
var name string
if t.noCompression {
if t.config.snappy {
name = fmt.Sprintf("%s.%04d.rdat", t.name, num)
} else {
name = fmt.Sprintf("%s.%04d.cdat", t.name, num)
@ -987,13 +987,13 @@ func (t *freezerTable) RetrieveItems(start, count, maxBytes uint64) ([][]byte, e
item := diskData[offset : offset+diskSize]
offset += diskSize
decompressedSize := diskSize
if !t.noCompression {
if !t.config.snappy {
decompressedSize, _ = snappy.DecodedLen(item)
}
if i > 0 && maxBytes != 0 && uint64(outputSize+decompressedSize) > maxBytes {
break
}
if !t.noCompression {
if !t.config.snappy {
data, err := snappy.Decode(nil, item)
if err != nil {
return nil, err

94
core/rawdb/freezer_table_test.go
View file

@ -39,7 +39,7 @@ func TestFreezerBasics(t *testing.T) {
// set cutoff at 50 bytes
f, err := newTable(os.TempDir(),
fmt.Sprintf("unittest-%d", rand.Uint64()),
metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, false)
metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, freezerTableConfig{snappy: true}, false)
if err != nil {
t.Fatal(err)
}
@ -84,7 +84,7 @@ func TestFreezerBasicsClosing(t *testing.T) {
f *freezerTable
err error
)
f, err = newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
f, err = newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{snappy: true}, false)
if err != nil {
t.Fatal(err)
}
@ -98,7 +98,7 @@ func TestFreezerBasicsClosing(t *testing.T) {
require.NoError(t, batch.commit())
f.Close()
f, err = newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
f, err = newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{snappy: true}, false)
if err != nil {
t.Fatal(err)
}
@ -115,7 +115,7 @@ func TestFreezerBasicsClosing(t *testing.T) {
t.Fatalf("test %d, got \n%x != \n%x", y, got, exp)
}
f.Close()
f, err = newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
f, err = newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{snappy: true}, false)
if err != nil {
t.Fatal(err)
}
@ -130,7 +130,7 @@ func TestFreezerRepairDanglingHead(t *testing.T) {
// Fill table
{
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{snappy: true}, false)
if err != nil {
t.Fatal(err)
}
@ -159,7 +159,7 @@ func TestFreezerRepairDanglingHead(t *testing.T) {
// Now open it again
{
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{snappy: true}, false)
if err != nil {
t.Fatal(err)
}
@ -182,7 +182,7 @@ func TestFreezerRepairDanglingHeadLarge(t *testing.T) {
// Fill a table and close it
{
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{snappy: true}, false)
if err != nil {
t.Fatal(err)
}
@ -208,7 +208,7 @@ func TestFreezerRepairDanglingHeadLarge(t *testing.T) {
// Now open it again
{
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{snappy: true}, false)
if err != nil {
t.Fatal(err)
}
@ -231,7 +231,7 @@ func TestFreezerRepairDanglingHeadLarge(t *testing.T) {
// And if we open it, we should now be able to read all of them (new values)
{
f, _ := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
f, _ := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{snappy: true}, false)
for y := 1; y < 255; y++ {
exp := getChunk(15, ^y)
got, err := f.Retrieve(uint64(y))
@ -253,7 +253,7 @@ func TestSnappyDetection(t *testing.T) {
// Open with snappy
{
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{snappy: true}, false)
if err != nil {
t.Fatal(err)
}
@ -264,7 +264,7 @@ func TestSnappyDetection(t *testing.T) {
// Open with snappy
{
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{snappy: true}, false)
if err != nil {
t.Fatal(err)
}
@ -277,7 +277,7 @@ func TestSnappyDetection(t *testing.T) {
// Open without snappy
{
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, false, false)
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{snappy: false}, false)
if err != nil {
t.Fatal(err)
}
@ -308,7 +308,7 @@ func TestFreezerRepairDanglingIndex(t *testing.T) {
// Fill a table and close it
{
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{snappy: true}, false)
if err != nil {
t.Fatal(err)
}
@ -344,7 +344,7 @@ func TestFreezerRepairDanglingIndex(t *testing.T) {
// 45, 45, 15
// with 3+3+1 items
{
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{snappy: true}, false)
if err != nil {
t.Fatal(err)
}
@ -365,7 +365,7 @@ func TestFreezerTruncate(t *testing.T) {
// Fill table
{
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{snappy: true}, false)
if err != nil {
t.Fatal(err)
}
@ -381,7 +381,7 @@ func TestFreezerTruncate(t *testing.T) {
// Reopen, truncate
{
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{snappy: true}, false)
if err != nil {
t.Fatal(err)
}
@ -406,7 +406,7 @@ func TestFreezerRepairFirstFile(t *testing.T) {
// Fill table
{
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{snappy: true}, false)
if err != nil {
t.Fatal(err)
}
@ -439,7 +439,7 @@ func TestFreezerRepairFirstFile(t *testing.T) {
// Reopen
{
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{snappy: true}, false)
if err != nil {
t.Fatal(err)
}
@ -474,7 +474,7 @@ func TestFreezerReadAndTruncate(t *testing.T) {
// Fill table
{
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{snappy: true}, false)
if err != nil {
t.Fatal(err)
}
@ -490,7 +490,7 @@ func TestFreezerReadAndTruncate(t *testing.T) {
// Reopen and read all files
{
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{snappy: true}, false)
if err != nil {
t.Fatal(err)
}
@ -521,7 +521,7 @@ func TestFreezerOffset(t *testing.T) {
fname := fmt.Sprintf("offset-%d", rand.Uint64())
// Fill table
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false)
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, freezerTableConfig{snappy: true}, false)
if err != nil {
t.Fatal(err)
}
@ -545,7 +545,7 @@ func TestFreezerOffset(t *testing.T) {
f.Close()
// Now open again
f, err = newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false)
f, err = newTable(os.TempDir(), fname, rm, wm, sg, 40, freezerTableConfig{snappy: true}, false)
if err != nil {
t.Fatal(err)
}
@ -597,7 +597,7 @@ func TestFreezerOffset(t *testing.T) {
// Check that existing items have been moved to index 1M.
{
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false)
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, freezerTableConfig{snappy: true}, false)
if err != nil {
t.Fatal(err)
}
@ -631,7 +631,7 @@ func TestTruncateTail(t *testing.T) {
fname := fmt.Sprintf("truncate-tail-%d", rand.Uint64())
// Fill table
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false)
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, freezerTableConfig{snappy: true}, false)
if err != nil {
t.Fatal(err)
}
@ -682,7 +682,7 @@ func TestTruncateTail(t *testing.T) {
// Reopen the table, the deletion information should be persisted as well
f.Close()
f, err = newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false)
f, err = newTable(os.TempDir(), fname, rm, wm, sg, 40, freezerTableConfig{snappy: true}, false)
if err != nil {
t.Fatal(err)
}
@ -716,7 +716,7 @@ func TestTruncateTail(t *testing.T) {
// Reopen the table, the above testing should still pass
f.Close()
f, err = newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false)
f, err = newTable(os.TempDir(), fname, rm, wm, sg, 40, freezerTableConfig{snappy: true}, false)
if err != nil {
t.Fatal(err)
}
@ -772,7 +772,7 @@ func TestTruncateHead(t *testing.T) {
fname := fmt.Sprintf("truncate-head-blow-tail-%d", rand.Uint64())
// Fill table
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false)
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, freezerTableConfig{snappy: true}, false)
if err != nil {
t.Fatal(err)
}
@ -883,7 +883,7 @@ func TestSequentialRead(t *testing.T) {
rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge()
fname := fmt.Sprintf("batchread-%d", rand.Uint64())
{ // Fill table
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{snappy: true}, false)
if err != nil {
t.Fatal(err)
}
@ -893,7 +893,7 @@ func TestSequentialRead(t *testing.T) {
f.Close()
}
{ // Open it, iterate, verify iteration
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{snappy: true}, false)
if err != nil {
t.Fatal(err)
}
@ -914,7 +914,7 @@ func TestSequentialRead(t *testing.T) {
}
{ // Open it, iterate, verify byte limit. The byte limit is less than item
// size, so each lookup should only return one item
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false)
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, freezerTableConfig{snappy: true}, false)
if err != nil {
t.Fatal(err)
}
@ -943,7 +943,7 @@ func TestSequentialReadByteLimit(t *testing.T) {
rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge()
fname := fmt.Sprintf("batchread-2-%d", rand.Uint64())
{ // Fill table
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, true, false)
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, freezerTableConfig{snappy: true}, false)
if err != nil {
t.Fatal(err)
}
@ -965,7 +965,7 @@ func TestSequentialReadByteLimit(t *testing.T) {
{100, 109, 10},
} {
{
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, true, false)
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, freezerTableConfig{snappy: true}, false)
if err != nil {
t.Fatal(err)
}
@ -993,7 +993,7 @@ func TestSequentialReadNoByteLimit(t *testing.T) {
rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge()
fname := fmt.Sprintf("batchread-3-%d", rand.Uint64())
{ // Fill table
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, true, false)
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, freezerTableConfig{snappy: true}, false)
if err != nil {
t.Fatal(err)
}
@ -1011,7 +1011,7 @@ func TestSequentialReadNoByteLimit(t *testing.T) {
{31, 30},
} {
{
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, true, false)
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, freezerTableConfig{snappy: true}, false)
if err != nil {
t.Fatal(err)
}
@ -1038,7 +1038,7 @@ func TestFreezerReadonly(t *testing.T) {
// Case 1: Check it fails on non-existent file.
_, err := newTable(tmpdir,
fmt.Sprintf("readonlytest-%d", rand.Uint64()),
metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, true)
metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, freezerTableConfig{snappy: true}, true)
if err == nil {
t.Fatal("readonly table instantiation should fail for non-existent table")
}
@ -1053,7 +1053,7 @@ func TestFreezerReadonly(t *testing.T) {
idxFile.Write(make([]byte, 17))
idxFile.Close()
_, err = newTable(tmpdir, fname,
metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, true)
metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, freezerTableConfig{snappy: true}, true)
if err == nil {
t.Errorf("readonly table instantiation should fail for invalid index size")
}
@ -1063,7 +1063,7 @@ func TestFreezerReadonly(t *testing.T) {
// again in readonly triggers an error.
fname = fmt.Sprintf("readonlytest-%d", rand.Uint64())
f, err := newTable(tmpdir, fname,
metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, false)
metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, freezerTableConfig{snappy: true}, false)
if err != nil {
t.Fatalf("failed to instantiate table: %v", err)
}
@ -1076,7 +1076,7 @@ func TestFreezerReadonly(t *testing.T) {
t.Fatal(err)
}
_, err = newTable(tmpdir, fname,
metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, true)
metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, freezerTableConfig{snappy: true}, true)
if err == nil {
t.Errorf("readonly table instantiation should fail for corrupt table file")
}
@ -1085,7 +1085,7 @@ func TestFreezerReadonly(t *testing.T) {
// Should be successful.
fname = fmt.Sprintf("readonlytest-%d", rand.Uint64())
f, err = newTable(tmpdir, fname,
metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, false)
metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, freezerTableConfig{snappy: true}, false)
if err != nil {
t.Fatalf("failed to instantiate table: %v\n", err)
}
@ -1094,7 +1094,7 @@ func TestFreezerReadonly(t *testing.T) {
t.Fatal(err)
}
f, err = newTable(tmpdir, fname,
metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, true)
metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, freezerTableConfig{snappy: true}, true)
if err != nil {
t.Fatal(err)
}
@ -1234,7 +1234,7 @@ func (randTest) Generate(r *rand.Rand, size int) reflect.Value {
func runRandTest(rt randTest) bool {
fname := fmt.Sprintf("randtest-%d", rand.Uint64())
f, err := newTable(os.TempDir(), fname, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, false)
f, err := newTable(os.TempDir(), fname, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, freezerTableConfig{snappy: true}, false)
if err != nil {
panic("failed to initialize table")
}
@ -1243,7 +1243,7 @@ func runRandTest(rt randTest) bool {
switch step.op {
case opReload:
f.Close()
f, err = newTable(os.TempDir(), fname, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, false)
f, err = newTable(os.TempDir(), fname, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, freezerTableConfig{snappy: true}, false)
if err != nil {
rt[i].err = fmt.Errorf("failed to reload table %v", err)
}
@ -1381,7 +1381,7 @@ func TestIndexValidation(t *testing.T) {
}
for _, c := range cases {
fn := fmt.Sprintf("t-%d", rand.Uint64())
f, err := newTable(os.TempDir(), fn, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 10*dataSize, true, false)
f, err := newTable(os.TempDir(), fn, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 10*dataSize, freezerTableConfig{snappy: true}, false)
if err != nil {
t.Fatal(err)
}
@ -1392,7 +1392,7 @@ func TestIndexValidation(t *testing.T) {
f.Close()
// reopen the table, corruption should be truncated
f, err = newTable(os.TempDir(), fn, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 100, true, false)
f, err = newTable(os.TempDir(), fn, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 100, freezerTableConfig{snappy: true}, false)
if err != nil {
t.Fatal(err)
}
@ -1422,7 +1422,7 @@ func TestFlushOffsetTracking(t *testing.T) {
fileSize = 100
)
fn := fmt.Sprintf("t-%d", rand.Uint64())
f, err := newTable(os.TempDir(), fn, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), fileSize, true, false)
f, err := newTable(os.TempDir(), fn, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), fileSize, freezerTableConfig{snappy: true}, false)
if err != nil {
t.Fatal(err)
}
@ -1533,7 +1533,7 @@ func TestTailTruncationCrash(t *testing.T) {
fileSize = 100
)
fn := fmt.Sprintf("t-%d", rand.Uint64())
f, err := newTable(os.TempDir(), fn, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), fileSize, true, false)
f, err := newTable(os.TempDir(), fn, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), fileSize, freezerTableConfig{snappy: true}, false)
if err != nil {
t.Fatal(err)
}
@ -1563,7 +1563,7 @@ func TestTailTruncationCrash(t *testing.T) {
// the offset
f.metadata.setFlushOffset(31*indexEntrySize, true)
f, err = newTable(os.TempDir(), fn, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), fileSize, true, false)
f, err = newTable(os.TempDir(), fn, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), fileSize, freezerTableConfig{snappy: true}, false)
if err != nil {
t.Fatal(err)
}

22
core/rawdb/freezer_test.go
View file

@ -31,7 +31,7 @@ import (
"github.com/stretchr/testify/require"
)
var freezerTestTableDef = map[string]bool{"test": true}
var freezerTestTableDef = map[string]freezerTableConfig{"test": {snappy: true}}
func TestFreezerModify(t *testing.T) {
t.Parallel()
@ -47,7 +47,7 @@ func TestFreezerModify(t *testing.T) {
valuesRLP = append(valuesRLP, iv)
}
tables := map[string]bool{"raw": true, "rlp": false}
tables := map[string]freezerTableConfig{"raw": {snappy: true}, "rlp": {snappy: false}}
f, _ := newFreezerForTesting(t, tables)
defer f.Close()
@ -111,7 +111,7 @@ func TestFreezerModifyRollback(t *testing.T) {
f.Close()
// Reopen and check that the rolled-back data doesn't reappear.
tables := map[string]bool{"test": true}
tables := map[string]freezerTableConfig{"test": {snappy: true}}
f2, err := NewFreezer(dir, "", false, 2049, tables)
if err != nil {
t.Fatalf("can't reopen freezer after failed ModifyAncients: %v", err)
@ -249,7 +249,7 @@ func TestFreezerConcurrentModifyTruncate(t *testing.T) {
}
func TestFreezerReadonlyValidate(t *testing.T) {
tables := map[string]bool{"a": true, "b": true}
tables := map[string]freezerTableConfig{"a": {snappy: true}, "b": {snappy: true}}
dir := t.TempDir()
// Open non-readonly freezer and fill individual tables
// with different amount of data.
@ -285,7 +285,7 @@ func TestFreezerReadonlyValidate(t *testing.T) {
func TestFreezerConcurrentReadonly(t *testing.T) {
t.Parallel()
tables := map[string]bool{"a": true}
tables := map[string]freezerTableConfig{"a": {snappy: true}}
dir := t.TempDir()
f, err := NewFreezer(dir, "", false, 2049, tables)
@ -333,7 +333,7 @@ func TestFreezerConcurrentReadonly(t *testing.T) {
}
}
func newFreezerForTesting(t *testing.T, tables map[string]bool) (*Freezer, string) {
func newFreezerForTesting(t *testing.T, tables map[string]freezerTableConfig) (*Freezer, string) {
t.Helper()
dir := t.TempDir()
@ -379,7 +379,7 @@ func checkAncientCount(t *testing.T, f *Freezer, kind string, n uint64) {
func TestFreezerCloseSync(t *testing.T) {
t.Parallel()
f, _ := newFreezerForTesting(t, map[string]bool{"a": true, "b": true})
f, _ := newFreezerForTesting(t, map[string]freezerTableConfig{"a": {snappy: true}, "b": {snappy: true}})
defer f.Close()
// Now, close and sync. This mimics the behaviour if the node is shut down,
@ -401,17 +401,17 @@ func TestFreezerCloseSync(t *testing.T) {
func TestFreezerSuite(t *testing.T) {
ancienttest.TestAncientSuite(t, func(kinds []string) ethdb.AncientStore {
tables := make(map[string]bool)
tables := make(map[string]freezerTableConfig)
for _, kind := range kinds {
tables[kind] = true
tables[kind] = freezerTableConfig{snappy: true}
}
f, _ := newFreezerForTesting(t, tables)
return f
})
ancienttest.TestResettableAncientSuite(t, func(kinds []string) ethdb.ResettableAncientStore {
tables := make(map[string]bool)
tables := make(map[string]freezerTableConfig)
for _, kind := range kinds {
tables[kind] = true
tables[kind] = freezerTableConfig{snappy: true}
}
f, _ := newResettableFreezer(t.TempDir(), "", false, 2048, tables)
return f