forked from forks/go-ethereum
core/rawdb: allow for truncation in the freezer (#31362)
Here we add the notion of prunable tables for the `TruncateTail` operation in the freezer. TruncateTail for the chain freezer now only truncates the body and receipts tables, leaving headers and hashes as-is. This change also requires changing the validation/repair at startup to allow for tables with different tail. For the header and hash tables, we now require them to start at number zero. --------- Co-authored-by: Felix Lange <fjl@twurst.com> Co-authored-by: Gary Rong <garyrong0905@gmail.com>
This commit is contained in:
Marius van der Wijden
GitHub
parent
64bd21393e
commit
0f06e35115
12 changed files with 208 additions and 156 deletions
|
|
@ -37,13 +37,21 @@ const (
|
|||
ChainFreezerReceiptTable = "receipts"
|
||||
)
|
||||
|
||||
// chainFreezerNoSnappy configures whether compression is disabled for the ancient-tables.
|
||||
// Hashes and difficulties don't compress well.
|
||||
var chainFreezerNoSnappy = map[string]bool{
|
||||
ChainFreezerHeaderTable: false,
|
||||
ChainFreezerHashTable: true,
|
||||
ChainFreezerBodiesTable: false,
|
||||
ChainFreezerReceiptTable: false,
|
||||
// chainFreezerTableConfigs configures the settings for tables in the chain freezer.
|
||||
// Compression is disabled for hashes as they don't compress well. Additionally,
|
||||
// tail truncation is disabled for the header and hash tables, as these are intended
|
||||
// to be retained long-term.
|
||||
var chainFreezerTableConfigs = map[string]freezerTableConfig{
|
||||
ChainFreezerHeaderTable: {noSnappy: false, prunable: false},
|
||||
ChainFreezerHashTable: {noSnappy: true, prunable: false},
|
||||
ChainFreezerBodiesTable: {noSnappy: false, prunable: true},
|
||||
ChainFreezerReceiptTable: {noSnappy: false, prunable: true},
|
||||
}
|
||||
|
||||
// freezerTableConfig contains the settings for a freezer table.
|
||||
type freezerTableConfig struct {
|
||||
noSnappy bool // disables item compression
|
||||
prunable bool // true for tables that can be pruned by TruncateTail
|
||||
}
|
||||
|
||||
const (
|
||||
|
|
@ -58,13 +66,13 @@ const (
|
|||
stateHistoryStorageData = "storage.data"
|
||||
)
|
||||
|
||||
// stateFreezerNoSnappy configures whether compression is disabled for the state freezer.
|
||||
var stateFreezerNoSnappy = map[string]bool{
|
||||
stateHistoryMeta: true,
|
||||
stateHistoryAccountIndex: false,
|
||||
stateHistoryStorageIndex: false,
|
||||
stateHistoryAccountData: false,
|
||||
stateHistoryStorageData: false,
|
||||
// stateFreezerTableConfigs configures the settings for tables in the state freezer.
|
||||
var stateFreezerTableConfigs = map[string]freezerTableConfig{
|
||||
stateHistoryMeta: {noSnappy: true, prunable: true},
|
||||
stateHistoryAccountIndex: {noSnappy: false, prunable: true},
|
||||
stateHistoryStorageIndex: {noSnappy: false, prunable: true},
|
||||
stateHistoryAccountData: {noSnappy: false, prunable: true},
|
||||
stateHistoryStorageData: {noSnappy: false, prunable: true},
|
||||
}
|
||||
|
||||
// The list of identifiers of ancient stores.
|
||||
|
|
@ -85,7 +93,7 @@ var freezers = []string{ChainFreezerName, MerkleStateFreezerName, VerkleStateFre
|
|||
// state freezer.
|
||||
func NewStateFreezer(ancientDir string, verkle bool, readOnly bool) (ethdb.ResettableAncientStore, error) {
|
||||
if ancientDir == "" {
|
||||
return NewMemoryFreezer(readOnly, stateFreezerNoSnappy), nil
|
||||
return NewMemoryFreezer(readOnly, stateFreezerTableConfigs), nil
|
||||
}
|
||||
var name string
|
||||
if verkle {
|
||||
|
|
@ -93,5 +101,5 @@ func NewStateFreezer(ancientDir string, verkle bool, readOnly bool) (ethdb.Reset
|
|||
} else {
|
||||
name = filepath.Join(ancientDir, MerkleStateFreezerName)
|
||||
}
|
||||
return newResettableFreezer(name, "eth/db/state", readOnly, stateHistoryTableSize, stateFreezerNoSnappy)
|
||||
return newResettableFreezer(name, "eth/db/state", readOnly, stateHistoryTableSize, stateFreezerTableConfigs)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -51,7 +51,7 @@ func (info *freezerInfo) size() common.StorageSize {
|
|||
return total
|
||||
}
|
||||
|
||||
func inspect(name string, order map[string]bool, reader ethdb.AncientReader) (freezerInfo, error) {
|
||||
func inspect(name string, order map[string]freezerTableConfig, reader ethdb.AncientReader) (freezerInfo, error) {
|
||||
info := freezerInfo{name: name}
|
||||
for t := range order {
|
||||
size, err := reader.AncientSize(t)
|
||||
|
|
@ -82,7 +82,7 @@ func inspectFreezers(db ethdb.Database) ([]freezerInfo, error) {
|
|||
for _, freezer := range freezers {
|
||||
switch freezer {
|
||||
case ChainFreezerName:
|
||||
info, err := inspect(ChainFreezerName, chainFreezerNoSnappy, db)
|
||||
info, err := inspect(ChainFreezerName, chainFreezerTableConfigs, db)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -99,7 +99,7 @@ func inspectFreezers(db ethdb.Database) ([]freezerInfo, error) {
|
|||
}
|
||||
defer f.Close()
|
||||
|
||||
info, err := inspect(freezer, stateFreezerNoSnappy, f)
|
||||
info, err := inspect(freezer, stateFreezerTableConfigs, f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -119,13 +119,13 @@ func inspectFreezers(db ethdb.Database) ([]freezerInfo, error) {
|
|||
func InspectFreezerTable(ancient string, freezerName string, tableName string, start, end int64) error {
|
||||
var (
|
||||
path string
|
||||
tables map[string]bool
|
||||
tables map[string]freezerTableConfig
|
||||
)
|
||||
switch freezerName {
|
||||
case ChainFreezerName:
|
||||
path, tables = resolveChainFreezerDir(ancient), chainFreezerNoSnappy
|
||||
path, tables = resolveChainFreezerDir(ancient), chainFreezerTableConfigs
|
||||
case MerkleStateFreezerName, VerkleStateFreezerName:
|
||||
path, tables = filepath.Join(ancient, freezerName), stateFreezerNoSnappy
|
||||
path, tables = filepath.Join(ancient, freezerName), stateFreezerTableConfigs
|
||||
default:
|
||||
return fmt.Errorf("unknown freezer, supported ones: %v", freezers)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -62,9 +62,9 @@ func newChainFreezer(datadir string, namespace string, readonly bool) (*chainFre
|
|||
freezer ethdb.AncientStore
|
||||
)
|
||||
if datadir == "" {
|
||||
freezer = NewMemoryFreezer(readonly, chainFreezerNoSnappy)
|
||||
freezer = NewMemoryFreezer(readonly, chainFreezerTableConfigs)
|
||||
} else {
|
||||
freezer, err = NewFreezer(datadir, namespace, readonly, freezerTableSize, chainFreezerNoSnappy)
|
||||
freezer, err = NewFreezer(datadir, namespace, readonly, freezerTableSize, chainFreezerTableConfigs)
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
|
|||
|
|
@ -78,7 +78,7 @@ type Freezer struct {
|
|||
//
|
||||
// The 'tables' argument defines the data tables. If the value of a map
|
||||
// entry is true, snappy compression is disabled for the table.
|
||||
func NewFreezer(datadir string, namespace string, readonly bool, maxTableSize uint32, tables map[string]bool) (*Freezer, error) {
|
||||
func NewFreezer(datadir string, namespace string, readonly bool, maxTableSize uint32, tables map[string]freezerTableConfig) (*Freezer, error) {
|
||||
// Create the initial freezer object
|
||||
var (
|
||||
readMeter = metrics.NewRegisteredMeter(namespace+"ancient/read", nil)
|
||||
|
|
@ -121,8 +121,8 @@ func NewFreezer(datadir string, namespace string, readonly bool, maxTableSize ui
|
|||
}
|
||||
|
||||
// Create the tables.
|
||||
for name, disableSnappy := range tables {
|
||||
table, err := newTable(datadir, name, readMeter, writeMeter, sizeGauge, maxTableSize, disableSnappy, readonly)
|
||||
for name, config := range tables {
|
||||
table, err := newTable(datadir, name, readMeter, writeMeter, sizeGauge, maxTableSize, config, readonly)
|
||||
if err != nil {
|
||||
for _, table := range freezer.tables {
|
||||
table.Close()
|
||||
|
|
@ -301,7 +301,8 @@ func (f *Freezer) TruncateHead(items uint64) (uint64, error) {
|
|||
return oitems, nil
|
||||
}
|
||||
|
||||
// TruncateTail discards any recent data below the provided threshold number.
|
||||
// TruncateTail discards all data below the specified threshold. Note that only
|
||||
// 'prunable' tables will be truncated.
|
||||
func (f *Freezer) TruncateTail(tail uint64) (uint64, error) {
|
||||
if f.readonly {
|
||||
return 0, errReadOnly
|
||||
|
|
@ -314,8 +315,10 @@ func (f *Freezer) TruncateTail(tail uint64) (uint64, error) {
|
|||
return old, nil
|
||||
}
|
||||
for _, table := range f.tables {
|
||||
if err := table.truncateTail(tail); err != nil {
|
||||
return 0, err
|
||||
if table.config.prunable {
|
||||
if err := table.truncateTail(tail); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
}
|
||||
f.tail.Store(tail)
|
||||
|
|
@ -343,56 +346,77 @@ func (f *Freezer) validate() error {
|
|||
return nil
|
||||
}
|
||||
var (
|
||||
head uint64
|
||||
tail uint64
|
||||
name string
|
||||
head uint64
|
||||
prunedTail *uint64
|
||||
)
|
||||
// Hack to get boundary of any table
|
||||
for kind, table := range f.tables {
|
||||
// get any head value
|
||||
for _, table := range f.tables {
|
||||
head = table.items.Load()
|
||||
tail = table.itemHidden.Load()
|
||||
name = kind
|
||||
break
|
||||
}
|
||||
// Now check every table against those boundaries.
|
||||
for kind, table := range f.tables {
|
||||
// all tables have to have the same head
|
||||
if head != table.items.Load() {
|
||||
return fmt.Errorf("freezer tables %s and %s have differing head: %d != %d", kind, name, table.items.Load(), head)
|
||||
return fmt.Errorf("freezer table %s has a differing head: %d != %d", kind, table.items.Load(), head)
|
||||
}
|
||||
if tail != table.itemHidden.Load() {
|
||||
return fmt.Errorf("freezer tables %s and %s have differing tail: %d != %d", kind, name, table.itemHidden.Load(), tail)
|
||||
if !table.config.prunable {
|
||||
// non-prunable tables have to start at 0
|
||||
if table.itemHidden.Load() != 0 {
|
||||
return fmt.Errorf("non-prunable freezer table '%s' has a non-zero tail: %d", kind, table.itemHidden.Load())
|
||||
}
|
||||
} else {
|
||||
// prunable tables have to have the same length
|
||||
if prunedTail == nil {
|
||||
tmp := table.itemHidden.Load()
|
||||
prunedTail = &tmp
|
||||
}
|
||||
if *prunedTail != table.itemHidden.Load() {
|
||||
return fmt.Errorf("freezer table %s has differing tail: %d != %d", kind, table.itemHidden.Load(), *prunedTail)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if prunedTail == nil {
|
||||
tmp := uint64(0)
|
||||
prunedTail = &tmp
|
||||
}
|
||||
|
||||
f.frozen.Store(head)
|
||||
f.tail.Store(tail)
|
||||
f.tail.Store(*prunedTail)
|
||||
return nil
|
||||
}
|
||||
|
||||
// repair truncates all data tables to the same length.
|
||||
func (f *Freezer) repair() error {
|
||||
var (
|
||||
head = uint64(math.MaxUint64)
|
||||
tail = uint64(0)
|
||||
head = uint64(math.MaxUint64)
|
||||
prunedTail = uint64(0)
|
||||
)
|
||||
// get the minimal head and the maximum tail
|
||||
for _, table := range f.tables {
|
||||
items := table.items.Load()
|
||||
if head > items {
|
||||
head = items
|
||||
}
|
||||
hidden := table.itemHidden.Load()
|
||||
if hidden > tail {
|
||||
tail = hidden
|
||||
}
|
||||
head = min(head, table.items.Load())
|
||||
prunedTail = max(prunedTail, table.itemHidden.Load())
|
||||
}
|
||||
for _, table := range f.tables {
|
||||
// apply the pruning
|
||||
for kind, table := range f.tables {
|
||||
// all tables need to have the same head
|
||||
if err := table.truncateHead(head); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := table.truncateTail(tail); err != nil {
|
||||
return err
|
||||
if !table.config.prunable {
|
||||
// non-prunable tables have to start at 0
|
||||
if table.itemHidden.Load() != 0 {
|
||||
panic(fmt.Sprintf("non-prunable freezer table %s has non-zero tail: %v", kind, table.itemHidden.Load()))
|
||||
}
|
||||
} else {
|
||||
// prunable tables have to have the same length
|
||||
if err := table.truncateTail(prunedTail); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
f.frozen.Store(head)
|
||||
f.tail.Store(tail)
|
||||
f.tail.Store(prunedTail)
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -96,7 +96,7 @@ type freezerTableBatch struct {
|
|||
// newBatch creates a new batch for the freezer table.
|
||||
func (t *freezerTable) newBatch() *freezerTableBatch {
|
||||
batch := &freezerTableBatch{t: t}
|
||||
if !t.noCompression {
|
||||
if !t.config.noSnappy {
|
||||
batch.sb = new(snappyBuffer)
|
||||
}
|
||||
batch.reset()
|
||||
|
|
|
|||
|
|
@ -30,17 +30,19 @@ import (
|
|||
|
||||
// memoryTable is used to store a list of sequential items in memory.
|
||||
type memoryTable struct {
|
||||
name string // Table name
|
||||
items uint64 // Number of stored items in the table, including the deleted ones
|
||||
offset uint64 // Number of deleted items from the table
|
||||
data [][]byte // List of rlp-encoded items, sort in order
|
||||
size uint64 // Total memory size occupied by the table
|
||||
lock sync.RWMutex
|
||||
|
||||
name string
|
||||
config freezerTableConfig
|
||||
}
|
||||
|
||||
// newMemoryTable initializes the memory table.
|
||||
func newMemoryTable(name string) *memoryTable {
|
||||
return &memoryTable{name: name}
|
||||
func newMemoryTable(name string, config freezerTableConfig) *memoryTable {
|
||||
return &memoryTable{name: name, config: config}
|
||||
}
|
||||
|
||||
// has returns an indicator whether the specified data exists.
|
||||
|
|
@ -218,10 +220,10 @@ type MemoryFreezer struct {
|
|||
}
|
||||
|
||||
// NewMemoryFreezer initializes an in-memory freezer instance.
|
||||
func NewMemoryFreezer(readonly bool, tableName map[string]bool) *MemoryFreezer {
|
||||
func NewMemoryFreezer(readonly bool, tableName map[string]freezerTableConfig) *MemoryFreezer {
|
||||
tables := make(map[string]*memoryTable)
|
||||
for name := range tableName {
|
||||
tables[name] = newMemoryTable(name)
|
||||
for name, cfg := range tableName {
|
||||
tables[name] = newMemoryTable(name, cfg)
|
||||
}
|
||||
return &MemoryFreezer{
|
||||
writeBatch: newMemoryBatch(),
|
||||
|
|
@ -368,7 +370,9 @@ func (f *MemoryFreezer) TruncateHead(items uint64) (uint64, error) {
|
|||
return old, nil
|
||||
}
|
||||
|
||||
// TruncateTail discards any recent data below the provided threshold number.
|
||||
// TruncateTail discards all data below the provided threshold number.
|
||||
// Note this will only truncate 'prunable' tables. Block headers and canonical
|
||||
// hashes cannot be truncated at this time.
|
||||
func (f *MemoryFreezer) TruncateTail(tail uint64) (uint64, error) {
|
||||
f.lock.Lock()
|
||||
defer f.lock.Unlock()
|
||||
|
|
@ -381,8 +385,10 @@ func (f *MemoryFreezer) TruncateTail(tail uint64) (uint64, error) {
|
|||
return old, nil
|
||||
}
|
||||
for _, table := range f.tables {
|
||||
if err := table.truncateTail(tail); err != nil {
|
||||
return 0, err
|
||||
if table.config.prunable {
|
||||
if err := table.truncateTail(tail); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
}
|
||||
}
|
||||
f.tail = tail
|
||||
|
|
@ -412,8 +418,8 @@ func (f *MemoryFreezer) Reset() error {
|
|||
defer f.lock.Unlock()
|
||||
|
||||
tables := make(map[string]*memoryTable)
|
||||
for name := range f.tables {
|
||||
tables[name] = newMemoryTable(name)
|
||||
for name, table := range f.tables {
|
||||
tables[name] = newMemoryTable(name, table.config)
|
||||
}
|
||||
f.tables = tables
|
||||
f.items, f.tail = 0, 0
|
||||
|
|
|
|||
|
|
@ -25,16 +25,22 @@ import (
|
|||
|
||||
func TestMemoryFreezer(t *testing.T) {
|
||||
ancienttest.TestAncientSuite(t, func(kinds []string) ethdb.AncientStore {
|
||||
tables := make(map[string]bool)
|
||||
tables := make(map[string]freezerTableConfig)
|
||||
for _, kind := range kinds {
|
||||
tables[kind] = true
|
||||
tables[kind] = freezerTableConfig{
|
||||
noSnappy: true,
|
||||
prunable: true,
|
||||
}
|
||||
}
|
||||
return NewMemoryFreezer(false, tables)
|
||||
})
|
||||
ancienttest.TestResettableAncientSuite(t, func(kinds []string) ethdb.ResettableAncientStore {
|
||||
tables := make(map[string]bool)
|
||||
tables := make(map[string]freezerTableConfig)
|
||||
for _, kind := range kinds {
|
||||
tables[kind] = true
|
||||
tables[kind] = freezerTableConfig{
|
||||
noSnappy: true,
|
||||
prunable: true,
|
||||
}
|
||||
}
|
||||
return NewMemoryFreezer(false, tables)
|
||||
})
|
||||
|
|
|
|||
|
|
@ -49,7 +49,7 @@ type resettableFreezer struct {
|
|||
//
|
||||
// The reset function will delete directory atomically and re-create the
|
||||
// freezer from scratch.
|
||||
func newResettableFreezer(datadir string, namespace string, readonly bool, maxTableSize uint32, tables map[string]bool) (*resettableFreezer, error) {
|
||||
func newResettableFreezer(datadir string, namespace string, readonly bool, maxTableSize uint32, tables map[string]freezerTableConfig) (*resettableFreezer, error) {
|
||||
if err := cleanup(datadir); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -100,11 +100,11 @@ type freezerTable struct {
|
|||
// should never be lower than itemOffset.
|
||||
itemHidden atomic.Uint64
|
||||
|
||||
noCompression bool // if true, disables snappy compression. Note: does not work retroactively
|
||||
readonly bool
|
||||
maxFileSize uint32 // Max file size for data-files
|
||||
name string
|
||||
path string
|
||||
config freezerTableConfig // if true, disables snappy compression. Note: does not work retroactively
|
||||
readonly bool
|
||||
maxFileSize uint32 // Max file size for data-files
|
||||
name string
|
||||
path string
|
||||
|
||||
head *os.File // File descriptor for the data head of the table
|
||||
index *os.File // File descriptor for the indexEntry file of the table
|
||||
|
|
@ -125,20 +125,20 @@ type freezerTable struct {
|
|||
}
|
||||
|
||||
// newFreezerTable opens the given path as a freezer table.
|
||||
func newFreezerTable(path, name string, disableSnappy, readonly bool) (*freezerTable, error) {
|
||||
return newTable(path, name, metrics.NewInactiveMeter(), metrics.NewInactiveMeter(), metrics.NewGauge(), freezerTableSize, disableSnappy, readonly)
|
||||
func newFreezerTable(path, name string, config freezerTableConfig, readonly bool) (*freezerTable, error) {
|
||||
return newTable(path, name, metrics.NewInactiveMeter(), metrics.NewInactiveMeter(), metrics.NewGauge(), freezerTableSize, config, readonly)
|
||||
}
|
||||
|
||||
// newTable opens a freezer table, creating the data and index files if they are
|
||||
// non-existent. Both files are truncated to the shortest common length to ensure
|
||||
// they don't go out of sync.
|
||||
func newTable(path string, name string, readMeter, writeMeter *metrics.Meter, sizeGauge *metrics.Gauge, maxFilesize uint32, noCompression, readonly bool) (*freezerTable, error) {
|
||||
func newTable(path string, name string, readMeter, writeMeter *metrics.Meter, sizeGauge *metrics.Gauge, maxFilesize uint32, config freezerTableConfig, readonly bool) (*freezerTable, error) {
|
||||
// Ensure the containing directory exists and open the indexEntry file
|
||||
if err := os.MkdirAll(path, 0755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var idxName string
|
||||
if noCompression {
|
||||
if config.noSnappy {
|
||||
idxName = fmt.Sprintf("%s.ridx", name) // raw index file
|
||||
} else {
|
||||
idxName = fmt.Sprintf("%s.cidx", name) // compressed index file
|
||||
|
|
@ -176,19 +176,19 @@ func newTable(path string, name string, readMeter, writeMeter *metrics.Meter, si
|
|||
}
|
||||
// Create the table and repair any past inconsistency
|
||||
tab := &freezerTable{
|
||||
index: index,
|
||||
metadata: metadata,
|
||||
lastSync: time.Now(),
|
||||
files: make(map[uint32]*os.File),
|
||||
readMeter: readMeter,
|
||||
writeMeter: writeMeter,
|
||||
sizeGauge: sizeGauge,
|
||||
name: name,
|
||||
path: path,
|
||||
logger: log.New("database", path, "table", name),
|
||||
noCompression: noCompression,
|
||||
readonly: readonly,
|
||||
maxFileSize: maxFilesize,
|
||||
index: index,
|
||||
metadata: metadata,
|
||||
lastSync: time.Now(),
|
||||
files: make(map[uint32]*os.File),
|
||||
readMeter: readMeter,
|
||||
writeMeter: writeMeter,
|
||||
sizeGauge: sizeGauge,
|
||||
name: name,
|
||||
path: path,
|
||||
logger: log.New("database", path, "table", name),
|
||||
config: config,
|
||||
readonly: readonly,
|
||||
maxFileSize: maxFilesize,
|
||||
}
|
||||
if err := tab.repair(); err != nil {
|
||||
tab.Close()
|
||||
|
|
@ -871,7 +871,7 @@ func (t *freezerTable) openFile(num uint32, opener func(string) (*os.File, error
|
|||
var exist bool
|
||||
if f, exist = t.files[num]; !exist {
|
||||
var name string
|
||||
if t.noCompression {
|
||||
if t.config.noSnappy {
|
||||
name = fmt.Sprintf("%s.%04d.rdat", t.name, num)
|
||||
} else {
|
||||
name = fmt.Sprintf("%s.%04d.cdat", t.name, num)
|
||||
|
|
@ -987,13 +987,13 @@ func (t *freezerTable) RetrieveItems(start, count, maxBytes uint64) ([][]byte, e
|
|||
item := diskData[offset : offset+diskSize]
|
||||
offset += diskSize
|
||||
decompressedSize := diskSize
|
||||
if !t.noCompression {
|
||||
if !t.config.noSnappy {
|
||||
decompressedSize, _ = snappy.DecodedLen(item)
|
||||
}
|
||||
if i > 0 && maxBytes != 0 && uint64(outputSize+decompressedSize) > maxBytes {
|
||||
break
|
||||
}
|
||||
if !t.noCompression {
|
||||
if !t.config.noSnappy {
|
||||
data, err := snappy.Decode(nil, item)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ func TestFreezerBasics(t *testing.T) {
|
|||
// set cutoff at 50 bytes
|
||||
f, err := newTable(os.TempDir(),
|
||||
fmt.Sprintf("unittest-%d", rand.Uint64()),
|
||||
metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, false)
|
||||
metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, freezerTableConfig{noSnappy: true}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -84,7 +84,7 @@ func TestFreezerBasicsClosing(t *testing.T) {
|
|||
f *freezerTable
|
||||
err error
|
||||
)
|
||||
f, err = newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
|
||||
f, err = newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{noSnappy: true}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -98,7 +98,7 @@ func TestFreezerBasicsClosing(t *testing.T) {
|
|||
require.NoError(t, batch.commit())
|
||||
f.Close()
|
||||
|
||||
f, err = newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
|
||||
f, err = newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{noSnappy: true}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -115,7 +115,7 @@ func TestFreezerBasicsClosing(t *testing.T) {
|
|||
t.Fatalf("test %d, got \n%x != \n%x", y, got, exp)
|
||||
}
|
||||
f.Close()
|
||||
f, err = newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
|
||||
f, err = newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{noSnappy: true}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -130,7 +130,7 @@ func TestFreezerRepairDanglingHead(t *testing.T) {
|
|||
|
||||
// Fill table
|
||||
{
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{noSnappy: true}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -159,7 +159,7 @@ func TestFreezerRepairDanglingHead(t *testing.T) {
|
|||
|
||||
// Now open it again
|
||||
{
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{noSnappy: true}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -182,7 +182,7 @@ func TestFreezerRepairDanglingHeadLarge(t *testing.T) {
|
|||
|
||||
// Fill a table and close it
|
||||
{
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{noSnappy: true}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -208,7 +208,7 @@ func TestFreezerRepairDanglingHeadLarge(t *testing.T) {
|
|||
|
||||
// Now open it again
|
||||
{
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{noSnappy: true}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -231,7 +231,7 @@ func TestFreezerRepairDanglingHeadLarge(t *testing.T) {
|
|||
|
||||
// And if we open it, we should now be able to read all of them (new values)
|
||||
{
|
||||
f, _ := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
|
||||
f, _ := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{noSnappy: true}, false)
|
||||
for y := 1; y < 255; y++ {
|
||||
exp := getChunk(15, ^y)
|
||||
got, err := f.Retrieve(uint64(y))
|
||||
|
|
@ -253,7 +253,7 @@ func TestSnappyDetection(t *testing.T) {
|
|||
|
||||
// Open with snappy
|
||||
{
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{noSnappy: true}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -264,7 +264,7 @@ func TestSnappyDetection(t *testing.T) {
|
|||
|
||||
// Open with snappy
|
||||
{
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{noSnappy: true}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -277,7 +277,7 @@ func TestSnappyDetection(t *testing.T) {
|
|||
|
||||
// Open without snappy
|
||||
{
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, false, false)
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{noSnappy: false}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -308,7 +308,7 @@ func TestFreezerRepairDanglingIndex(t *testing.T) {
|
|||
|
||||
// Fill a table and close it
|
||||
{
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{noSnappy: true}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -344,7 +344,7 @@ func TestFreezerRepairDanglingIndex(t *testing.T) {
|
|||
// 45, 45, 15
|
||||
// with 3+3+1 items
|
||||
{
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{noSnappy: true}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -365,7 +365,7 @@ func TestFreezerTruncate(t *testing.T) {
|
|||
|
||||
// Fill table
|
||||
{
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{noSnappy: true}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -381,7 +381,7 @@ func TestFreezerTruncate(t *testing.T) {
|
|||
|
||||
// Reopen, truncate
|
||||
{
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{noSnappy: true}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -406,7 +406,7 @@ func TestFreezerRepairFirstFile(t *testing.T) {
|
|||
|
||||
// Fill table
|
||||
{
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{noSnappy: true}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -439,7 +439,7 @@ func TestFreezerRepairFirstFile(t *testing.T) {
|
|||
|
||||
// Reopen
|
||||
{
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{noSnappy: true}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -474,7 +474,7 @@ func TestFreezerReadAndTruncate(t *testing.T) {
|
|||
|
||||
// Fill table
|
||||
{
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{noSnappy: true}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -490,7 +490,7 @@ func TestFreezerReadAndTruncate(t *testing.T) {
|
|||
|
||||
// Reopen and read all files
|
||||
{
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{noSnappy: true}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -521,7 +521,7 @@ func TestFreezerOffset(t *testing.T) {
|
|||
fname := fmt.Sprintf("offset-%d", rand.Uint64())
|
||||
|
||||
// Fill table
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false)
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, freezerTableConfig{noSnappy: true}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -545,7 +545,7 @@ func TestFreezerOffset(t *testing.T) {
|
|||
f.Close()
|
||||
|
||||
// Now open again
|
||||
f, err = newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false)
|
||||
f, err = newTable(os.TempDir(), fname, rm, wm, sg, 40, freezerTableConfig{noSnappy: true}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -597,7 +597,7 @@ func TestFreezerOffset(t *testing.T) {
|
|||
|
||||
// Check that existing items have been moved to index 1M.
|
||||
{
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false)
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, freezerTableConfig{noSnappy: true}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -631,7 +631,7 @@ func TestTruncateTail(t *testing.T) {
|
|||
fname := fmt.Sprintf("truncate-tail-%d", rand.Uint64())
|
||||
|
||||
// Fill table
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false)
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, freezerTableConfig{noSnappy: true}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -682,7 +682,7 @@ func TestTruncateTail(t *testing.T) {
|
|||
|
||||
// Reopen the table, the deletion information should be persisted as well
|
||||
f.Close()
|
||||
f, err = newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false)
|
||||
f, err = newTable(os.TempDir(), fname, rm, wm, sg, 40, freezerTableConfig{noSnappy: true}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -716,7 +716,7 @@ func TestTruncateTail(t *testing.T) {
|
|||
|
||||
// Reopen the table, the above testing should still pass
|
||||
f.Close()
|
||||
f, err = newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false)
|
||||
f, err = newTable(os.TempDir(), fname, rm, wm, sg, 40, freezerTableConfig{noSnappy: true}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -772,7 +772,7 @@ func TestTruncateHead(t *testing.T) {
|
|||
fname := fmt.Sprintf("truncate-head-blow-tail-%d", rand.Uint64())
|
||||
|
||||
// Fill table
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false)
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, freezerTableConfig{noSnappy: true}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -883,7 +883,7 @@ func TestSequentialRead(t *testing.T) {
|
|||
rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge()
|
||||
fname := fmt.Sprintf("batchread-%d", rand.Uint64())
|
||||
{ // Fill table
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{noSnappy: true}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -893,7 +893,7 @@ func TestSequentialRead(t *testing.T) {
|
|||
f.Close()
|
||||
}
|
||||
{ // Open it, iterate, verify iteration
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true, false)
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, freezerTableConfig{noSnappy: true}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -914,7 +914,7 @@ func TestSequentialRead(t *testing.T) {
|
|||
}
|
||||
{ // Open it, iterate, verify byte limit. The byte limit is less than item
|
||||
// size, so each lookup should only return one item
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true, false)
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, freezerTableConfig{noSnappy: true}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -943,7 +943,7 @@ func TestSequentialReadByteLimit(t *testing.T) {
|
|||
rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge()
|
||||
fname := fmt.Sprintf("batchread-2-%d", rand.Uint64())
|
||||
{ // Fill table
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, true, false)
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, freezerTableConfig{noSnappy: true}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -965,7 +965,7 @@ func TestSequentialReadByteLimit(t *testing.T) {
|
|||
{100, 109, 10},
|
||||
} {
|
||||
{
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, true, false)
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, freezerTableConfig{noSnappy: true}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -993,7 +993,7 @@ func TestSequentialReadNoByteLimit(t *testing.T) {
|
|||
rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge()
|
||||
fname := fmt.Sprintf("batchread-3-%d", rand.Uint64())
|
||||
{ // Fill table
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, true, false)
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, freezerTableConfig{noSnappy: true}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -1011,7 +1011,7 @@ func TestSequentialReadNoByteLimit(t *testing.T) {
|
|||
{31, 30},
|
||||
} {
|
||||
{
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, true, false)
|
||||
f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, freezerTableConfig{noSnappy: true}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -1038,7 +1038,7 @@ func TestFreezerReadonly(t *testing.T) {
|
|||
// Case 1: Check it fails on non-existent file.
|
||||
_, err := newTable(tmpdir,
|
||||
fmt.Sprintf("readonlytest-%d", rand.Uint64()),
|
||||
metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, true)
|
||||
metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, freezerTableConfig{noSnappy: true}, true)
|
||||
if err == nil {
|
||||
t.Fatal("readonly table instantiation should fail for non-existent table")
|
||||
}
|
||||
|
|
@ -1053,7 +1053,7 @@ func TestFreezerReadonly(t *testing.T) {
|
|||
idxFile.Write(make([]byte, 17))
|
||||
idxFile.Close()
|
||||
_, err = newTable(tmpdir, fname,
|
||||
metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, true)
|
||||
metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, freezerTableConfig{noSnappy: true}, true)
|
||||
if err == nil {
|
||||
t.Errorf("readonly table instantiation should fail for invalid index size")
|
||||
}
|
||||
|
|
@ -1063,7 +1063,7 @@ func TestFreezerReadonly(t *testing.T) {
|
|||
// again in readonly triggers an error.
|
||||
fname = fmt.Sprintf("readonlytest-%d", rand.Uint64())
|
||||
f, err := newTable(tmpdir, fname,
|
||||
metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, false)
|
||||
metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, freezerTableConfig{noSnappy: true}, false)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to instantiate table: %v", err)
|
||||
}
|
||||
|
|
@ -1076,7 +1076,7 @@ func TestFreezerReadonly(t *testing.T) {
|
|||
t.Fatal(err)
|
||||
}
|
||||
_, err = newTable(tmpdir, fname,
|
||||
metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, true)
|
||||
metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, freezerTableConfig{noSnappy: true}, true)
|
||||
if err == nil {
|
||||
t.Errorf("readonly table instantiation should fail for corrupt table file")
|
||||
}
|
||||
|
|
@ -1085,7 +1085,7 @@ func TestFreezerReadonly(t *testing.T) {
|
|||
// Should be successful.
|
||||
fname = fmt.Sprintf("readonlytest-%d", rand.Uint64())
|
||||
f, err = newTable(tmpdir, fname,
|
||||
metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, false)
|
||||
metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, freezerTableConfig{noSnappy: true}, false)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to instantiate table: %v\n", err)
|
||||
}
|
||||
|
|
@ -1094,7 +1094,7 @@ func TestFreezerReadonly(t *testing.T) {
|
|||
t.Fatal(err)
|
||||
}
|
||||
f, err = newTable(tmpdir, fname,
|
||||
metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, true)
|
||||
metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, freezerTableConfig{noSnappy: true}, true)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -1234,7 +1234,7 @@ func (randTest) Generate(r *rand.Rand, size int) reflect.Value {
|
|||
|
||||
func runRandTest(rt randTest) bool {
|
||||
fname := fmt.Sprintf("randtest-%d", rand.Uint64())
|
||||
f, err := newTable(os.TempDir(), fname, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, false)
|
||||
f, err := newTable(os.TempDir(), fname, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, freezerTableConfig{noSnappy: true}, false)
|
||||
if err != nil {
|
||||
panic("failed to initialize table")
|
||||
}
|
||||
|
|
@ -1243,7 +1243,7 @@ func runRandTest(rt randTest) bool {
|
|||
switch step.op {
|
||||
case opReload:
|
||||
f.Close()
|
||||
f, err = newTable(os.TempDir(), fname, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true, false)
|
||||
f, err = newTable(os.TempDir(), fname, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, freezerTableConfig{noSnappy: true}, false)
|
||||
if err != nil {
|
||||
rt[i].err = fmt.Errorf("failed to reload table %v", err)
|
||||
}
|
||||
|
|
@ -1381,7 +1381,7 @@ func TestIndexValidation(t *testing.T) {
|
|||
}
|
||||
for _, c := range cases {
|
||||
fn := fmt.Sprintf("t-%d", rand.Uint64())
|
||||
f, err := newTable(os.TempDir(), fn, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 10*dataSize, true, false)
|
||||
f, err := newTable(os.TempDir(), fn, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 10*dataSize, freezerTableConfig{noSnappy: true}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -1392,7 +1392,7 @@ func TestIndexValidation(t *testing.T) {
|
|||
f.Close()
|
||||
|
||||
// reopen the table, corruption should be truncated
|
||||
f, err = newTable(os.TempDir(), fn, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 100, true, false)
|
||||
f, err = newTable(os.TempDir(), fn, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 100, freezerTableConfig{noSnappy: true}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -1422,7 +1422,7 @@ func TestFlushOffsetTracking(t *testing.T) {
|
|||
fileSize = 100
|
||||
)
|
||||
fn := fmt.Sprintf("t-%d", rand.Uint64())
|
||||
f, err := newTable(os.TempDir(), fn, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), fileSize, true, false)
|
||||
f, err := newTable(os.TempDir(), fn, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), fileSize, freezerTableConfig{noSnappy: true}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -1533,7 +1533,7 @@ func TestTailTruncationCrash(t *testing.T) {
|
|||
fileSize = 100
|
||||
)
|
||||
fn := fmt.Sprintf("t-%d", rand.Uint64())
|
||||
f, err := newTable(os.TempDir(), fn, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), fileSize, true, false)
|
||||
f, err := newTable(os.TempDir(), fn, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), fileSize, freezerTableConfig{noSnappy: true}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
@ -1563,7 +1563,7 @@ func TestTailTruncationCrash(t *testing.T) {
|
|||
// the offset
|
||||
f.metadata.setFlushOffset(31*indexEntrySize, true)
|
||||
|
||||
f, err = newTable(os.TempDir(), fn, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), fileSize, true, false)
|
||||
f, err = newTable(os.TempDir(), fn, metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), fileSize, freezerTableConfig{noSnappy: true}, false)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@ import (
|
|||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
var freezerTestTableDef = map[string]bool{"test": true}
|
||||
var freezerTestTableDef = map[string]freezerTableConfig{"test": {noSnappy: true}}
|
||||
|
||||
func TestFreezerModify(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
|
@ -47,7 +47,7 @@ func TestFreezerModify(t *testing.T) {
|
|||
valuesRLP = append(valuesRLP, iv)
|
||||
}
|
||||
|
||||
tables := map[string]bool{"raw": true, "rlp": false}
|
||||
tables := map[string]freezerTableConfig{"raw": {noSnappy: true}, "rlp": {noSnappy: false}}
|
||||
f, _ := newFreezerForTesting(t, tables)
|
||||
defer f.Close()
|
||||
|
||||
|
|
@ -111,7 +111,7 @@ func TestFreezerModifyRollback(t *testing.T) {
|
|||
f.Close()
|
||||
|
||||
// Reopen and check that the rolled-back data doesn't reappear.
|
||||
tables := map[string]bool{"test": true}
|
||||
tables := map[string]freezerTableConfig{"test": {noSnappy: true}}
|
||||
f2, err := NewFreezer(dir, "", false, 2049, tables)
|
||||
if err != nil {
|
||||
t.Fatalf("can't reopen freezer after failed ModifyAncients: %v", err)
|
||||
|
|
@ -249,7 +249,7 @@ func TestFreezerConcurrentModifyTruncate(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestFreezerReadonlyValidate(t *testing.T) {
|
||||
tables := map[string]bool{"a": true, "b": true}
|
||||
tables := map[string]freezerTableConfig{"a": {noSnappy: true}, "b": {noSnappy: true}}
|
||||
dir := t.TempDir()
|
||||
// Open non-readonly freezer and fill individual tables
|
||||
// with different amount of data.
|
||||
|
|
@ -285,7 +285,7 @@ func TestFreezerReadonlyValidate(t *testing.T) {
|
|||
func TestFreezerConcurrentReadonly(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
tables := map[string]bool{"a": true}
|
||||
tables := map[string]freezerTableConfig{"a": {noSnappy: true}}
|
||||
dir := t.TempDir()
|
||||
|
||||
f, err := NewFreezer(dir, "", false, 2049, tables)
|
||||
|
|
@ -333,7 +333,7 @@ func TestFreezerConcurrentReadonly(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func newFreezerForTesting(t *testing.T, tables map[string]bool) (*Freezer, string) {
|
||||
func newFreezerForTesting(t *testing.T, tables map[string]freezerTableConfig) (*Freezer, string) {
|
||||
t.Helper()
|
||||
|
||||
dir := t.TempDir()
|
||||
|
|
@ -379,7 +379,7 @@ func checkAncientCount(t *testing.T, f *Freezer, kind string, n uint64) {
|
|||
|
||||
func TestFreezerCloseSync(t *testing.T) {
|
||||
t.Parallel()
|
||||
f, _ := newFreezerForTesting(t, map[string]bool{"a": true, "b": true})
|
||||
f, _ := newFreezerForTesting(t, map[string]freezerTableConfig{"a": {noSnappy: true}, "b": {noSnappy: true}})
|
||||
defer f.Close()
|
||||
|
||||
// Now, close and sync. This mimics the behaviour if the node is shut down,
|
||||
|
|
@ -401,17 +401,23 @@ func TestFreezerCloseSync(t *testing.T) {
|
|||
|
||||
func TestFreezerSuite(t *testing.T) {
|
||||
ancienttest.TestAncientSuite(t, func(kinds []string) ethdb.AncientStore {
|
||||
tables := make(map[string]bool)
|
||||
tables := make(map[string]freezerTableConfig)
|
||||
for _, kind := range kinds {
|
||||
tables[kind] = true
|
||||
tables[kind] = freezerTableConfig{
|
||||
noSnappy: true,
|
||||
prunable: true,
|
||||
}
|
||||
}
|
||||
f, _ := newFreezerForTesting(t, tables)
|
||||
return f
|
||||
})
|
||||
ancienttest.TestResettableAncientSuite(t, func(kinds []string) ethdb.ResettableAncientStore {
|
||||
tables := make(map[string]bool)
|
||||
tables := make(map[string]freezerTableConfig)
|
||||
for _, kind := range kinds {
|
||||
tables[kind] = true
|
||||
tables[kind] = freezerTableConfig{
|
||||
noSnappy: true,
|
||||
prunable: true,
|
||||
}
|
||||
}
|
||||
f, _ := newResettableFreezer(t.TempDir(), "", false, 2048, tables)
|
||||
return f
|
||||
|
|
|
|||
|
|
@ -128,6 +128,8 @@ type AncientWriter interface {
|
|||
// is item_n(start from 0). The deleted items may not be removed from the ancient store
|
||||
// immediately, but only when the accumulated deleted data reach the threshold then
|
||||
// will be removed all together.
|
||||
//
|
||||
// Note that data marked as non-prunable will still be retained and remain accessible.
|
||||
TruncateTail(n uint64) (uint64, error)
|
||||
|
||||
// Sync flushes all in-memory ancient store data to disk.
|
||||
|
|
|
|||
Loading…
Reference in a new issue