mirror of
https://github.com/ethereum/go-ethereum.git
synced 2026-06-02 21:18:40 +00:00
Merge ff247428af into 12eabbd76d
This commit is contained in:
commit
9551e226da
24 changed files with 547 additions and 198 deletions
|
|
@ -744,7 +744,7 @@ func pruneHistory(ctx *cli.Context) error {
|
||||||
)
|
)
|
||||||
|
|
||||||
// Check the current freezer tail to see if pruning is needed/possible.
|
// Check the current freezer tail to see if pruning is needed/possible.
|
||||||
freezerTail, _ := chaindb.Tail()
|
freezerTail, _ := chaindb.Tail(rawdb.ChainFreezerBlockDataGroup)
|
||||||
if freezerTail > 0 {
|
if freezerTail > 0 {
|
||||||
if freezerTail == targetBlock {
|
if freezerTail == targetBlock {
|
||||||
log.Info("Database already pruned to target block", "tail", freezerTail)
|
log.Info("Database already pruned to target block", "tail", freezerTail)
|
||||||
|
|
@ -776,7 +776,7 @@ func pruneHistory(ctx *cli.Context) error {
|
||||||
log.Info("Starting history pruning", "head", currentHeader.Number, "target", targetBlock, "targetHash", targetBlockHash.Hex())
|
log.Info("Starting history pruning", "head", currentHeader.Number, "target", targetBlock, "targetHash", targetBlockHash.Hex())
|
||||||
start := time.Now()
|
start := time.Now()
|
||||||
rawdb.PruneTransactionIndex(chaindb, targetBlock)
|
rawdb.PruneTransactionIndex(chaindb, targetBlock)
|
||||||
if _, err := chaindb.TruncateTail(targetBlock); err != nil {
|
if _, err := chaindb.TruncateTail(rawdb.ChainFreezerBlockDataGroup, targetBlock); err != nil {
|
||||||
return fmt.Errorf("failed to truncate ancient data: %v", err)
|
return fmt.Errorf("failed to truncate ancient data: %v", err)
|
||||||
}
|
}
|
||||||
log.Info("History pruning completed", "tail", targetBlock, "elapsed", common.PrettyDuration(time.Since(start)))
|
log.Info("History pruning completed", "tail", targetBlock, "elapsed", common.PrettyDuration(time.Since(start)))
|
||||||
|
|
|
||||||
|
|
@ -720,7 +720,7 @@ func (bc *BlockChain) loadLastState() error {
|
||||||
|
|
||||||
// initializeHistoryPruning sets bc.historyPrunePoint.
|
// initializeHistoryPruning sets bc.historyPrunePoint.
|
||||||
func (bc *BlockChain) initializeHistoryPruning(latest uint64) error {
|
func (bc *BlockChain) initializeHistoryPruning(latest uint64) error {
|
||||||
freezerTail, _ := bc.db.Tail()
|
freezerTail, _ := bc.db.Tail(rawdb.ChainFreezerBlockDataGroup)
|
||||||
policy := bc.cfg.HistoryPolicy
|
policy := bc.cfg.HistoryPolicy
|
||||||
|
|
||||||
switch policy.Mode {
|
switch policy.Mode {
|
||||||
|
|
@ -2965,7 +2965,7 @@ func (bc *BlockChain) InsertHeadersBeforeCutoff(headers []*types.Header) (int, e
|
||||||
}
|
}
|
||||||
// Truncate the useless chain segment (zero bodies and receipts) in the
|
// Truncate the useless chain segment (zero bodies and receipts) in the
|
||||||
// ancient store.
|
// ancient store.
|
||||||
if _, err := bc.db.TruncateTail(last.Number.Uint64() + 1); err != nil {
|
if _, err := bc.db.TruncateTail(rawdb.ChainFreezerBlockDataGroup, last.Number.Uint64()+1); err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
// Last step update all in-memory markers
|
// Last step update all in-memory markers
|
||||||
|
|
|
||||||
|
|
@ -4386,7 +4386,7 @@ func testInsertChainWithCutoff(t *testing.T, cutoff uint64, ancientLimit uint64,
|
||||||
if header.Hash() != hash {
|
if header.Hash() != hash {
|
||||||
t.Errorf("block #%d: header mismatch: want: %v, got: %v", num, hash, header.Hash())
|
t.Errorf("block #%d: header mismatch: want: %v, got: %v", num, hash, header.Hash())
|
||||||
}
|
}
|
||||||
tail, err := db.Tail()
|
tail, err := db.Tail(rawdb.ChainFreezerBlockDataGroup)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Failed to get chain tail, %v", err)
|
t.Fatalf("Failed to get chain tail, %v", err)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -614,9 +614,18 @@ func HasAccessList(db ethdb.Reader, hash common.Hash, number uint64) bool {
|
||||||
return has
|
return has
|
||||||
}
|
}
|
||||||
|
|
||||||
// ReadAccessListRLP retrieves the RLP-encoded block access list for a block from KV.
|
// ReadAccessListRLP retrieves the RLP-encoded block access list for a block.
|
||||||
func ReadAccessListRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue {
|
func ReadAccessListRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue {
|
||||||
data, _ := db.Get(accessListKey(number, hash))
|
var data []byte
|
||||||
|
db.ReadAncients(func(reader ethdb.AncientReaderOp) error {
|
||||||
|
data, _ = reader.Ancient(ChainFreezerBALTable, number)
|
||||||
|
if len(data) > 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
// Block is not in ancients, read from key-value store by hash and number.
|
||||||
|
data, _ = db.Get(accessListKey(number, hash))
|
||||||
|
return nil
|
||||||
|
})
|
||||||
return data
|
return data
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -759,6 +768,13 @@ func writeAncientBlock(op ethdb.AncientWriteOp, block *types.Block, header *type
|
||||||
if err := op.Append(ChainFreezerReceiptTable, num, receipts); err != nil {
|
if err := op.Append(ChainFreezerReceiptTable, num, receipts); err != nil {
|
||||||
return fmt.Errorf("can't append block %d receipts: %v", num, err)
|
return fmt.Errorf("can't append block %d receipts: %v", num, err)
|
||||||
}
|
}
|
||||||
|
// The assumption is held that BAL of ancient block is no longer available
|
||||||
|
// (it may still reachable, but it's not worthwhile to even retrieve it
|
||||||
|
// from the network). A nil entry is stored in the BAL table as the absence
|
||||||
|
// placeholder.
|
||||||
|
if err := op.AppendRaw(ChainFreezerBALTable, num, nil); err != nil {
|
||||||
|
return fmt.Errorf("can't append block %d bals: %v", num, err)
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -781,6 +797,13 @@ func WriteAncientHeaderChain(db ethdb.AncientWriter, headers []*types.Header) (i
|
||||||
if err := op.AppendRaw(ChainFreezerReceiptTable, num, nil); err != nil {
|
if err := op.AppendRaw(ChainFreezerReceiptTable, num, nil); err != nil {
|
||||||
return fmt.Errorf("can't append block %d receipts: %v", num, err)
|
return fmt.Errorf("can't append block %d receipts: %v", num, err)
|
||||||
}
|
}
|
||||||
|
// The assumption is held that BAL of ancient block is no longer available
|
||||||
|
// (it may still reachable, but it's not worthwhile to even retrieve it
|
||||||
|
// from the network). A nil entry is stored in the BAL table as the absence
|
||||||
|
// placeholder.
|
||||||
|
if err := op.AppendRaw(ChainFreezerBALTable, num, nil); err != nil {
|
||||||
|
return fmt.Errorf("can't append block %d bals: %v", num, err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
})
|
})
|
||||||
|
|
@ -791,6 +814,7 @@ func DeleteBlock(db ethdb.KeyValueWriter, hash common.Hash, number uint64) {
|
||||||
DeleteReceipts(db, hash, number)
|
DeleteReceipts(db, hash, number)
|
||||||
DeleteHeader(db, hash, number)
|
DeleteHeader(db, hash, number)
|
||||||
DeleteBody(db, hash, number)
|
DeleteBody(db, hash, number)
|
||||||
|
DeleteAccessList(db, hash, number)
|
||||||
}
|
}
|
||||||
|
|
||||||
// DeleteBlockWithoutNumber removes all block data associated with a hash, except
|
// DeleteBlockWithoutNumber removes all block data associated with a hash, except
|
||||||
|
|
@ -799,6 +823,7 @@ func DeleteBlockWithoutNumber(db ethdb.KeyValueWriter, hash common.Hash, number
|
||||||
DeleteReceipts(db, hash, number)
|
DeleteReceipts(db, hash, number)
|
||||||
deleteHeaderWithoutNumber(db, hash, number)
|
deleteHeaderWithoutNumber(db, hash, number)
|
||||||
DeleteBody(db, hash, number)
|
DeleteBody(db, hash, number)
|
||||||
|
DeleteAccessList(db, hash, number)
|
||||||
}
|
}
|
||||||
|
|
||||||
const badBlockToKeep = 10
|
const badBlockToKeep = 10
|
||||||
|
|
|
||||||
|
|
@ -926,6 +926,47 @@ func makeTestBAL(t *testing.T) (rlp.RawValue, *bal.BlockAccessList) {
|
||||||
return encoded, &decoded
|
return encoded, &decoded
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestWriteAncientBlocksNilBAL ensures that freezing a block with no block
|
||||||
|
// access list produces an empty entry in the BAL ancient table and that
|
||||||
|
// ReadAccessList returns nil afterwards (i.e. the empty entry is not surfaced
|
||||||
|
// as a malformed BAL).
|
||||||
|
func TestWriteAncientBlocksNilBAL(t *testing.T) {
|
||||||
|
db, err := Open(NewMemoryDatabase(), OpenOptions{Ancient: t.TempDir()})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("failed to create database with ancient backend: %v", err)
|
||||||
|
}
|
||||||
|
defer db.Close()
|
||||||
|
|
||||||
|
block := types.NewBlockWithHeader(&types.Header{
|
||||||
|
Number: big.NewInt(0),
|
||||||
|
Extra: []byte("nil-bal block"),
|
||||||
|
UncleHash: types.EmptyUncleHash,
|
||||||
|
TxHash: types.EmptyTxsHash,
|
||||||
|
ReceiptHash: types.EmptyReceiptsHash,
|
||||||
|
})
|
||||||
|
if block.AccessList() != nil {
|
||||||
|
t.Fatalf("test precondition: block must have nil access list")
|
||||||
|
}
|
||||||
|
if _, err := WriteAncientBlocks(db, []*types.Block{block}, types.EncodeBlockReceiptLists([]types.Receipts{nil})); err != nil {
|
||||||
|
t.Fatalf("WriteAncientBlocks failed: %v", err)
|
||||||
|
}
|
||||||
|
hash, number := block.Hash(), block.NumberU64()
|
||||||
|
|
||||||
|
// The BAL ancient entry should exist as an empty blob.
|
||||||
|
if blob := ReadAccessListRLP(db, hash, number); len(blob) != 0 {
|
||||||
|
t.Fatalf("ReadAccessListRLP: got %x, want empty", blob)
|
||||||
|
}
|
||||||
|
// ReadAccessList must surface nil rather than attempting to RLP-decode
|
||||||
|
// the empty payload.
|
||||||
|
if b := ReadAccessList(db, hash, number); b != nil {
|
||||||
|
t.Fatalf("ReadAccessList: got %v, want nil", b)
|
||||||
|
}
|
||||||
|
// HasAccessList only consults the KV store and there's nothing there.
|
||||||
|
if HasAccessList(db, hash, number) {
|
||||||
|
t.Fatal("HasAccessList returned true for absent BAL")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// TestBALStorage tests write/read/delete of BALs in the KV store.
|
// TestBALStorage tests write/read/delete of BALs in the KV store.
|
||||||
func TestBALStorage(t *testing.T) {
|
func TestBALStorage(t *testing.T) {
|
||||||
db := NewMemoryDatabase()
|
db := NewMemoryDatabase()
|
||||||
|
|
|
||||||
|
|
@ -35,6 +35,23 @@ const (
|
||||||
|
|
||||||
// ChainFreezerReceiptTable indicates the name of the freezer receipts table.
|
// ChainFreezerReceiptTable indicates the name of the freezer receipts table.
|
||||||
ChainFreezerReceiptTable = "receipts"
|
ChainFreezerReceiptTable = "receipts"
|
||||||
|
|
||||||
|
// ChainFreezerBALTable indicates the name of the freezer block access list
|
||||||
|
// table introduced by EIP-7928.
|
||||||
|
ChainFreezerBALTable = "bals"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Identifiers of tail groups used by the chain freezer.
|
||||||
|
const (
|
||||||
|
// ChainFreezerBlockDataGroup is the tail group shared by the body and
|
||||||
|
// receipt tables. The two tables are pruned together and therefore have
|
||||||
|
// the same tail position.
|
||||||
|
ChainFreezerBlockDataGroup = "blockdata"
|
||||||
|
|
||||||
|
// ChainFreezerBALGroup is the tail group for the block access list table.
|
||||||
|
// BAL is only populated after EIP-7928 activates, so it generally has a
|
||||||
|
// higher tail than the block-data group and is pruned independently.
|
||||||
|
ChainFreezerBALGroup = "bal"
|
||||||
)
|
)
|
||||||
|
|
||||||
// chainFreezerTableConfigs configures the settings for tables in the chain freezer.
|
// chainFreezerTableConfigs configures the settings for tables in the chain freezer.
|
||||||
|
|
@ -42,16 +59,23 @@ const (
|
||||||
// tail truncation is disabled for the header and hash tables, as these are intended
|
// tail truncation is disabled for the header and hash tables, as these are intended
|
||||||
// to be retained long-term.
|
// to be retained long-term.
|
||||||
var chainFreezerTableConfigs = map[string]freezerTableConfig{
|
var chainFreezerTableConfigs = map[string]freezerTableConfig{
|
||||||
ChainFreezerHeaderTable: {noSnappy: false, prunable: false},
|
ChainFreezerHeaderTable: {noSnappy: false},
|
||||||
ChainFreezerHashTable: {noSnappy: true, prunable: false},
|
ChainFreezerHashTable: {noSnappy: true},
|
||||||
ChainFreezerBodiesTable: {noSnappy: false, prunable: true},
|
ChainFreezerBodiesTable: {noSnappy: false, tailGroup: ChainFreezerBlockDataGroup},
|
||||||
ChainFreezerReceiptTable: {noSnappy: false, prunable: true},
|
ChainFreezerReceiptTable: {noSnappy: false, tailGroup: ChainFreezerBlockDataGroup},
|
||||||
|
ChainFreezerBALTable: {noSnappy: false, tailGroup: ChainFreezerBALGroup},
|
||||||
}
|
}
|
||||||
|
|
||||||
// freezerTableConfig contains the settings for a freezer table.
|
// freezerTableConfig contains the settings for a freezer table.
|
||||||
type freezerTableConfig struct {
|
type freezerTableConfig struct {
|
||||||
noSnappy bool // disables item compression
|
// noSnappy disables item compression when true.
|
||||||
prunable bool // true for tables that can be pruned by TruncateTail
|
noSnappy bool
|
||||||
|
|
||||||
|
// tailGroup names a logical group of tables that share the same tail
|
||||||
|
// position. Tables in the same group are pruned together and must agree
|
||||||
|
// on their tail. An empty value means the table is not prunable; its
|
||||||
|
// tail is always 0.
|
||||||
|
tailGroup string
|
||||||
}
|
}
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
|
@ -66,13 +90,17 @@ const (
|
||||||
stateHistoryStorageData = "storage.data"
|
stateHistoryStorageData = "storage.data"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// DefaultHistoryGroup is the tail group shared by all state/trienode history
|
||||||
|
// tables with tail pruning enabled.
|
||||||
|
const DefaultHistoryGroup = "history"
|
||||||
|
|
||||||
// stateFreezerTableConfigs configures the settings for tables in the state freezer.
|
// stateFreezerTableConfigs configures the settings for tables in the state freezer.
|
||||||
var stateFreezerTableConfigs = map[string]freezerTableConfig{
|
var stateFreezerTableConfigs = map[string]freezerTableConfig{
|
||||||
stateHistoryMeta: {noSnappy: true, prunable: true},
|
stateHistoryMeta: {noSnappy: true, tailGroup: DefaultHistoryGroup},
|
||||||
stateHistoryAccountIndex: {noSnappy: false, prunable: true},
|
stateHistoryAccountIndex: {noSnappy: false, tailGroup: DefaultHistoryGroup},
|
||||||
stateHistoryStorageIndex: {noSnappy: false, prunable: true},
|
stateHistoryStorageIndex: {noSnappy: false, tailGroup: DefaultHistoryGroup},
|
||||||
stateHistoryAccountData: {noSnappy: false, prunable: true},
|
stateHistoryAccountData: {noSnappy: false, tailGroup: DefaultHistoryGroup},
|
||||||
stateHistoryStorageData: {noSnappy: false, prunable: true},
|
stateHistoryStorageData: {noSnappy: false, tailGroup: DefaultHistoryGroup},
|
||||||
}
|
}
|
||||||
|
|
||||||
const (
|
const (
|
||||||
|
|
@ -83,13 +111,13 @@ const (
|
||||||
|
|
||||||
// trienodeFreezerTableConfigs configures the settings for tables in the trienode freezer.
|
// trienodeFreezerTableConfigs configures the settings for tables in the trienode freezer.
|
||||||
var trienodeFreezerTableConfigs = map[string]freezerTableConfig{
|
var trienodeFreezerTableConfigs = map[string]freezerTableConfig{
|
||||||
trienodeHistoryHeaderTable: {noSnappy: false, prunable: true},
|
trienodeHistoryHeaderTable: {noSnappy: false, tailGroup: DefaultHistoryGroup},
|
||||||
|
|
||||||
// Disable snappy compression to allow efficient partial read.
|
// Disable snappy compression to allow efficient partial read.
|
||||||
trienodeHistoryKeySectionTable: {noSnappy: true, prunable: true},
|
trienodeHistoryKeySectionTable: {noSnappy: true, tailGroup: DefaultHistoryGroup},
|
||||||
|
|
||||||
// Disable snappy compression to allow efficient partial read.
|
// Disable snappy compression to allow efficient partial read.
|
||||||
trienodeHistoryValueSectionTable: {noSnappy: true, prunable: true},
|
trienodeHistoryValueSectionTable: {noSnappy: true, tailGroup: DefaultHistoryGroup},
|
||||||
}
|
}
|
||||||
|
|
||||||
// The list of identifiers of ancient stores.
|
// The list of identifiers of ancient stores.
|
||||||
|
|
|
||||||
|
|
@ -24,24 +24,23 @@ import (
|
||||||
"github.com/ethereum/go-ethereum/ethdb"
|
"github.com/ethereum/go-ethereum/ethdb"
|
||||||
)
|
)
|
||||||
|
|
||||||
type tableSize struct {
|
type tableInfo struct {
|
||||||
name string
|
name string
|
||||||
size common.StorageSize
|
size common.StorageSize
|
||||||
|
count uint64
|
||||||
}
|
}
|
||||||
|
|
||||||
// freezerInfo contains the basic information of the freezer.
|
// freezerInfo contains the basic information of the freezer.
|
||||||
type freezerInfo struct {
|
type freezerInfo struct {
|
||||||
name string // The identifier of freezer
|
name string // The identifier of freezer
|
||||||
head uint64 // The number of last stored item in the freezer
|
head uint64 // The number of last stored item in the freezer
|
||||||
tail uint64 // The number of first stored item in the freezer
|
tables []tableInfo // Per-table storage size and item count
|
||||||
count uint64 // The number of stored items in the freezer
|
|
||||||
sizes []tableSize // The storage size per table
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// size returns the storage size of the entire freezer.
|
// size returns the storage size of the entire freezer.
|
||||||
func (info *freezerInfo) size() common.StorageSize {
|
func (info *freezerInfo) size() common.StorageSize {
|
||||||
var total common.StorageSize
|
var total common.StorageSize
|
||||||
for _, table := range info.sizes {
|
for _, table := range info.tables {
|
||||||
total += table.size
|
total += table.size
|
||||||
}
|
}
|
||||||
return total
|
return total
|
||||||
|
|
@ -49,35 +48,41 @@ func (info *freezerInfo) size() common.StorageSize {
|
||||||
|
|
||||||
func inspect(name string, order map[string]freezerTableConfig, reader ethdb.AncientReader) (freezerInfo, error) {
|
func inspect(name string, order map[string]freezerTableConfig, reader ethdb.AncientReader) (freezerInfo, error) {
|
||||||
info := freezerInfo{name: name}
|
info := freezerInfo{name: name}
|
||||||
for t := range order {
|
|
||||||
size, err := reader.AncientSize(t)
|
// Retrieve the number of last stored item.
|
||||||
if err != nil {
|
|
||||||
return freezerInfo{}, err
|
|
||||||
}
|
|
||||||
info.sizes = append(info.sizes, tableSize{name: t, size: common.StorageSize(size)})
|
|
||||||
}
|
|
||||||
// Retrieve the number of last stored item
|
|
||||||
ancients, err := reader.Ancients()
|
ancients, err := reader.Ancients()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return freezerInfo{}, err
|
return freezerInfo{}, err
|
||||||
}
|
}
|
||||||
if ancients > 0 {
|
if ancients > 0 {
|
||||||
info.head = ancients - 1
|
info.head = ancients - 1
|
||||||
} else {
|
|
||||||
info.head = 0
|
|
||||||
}
|
}
|
||||||
|
// Resolve per-group tails so each table can report its own item count.
|
||||||
// Retrieve the number of first stored item
|
groupTails := make(map[string]uint64)
|
||||||
tail, err := reader.Tail()
|
for _, cfg := range order {
|
||||||
if err != nil {
|
if cfg.tailGroup == "" {
|
||||||
return freezerInfo{}, err
|
continue
|
||||||
|
}
|
||||||
|
if _, ok := groupTails[cfg.tailGroup]; ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
t, err := reader.Tail(cfg.tailGroup)
|
||||||
|
if err != nil {
|
||||||
|
return freezerInfo{}, err
|
||||||
|
}
|
||||||
|
groupTails[cfg.tailGroup] = t
|
||||||
}
|
}
|
||||||
info.tail = tail
|
for t, cfg := range order {
|
||||||
|
size, err := reader.AncientSize(t)
|
||||||
if ancients == 0 {
|
if err != nil {
|
||||||
info.count = 0
|
return freezerInfo{}, err
|
||||||
} else {
|
}
|
||||||
info.count = info.head - info.tail + 1
|
var count uint64
|
||||||
|
if ancients > 0 {
|
||||||
|
tail := groupTails[cfg.tailGroup] // 0 for non-prunable tables
|
||||||
|
count = ancients - tail
|
||||||
|
}
|
||||||
|
info.tables = append(info.tables, tableInfo{name: t, size: common.StorageSize(size), count: count})
|
||||||
}
|
}
|
||||||
return info, nil
|
return info, nil
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -25,6 +25,11 @@ import (
|
||||||
"github.com/ethereum/go-ethereum/internal/testrand"
|
"github.com/ethereum/go-ethereum/internal/testrand"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// TailGroup is the tail group used by tables created in this test suite. The
|
||||||
|
// store factory passed to TestAncientSuite must wire its tables to this group
|
||||||
|
// so that the suite can query the freezer's tail consistently.
|
||||||
|
const TailGroup = "test"
|
||||||
|
|
||||||
// TestAncientSuite runs a suite of tests against an ancient database
|
// TestAncientSuite runs a suite of tests against an ancient database
|
||||||
// implementation.
|
// implementation.
|
||||||
func TestAncientSuite(t *testing.T, newFn func(kinds []string) ethdb.AncientStore) {
|
func TestAncientSuite(t *testing.T, newFn func(kinds []string) ethdb.AncientStore) {
|
||||||
|
|
@ -58,11 +63,11 @@ func basicRead(t *testing.T, newFn func(kinds []string) ethdb.AncientStore) {
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
t.Fatalf("Failed to write ancient data %v", err)
|
t.Fatalf("Failed to write ancient data %v", err)
|
||||||
}
|
}
|
||||||
db.TruncateTail(10)
|
db.TruncateTail(TailGroup, 10)
|
||||||
db.TruncateHead(90)
|
db.TruncateHead(90)
|
||||||
|
|
||||||
// Test basic tail and head retrievals
|
// Test basic tail and head retrievals
|
||||||
tail, err := db.Tail()
|
tail, err := db.Tail(TailGroup)
|
||||||
if err != nil || tail != 10 {
|
if err != nil || tail != 10 {
|
||||||
t.Fatal("Failed to retrieve tail")
|
t.Fatal("Failed to retrieve tail")
|
||||||
}
|
}
|
||||||
|
|
@ -123,7 +128,7 @@ func batchRead(t *testing.T, newFn func(kinds []string) ethdb.AncientStore) {
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
t.Fatalf("Failed to write ancient data %v", err)
|
t.Fatalf("Failed to write ancient data %v", err)
|
||||||
}
|
}
|
||||||
db.TruncateTail(10)
|
db.TruncateTail(TailGroup, 10)
|
||||||
db.TruncateHead(90)
|
db.TruncateHead(90)
|
||||||
|
|
||||||
// Test the items in range should be reachable
|
// Test the items in range should be reachable
|
||||||
|
|
@ -262,12 +267,12 @@ func basicWrite(t *testing.T, newFn func(kinds []string) ethdb.AncientStore) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Write should work after truncating from tail but over the head
|
// Write should work after truncating from tail but over the head
|
||||||
db.TruncateTail(200)
|
db.TruncateTail(TailGroup, 200)
|
||||||
head, err := db.Ancients()
|
head, err := db.Ancients()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Failed to retrieve head ancients %v", err)
|
t.Fatalf("Failed to retrieve head ancients %v", err)
|
||||||
}
|
}
|
||||||
tail, err := db.Tail()
|
tail, err := db.Tail(TailGroup)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Failed to retrieve tail ancients %v", err)
|
t.Fatalf("Failed to retrieve tail ancients %v", err)
|
||||||
}
|
}
|
||||||
|
|
@ -293,7 +298,7 @@ func basicWrite(t *testing.T, newFn func(kinds []string) ethdb.AncientStore) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Failed to retrieve head ancients %v", err)
|
t.Fatalf("Failed to retrieve head ancients %v", err)
|
||||||
}
|
}
|
||||||
tail, err = db.Tail()
|
tail, err = db.Tail(TailGroup)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("Failed to retrieve tail ancients %v", err)
|
t.Fatalf("Failed to retrieve tail ancients %v", err)
|
||||||
}
|
}
|
||||||
|
|
@ -351,7 +356,7 @@ func TestResettableAncientSuite(t *testing.T, newFn func(kinds []string) ethdb.R
|
||||||
}); err != nil {
|
}); err != nil {
|
||||||
t.Fatalf("Failed to write ancient data %v", err)
|
t.Fatalf("Failed to write ancient data %v", err)
|
||||||
}
|
}
|
||||||
db.TruncateTail(10)
|
db.TruncateTail(TailGroup, 10)
|
||||||
db.TruncateHead(90)
|
db.TruncateHead(90)
|
||||||
|
|
||||||
// Ancient write should work after resetting
|
// Ancient write should work after resetting
|
||||||
|
|
|
||||||
|
|
@ -45,9 +45,7 @@ const (
|
||||||
// key-value database to flat files for saving space on live database.
|
// key-value database to flat files for saving space on live database.
|
||||||
type chainFreezer struct {
|
type chainFreezer struct {
|
||||||
ancients ethdb.AncientStore // Ancient store for storing cold chain segment
|
ancients ethdb.AncientStore // Ancient store for storing cold chain segment
|
||||||
|
eradb *eradb.Store // Optional Era database used as a backup for the pruned chain
|
||||||
// Optional Era database used as a backup for the pruned chain.
|
|
||||||
eradb *eradb.Store
|
|
||||||
|
|
||||||
quit chan struct{}
|
quit chan struct{}
|
||||||
wg sync.WaitGroup
|
wg sync.WaitGroup
|
||||||
|
|
@ -327,6 +325,16 @@ func (f *chainFreezer) freezeRange(nfdb *nofreezedb, number, limit uint64) (hash
|
||||||
if len(receipts) == 0 {
|
if len(receipts) == 0 {
|
||||||
return fmt.Errorf("block receipts missing, can't freeze block %d", number)
|
return fmt.Errorf("block receipts missing, can't freeze block %d", number)
|
||||||
}
|
}
|
||||||
|
// An empty block access list is allowed and may occur in multiple
|
||||||
|
// scenarios, such as:
|
||||||
|
// - pre-Amsterdam blocks
|
||||||
|
// - post-Amsterdam blocks with the BAL absent (e.g. pruned by network)
|
||||||
|
// - post-Amsterdam blocks with an explicitly empty BAL
|
||||||
|
//
|
||||||
|
// In these cases, a nil entry will be stored in the BAL table as the
|
||||||
|
// absence placeholder.
|
||||||
|
bals := ReadAccessListRLP(nfdb, hash, number)
|
||||||
|
|
||||||
// Write to the batch.
|
// Write to the batch.
|
||||||
if err := op.AppendRaw(ChainFreezerHashTable, number, hash[:]); err != nil {
|
if err := op.AppendRaw(ChainFreezerHashTable, number, hash[:]); err != nil {
|
||||||
return fmt.Errorf("can't write hash to Freezer: %v", err)
|
return fmt.Errorf("can't write hash to Freezer: %v", err)
|
||||||
|
|
@ -340,6 +348,9 @@ func (f *chainFreezer) freezeRange(nfdb *nofreezedb, number, limit uint64) (hash
|
||||||
if err := op.AppendRaw(ChainFreezerReceiptTable, number, receipts); err != nil {
|
if err := op.AppendRaw(ChainFreezerReceiptTable, number, receipts); err != nil {
|
||||||
return fmt.Errorf("can't write receipts to Freezer: %v", err)
|
return fmt.Errorf("can't write receipts to Freezer: %v", err)
|
||||||
}
|
}
|
||||||
|
if err := op.AppendRaw(ChainFreezerBALTable, number, bals); err != nil {
|
||||||
|
return fmt.Errorf("can't write bals to Freezer: %v", err)
|
||||||
|
}
|
||||||
hashes = append(hashes, hash)
|
hashes = append(hashes, hash)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
|
|
@ -354,7 +365,11 @@ func (f *chainFreezer) Ancient(kind string, number uint64) ([]byte, error) {
|
||||||
if kind == ChainFreezerHeaderTable || kind == ChainFreezerHashTable {
|
if kind == ChainFreezerHeaderTable || kind == ChainFreezerHashTable {
|
||||||
return f.ancients.Ancient(kind, number)
|
return f.ancients.Ancient(kind, number)
|
||||||
}
|
}
|
||||||
tail, err := f.ancients.Tail()
|
group, err := tableTailGroup(kind)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
tail, err := f.ancients.Tail(group)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
@ -371,10 +386,20 @@ func (f *chainFreezer) Ancient(kind string, number uint64) ([]byte, error) {
|
||||||
return f.eradb.GetRawBody(number)
|
return f.eradb.GetRawBody(number)
|
||||||
case ChainFreezerReceiptTable:
|
case ChainFreezerReceiptTable:
|
||||||
return f.eradb.GetRawReceipts(number)
|
return f.eradb.GetRawReceipts(number)
|
||||||
|
case ChainFreezerBALTable:
|
||||||
|
return nil, errOutOfBounds
|
||||||
}
|
}
|
||||||
return nil, errUnknownTable
|
return nil, errUnknownTable
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// tableTailGroup returns the tail group identifier for a chain freezer table.
|
||||||
|
func tableTailGroup(kind string) (string, error) {
|
||||||
|
if cfg, ok := chainFreezerTableConfigs[kind]; ok {
|
||||||
|
return cfg.tailGroup, nil
|
||||||
|
}
|
||||||
|
return "", errUnknownTable
|
||||||
|
}
|
||||||
|
|
||||||
// ReadAncients executes an operation while preventing mutations to the freezer,
|
// ReadAncients executes an operation while preventing mutations to the freezer,
|
||||||
// i.e. if fn performs multiple reads, they will be consistent with each other.
|
// i.e. if fn performs multiple reads, they will be consistent with each other.
|
||||||
func (f *chainFreezer) ReadAncients(fn func(ethdb.AncientReaderOp) error) (err error) {
|
func (f *chainFreezer) ReadAncients(fn func(ethdb.AncientReaderOp) error) (err error) {
|
||||||
|
|
@ -391,8 +416,8 @@ func (f *chainFreezer) Ancients() (uint64, error) {
|
||||||
return f.ancients.Ancients()
|
return f.ancients.Ancients()
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *chainFreezer) Tail() (uint64, error) {
|
func (f *chainFreezer) Tail(group string) (uint64, error) {
|
||||||
return f.ancients.Tail()
|
return f.ancients.Tail(group)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *chainFreezer) AncientSize(kind string) (uint64, error) {
|
func (f *chainFreezer) AncientSize(kind string) (uint64, error) {
|
||||||
|
|
@ -415,8 +440,8 @@ func (f *chainFreezer) TruncateHead(items uint64) (uint64, error) {
|
||||||
return f.ancients.TruncateHead(items)
|
return f.ancients.TruncateHead(items)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *chainFreezer) TruncateTail(items uint64) (uint64, error) {
|
func (f *chainFreezer) TruncateTail(group string, items uint64) (uint64, error) {
|
||||||
return f.ancients.TruncateTail(items)
|
return f.ancients.TruncateTail(group, items)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *chainFreezer) SyncAncient() error {
|
func (f *chainFreezer) SyncAncient() error {
|
||||||
|
|
|
||||||
|
|
@ -113,7 +113,7 @@ func (db *nofreezedb) Ancients() (uint64, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Tail returns an error as we don't have a backing chain freezer.
|
// Tail returns an error as we don't have a backing chain freezer.
|
||||||
func (db *nofreezedb) Tail() (uint64, error) {
|
func (db *nofreezedb) Tail(group string) (uint64, error) {
|
||||||
return 0, errNotSupported
|
return 0, errNotSupported
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -133,7 +133,7 @@ func (db *nofreezedb) TruncateHead(items uint64) (uint64, error) {
|
||||||
}
|
}
|
||||||
|
|
||||||
// TruncateTail returns an error as we don't have a backing chain freezer.
|
// TruncateTail returns an error as we don't have a backing chain freezer.
|
||||||
func (db *nofreezedb) TruncateTail(items uint64) (uint64, error) {
|
func (db *nofreezedb) TruncateTail(group string, items uint64) (uint64, error) {
|
||||||
return 0, errNotSupported
|
return 0, errNotSupported
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -658,12 +658,12 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
for _, ancient := range ancients {
|
for _, ancient := range ancients {
|
||||||
for _, table := range ancient.sizes {
|
for _, table := range ancient.tables {
|
||||||
stats = append(stats, []string{
|
stats = append(stats, []string{
|
||||||
fmt.Sprintf("Ancient store (%s)", strings.Title(ancient.name)),
|
fmt.Sprintf("Ancient store (%s)", strings.Title(ancient.name)),
|
||||||
strings.Title(table.name),
|
strings.Title(table.name),
|
||||||
table.size.String(),
|
table.size.String(),
|
||||||
fmt.Sprintf("%d", ancient.count),
|
fmt.Sprintf("%d", table.count),
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
total.Add(uint64(ancient.size()))
|
total.Add(uint64(ancient.size()))
|
||||||
|
|
|
||||||
|
|
@ -59,8 +59,8 @@ const freezerTableSize = 2 * 1000 * 1000 * 1000
|
||||||
// - The in-order data ensures that disk reads are always optimized.
|
// - The in-order data ensures that disk reads are always optimized.
|
||||||
type Freezer struct {
|
type Freezer struct {
|
||||||
datadir string
|
datadir string
|
||||||
head atomic.Uint64 // Number of items stored (including items removed from tail)
|
head atomic.Uint64 // Number of items stored (including items removed from tail)
|
||||||
tail atomic.Uint64 // Number of the first stored item in the freezer
|
tails map[string]*atomic.Uint64 // Per-group tail cache, keyed by tail group name
|
||||||
|
|
||||||
// This lock synchronizes writers and the truncate operation, as well as
|
// This lock synchronizes writers and the truncate operation, as well as
|
||||||
// the "atomic" (batched) read operations.
|
// the "atomic" (batched) read operations.
|
||||||
|
|
@ -77,8 +77,8 @@ type Freezer struct {
|
||||||
// data according to the given parameters.
|
// data according to the given parameters.
|
||||||
//
|
//
|
||||||
// The 'tables' argument defines the freezer tables and their configuration.
|
// The 'tables' argument defines the freezer tables and their configuration.
|
||||||
// Each value is a freezerTableConfig specifying whether snappy compression is
|
// Each value is a freezerTableConfig describing whether Snappy compression
|
||||||
// disabled (noSnappy) and whether the table is prunable (prunable).
|
// is disabled (noSnappy) and which tail group the table belongs to.
|
||||||
func NewFreezer(datadir string, namespace string, readonly bool, maxTableSize uint32, tables map[string]freezerTableConfig) (*Freezer, error) {
|
func NewFreezer(datadir string, namespace string, readonly bool, maxTableSize uint32, tables map[string]freezerTableConfig) (*Freezer, error) {
|
||||||
// Create the initial freezer object
|
// Create the initial freezer object
|
||||||
var (
|
var (
|
||||||
|
|
@ -118,6 +118,7 @@ func NewFreezer(datadir string, namespace string, readonly bool, maxTableSize ui
|
||||||
datadir: datadir,
|
datadir: datadir,
|
||||||
readonly: readonly,
|
readonly: readonly,
|
||||||
tables: make(map[string]*freezerTable),
|
tables: make(map[string]*freezerTable),
|
||||||
|
tails: make(map[string]*atomic.Uint64),
|
||||||
instanceLock: lock,
|
instanceLock: lock,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -216,9 +217,19 @@ func (f *Freezer) Ancients() (uint64, error) {
|
||||||
return f.head.Load(), nil
|
return f.head.Load(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Tail returns the number of first stored item in the freezer.
|
// Tail returns the lowest accessible item index for the given tail group.
|
||||||
func (f *Freezer) Tail() (uint64, error) {
|
// All tables sharing this group agree on the tail; an empty group name
|
||||||
return f.tail.Load(), nil
|
// refers to non-prunable tables and always returns 0. Unknown groups return
|
||||||
|
// an error.
|
||||||
|
func (f *Freezer) Tail(group string) (uint64, error) {
|
||||||
|
if group == "" {
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
tail, ok := f.tails[group]
|
||||||
|
if !ok {
|
||||||
|
return 0, fmt.Errorf("unknown tail group: %q", group)
|
||||||
|
}
|
||||||
|
return tail.Load(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// AncientSize returns the ancient size of the specified category.
|
// AncientSize returns the ancient size of the specified category.
|
||||||
|
|
@ -299,33 +310,43 @@ func (f *Freezer) TruncateHead(items uint64) (uint64, error) {
|
||||||
return oitems, nil
|
return oitems, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// TruncateTail discards all data below the specified threshold. Note that only
|
// TruncateTail discards all data below the specified threshold across every
|
||||||
// 'prunable' tables will be truncated.
|
// table that belongs to the named tail group. Tables that are already past
|
||||||
func (f *Freezer) TruncateTail(tail uint64) (uint64, error) {
|
// the threshold are left untouched. The previous tail of the group is
|
||||||
|
// returned. An empty group name or an unknown group name returns an error.
|
||||||
|
func (f *Freezer) TruncateTail(group string, tail uint64) (uint64, error) {
|
||||||
if f.readonly {
|
if f.readonly {
|
||||||
return 0, errReadOnly
|
return 0, errReadOnly
|
||||||
}
|
}
|
||||||
|
if group == "" {
|
||||||
|
return 0, errors.New("empty tail group")
|
||||||
|
}
|
||||||
|
cached, ok := f.tails[group]
|
||||||
|
if !ok {
|
||||||
|
return 0, fmt.Errorf("unknown tail group: %q", group)
|
||||||
|
}
|
||||||
f.writeLock.Lock()
|
f.writeLock.Lock()
|
||||||
defer f.writeLock.Unlock()
|
defer f.writeLock.Unlock()
|
||||||
|
|
||||||
old := f.tail.Load()
|
prev := cached.Load()
|
||||||
if old >= tail {
|
if prev >= tail {
|
||||||
return old, nil
|
return prev, nil
|
||||||
}
|
}
|
||||||
for _, table := range f.tables {
|
for _, table := range f.tables {
|
||||||
if table.config.prunable {
|
if table.config.tailGroup != group {
|
||||||
if err := table.truncateTail(tail); err != nil {
|
continue
|
||||||
return 0, err
|
}
|
||||||
}
|
if err := table.truncateTail(tail); err != nil {
|
||||||
|
return 0, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
f.tail.Store(tail)
|
cached.Store(tail)
|
||||||
|
|
||||||
// Update the head if the requested tail exceeds the current head
|
// Update the head if the requested tail exceeds the current head.
|
||||||
if f.head.Load() < tail {
|
if f.head.Load() < tail {
|
||||||
f.head.Store(tail)
|
f.head.Store(tail)
|
||||||
}
|
}
|
||||||
return old, nil
|
return prev, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// SyncAncient flushes all data tables to disk.
|
// SyncAncient flushes all data tables to disk.
|
||||||
|
|
@ -342,84 +363,123 @@ func (f *Freezer) SyncAncient() error {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// validate checks that every table has the same boundary.
|
// validate checks that every table has the same head and that tables sharing
|
||||||
// Used instead of `repair` in readonly mode.
|
// a tail group also share a tail. Used instead of `repair` in readonly mode.
|
||||||
func (f *Freezer) validate() error {
|
func (f *Freezer) validate() error {
|
||||||
if len(f.tables) == 0 {
|
if len(f.tables) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
var (
|
var (
|
||||||
head uint64
|
head uint64
|
||||||
prunedTail *uint64
|
headSet bool
|
||||||
|
tails = make(map[string]uint64)
|
||||||
)
|
)
|
||||||
// get any head value
|
|
||||||
for _, table := range f.tables {
|
|
||||||
head = table.items.Load()
|
|
||||||
break
|
|
||||||
}
|
|
||||||
for kind, table := range f.tables {
|
for kind, table := range f.tables {
|
||||||
// all tables have to have the same head
|
// Validate the table head
|
||||||
if head != table.items.Load() {
|
items := table.items.Load()
|
||||||
return fmt.Errorf("freezer table %s has a differing head: %d != %d", kind, table.items.Load(), head)
|
if !headSet {
|
||||||
|
head = items
|
||||||
|
headSet = true
|
||||||
|
} else if items != head {
|
||||||
|
return fmt.Errorf("freezer table %s has a differing head: %d != %d", kind, items, head)
|
||||||
}
|
}
|
||||||
if !table.config.prunable {
|
// Validate the table tail
|
||||||
// non-prunable tables have to start at 0
|
if table.config.tailGroup == "" {
|
||||||
if table.itemHidden.Load() != 0 {
|
if table.itemHidden.Load() != 0 {
|
||||||
return fmt.Errorf("non-prunable freezer table '%s' has a non-zero tail: %d", kind, table.itemHidden.Load())
|
return fmt.Errorf("non-prunable freezer table '%s' has a non-zero tail: %d", kind, table.itemHidden.Load())
|
||||||
}
|
}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
hidden := table.itemHidden.Load()
|
||||||
|
if t, ok := tails[table.config.tailGroup]; ok {
|
||||||
|
if t != hidden {
|
||||||
|
return fmt.Errorf("freezer table %s has differing tail in group %q: %d != %d", kind, table.config.tailGroup, hidden, t)
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
// prunable tables have to have the same length
|
tails[table.config.tailGroup] = hidden
|
||||||
if prunedTail == nil {
|
|
||||||
tmp := table.itemHidden.Load()
|
|
||||||
prunedTail = &tmp
|
|
||||||
}
|
|
||||||
if *prunedTail != table.itemHidden.Load() {
|
|
||||||
return fmt.Errorf("freezer table %s has differing tail: %d != %d", kind, table.itemHidden.Load(), *prunedTail)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if prunedTail == nil {
|
|
||||||
tmp := uint64(0)
|
|
||||||
prunedTail = &tmp
|
|
||||||
}
|
|
||||||
|
|
||||||
f.head.Store(head)
|
f.head.Store(head)
|
||||||
f.tail.Store(*prunedTail)
|
|
||||||
|
for group, tail := range tails {
|
||||||
|
counter := new(atomic.Uint64)
|
||||||
|
counter.Store(tail)
|
||||||
|
f.tails[group] = counter
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// repair truncates all data tables to the same length.
|
// repair brings every table into a consistent state. The common head is taken
|
||||||
|
// as the minimum item count among non-empty tables; freshly added empty tables
|
||||||
|
// are fast-forwarded to that head via tail truncation. Within each tail group
|
||||||
|
// the maximum tail wins, and prunable tables are truncated to it.
|
||||||
func (f *Freezer) repair() error {
|
func (f *Freezer) repair() error {
|
||||||
|
// Determine the common head from non-empty tables. Empty tables are
|
||||||
|
// excluded so that a freshly added table cannot drag the existing head
|
||||||
|
// down to zero on first cold-start.
|
||||||
var (
|
var (
|
||||||
head = uint64(math.MaxUint64)
|
hasNonEmpty bool
|
||||||
prunedTail = uint64(0)
|
head uint64 = math.MaxUint64
|
||||||
)
|
)
|
||||||
// get the minimal head and the maximum tail
|
|
||||||
for _, table := range f.tables {
|
for _, table := range f.tables {
|
||||||
head = min(head, table.items.Load())
|
if table.items.Load() == 0 {
|
||||||
prunedTail = max(prunedTail, table.itemHidden.Load())
|
continue
|
||||||
|
}
|
||||||
|
if items := table.items.Load(); items < head {
|
||||||
|
head = items
|
||||||
|
}
|
||||||
|
hasNonEmpty = true
|
||||||
}
|
}
|
||||||
// apply the pruning
|
if !hasNonEmpty {
|
||||||
for kind, table := range f.tables {
|
head = 0
|
||||||
// all tables need to have the same head
|
}
|
||||||
|
// Align newly added empty tables to the common head. truncateTail
|
||||||
|
// internally calls resetTo when the requested tail exceeds the current
|
||||||
|
// head, which is exactly what we need here.
|
||||||
|
if head > 0 {
|
||||||
|
for _, table := range f.tables {
|
||||||
|
if table.items.Load() == 0 {
|
||||||
|
if err := table.truncateTail(head); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Truncate every table to the common head.
|
||||||
|
for _, table := range f.tables {
|
||||||
if err := table.truncateHead(head); err != nil {
|
if err := table.truncateHead(head); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if !table.config.prunable {
|
}
|
||||||
// non-prunable tables have to start at 0
|
// Per-group tail alignment: take the maximum tail in each group and apply
|
||||||
|
// it to all members. Non-prunable tables must remain at tail 0.
|
||||||
|
tails := make(map[string]uint64)
|
||||||
|
for kind, table := range f.tables {
|
||||||
|
if table.config.tailGroup == "" {
|
||||||
if table.itemHidden.Load() != 0 {
|
if table.itemHidden.Load() != 0 {
|
||||||
panic(fmt.Sprintf("non-prunable freezer table %s has non-zero tail: %v", kind, table.itemHidden.Load()))
|
panic(fmt.Sprintf("non-prunable freezer table %s has non-zero tail: %v", kind, table.itemHidden.Load()))
|
||||||
}
|
}
|
||||||
} else {
|
continue
|
||||||
// prunable tables have to have the same length
|
}
|
||||||
if err := table.truncateTail(prunedTail); err != nil {
|
hidden := table.itemHidden.Load()
|
||||||
return err
|
if t, ok := tails[table.config.tailGroup]; !ok || hidden > t {
|
||||||
}
|
tails[table.config.tailGroup] = hidden
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for _, table := range f.tables {
|
||||||
|
if table.config.tailGroup == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if err := table.truncateTail(tails[table.config.tailGroup]); err != nil {
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
f.head.Store(head)
|
f.head.Store(head)
|
||||||
f.tail.Store(prunedTail)
|
|
||||||
|
for group, tail := range tails {
|
||||||
|
counter := new(atomic.Uint64)
|
||||||
|
counter.Store(tail)
|
||||||
|
f.tails[group] = counter
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -228,7 +228,7 @@ func (b *memoryBatch) commit(freezer *MemoryFreezer) (items uint64, writeSize in
|
||||||
// interface and can be used along with ephemeral key-value store.
|
// interface and can be used along with ephemeral key-value store.
|
||||||
type MemoryFreezer struct {
|
type MemoryFreezer struct {
|
||||||
items uint64 // Number of items stored
|
items uint64 // Number of items stored
|
||||||
tail uint64 // Number of the first stored item in the freezer
|
tails map[string]uint64 // Per-group tail cache; access serialized by lock
|
||||||
readonly bool // Flag if the freezer is only for reading
|
readonly bool // Flag if the freezer is only for reading
|
||||||
lock sync.RWMutex // Lock to protect fields
|
lock sync.RWMutex // Lock to protect fields
|
||||||
tables map[string]*memoryTable // Tables for storing everything
|
tables map[string]*memoryTable // Tables for storing everything
|
||||||
|
|
@ -237,14 +237,21 @@ type MemoryFreezer struct {
|
||||||
|
|
||||||
// NewMemoryFreezer initializes an in-memory freezer instance.
|
// NewMemoryFreezer initializes an in-memory freezer instance.
|
||||||
func NewMemoryFreezer(readonly bool, tableName map[string]freezerTableConfig) *MemoryFreezer {
|
func NewMemoryFreezer(readonly bool, tableName map[string]freezerTableConfig) *MemoryFreezer {
|
||||||
tables := make(map[string]*memoryTable)
|
var (
|
||||||
|
tables = make(map[string]*memoryTable)
|
||||||
|
tails = make(map[string]uint64)
|
||||||
|
)
|
||||||
for name, cfg := range tableName {
|
for name, cfg := range tableName {
|
||||||
tables[name] = newMemoryTable(name, cfg)
|
tables[name] = newMemoryTable(name, cfg)
|
||||||
|
if cfg.tailGroup != "" {
|
||||||
|
tails[cfg.tailGroup] = 0
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return &MemoryFreezer{
|
return &MemoryFreezer{
|
||||||
writeBatch: newMemoryBatch(),
|
writeBatch: newMemoryBatch(),
|
||||||
readonly: readonly,
|
readonly: readonly,
|
||||||
tables: tables,
|
tables: tables,
|
||||||
|
tails: tails,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -289,13 +296,21 @@ func (f *MemoryFreezer) Ancients() (uint64, error) {
|
||||||
return f.items, nil
|
return f.items, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// Tail returns the number of first stored item in the freezer.
|
// Tail returns the lowest accessible item index for the given tail group.
|
||||||
// This number can also be interpreted as the total deleted item numbers.
|
// All tables sharing the group agree on the tail; an empty group name
|
||||||
func (f *MemoryFreezer) Tail() (uint64, error) {
|
// refers to non-prunable tables and always returns 0.
|
||||||
|
func (f *MemoryFreezer) Tail(group string) (uint64, error) {
|
||||||
f.lock.RLock()
|
f.lock.RLock()
|
||||||
defer f.lock.RUnlock()
|
defer f.lock.RUnlock()
|
||||||
|
|
||||||
return f.tail, nil
|
if group == "" {
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
tail, ok := f.tails[group]
|
||||||
|
if !ok {
|
||||||
|
return 0, fmt.Errorf("unknown tail group: %q", group)
|
||||||
|
}
|
||||||
|
return tail, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// AncientSize returns the ancient size of the specified category.
|
// AncientSize returns the ancient size of the specified category.
|
||||||
|
|
@ -375,32 +390,39 @@ func (f *MemoryFreezer) TruncateHead(items uint64) (uint64, error) {
|
||||||
return old, nil
|
return old, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// TruncateTail discards all data below the provided threshold number.
|
// TruncateTail discards all data below the provided threshold across every
|
||||||
// Note this will only truncate 'prunable' tables. Block headers and canonical
|
// table that belongs to the named tail group. Tables already past the
|
||||||
// hashes cannot be truncated at this time.
|
// threshold are left untouched. The previous tail of the group is returned.
|
||||||
func (f *MemoryFreezer) TruncateTail(tail uint64) (uint64, error) {
|
func (f *MemoryFreezer) TruncateTail(group string, tail uint64) (uint64, error) {
|
||||||
f.lock.Lock()
|
f.lock.Lock()
|
||||||
defer f.lock.Unlock()
|
defer f.lock.Unlock()
|
||||||
|
|
||||||
if f.readonly {
|
if f.readonly {
|
||||||
return 0, errReadOnly
|
return 0, errReadOnly
|
||||||
}
|
}
|
||||||
old := f.tail
|
if group == "" {
|
||||||
if old >= tail {
|
return 0, errors.New("empty tail group")
|
||||||
return old, nil
|
}
|
||||||
|
prev, ok := f.tails[group]
|
||||||
|
if !ok {
|
||||||
|
return 0, fmt.Errorf("unknown tail group: %q", group)
|
||||||
|
}
|
||||||
|
if prev >= tail {
|
||||||
|
return prev, nil
|
||||||
}
|
}
|
||||||
for _, table := range f.tables {
|
for _, table := range f.tables {
|
||||||
if table.config.prunable {
|
if table.config.tailGroup != group {
|
||||||
if err := table.truncateTail(tail); err != nil {
|
continue
|
||||||
return 0, err
|
}
|
||||||
}
|
if err := table.truncateTail(tail); err != nil {
|
||||||
|
return 0, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
f.tail = tail
|
f.tails[group] = tail
|
||||||
if f.items < tail {
|
if f.items < tail {
|
||||||
f.items = tail
|
f.items = tail
|
||||||
}
|
}
|
||||||
return old, nil
|
return prev, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// SyncAncient flushes all data tables to disk.
|
// SyncAncient flushes all data tables to disk.
|
||||||
|
|
@ -426,11 +448,16 @@ func (f *MemoryFreezer) Reset() error {
|
||||||
defer f.lock.Unlock()
|
defer f.lock.Unlock()
|
||||||
|
|
||||||
tables := make(map[string]*memoryTable)
|
tables := make(map[string]*memoryTable)
|
||||||
|
tails := make(map[string]uint64)
|
||||||
for name, table := range f.tables {
|
for name, table := range f.tables {
|
||||||
tables[name] = newMemoryTable(name, table.config)
|
tables[name] = newMemoryTable(name, table.config)
|
||||||
|
if table.config.tailGroup != "" {
|
||||||
|
tails[table.config.tailGroup] = 0
|
||||||
|
}
|
||||||
}
|
}
|
||||||
f.tables = tables
|
f.tables = tables
|
||||||
f.items, f.tail = 0, 0
|
f.tails = tails
|
||||||
|
f.items = 0
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -28,8 +28,8 @@ func TestMemoryFreezer(t *testing.T) {
|
||||||
tables := make(map[string]freezerTableConfig)
|
tables := make(map[string]freezerTableConfig)
|
||||||
for _, kind := range kinds {
|
for _, kind := range kinds {
|
||||||
tables[kind] = freezerTableConfig{
|
tables[kind] = freezerTableConfig{
|
||||||
noSnappy: true,
|
noSnappy: true,
|
||||||
prunable: true,
|
tailGroup: ancienttest.TailGroup,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return NewMemoryFreezer(false, tables)
|
return NewMemoryFreezer(false, tables)
|
||||||
|
|
@ -38,8 +38,8 @@ func TestMemoryFreezer(t *testing.T) {
|
||||||
tables := make(map[string]freezerTableConfig)
|
tables := make(map[string]freezerTableConfig)
|
||||||
for _, kind := range kinds {
|
for _, kind := range kinds {
|
||||||
tables[kind] = freezerTableConfig{
|
tables[kind] = freezerTableConfig{
|
||||||
noSnappy: true,
|
noSnappy: true,
|
||||||
prunable: true,
|
tailGroup: ancienttest.TailGroup,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return NewMemoryFreezer(false, tables)
|
return NewMemoryFreezer(false, tables)
|
||||||
|
|
|
||||||
|
|
@ -143,12 +143,12 @@ func (f *resettableFreezer) Ancients() (uint64, error) {
|
||||||
return f.freezer.Ancients()
|
return f.freezer.Ancients()
|
||||||
}
|
}
|
||||||
|
|
||||||
// Tail returns the number of first stored item in the freezer.
|
// Tail returns the lowest accessible item index for the given tail group.
|
||||||
func (f *resettableFreezer) Tail() (uint64, error) {
|
func (f *resettableFreezer) Tail(group string) (uint64, error) {
|
||||||
f.lock.RLock()
|
f.lock.RLock()
|
||||||
defer f.lock.RUnlock()
|
defer f.lock.RUnlock()
|
||||||
|
|
||||||
return f.freezer.Tail()
|
return f.freezer.Tail(group)
|
||||||
}
|
}
|
||||||
|
|
||||||
// AncientSize returns the ancient size of the specified category.
|
// AncientSize returns the ancient size of the specified category.
|
||||||
|
|
@ -185,13 +185,13 @@ func (f *resettableFreezer) TruncateHead(items uint64) (uint64, error) {
|
||||||
return f.freezer.TruncateHead(items)
|
return f.freezer.TruncateHead(items)
|
||||||
}
|
}
|
||||||
|
|
||||||
// TruncateTail discards any recent data below the provided threshold number.
|
// TruncateTail discards data below the provided threshold for the named tail
|
||||||
// It returns the previous value
|
// group. It returns the previous tail of the group.
|
||||||
func (f *resettableFreezer) TruncateTail(tail uint64) (uint64, error) {
|
func (f *resettableFreezer) TruncateTail(group string, tail uint64) (uint64, error) {
|
||||||
f.lock.RLock()
|
f.lock.RLock()
|
||||||
defer f.lock.RUnlock()
|
defer f.lock.RUnlock()
|
||||||
|
|
||||||
return f.freezer.TruncateTail(tail)
|
return f.freezer.TruncateTail(group, tail)
|
||||||
}
|
}
|
||||||
|
|
||||||
// SyncAncient flushes all data tables to disk.
|
// SyncAncient flushes all data tables to disk.
|
||||||
|
|
|
||||||
|
|
@ -371,6 +371,133 @@ func checkAncientCount(t *testing.T, f *Freezer, kind string, n uint64) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestChainFreezerBALAlignment exercises the new-table alignment path: a chain
|
||||||
|
// freezer is first opened with the legacy table set (no BAL), populated with a
|
||||||
|
// few blocks and closed. It is then re-opened with the full chain freezer
|
||||||
|
// table set (which includes the BAL column). The expectation is that the BAL
|
||||||
|
// table is fast-forwarded to the existing head without disturbing the body /
|
||||||
|
// receipt tables, and that subsequent writes append cleanly across all tables.
|
||||||
|
func TestChainFreezerBALAlignment(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
|
||||||
|
// Build a "legacy" subset of the chain freezer table set, omitting BAL.
|
||||||
|
legacyTables := make(map[string]freezerTableConfig)
|
||||||
|
for name, cfg := range chainFreezerTableConfigs {
|
||||||
|
if name == ChainFreezerBALTable {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
legacyTables[name] = cfg
|
||||||
|
}
|
||||||
|
|
||||||
|
// First open: legacy config. Fill in `items` blocks of dummy data.
|
||||||
|
const items = uint64(10)
|
||||||
|
payload := bytes.Repeat([]byte{0xab}, 64)
|
||||||
|
|
||||||
|
f, err := NewFreezer(dir, "", false, 2049, legacyTables)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("can't open legacy freezer: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := f.ModifyAncients(func(op ethdb.AncientWriteOp) error {
|
||||||
|
for i := uint64(0); i < items; i++ {
|
||||||
|
if err := op.AppendRaw(ChainFreezerHashTable, i, payload); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := op.AppendRaw(ChainFreezerHeaderTable, i, payload); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := op.AppendRaw(ChainFreezerBodiesTable, i, payload); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := op.AppendRaw(ChainFreezerReceiptTable, i, payload); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("legacy write failed: %v", err)
|
||||||
|
}
|
||||||
|
if got, _ := f.Ancients(); got != items {
|
||||||
|
t.Fatalf("legacy head: got %d, want %d", got, items)
|
||||||
|
}
|
||||||
|
require.NoError(t, f.Close())
|
||||||
|
|
||||||
|
// Re-open with the full chain freezer table set, which now includes BAL.
|
||||||
|
// repair() should detect the empty BAL table and fast-forward it to the
|
||||||
|
// existing head rather than truncating everyone down to zero.
|
||||||
|
f, err = NewFreezer(dir, "", false, 2049, chainFreezerTableConfigs)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("can't re-open freezer with BAL added: %v", err)
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
|
||||||
|
// The head must be preserved.
|
||||||
|
if got, _ := f.Ancients(); got != items {
|
||||||
|
t.Fatalf("head after re-open: got %d, want %d", got, items)
|
||||||
|
}
|
||||||
|
// Existing data must still be readable in full.
|
||||||
|
for i := uint64(0); i < items; i++ {
|
||||||
|
for _, kind := range []string{
|
||||||
|
ChainFreezerHashTable, ChainFreezerHeaderTable,
|
||||||
|
ChainFreezerBodiesTable, ChainFreezerReceiptTable,
|
||||||
|
} {
|
||||||
|
got, err := f.Ancient(kind, i)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("read %s[%d]: %v", kind, i, err)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(got, payload) {
|
||||||
|
t.Fatalf("read %s[%d]: payload mismatch", kind, i)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// The block-data tail must be unchanged (no spurious tail bump).
|
||||||
|
if tail, err := f.Tail(ChainFreezerBlockDataGroup); err != nil || tail != 0 {
|
||||||
|
t.Fatalf("blockdata tail: got %d (err %v), want 0", tail, err)
|
||||||
|
}
|
||||||
|
// The BAL tail should equal the head — the table is empty but aligned.
|
||||||
|
if tail, err := f.Tail(ChainFreezerBALGroup); err != nil || tail != items {
|
||||||
|
t.Fatalf("BAL tail: got %d (err %v), want %d", tail, err, items)
|
||||||
|
}
|
||||||
|
// Reads to BAL for any pre-alignment block must report out-of-bounds.
|
||||||
|
for i := uint64(0); i < items; i++ {
|
||||||
|
if _, err := f.Ancient(ChainFreezerBALTable, i); err == nil {
|
||||||
|
t.Fatalf("reading BAL[%d] succeeded; want error (out of bounds)", i)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// A subsequent batch must append uniformly to every table, BAL included.
|
||||||
|
balPayload := []byte("real-bal")
|
||||||
|
if _, err := f.ModifyAncients(func(op ethdb.AncientWriteOp) error {
|
||||||
|
i := items
|
||||||
|
if err := op.AppendRaw(ChainFreezerHashTable, i, payload); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := op.AppendRaw(ChainFreezerHeaderTable, i, payload); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := op.AppendRaw(ChainFreezerBodiesTable, i, payload); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := op.AppendRaw(ChainFreezerReceiptTable, i, payload); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
if err := op.AppendRaw(ChainFreezerBALTable, i, balPayload); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("post-alignment write failed: %v", err)
|
||||||
|
}
|
||||||
|
if got, _ := f.Ancients(); got != items+1 {
|
||||||
|
t.Fatalf("head after post-alignment write: got %d, want %d", got, items+1)
|
||||||
|
}
|
||||||
|
got, err := f.Ancient(ChainFreezerBALTable, items)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("BAL[%d]: %v", items, err)
|
||||||
|
}
|
||||||
|
if !bytes.Equal(got, balPayload) {
|
||||||
|
t.Fatalf("BAL[%d]: got %x, want %x", items, got, balPayload)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestFreezerCloseSync(t *testing.T) {
|
func TestFreezerCloseSync(t *testing.T) {
|
||||||
t.Parallel()
|
t.Parallel()
|
||||||
f, _ := newFreezerForTesting(t, map[string]freezerTableConfig{"a": {noSnappy: true}, "b": {noSnappy: true}})
|
f, _ := newFreezerForTesting(t, map[string]freezerTableConfig{"a": {noSnappy: true}, "b": {noSnappy: true}})
|
||||||
|
|
@ -398,8 +525,8 @@ func TestFreezerSuite(t *testing.T) {
|
||||||
tables := make(map[string]freezerTableConfig)
|
tables := make(map[string]freezerTableConfig)
|
||||||
for _, kind := range kinds {
|
for _, kind := range kinds {
|
||||||
tables[kind] = freezerTableConfig{
|
tables[kind] = freezerTableConfig{
|
||||||
noSnappy: true,
|
noSnappy: true,
|
||||||
prunable: true,
|
tailGroup: ancienttest.TailGroup,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
f, _ := newFreezerForTesting(t, tables)
|
f, _ := newFreezerForTesting(t, tables)
|
||||||
|
|
@ -409,8 +536,8 @@ func TestFreezerSuite(t *testing.T) {
|
||||||
tables := make(map[string]freezerTableConfig)
|
tables := make(map[string]freezerTableConfig)
|
||||||
for _, kind := range kinds {
|
for _, kind := range kinds {
|
||||||
tables[kind] = freezerTableConfig{
|
tables[kind] = freezerTableConfig{
|
||||||
noSnappy: true,
|
noSnappy: true,
|
||||||
prunable: true,
|
tailGroup: ancienttest.TailGroup,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
f, _ := newResettableFreezer(t.TempDir(), "", false, 2048, tables)
|
f, _ := newResettableFreezer(t.TempDir(), "", false, 2048, tables)
|
||||||
|
|
|
||||||
|
|
@ -76,8 +76,8 @@ func (t *table) Ancients() (uint64, error) {
|
||||||
|
|
||||||
// Tail is a noop passthrough that just forwards the request to the underlying
|
// Tail is a noop passthrough that just forwards the request to the underlying
|
||||||
// database.
|
// database.
|
||||||
func (t *table) Tail() (uint64, error) {
|
func (t *table) Tail(group string) (uint64, error) {
|
||||||
return t.db.Tail()
|
return t.db.Tail(group)
|
||||||
}
|
}
|
||||||
|
|
||||||
// AncientSize is a noop passthrough that just forwards the request to the underlying
|
// AncientSize is a noop passthrough that just forwards the request to the underlying
|
||||||
|
|
@ -103,8 +103,8 @@ func (t *table) TruncateHead(items uint64) (uint64, error) {
|
||||||
|
|
||||||
// TruncateTail is a noop passthrough that just forwards the request to the underlying
|
// TruncateTail is a noop passthrough that just forwards the request to the underlying
|
||||||
// database.
|
// database.
|
||||||
func (t *table) TruncateTail(items uint64) (uint64, error) {
|
func (t *table) TruncateTail(group string, items uint64) (uint64, error) {
|
||||||
return t.db.TruncateTail(items)
|
return t.db.TruncateTail(group, items)
|
||||||
}
|
}
|
||||||
|
|
||||||
// SyncAncient is a noop passthrough that just forwards the request to the underlying
|
// SyncAncient is a noop passthrough that just forwards the request to the underlying
|
||||||
|
|
|
||||||
|
|
@ -128,9 +128,12 @@ type AncientReaderOp interface {
|
||||||
// Ancients returns the ancient item numbers in the ancient store.
|
// Ancients returns the ancient item numbers in the ancient store.
|
||||||
Ancients() (uint64, error)
|
Ancients() (uint64, error)
|
||||||
|
|
||||||
// Tail returns the number of first stored item in the ancient store.
|
// Tail returns the lowest accessible item index for the given tail group.
|
||||||
// This number can also be interpreted as the total deleted items.
|
// This number can also be interpreted as the total deleted items in the
|
||||||
Tail() (uint64, error)
|
// group. Tables sharing a group are pruned together and therefore agree
|
||||||
|
// on the value. An empty group name refers to non-prunable tables and
|
||||||
|
// always returns 0.
|
||||||
|
Tail(group string) (uint64, error)
|
||||||
|
|
||||||
// AncientSize returns the ancient size of the specified category.
|
// AncientSize returns the ancient size of the specified category.
|
||||||
AncientSize(kind string) (uint64, error)
|
AncientSize(kind string) (uint64, error)
|
||||||
|
|
@ -159,14 +162,16 @@ type AncientWriter interface {
|
||||||
// After the truncation, the latest item can be accessed it item_n-1(start from 0).
|
// After the truncation, the latest item can be accessed it item_n-1(start from 0).
|
||||||
TruncateHead(n uint64) (uint64, error)
|
TruncateHead(n uint64) (uint64, error)
|
||||||
|
|
||||||
// TruncateTail discards the first n ancient data from the ancient store. The already
|
// TruncateTail discards the first n items from every table belonging to
|
||||||
// deleted items are ignored. After the truncation, the earliest item can be accessed
|
// the named tail group. Already-deleted items are ignored. After the
|
||||||
// is item_n(start from 0). The deleted items may not be removed from the ancient store
|
// truncation, the earliest accessible item in the group is item_n
|
||||||
// immediately, but only when the accumulated deleted data reach the threshold then
|
// (starting from 0). Deleted items may not be removed from disk
|
||||||
// will be removed all together.
|
// immediately, but only once the accumulated deleted data reaches the
|
||||||
|
// threshold, at which point they are removed all together.
|
||||||
//
|
//
|
||||||
// Note that data marked as non-prunable will still be retained and remain accessible.
|
// The previous tail of the group is returned. Tables outside the group
|
||||||
TruncateTail(n uint64) (uint64, error)
|
// (including non-prunable ones) are untouched.
|
||||||
|
TruncateTail(group string, n uint64) (uint64, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
// AncientWriteOp is given to the function argument of ModifyAncients.
|
// AncientWriteOp is given to the function argument of ModifyAncients.
|
||||||
|
|
|
||||||
|
|
@ -67,7 +67,7 @@ func (db *Database) Ancients() (uint64, error) {
|
||||||
return resp, err
|
return resp, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func (db *Database) Tail() (uint64, error) {
|
func (db *Database) Tail(group string) (uint64, error) {
|
||||||
panic("not supported")
|
panic("not supported")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -99,7 +99,7 @@ func (db *Database) TruncateHead(n uint64) (uint64, error) {
|
||||||
panic("not supported")
|
panic("not supported")
|
||||||
}
|
}
|
||||||
|
|
||||||
func (db *Database) TruncateTail(n uint64) (uint64, error) {
|
func (db *Database) TruncateTail(group string, n uint64) (uint64, error) {
|
||||||
panic("not supported")
|
panic("not supported")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -378,7 +378,7 @@ func (dl *diskLayer) writeHistory(typ historyType, diff *diffLayer) (bool, error
|
||||||
if limit == 0 {
|
if limit == 0 {
|
||||||
return false, nil
|
return false, nil
|
||||||
}
|
}
|
||||||
tail, err := freezer.Tail()
|
tail, err := freezer.Tail(rawdb.DefaultHistoryGroup)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, err
|
return false, err
|
||||||
} // firstID = tail+1
|
} // firstID = tail+1
|
||||||
|
|
|
||||||
|
|
@ -273,7 +273,7 @@ func truncateFromHead(store ethdb.AncientStore, typ historyType, nhead uint64) (
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
otail, err := store.Tail()
|
otail, err := store.Tail(rawdb.DefaultHistoryGroup)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
|
|
@ -303,7 +303,7 @@ func truncateFromTail(store ethdb.AncientStore, typ historyType, ntail uint64) (
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
otail, err := store.Tail()
|
otail, err := store.Tail(rawdb.DefaultHistoryGroup)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
|
|
@ -315,7 +315,7 @@ func truncateFromTail(store ethdb.AncientStore, typ historyType, ntail uint64) (
|
||||||
if otail == ntail {
|
if otail == ntail {
|
||||||
return 0, nil
|
return 0, nil
|
||||||
}
|
}
|
||||||
otail, err = store.TruncateTail(ntail)
|
otail, err = store.TruncateTail(rawdb.DefaultHistoryGroup, ntail)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
|
|
@ -430,7 +430,7 @@ func repairHistory(db ethdb.Database, isUBT bool, readOnly bool, stateID uint64,
|
||||||
truncTo = min(truncTo, thead)
|
truncTo = min(truncTo, thead)
|
||||||
} else {
|
} else {
|
||||||
if thead == 0 {
|
if thead == 0 {
|
||||||
_, err = trienodes.TruncateTail(stateID)
|
_, err = trienodes.TruncateTail(rawdb.DefaultHistoryGroup, stateID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -542,7 +542,7 @@ func (i *indexIniter) run(recover bool) {
|
||||||
|
|
||||||
// next returns the ID of the next state history to be indexed.
|
// next returns the ID of the next state history to be indexed.
|
||||||
func (i *indexIniter) next() (uint64, error) {
|
func (i *indexIniter) next() (uint64, error) {
|
||||||
tail, err := i.freezer.Tail()
|
tail, err := i.freezer.Tail(rawdb.DefaultHistoryGroup)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -21,6 +21,7 @@ import (
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/ethereum/go-ethereum/common"
|
"github.com/ethereum/go-ethereum/common"
|
||||||
|
"github.com/ethereum/go-ethereum/core/rawdb"
|
||||||
"github.com/ethereum/go-ethereum/crypto"
|
"github.com/ethereum/go-ethereum/crypto"
|
||||||
"github.com/ethereum/go-ethereum/ethdb"
|
"github.com/ethereum/go-ethereum/ethdb"
|
||||||
"github.com/ethereum/go-ethereum/log"
|
"github.com/ethereum/go-ethereum/log"
|
||||||
|
|
@ -37,7 +38,7 @@ type HistoryStats struct {
|
||||||
// sanitizeRange limits the given range to fit within the local history store.
|
// sanitizeRange limits the given range to fit within the local history store.
|
||||||
func sanitizeRange(start, end uint64, freezer ethdb.AncientReader) (uint64, uint64, error) {
|
func sanitizeRange(start, end uint64, freezer ethdb.AncientReader) (uint64, uint64, error) {
|
||||||
// Load the id of the first history object in local store.
|
// Load the id of the first history object in local store.
|
||||||
tail, err := freezer.Tail()
|
tail, err := freezer.Tail(rawdb.DefaultHistoryGroup)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, 0, err
|
return 0, 0, err
|
||||||
}
|
}
|
||||||
|
|
@ -132,7 +133,7 @@ func storageHistory(freezer ethdb.AncientReader, address common.Address, slot co
|
||||||
// historyRange returns the block number range of local state histories.
|
// historyRange returns the block number range of local state histories.
|
||||||
func historyRange(freezer ethdb.AncientReader) (uint64, uint64, error) {
|
func historyRange(freezer ethdb.AncientReader) (uint64, uint64, error) {
|
||||||
// Load the id of the first history object in local store.
|
// Load the id of the first history object in local store.
|
||||||
tail, err := freezer.Tail()
|
tail, err := freezer.Tail(rawdb.DefaultHistoryGroup)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, 0, err
|
return 0, 0, err
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -470,7 +470,7 @@ func checkStateAvail(state stateIdent, exptyp historyType, freezer ethdb.Ancient
|
||||||
return 0, fmt.Errorf("unsupported history type: %d, want: %v", toHistoryType(state.typ), exptyp)
|
return 0, fmt.Errorf("unsupported history type: %d, want: %v", toHistoryType(state.typ), exptyp)
|
||||||
}
|
}
|
||||||
// firstID = tail+1
|
// firstID = tail+1
|
||||||
tail, err := freezer.Tail()
|
tail, err := freezer.Tail(rawdb.DefaultHistoryGroup)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -237,7 +237,7 @@ func TestTruncateOutOfRange(t *testing.T) {
|
||||||
|
|
||||||
// Ensure of-out-range truncations are rejected correctly.
|
// Ensure of-out-range truncations are rejected correctly.
|
||||||
head, _ := freezer.Ancients()
|
head, _ := freezer.Ancients()
|
||||||
tail, _ := freezer.Tail()
|
tail, _ := freezer.Tail(rawdb.DefaultHistoryGroup)
|
||||||
|
|
||||||
cases := []struct {
|
cases := []struct {
|
||||||
mode int
|
mode int
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue