mirror of
https://github.com/ethereum/go-ethereum.git
synced 2026-08-20 10:52:25 +00:00
core/rawdb: rewind KV to Ancient in case chain gap detected
If a gap in the chain is detected, geth will fail to start. In this case, the only solution is to rm -rf chaindata/ and resync. This change introduces a feature which rewinds the KV chain head, purging all data along the way, to the current height of the Ancient data. Signed-off-by: meows <b5c6@protonmail.com>
This commit is contained in:
meows
parent
556888c4a9
commit
7b3ed2a157
1 changed files with 138 additions and 42 deletions
|
|
@ -21,6 +21,7 @@ import (
|
|||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
|
|
@ -104,9 +105,9 @@ func NewDatabase(db ethdb.KeyValueStore) ethdb.Database {
|
|||
// NewDatabaseWithFreezer creates a high level database on top of a given key-
|
||||
// value data store with a freezer moving immutable chain segments into cold
|
||||
// storage.
|
||||
func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace string) (ethdb.Database, error) {
|
||||
func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezerStr string, namespace string) (ethdb.Database, error) {
|
||||
// Create the idle freezer instance
|
||||
frdb, err := newFreezer(freezer, namespace)
|
||||
frdb, err := newFreezer(freezerStr, namespace)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -128,49 +129,39 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace st
|
|||
// upgrading to the freezer release, or we might have had a small chain and
|
||||
// not frozen anything yet. Ensure that no blocks are missing yet from the
|
||||
// key-value store, since that would mean we already had an old freezer.
|
||||
validateErr := validateFreezerVsKV(frdb, db)
|
||||
if validateErr != nil {
|
||||
|
||||
// If the genesis hash is empty, we have a new key-value store, so nothing to
|
||||
// validate in this method. If, however, the genesis hash is not nil, compare
|
||||
// it to the freezer content.
|
||||
if kvgenesis, _ := db.Get(headerHashKey(0)); len(kvgenesis) > 0 {
|
||||
if frozen, _ := frdb.Ancients(); frozen > 0 {
|
||||
// If the freezer already contains something, ensure that the genesis blocks
|
||||
// match, otherwise we might mix up freezers across chains and destroy both
|
||||
// the freezer and the key-value store.
|
||||
if frgenesis, _ := frdb.Ancient(freezerHashTable, 0); !bytes.Equal(kvgenesis, frgenesis) {
|
||||
return nil, fmt.Errorf("genesis mismatch: %#x (leveldb) != %#x (ancients)", kvgenesis, frgenesis)
|
||||
}
|
||||
// Key-value store and freezer belong to the same network. Ensure that they
|
||||
// are contiguous, otherwise we might end up with a non-functional freezer.
|
||||
if kvhash, _ := db.Get(headerHashKey(frozen)); len(kvhash) == 0 {
|
||||
// Subsequent header after the freezer limit is missing from the database.
|
||||
// Reject startup is the database has a more recent head.
|
||||
if *ReadHeaderNumber(db, ReadHeadHeaderHash(db)) > frozen-1 {
|
||||
return nil, fmt.Errorf("gap (#%d) in the chain between ancients and leveldb", frozen)
|
||||
}
|
||||
// Database contains only older data than the freezer, this happens if the
|
||||
// state was wiped and reinited from an existing freezer.
|
||||
}
|
||||
// Otherwise, key-value store continues where the freezer left off, all is fine.
|
||||
// We might have duplicate blocks (crash after freezer write but before key-value
|
||||
// store deletion, but that's fine).
|
||||
} else {
|
||||
// If the freezer is empty, ensure nothing was moved yet from the key-value
|
||||
// store, otherwise we'll end up missing data. We check block #1 to decide
|
||||
// if we froze anything previously or not, but do take care of databases with
|
||||
// only the genesis block.
|
||||
if ReadHeadHeaderHash(db) != common.BytesToHash(kvgenesis) {
|
||||
// Key-value store contains more data than the genesis block, make sure we
|
||||
// didn't freeze anything yet.
|
||||
if kvblob, _ := db.Get(headerHashKey(1)); len(kvblob) == 0 {
|
||||
return nil, errors.New("ancient chain segments already extracted, please set --datadir.ancient to the correct path")
|
||||
}
|
||||
// Block #1 is still in the database, we're allowed to init a new feezer
|
||||
}
|
||||
// Otherwise, the head header is still the genesis, we're allowed to init a new
|
||||
// feezer.
|
||||
log.Warn("Freezer/KV validation error, attempting freezer repair", "error", validateErr)
|
||||
if reperr := frdb.repair(); reperr != nil {
|
||||
log.Warn("Freezer repair errored", "error", reperr)
|
||||
|
||||
// Repair did error, AND the validation errored, so return both together because that's double bad.
|
||||
return nil, fmt.Errorf("freezer/kv error=%v freezer repair error=%v", validateErr, reperr)
|
||||
}
|
||||
|
||||
log.Warn("Freezer repair OK")
|
||||
|
||||
truncateKVtoFreezer(frdb, db)
|
||||
|
||||
// Re-validate the ancient/kv dbs.
|
||||
// If still a gap, try removing the kv data back to the ancient level.
|
||||
validateErr = validateFreezerVsKV(frdb, db)
|
||||
}
|
||||
|
||||
if validateErr != nil && strings.Contains(validateErr.Error(), "gap") {
|
||||
// Re-validate again.
|
||||
validateErr = validateFreezerVsKV(frdb, db)
|
||||
} else if validateErr != nil {
|
||||
return nil, validateErr
|
||||
}
|
||||
|
||||
if validateErr != nil {
|
||||
// If this fails, there's nothing left for us to do.
|
||||
log.Warn("KV truncation failed to resuscitate Freezer/KV db gap.")
|
||||
return nil, validateErr
|
||||
}
|
||||
|
||||
// Freezer is consistent with the key-value database, permit combining the two
|
||||
go frdb.freeze(db)
|
||||
|
||||
|
|
@ -352,3 +343,108 @@ func InspectDatabase(db ethdb.Database) error {
|
|||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateFreezerVsKV(freezerdb *freezer, db ethdb.KeyValueStore) error {
|
||||
// If the genesis hash is empty, we have a new key-value store, so nothing to
|
||||
// validate in this method. If, however, the genesis hash is not nil, compare
|
||||
// it to the freezer content.
|
||||
if kvgenesis, _ := db.Get(headerHashKey(0)); len(kvgenesis) > 0 {
|
||||
if frozen, _ := freezerdb.Ancients(); frozen > 0 {
|
||||
// If the freezer already contains something, ensure that the genesis blocks
|
||||
// match, otherwise we might mix up freezers across chains and destroy both
|
||||
// the freezer and the key-value store.
|
||||
if frgenesis, _ := freezerdb.Ancient(freezerHashTable, 0); !bytes.Equal(kvgenesis, frgenesis) {
|
||||
return fmt.Errorf("genesis mismatch: %#x (leveldb) != %#x (ancients)", kvgenesis, frgenesis)
|
||||
}
|
||||
// Key-value store and freezer belong to the same network. Ensure that they
|
||||
// are contiguous, otherwise we might end up with a non-functional freezer.
|
||||
if kvhash, _ := db.Get(headerHashKey(frozen)); len(kvhash) == 0 {
|
||||
// Subsequent header after the freezer limit is missing from the database.
|
||||
// Reject startup is the database has a more recent head.
|
||||
if headHeaderN := *ReadHeaderNumber(db, ReadHeadHeaderHash(db)); headHeaderN > frozen-1 {
|
||||
return fmt.Errorf("gap (chaindb=#%d frozen=#%d) in the chain between ancients and leveldb", headHeaderN, frozen)
|
||||
}
|
||||
// Database contains only older data than the freezer, this happens if the
|
||||
// state was wiped and reinited from an existing freezer.
|
||||
}
|
||||
// Otherwise, key-value store continues where the freezer left off, all is fine.
|
||||
// We might have duplicate blocks (crash after freezer write but before key-value
|
||||
// store deletion, but that's fine).
|
||||
} else {
|
||||
// If the freezer is empty, ensure nothing was moved yet from the key-value
|
||||
// store, otherwise we'll end up missing data. We check block #1 to decide
|
||||
// if we froze anything previously or not, but do take care of databases with
|
||||
// only the genesis block.
|
||||
if ReadHeadHeaderHash(db) != common.BytesToHash(kvgenesis) {
|
||||
// Key-value store contains more data than the genesis block, make sure we
|
||||
// didn't freeze anything yet.
|
||||
if kvblob, _ := db.Get(headerHashKey(1)); len(kvblob) == 0 {
|
||||
return errors.New("ancient chain segments already extracted, please set --datadir.ancient to the correct path")
|
||||
}
|
||||
// Block #1 is still in the database, we're allowed to init a new freezer.
|
||||
}
|
||||
// Otherwise, the head header is still the genesis, we're allowed to init a new
|
||||
// freezer.
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func truncateKVtoFreezer(freezerdb *freezer, db ethdb.KeyValueStore) {
|
||||
hhh := ReadHeadHeaderHash(db)
|
||||
n := *ReadHeaderNumber(db, hhh)
|
||||
frozen, _ := freezerdb.Ancients()
|
||||
|
||||
normalize := func(n *uint64) uint64 {
|
||||
if n == nil {
|
||||
return 0
|
||||
}
|
||||
return *n
|
||||
}
|
||||
|
||||
headHeaderHash := ReadHeadHeaderHash(db)
|
||||
headHeader := normalize(ReadHeaderNumber(db, headHeaderHash))
|
||||
|
||||
headFastHash := ReadHeadFastBlockHash(db)
|
||||
headFast := normalize(ReadHeaderNumber(db, headFastHash))
|
||||
|
||||
headFullHash := ReadHeadBlockHash(db)
|
||||
headFull := normalize(ReadHeaderNumber(db, headFullHash))
|
||||
|
||||
log.Warn("Head header", "number", headHeader, "hash", headHeaderHash)
|
||||
log.Warn("Head fast", "number", headFast, "hash", headFastHash)
|
||||
log.Warn("Head full", "number", headFull, "hash", headFullHash)
|
||||
|
||||
log.Warn("Persistent Freezer/KV gap: Truncating KV database to freezer height", "ancients", frozen, "kv.head_header_number", n, "kv.head_header_hash", hhh)
|
||||
|
||||
for ; n > frozen-1 && n != 0; n-- {
|
||||
for _, hash := range ReadAllHashes(db, n) {
|
||||
if n%10000 == 0 {
|
||||
log.Warn("Removing KV block data", "n", n, "hash", hash.String())
|
||||
}
|
||||
DeleteBlock(db, hash, n)
|
||||
DeleteCanonicalHash(db, n)
|
||||
}
|
||||
}
|
||||
log.Warn("Finished KV truncation")
|
||||
|
||||
data, _ := freezerdb.Ancient(freezerHashTable, n)
|
||||
h := common.BytesToHash(data)
|
||||
|
||||
// If h is the empty common hash, then when the headHeaderHash gets read, whoever's reading it isn't going to like that.
|
||||
// This logic doesn't check for that because there's really nothing that can be sensibly done in this scope,
|
||||
// and it seems reasonable to think that when a higher level function like `loadLastState` finds an empty hash in the
|
||||
// headHeaderHash value, it's going to bark pretty loudly and probably just roll the whole thing (database(s)) back since the
|
||||
// ancient database would appear to be screwy beyond repair since it lied about what frozen headers it had.
|
||||
// So we're just gonna write this sucker.
|
||||
log.Warn("Writing KV head header", "hash", h.String())
|
||||
WriteHeadHeaderHash(db, h)
|
||||
|
||||
// If we had nonzero values for full and/or fast blocks, infer that preceding states will still be valid.
|
||||
if headFast != 0 {
|
||||
WriteHeadFastBlockHash(db, h)
|
||||
}
|
||||
if headFull != 0 {
|
||||
WriteHeadBlockHash(db, h)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue