cmd, core, tests: final nail in the td coffin, remove the data

This commit is contained in:
Péter Szilágyi 2024-11-12 01:22:28 +07:00 committed by Gary Rong
parent 6539c399ae
commit 125b68a797
17 changed files with 56 additions and 291 deletions

View file

@ -25,6 +25,7 @@ import (
"errors"
"fmt"
"io"
"math/big"
"os"
"os/signal"
"path/filepath"
@ -422,6 +423,10 @@ func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64) er
buf = bytes.NewBuffer(nil)
checksums []string
)
td := new(big.Int)
for i := uint64(0); i < first; i++ {
td.Add(td, bc.GetHeaderByNumber(i).Difficulty)
}
for i := first; i <= last; i += step {
err := func() error {
filename := filepath.Join(dir, era.Filename(network, int(i/step), common.Hash{}))
@ -444,7 +449,7 @@ func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64) er
if receipts == nil {
return fmt.Errorf("export failed on #%d: receipts not found", n)
}
td := bc.GetTd(block.Hash(), block.NumberU64())
td.Add(td, block.Difficulty())
if td == nil {
return fmt.Errorf("export failed on #%d: total difficulty not found", n)
}

View file

@ -285,7 +285,6 @@ func makeChainForBench(db ethdb.Database, genesis *Genesis, full bool, count uin
rawdb.WriteHeader(db, header)
rawdb.WriteCanonicalHash(db, hash, n)
rawdb.WriteTd(db, hash, n, big.NewInt(int64(n+1)))
if n == 0 {
rawdb.WriteChainConfig(db, hash, genesis.Config)

View file

@ -543,21 +543,16 @@ func (bc *BlockChain) loadLastState() error {
var (
currentSnapBlock = bc.CurrentSnapBlock()
currentFinalBlock = bc.CurrentFinalBlock()
headerTd = bc.GetTd(headHeader.Hash(), headHeader.Number.Uint64())
blockTd = bc.GetTd(headBlock.Hash(), headBlock.NumberU64())
)
if headHeader.Hash() != headBlock.Hash() {
log.Info("Loaded most recent local header", "number", headHeader.Number, "hash", headHeader.Hash(), "td", headerTd, "age", common.PrettyAge(time.Unix(int64(headHeader.Time), 0)))
log.Info("Loaded most recent local header", "number", headHeader.Number, "hash", headHeader.Hash(), "age", common.PrettyAge(time.Unix(int64(headHeader.Time), 0)))
}
log.Info("Loaded most recent local block", "number", headBlock.Number(), "hash", headBlock.Hash(), "td", blockTd, "age", common.PrettyAge(time.Unix(int64(headBlock.Time()), 0)))
log.Info("Loaded most recent local block", "number", headBlock.Number(), "hash", headBlock.Hash(), "age", common.PrettyAge(time.Unix(int64(headBlock.Time()), 0)))
if headBlock.Hash() != currentSnapBlock.Hash() {
snapTd := bc.GetTd(currentSnapBlock.Hash(), currentSnapBlock.Number.Uint64())
log.Info("Loaded most recent local snap block", "number", currentSnapBlock.Number, "hash", currentSnapBlock.Hash(), "td", snapTd, "age", common.PrettyAge(time.Unix(int64(currentSnapBlock.Time), 0)))
log.Info("Loaded most recent local snap block", "number", currentSnapBlock.Number, "hash", currentSnapBlock.Hash(), "age", common.PrettyAge(time.Unix(int64(currentSnapBlock.Time), 0)))
}
if currentFinalBlock != nil {
finalTd := bc.GetTd(currentFinalBlock.Hash(), currentFinalBlock.Number.Uint64())
log.Info("Loaded most recent local finalized block", "number", currentFinalBlock.Number, "hash", currentFinalBlock.Hash(), "td", finalTd, "age", common.PrettyAge(time.Unix(int64(currentFinalBlock.Time), 0)))
log.Info("Loaded most recent local finalized block", "number", currentFinalBlock.Number, "hash", currentFinalBlock.Hash(), "age", common.PrettyAge(time.Unix(int64(currentFinalBlock.Time), 0)))
}
if pivot := rawdb.ReadLastPivotNumber(bc.db); pivot != nil {
log.Info("Loaded last snap-sync pivot marker", "number", *pivot)
@ -989,7 +984,6 @@ func (bc *BlockChain) ResetWithGenesisBlock(genesis *types.Block) error {
// Prepare the genesis block and reinitialise the chain
batch := bc.db.NewBatch()
rawdb.WriteTd(batch, genesis.Hash(), genesis.NumberU64(), genesis.Difficulty())
rawdb.WriteBlock(batch, genesis)
if err := batch.Write(); err != nil {
log.Crit("Failed to write genesis block", "err", err)
@ -1262,8 +1256,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
// Ensure genesis is in ancients.
if first.NumberU64() == 1 {
if frozen, _ := bc.db.Ancients(); frozen == 0 {
td := bc.genesisBlock.Difficulty()
writeSize, err := rawdb.WriteAncientBlocks(bc.db, []*types.Block{bc.genesisBlock}, []types.Receipts{nil}, td)
writeSize, err := rawdb.WriteAncientBlocks(bc.db, []*types.Block{bc.genesisBlock}, []types.Receipts{nil})
if err != nil {
log.Error("Error writing genesis to ancients", "err", err)
return 0, err
@ -1278,10 +1271,8 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
if !bc.HasHeader(last.Hash(), last.NumberU64()) {
return 0, fmt.Errorf("containing header #%d [%x..] unknown", last.Number(), last.Hash().Bytes()[:4])
}
// Write all chain data to ancients.
td := bc.GetTd(first.Hash(), first.NumberU64())
writeSize, err := rawdb.WriteAncientBlocks(bc.db, blockChain, receiptChain, td)
writeSize, err := rawdb.WriteAncientBlocks(bc.db, blockChain, receiptChain)
if err != nil {
log.Error("Error importing chain data to ancients", "err", err)
return 0, err
@ -1421,12 +1412,11 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [
// writeBlockWithoutState writes only the block and its metadata to the database,
// but does not write any state. This is used to construct competing side forks
// up to the point where they exceed the canonical total difficulty.
func (bc *BlockChain) writeBlockWithoutState(block *types.Block, td *big.Int) (err error) {
func (bc *BlockChain) writeBlockWithoutState(block *types.Block) (err error) {
if bc.insertStopped() {
return errInsertionInterrupted
}
batch := bc.db.NewBatch()
rawdb.WriteTd(batch, block.Hash(), block.NumberU64(), td)
rawdb.WriteBlock(batch, block)
if err := batch.Write(); err != nil {
log.Crit("Failed to write block into disk", "err", err)
@ -1450,20 +1440,14 @@ func (bc *BlockChain) writeKnownBlock(block *types.Block) error {
// writeBlockWithState writes block, metadata and corresponding state data to the
// database.
func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.Receipt, statedb *state.StateDB) error {
// Calculate the total difficulty of the block
ptd := bc.GetTd(block.ParentHash(), block.NumberU64()-1)
if ptd == nil {
if !bc.HasHeader(block.ParentHash(), block.NumberU64()-1) {
return consensus.ErrUnknownAncestor
}
// Make sure no inconsistent state is leaked during insertion
externTd := new(big.Int).Add(block.Difficulty(), ptd)
// Irrelevant of the canonical status, write the block itself to the database.
//
// Note all the components of block(td, hash->number map, header, body, receipts)
// Note all the components of block(hash->number map, header, body, receipts)
// should be written atomically. BlockBatch is used for containing all components.
blockBatch := bc.db.NewBatch()
rawdb.WriteTd(blockBatch, block.Hash(), block.NumberU64(), externTd)
rawdb.WriteBlock(blockBatch, block)
rawdb.WriteReceipts(blockBatch, block.Hash(), block.NumberU64(), receipts)
rawdb.WritePreimages(blockBatch, statedb.Preimages())
@ -1757,7 +1741,6 @@ func (bc *BlockChain) insertChain(chain types.Blocks, setHead bool, makeWitness
if bc.logger != nil && bc.logger.OnSkippedBlock != nil {
bc.logger.OnSkippedBlock(tracing.BlockEvent{
Block: block,
TD: bc.GetTd(block.ParentHash(), block.NumberU64()-1),
Finalized: bc.CurrentFinalBlock(),
Safe: bc.CurrentSafeBlock(),
})
@ -1882,10 +1865,8 @@ type blockProcessingResult struct {
// it writes the block and associated state to database.
func (bc *BlockChain) processBlock(block *types.Block, statedb *state.StateDB, start time.Time, setHead bool) (_ *blockProcessingResult, blockEndErr error) {
if bc.logger != nil && bc.logger.OnBlockStart != nil {
td := bc.GetTd(block.ParentHash(), block.NumberU64()-1)
bc.logger.OnBlockStart(tracing.BlockEvent{
Block: block,
TD: td,
Finalized: bc.CurrentFinalBlock(),
Safe: bc.CurrentSafeBlock(),
})
@ -1995,10 +1976,8 @@ func (bc *BlockChain) processBlock(block *types.Block, statedb *state.StateDB, s
// switch over to the new chain if the TD exceeded the current chain.
// insertSideChain is only used pre-merge.
func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator, makeWitness bool) (*stateless.Witness, int, error) {
var (
externTd *big.Int
current = bc.CurrentBlock()
)
var current = bc.CurrentBlock()
// The first sidechain block error is already verified to be ErrPrunedAncestor.
// Since we don't import them here, we expect ErrUnknownAncestor for the remaining
// ones. Any other errors means that the block is invalid, and should not be written
@ -2010,11 +1989,6 @@ func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator, ma
canonical := bc.GetBlockByNumber(number)
if canonical != nil && canonical.Hash() == block.Hash() {
// Not a sidechain block, this is a re-import of a canon block which has it's state pruned
// Collect the TD of the block. Since we know it's a canon one,
// we can get it directly, and not (like further below) use
// the parent and then add the block on top
externTd = bc.GetTd(block.Hash(), block.NumberU64())
continue
}
if canonical != nil && canonical.Root() == block.Root() {
@ -2033,14 +2007,9 @@ func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator, ma
return nil, it.index, errors.New("sidechain ghost-state attack")
}
}
if externTd == nil {
externTd = bc.GetTd(block.ParentHash(), block.NumberU64()-1)
}
externTd = new(big.Int).Add(externTd, block.Difficulty())
if !bc.HasBlock(block.Hash(), block.NumberU64()) {
start := time.Now()
if err := bc.writeBlockWithoutState(block, externTd); err != nil {
if err := bc.writeBlockWithoutState(block); err != nil {
return nil, it.index, err
}
log.Debug("Injected sidechain block", "number", block.Number(), "hash", block.Hash(),

View file

@ -18,7 +18,6 @@ package core
import (
"errors"
"math/big"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/consensus"
@ -307,12 +306,6 @@ func (bc *BlockChain) GetTransactionLookup(hash common.Hash) (*rawdb.LegacyTxLoo
return lookup, tx, nil
}
// GetTd retrieves a block's total difficulty in the canonical chain from the
// database by hash and number, caching it if found.
func (bc *BlockChain) GetTd(hash common.Hash, number uint64) *big.Int {
return bc.hc.GetTd(hash, number)
}
// HasState checks if state trie is fully present in the database or not.
func (bc *BlockChain) HasState(hash common.Hash) bool {
_, err := bc.statedb.OpenTrie(hash)

View file

@ -88,7 +88,7 @@ func newGwei(n int64) *big.Int {
}
// Test fork of length N starting from block i
func testFork(t *testing.T, blockchain *BlockChain, i, n int, full bool, comparator func(td1, td2 *big.Int), scheme string) {
func testFork(t *testing.T, blockchain *BlockChain, i, n int, full bool, scheme string) {
// Copy old chain up to #i into a new db
genDb, _, blockchain2, err := newCanonical(ethash.NewFaker(), i, full, scheme)
if err != nil {
@ -125,27 +125,15 @@ func testFork(t *testing.T, blockchain *BlockChain, i, n int, full bool, compara
}
}
// Sanity check that the forked chain can be imported into the original
var tdPre, tdPost *big.Int
if full {
cur := blockchain.CurrentBlock()
tdPre = blockchain.GetTd(cur.Hash(), cur.Number.Uint64())
if err := testBlockChainImport(blockChainB, blockchain); err != nil {
t.Fatalf("failed to import forked block chain: %v", err)
}
last := blockChainB[len(blockChainB)-1]
tdPost = blockchain.GetTd(last.Hash(), last.NumberU64())
} else {
cur := blockchain.CurrentHeader()
tdPre = blockchain.GetTd(cur.Hash(), cur.Number.Uint64())
if err := testHeaderChainImport(headerChainB, blockchain); err != nil {
t.Fatalf("failed to import forked header chain: %v", err)
}
last := headerChainB[len(headerChainB)-1]
tdPost = blockchain.GetTd(last.Hash(), last.Number.Uint64())
}
// Compare the total difficulties of the chains
comparator(tdPre, tdPost)
}
// testBlockChainImport tries to process a chain of blocks, writing them into
@ -179,7 +167,6 @@ func testBlockChainImport(chain types.Blocks, blockchain *BlockChain) error {
}
blockchain.chainmu.MustLock()
rawdb.WriteTd(blockchain.db, block.Hash(), block.NumberU64(), new(big.Int).Add(block.Difficulty(), blockchain.GetTd(block.ParentHash(), block.NumberU64()-1)))
rawdb.WriteBlock(blockchain.db, block)
statedb.Commit(block.NumberU64(), false, false)
blockchain.chainmu.Unlock()
@ -197,7 +184,6 @@ func testHeaderChainImport(chain []*types.Header, blockchain *BlockChain) error
}
// Manually insert the header into the database, but don't reorganise (allows subsequent testing)
blockchain.chainmu.MustLock()
rawdb.WriteTd(blockchain.db, header.Hash(), header.Number.Uint64(), new(big.Int).Add(header.Difficulty, blockchain.GetTd(header.ParentHash, header.Number.Uint64()-1)))
rawdb.WriteHeader(blockchain.db, header)
blockchain.chainmu.Unlock()
}
@ -294,17 +280,11 @@ func testExtendCanonical(t *testing.T, full bool, scheme string) {
}
defer processor.Stop()
// Define the difficulty comparator
better := func(td1, td2 *big.Int) {
if td2.Cmp(td1) <= 0 {
t.Errorf("total difficulty mismatch: have %v, expected more than %v", td2, td1)
}
}
// Start fork from current height
testFork(t, processor, length, 1, full, better, scheme)
testFork(t, processor, length, 2, full, better, scheme)
testFork(t, processor, length, 5, full, better, scheme)
testFork(t, processor, length, 10, full, better, scheme)
testFork(t, processor, length, 1, full, scheme)
testFork(t, processor, length, 2, full, scheme)
testFork(t, processor, length, 5, full, scheme)
testFork(t, processor, length, 10, full, scheme)
}
// Tests that given a starting canonical chain of a given size, it can be extended
@ -353,19 +333,13 @@ func testShorterFork(t *testing.T, full bool, scheme string) {
}
defer processor.Stop()
// Define the difficulty comparator
worse := func(td1, td2 *big.Int) {
if td2.Cmp(td1) >= 0 {
t.Errorf("total difficulty mismatch: have %v, expected less than %v", td2, td1)
}
}
// Sum of numbers must be less than `length` for this to be a shorter fork
testFork(t, processor, 0, 3, full, worse, scheme)
testFork(t, processor, 0, 7, full, worse, scheme)
testFork(t, processor, 1, 1, full, worse, scheme)
testFork(t, processor, 1, 7, full, worse, scheme)
testFork(t, processor, 5, 3, full, worse, scheme)
testFork(t, processor, 5, 4, full, worse, scheme)
testFork(t, processor, 0, 3, full, scheme)
testFork(t, processor, 0, 7, full, scheme)
testFork(t, processor, 1, 1, full, scheme)
testFork(t, processor, 1, 7, full, scheme)
testFork(t, processor, 5, 3, full, scheme)
testFork(t, processor, 5, 4, full, scheme)
}
// Tests that given a starting canonical chain of a given size, creating shorter
@ -476,19 +450,13 @@ func testEqualFork(t *testing.T, full bool, scheme string) {
}
defer processor.Stop()
// Define the difficulty comparator
equal := func(td1, td2 *big.Int) {
if td2.Cmp(td1) != 0 {
t.Errorf("total difficulty mismatch: have %v, want %v", td2, td1)
}
}
// Sum of numbers must be equal to `length` for this to be an equal fork
testFork(t, processor, 0, 10, full, equal, scheme)
testFork(t, processor, 1, 9, full, equal, scheme)
testFork(t, processor, 2, 8, full, equal, scheme)
testFork(t, processor, 5, 5, full, equal, scheme)
testFork(t, processor, 6, 4, full, equal, scheme)
testFork(t, processor, 9, 1, full, equal, scheme)
testFork(t, processor, 0, 10, full, scheme)
testFork(t, processor, 1, 9, full, scheme)
testFork(t, processor, 2, 8, full, scheme)
testFork(t, processor, 5, 5, full, scheme)
testFork(t, processor, 6, 4, full, scheme)
testFork(t, processor, 9, 1, full, scheme)
}
// Tests that given a starting canonical chain of a given size, creating equal
@ -647,19 +615,6 @@ func testReorg(t *testing.T, first, second []int64, td int64, full bool, scheme
}
}
}
// Make sure the chain total difficulty is the correct one
want := new(big.Int).Add(blockchain.genesisBlock.Difficulty(), big.NewInt(td))
if full {
cur := blockchain.CurrentBlock()
if have := blockchain.GetTd(cur.Hash(), cur.Number.Uint64()); have.Cmp(want) != 0 {
t.Errorf("total difficulty mismatch: have %v, want %v", have, want)
}
} else {
cur := blockchain.CurrentHeader()
if have := blockchain.GetTd(cur.Hash(), cur.Number.Uint64()); have.Cmp(want) != 0 {
t.Errorf("total difficulty mismatch: have %v, want %v", have, want)
}
}
}
// Tests chain insertions in the face of one entity containing an invalid nonce.
@ -809,12 +764,6 @@ func testFastVsFullChains(t *testing.T, scheme string) {
for i := 0; i < len(blocks); i++ {
num, hash, time := blocks[i].NumberU64(), blocks[i].Hash(), blocks[i].Time()
if ftd, atd := fast.GetTd(hash, num), archive.GetTd(hash, num); ftd.Cmp(atd) != 0 {
t.Errorf("block #%d [%x]: td mismatch: fastdb %v, archivedb %v", num, hash, ftd, atd)
}
if antd, artd := ancient.GetTd(hash, num), archive.GetTd(hash, num); antd.Cmp(artd) != 0 {
t.Errorf("block #%d [%x]: td mismatch: ancientdb %v, archivedb %v", num, hash, antd, artd)
}
if fheader, aheader := fast.GetHeaderByHash(hash), archive.GetHeaderByHash(hash); fheader.Hash() != aheader.Hash() {
t.Errorf("block #%d [%x]: header mismatch: fastdb %v, archivedb %v", num, hash, fheader, aheader)
}

View file

@ -530,7 +530,6 @@ func (g *Genesis) Commit(db ethdb.Database, triedb *triedb.Database) (*types.Blo
}
batch := db.NewBatch()
rawdb.WriteGenesisStateSpec(batch, block.Hash(), blob)
rawdb.WriteTd(batch, block.Hash(), block.NumberU64(), block.Difficulty())
rawdb.WriteBlock(batch, block)
rawdb.WriteReceipts(batch, block.Hash(), block.NumberU64(), nil)
rawdb.WriteCanonicalHash(batch, block.Hash(), block.NumberU64())

View file

@ -191,7 +191,7 @@ func TestGenesisHashes(t *testing.T) {
}
}
func TestGenesis_Commit(t *testing.T) {
func TestGenesisCommit(t *testing.T) {
genesis := &Genesis{
BaseFee: big.NewInt(params.InitialBaseFee),
Config: params.TestChainConfig,
@ -209,13 +209,6 @@ func TestGenesis_Commit(t *testing.T) {
if genesisBlock.Difficulty().Cmp(params.GenesisDifficulty) != 0 {
t.Errorf("assumption wrong: want: %d, got: %v", params.GenesisDifficulty, genesisBlock.Difficulty())
}
// Expect the stored total difficulty to be the difficulty of the genesis block.
stored := rawdb.ReadTd(db, genesisBlock.Hash(), genesisBlock.NumberU64())
if stored.Cmp(genesisBlock.Difficulty()) != 0 {
t.Errorf("inequal difficulty; stored: %v, genesisBlock: %v", stored, genesisBlock.Difficulty())
}
}
func TestReadWriteGenesisAlloc(t *testing.T) {

View file

@ -19,7 +19,6 @@ package core
import (
"errors"
"fmt"
"math/big"
"sync/atomic"
"time"
@ -36,7 +35,6 @@ import (
const (
headerCacheLimit = 512
tdCacheLimit = 1024
numberCacheLimit = 2048
)
@ -65,8 +63,7 @@ type HeaderChain struct {
currentHeaderHash common.Hash // Hash of the current head of the header chain (prevent recomputing all the time)
headerCache *lru.Cache[common.Hash, *types.Header]
tdCache *lru.Cache[common.Hash, *big.Int] // most recent total difficulties
numberCache *lru.Cache[common.Hash, uint64] // most recent block numbers
numberCache *lru.Cache[common.Hash, uint64] // most recent block numbers
procInterrupt func() bool
engine consensus.Engine
@ -79,7 +76,6 @@ func NewHeaderChain(chainDb ethdb.Database, config *params.ChainConfig, engine c
config: config,
chainDb: chainDb,
headerCache: lru.NewCache[common.Hash, *types.Header](headerCacheLimit),
tdCache: lru.NewCache[common.Hash, *big.Int](tdCacheLimit),
numberCache: lru.NewCache[common.Hash, uint64](numberCacheLimit),
procInterrupt: procInterrupt,
engine: engine,
@ -197,14 +193,12 @@ func (hc *HeaderChain) WriteHeaders(headers []*types.Header) (int, error) {
if len(headers) == 0 {
return 0, nil
}
ptd := hc.GetTd(headers[0].ParentHash, headers[0].Number.Uint64()-1)
if ptd == nil {
if !hc.HasHeader(headers[0].ParentHash, headers[0].Number.Uint64()-1) {
return 0, consensus.ErrUnknownAncestor
}
var (
newTD = new(big.Int).Set(ptd) // Total difficulty of inserted chain
inserted []rawdb.NumberHash // Ephemeral lookup of number/hash for the chain
parentKnown = true // Set to true to force hc.HasHeader check the first iteration
inserted []rawdb.NumberHash // Ephemeral lookup of number/hash for the chain
parentKnown = true // Set to true to force hc.HasHeader check the first iteration
batch = hc.chainDb.NewBatch()
)
for i, header := range headers {
@ -218,16 +212,11 @@ func (hc *HeaderChain) WriteHeaders(headers []*types.Header) (int, error) {
hash = header.Hash()
}
number := header.Number.Uint64()
newTD.Add(newTD, header.Difficulty)
// If the parent was not present, store it
// If the header is already known, skip it, otherwise store
alreadyKnown := parentKnown && hc.HasHeader(hash, number)
if !alreadyKnown {
// Irrelevant of the canonical status, write the TD and header to the database.
rawdb.WriteTd(batch, hash, number, newTD)
hc.tdCache.Add(hash, new(big.Int).Set(newTD))
rawdb.WriteHeader(batch, header)
inserted = append(inserted, rawdb.NumberHash{Number: number, Hash: hash})
hc.headerCache.Add(hash, header)
@ -392,22 +381,6 @@ func (hc *HeaderChain) GetAncestor(hash common.Hash, number, ancestor uint64, ma
return hash, number
}
// GetTd retrieves a block's total difficulty in the canonical chain from the
// database by hash and number, caching it if found.
func (hc *HeaderChain) GetTd(hash common.Hash, number uint64) *big.Int {
// Short circuit if the td's already in the cache, retrieve otherwise
if cached, ok := hc.tdCache.Get(hash); ok {
return cached
}
td := rawdb.ReadTd(hc.chainDb, hash, number)
if td == nil {
return nil
}
// Cache the found body for next time and return
hc.tdCache.Add(hash, td)
return td
}
// GetHeader retrieves a block header from the database by hash and number,
// caching it if found.
func (hc *HeaderChain) GetHeader(hash common.Hash, number uint64) *types.Header {
@ -620,7 +593,6 @@ func (hc *HeaderChain) setHead(headBlock uint64, headTime uint64, updateFn Updat
delFn(batch, hash, num)
}
rawdb.DeleteHeader(batch, hash, num)
rawdb.DeleteTd(batch, hash, num)
}
rawdb.DeleteCanonicalHash(batch, num)
}
@ -631,7 +603,6 @@ func (hc *HeaderChain) setHead(headBlock uint64, headTime uint64, updateFn Updat
}
// Clear out any stale content from the caches
hc.headerCache.Purge()
hc.tdCache.Purge()
hc.numberCache.Purge()
}

View file

@ -39,10 +39,6 @@ func verifyUnbrokenCanonchain(hc *HeaderChain) error {
return fmt.Errorf("Canon hash chain broken, block %d got %x, expected %x",
h.Number, canonHash[:8], exp[:8])
}
// Verify that we have the TD
if td := rawdb.ReadTd(hc.chainDb, canonHash, h.Number.Uint64()); td == nil {
return fmt.Errorf("Canon TD missing at block %d", h.Number)
}
if h.Number.Uint64() == 0 {
break
}

View file

@ -508,54 +508,6 @@ func DeleteBody(db ethdb.KeyValueWriter, hash common.Hash, number uint64) {
}
}
// ReadTdRLP retrieves a block's total difficulty corresponding to the hash in RLP encoding.
func ReadTdRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue {
var data []byte
db.ReadAncients(func(reader ethdb.AncientReaderOp) error {
// Check if the data is in ancients
if isCanon(reader, number, hash) {
data, _ = reader.Ancient(ChainFreezerDifficultyTable, number)
return nil
}
// If not, try reading from leveldb
data, _ = db.Get(headerTDKey(number, hash))
return nil
})
return data
}
// ReadTd retrieves a block's total difficulty corresponding to the hash.
func ReadTd(db ethdb.Reader, hash common.Hash, number uint64) *big.Int {
data := ReadTdRLP(db, hash, number)
if len(data) == 0 {
return nil
}
td := new(big.Int)
if err := rlp.DecodeBytes(data, td); err != nil {
log.Error("Invalid block total difficulty RLP", "hash", hash, "err", err)
return nil
}
return td
}
// WriteTd stores the total difficulty of a block into the database.
func WriteTd(db ethdb.KeyValueWriter, hash common.Hash, number uint64, td *big.Int) {
data, err := rlp.EncodeToBytes(td)
if err != nil {
log.Crit("Failed to RLP encode block total difficulty", "err", err)
}
if err := db.Put(headerTDKey(number, hash), data); err != nil {
log.Crit("Failed to store block total difficulty", "err", err)
}
}
// DeleteTd removes all block total difficulty data associated with a hash.
func DeleteTd(db ethdb.KeyValueWriter, hash common.Hash, number uint64) {
if err := db.Delete(headerTDKey(number, hash)); err != nil {
log.Crit("Failed to delete block total difficulty", "err", err)
}
}
// HasReceipts verifies the existence of all the transaction receipts belonging
// to a block.
func HasReceipts(db ethdb.Reader, hash common.Hash, number uint64) bool {
@ -741,11 +693,9 @@ func WriteBlock(db ethdb.KeyValueWriter, block *types.Block) {
}
// WriteAncientBlocks writes entire block data into ancient store and returns the total written size.
func WriteAncientBlocks(db ethdb.AncientWriter, blocks []*types.Block, receipts []types.Receipts, td *big.Int) (int64, error) {
var (
tdSum = new(big.Int).Set(td)
stReceipts []*types.ReceiptForStorage
)
func WriteAncientBlocks(db ethdb.AncientWriter, blocks []*types.Block, receipts []types.Receipts) (int64, error) {
var stReceipts []*types.ReceiptForStorage
return db.ModifyAncients(func(op ethdb.AncientWriteOp) error {
for i, block := range blocks {
// Convert receipts to storage format and sum up total difficulty.
@ -754,10 +704,7 @@ func WriteAncientBlocks(db ethdb.AncientWriter, blocks []*types.Block, receipts
stReceipts = append(stReceipts, (*types.ReceiptForStorage)(receipt))
}
header := block.Header()
if i > 0 {
tdSum.Add(tdSum, header.Difficulty)
}
if err := writeAncientBlock(op, block, header, stReceipts, tdSum); err != nil {
if err := writeAncientBlock(op, block, header, stReceipts); err != nil {
return err
}
}
@ -765,7 +712,7 @@ func WriteAncientBlocks(db ethdb.AncientWriter, blocks []*types.Block, receipts
})
}
func writeAncientBlock(op ethdb.AncientWriteOp, block *types.Block, header *types.Header, receipts []*types.ReceiptForStorage, td *big.Int) error {
func writeAncientBlock(op ethdb.AncientWriteOp, block *types.Block, header *types.Header, receipts []*types.ReceiptForStorage) error {
num := block.NumberU64()
if err := op.AppendRaw(ChainFreezerHashTable, num, block.Hash().Bytes()); err != nil {
return fmt.Errorf("can't add block %d hash: %v", num, err)
@ -779,9 +726,6 @@ func writeAncientBlock(op ethdb.AncientWriteOp, block *types.Block, header *type
if err := op.Append(ChainFreezerReceiptTable, num, receipts); err != nil {
return fmt.Errorf("can't append block %d receipts: %v", num, err)
}
if err := op.Append(ChainFreezerDifficultyTable, num, td); err != nil {
return fmt.Errorf("can't append block %d total difficulty: %v", num, err)
}
return nil
}
@ -790,7 +734,6 @@ func DeleteBlock(db ethdb.KeyValueWriter, hash common.Hash, number uint64) {
DeleteReceipts(db, hash, number)
DeleteHeader(db, hash, number)
DeleteBody(db, hash, number)
DeleteTd(db, hash, number)
}
// DeleteBlockWithoutNumber removes all block data associated with a hash, except
@ -799,7 +742,6 @@ func DeleteBlockWithoutNumber(db ethdb.KeyValueWriter, hash common.Hash, number
DeleteReceipts(db, hash, number)
deleteHeaderWithoutNumber(db, hash, number)
DeleteBody(db, hash, number)
DeleteTd(db, hash, number)
}
const badBlockToKeep = 10

View file

@ -258,29 +258,6 @@ func TestBadBlockStorage(t *testing.T) {
}
}
// Tests block total difficulty storage and retrieval operations.
func TestTdStorage(t *testing.T) {
db := NewMemoryDatabase()
// Create a test TD to move around the database and make sure it's really new
hash, td := common.Hash{}, big.NewInt(314)
if entry := ReadTd(db, hash, 0); entry != nil {
t.Fatalf("Non existent TD returned: %v", entry)
}
// Write and verify the TD in the database
WriteTd(db, hash, 0, td)
if entry := ReadTd(db, hash, 0); entry == nil {
t.Fatalf("Stored TD not found")
} else if entry.Cmp(td) != 0 {
t.Fatalf("Retrieved TD mismatch: have %v, want %v", entry, td)
}
// Delete the TD and verify the execution
DeleteTd(db, hash, 0)
if entry := ReadTd(db, hash, 0); entry != nil {
t.Fatalf("Deleted TD returned: %v", entry)
}
}
// Tests that canonical numbers can be mapped to hashes and retrieved.
func TestCanonicalMappingStorage(t *testing.T) {
db := NewMemoryDatabase()
@ -460,12 +437,9 @@ func TestAncientStorage(t *testing.T) {
if blob := ReadReceiptsRLP(db, hash, number); len(blob) > 0 {
t.Fatalf("non existent receipts returned")
}
if blob := ReadTdRLP(db, hash, number); len(blob) > 0 {
t.Fatalf("non existent td returned")
}
// Write and verify the header in the database
WriteAncientBlocks(db, []*types.Block{block}, []types.Receipts{nil}, big.NewInt(100))
WriteAncientBlocks(db, []*types.Block{block}, []types.Receipts{nil})
if blob := ReadHeaderRLP(db, hash, number); len(blob) == 0 {
t.Fatalf("no header returned")
@ -476,9 +450,6 @@ func TestAncientStorage(t *testing.T) {
if blob := ReadReceiptsRLP(db, hash, number); len(blob) == 0 {
t.Fatalf("no receipts returned")
}
if blob := ReadTdRLP(db, hash, number); len(blob) == 0 {
t.Fatalf("no td returned")
}
// Use a fake hash for data retrieval, nothing should be returned.
fakeHash := common.BytesToHash([]byte{0x01, 0x02, 0x03})
@ -491,9 +462,6 @@ func TestAncientStorage(t *testing.T) {
if blob := ReadReceiptsRLP(db, fakeHash, number); len(blob) != 0 {
t.Fatalf("invalid receipts returned")
}
if blob := ReadTdRLP(db, fakeHash, number); len(blob) != 0 {
t.Fatalf("invalid td returned")
}
}
func TestCanonicalHashIteration(t *testing.T) {
@ -518,7 +486,6 @@ func TestCanonicalHashIteration(t *testing.T) {
// Fill database with testing data.
for i := uint64(1); i <= 8; i++ {
WriteCanonicalHash(db, common.Hash{}, i)
WriteTd(db, common.Hash{}, i, big.NewInt(10)) // Write some interferential data
}
for i, c := range cases {
numbers, _ := ReadAllCanonicalHashes(db, c.from, c.to, c.limit)
@ -591,7 +558,6 @@ func BenchmarkWriteAncientBlocks(b *testing.B) {
// The benchmark loop writes batches of blocks, but note that the total block count is
// b.N. This means the resulting ns/op measurement is the time it takes to write a
// single block and its associated data.
var td = big.NewInt(55)
var totalSize int64
for i := 0; i < b.N; i += batchSize {
length := batchSize
@ -601,7 +567,7 @@ func BenchmarkWriteAncientBlocks(b *testing.B) {
blocks := allBlocks[i : i+length]
receipts := batchReceipts[:length]
writeSize, err := WriteAncientBlocks(db, blocks, receipts, td)
writeSize, err := WriteAncientBlocks(db, blocks, receipts)
if err != nil {
b.Fatal(err)
}
@ -900,7 +866,7 @@ func TestHeadersRLPStorage(t *testing.T) {
}
var receipts []types.Receipts = make([]types.Receipts, 100)
// Write first half to ancients
WriteAncientBlocks(db, chain[:50], receipts[:50], big.NewInt(100))
WriteAncientBlocks(db, chain[:50], receipts[:50])
// Write second half to db
for i := 50; i < 100; i++ {
WriteCanonicalHash(db, chain[i].Hash(), chain[i].NumberU64())

View file

@ -35,19 +35,15 @@ const (
// ChainFreezerReceiptTable indicates the name of the freezer receipts table.
ChainFreezerReceiptTable = "receipts"
// ChainFreezerDifficultyTable indicates the name of the freezer total difficulty table.
ChainFreezerDifficultyTable = "diffs"
)
// chainFreezerNoSnappy configures whether compression is disabled for the ancient-tables.
// Hashes and difficulties don't compress well.
var chainFreezerNoSnappy = map[string]bool{
ChainFreezerHeaderTable: false,
ChainFreezerHashTable: true,
ChainFreezerBodiesTable: false,
ChainFreezerReceiptTable: false,
ChainFreezerDifficultyTable: true,
ChainFreezerHeaderTable: false,
ChainFreezerHashTable: true,
ChainFreezerBodiesTable: false,
ChainFreezerReceiptTable: false,
}
const (

View file

@ -315,11 +315,6 @@ func (f *chainFreezer) freezeRange(nfdb *nofreezedb, number, limit uint64) (hash
if len(receipts) == 0 {
return fmt.Errorf("block receipts missing, can't freeze block %d", number)
}
td := ReadTdRLP(nfdb, hash, number)
if len(td) == 0 {
return fmt.Errorf("total difficulty missing, can't freeze block %d", number)
}
// Write to the batch.
if err := op.AppendRaw(ChainFreezerHashTable, number, hash[:]); err != nil {
return fmt.Errorf("can't write hash to Freezer: %v", err)
@ -333,9 +328,6 @@ func (f *chainFreezer) freezeRange(nfdb *nofreezedb, number, limit uint64) (hash
if err := op.AppendRaw(ChainFreezerReceiptTable, number, receipts); err != nil {
return fmt.Errorf("can't write receipts to Freezer: %v", err)
}
if err := op.AppendRaw(ChainFreezerDifficultyTable, number, td); err != nil {
return fmt.Errorf("can't write td to Freezer: %v", err)
}
hashes = append(hashes, hash)
}
return nil

View file

@ -408,8 +408,6 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
bodies.Add(size)
case bytes.HasPrefix(key, blockReceiptsPrefix) && len(key) == (len(blockReceiptsPrefix)+8+common.HashLength):
receipts.Add(size)
case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerTDSuffix):
tds.Add(size)
case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerHashSuffix):
numHashPairings.Add(size)
case bytes.HasPrefix(key, headerNumberPrefix) && len(key) == (len(headerNumberPrefix)+common.HashLength):

View file

@ -98,7 +98,6 @@ var (
// Data item prefixes (use single byte to avoid mixing data types, avoid `i`, used for indexes).
headerPrefix = []byte("h") // headerPrefix + num (uint64 big endian) + hash -> header
headerTDSuffix = []byte("t") // headerPrefix + num (uint64 big endian) + hash + headerTDSuffix -> td
headerHashSuffix = []byte("n") // headerPrefix + num (uint64 big endian) + headerHashSuffix -> hash
headerNumberPrefix = []byte("H") // headerNumberPrefix + hash -> num (uint64 big endian)
@ -174,11 +173,6 @@ func headerKey(number uint64, hash common.Hash) []byte {
return append(append(headerPrefix, encodeBlockNumber(number)...), hash.Bytes()...)
}
// headerTDKey = headerPrefix + num (uint64 big endian) + hash + headerTDSuffix
func headerTDKey(number uint64, hash common.Hash) []byte {
return append(headerKey(number, hash), headerTDSuffix...)
}
// headerHashKey = headerPrefix + num (uint64 big endian) + headerHashSuffix
func headerHashKey(number uint64) []byte {
return append(append(headerPrefix, encodeBlockNumber(number)...), headerHashSuffix...)

View file

@ -211,7 +211,7 @@ func TestTxIndexer(t *testing.T) {
}
for _, c := range cases {
db, _ := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), "", "", false)
rawdb.WriteAncientBlocks(db, append([]*types.Block{gspec.ToBlock()}, blocks...), append([]types.Receipts{{}}, receipts...), big.NewInt(0))
rawdb.WriteAncientBlocks(db, append([]*types.Block{gspec.ToBlock()}, blocks...), append([]types.Receipts{{}}, receipts...))
// Index the initial blocks from ancient store
indexer := &txIndexer{

View file

@ -60,6 +60,9 @@ func TestBlockchain(t *testing.T) {
bt.skipLoad(`.*bcFrontierToHomestead/blockChainFrontierWithLargerTDvsHomesteadBlockchain.json`)
bt.skipLoad(`.*bcFrontierToHomestead/blockChainFrontierWithLargerTDvsHomesteadBlockchain2.json`)
// With chain history removal, TDs become unavailable, this transition tests based on TTD are unrunnable
bt.skipLoad(`.*bcArrowGlacierToParis/powToPosBlockRejection.json`)
bt.walk(t, blockTestDir, func(t *testing.T, name string, test *BlockTest) {
execBlockTest(t, bt, test)
})