// Copyright 2018 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify // it under the terms of the GNU Lesser General Public License as published by // the Free Software Foundation, either version 3 of the License, or // (at your option) any later version. // // The go-ethereum library is distributed in the hope that it will be useful, // but WITHOUT ANY WARRANTY; without even the implied warranty of // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the // GNU Lesser General Public License for more details. // // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . package rawdb import ( "bytes" "encoding/binary" "errors" "math/big" "slices" "github.com/XinFinOrg/XDPoSChain/common" "github.com/XinFinOrg/XDPoSChain/core/types" "github.com/XinFinOrg/XDPoSChain/crypto" "github.com/XinFinOrg/XDPoSChain/ethdb" "github.com/XinFinOrg/XDPoSChain/log" "github.com/XinFinOrg/XDPoSChain/params" "github.com/XinFinOrg/XDPoSChain/rlp" ) // ReadCanonicalHash retrieves the hash assigned to a canonical block number. func ReadCanonicalHash(db ethdb.Reader, number uint64) common.Hash { data, _ := db.Ancient(freezerHashTable, number) if len(data) == 0 { data, _ = db.Get(headerHashKey(number)) // In the background freezer is moving data from leveldb to flatten files. // So during the first check for ancient db, the data is not yet in there, // but when we reach into leveldb, the data was already moved. That would // result in a not found error. if len(data) == 0 { data, _ = db.Ancient(freezerHashTable, number) } } if len(data) == 0 { return common.Hash{} } return common.BytesToHash(data) } // WriteCanonicalHash stores the hash assigned to a canonical block number. func WriteCanonicalHash(db ethdb.KeyValueWriter, hash common.Hash, number uint64) { if err := db.Put(headerHashKey(number), hash.Bytes()); err != nil { log.Crit("Failed to store number to hash mapping", "err", err) } } // DeleteCanonicalHash removes the number to hash canonical mapping. func DeleteCanonicalHash(db ethdb.KeyValueWriter, number uint64) { if err := db.Delete(headerHashKey(number)); err != nil { log.Crit("Failed to delete number to hash mapping", "err", err) } } // ReadHeaderNumber returns the header number assigned to a hash. func ReadHeaderNumber(db ethdb.KeyValueReader, hash common.Hash) *uint64 { data, _ := db.Get(headerNumberKey(hash)) if len(data) != 8 { return nil } number := binary.BigEndian.Uint64(data) return &number } // WriteHeaderNumber stores the hash->number mapping. func WriteHeaderNumber(db ethdb.KeyValueWriter, hash common.Hash, number uint64) { key := headerNumberKey(hash) enc := encodeBlockNumber(number) if err := db.Put(key, enc); err != nil { log.Crit("Failed to store hash to number mapping", "err", err) } } // DeleteHeaderNumber removes hash->number mapping. func DeleteHeaderNumber(db ethdb.KeyValueWriter, hash common.Hash) { if err := db.Delete(headerNumberKey(hash)); err != nil { log.Crit("Failed to delete hash to number mapping", "err", err) } } // ReadHeadHeaderHash retrieves the hash of the current canonical head block's // header. The difference between this and GetHeadBlockHash is that whereas the // last block hash is only updated upon a full block import, the last header // hash is updated already at header import, allowing head tracking for the // light synchronization mechanism. func ReadHeadHeaderHash(db ethdb.KeyValueReader) common.Hash { data, _ := db.Get(headHeaderKey) if len(data) == 0 { return common.Hash{} } return common.BytesToHash(data) } // WriteHeadHeaderHash stores the hash of the current canonical head header. func WriteHeadHeaderHash(db ethdb.KeyValueWriter, hash common.Hash) { if err := db.Put(headHeaderKey, hash.Bytes()); err != nil { log.Crit("Failed to store last header's hash", "err", err) } } // ReadHeadBlockHash retrieves the hash of the current canonical head block. func ReadHeadBlockHash(db ethdb.KeyValueReader) common.Hash { data, _ := db.Get(headBlockKey) if len(data) == 0 { return common.Hash{} } return common.BytesToHash(data) } // WriteHeadBlockHash stores the head block's hash. func WriteHeadBlockHash(db ethdb.KeyValueWriter, hash common.Hash) { if err := db.Put(headBlockKey, hash.Bytes()); err != nil { log.Crit("Failed to store last block's hash", "err", err) } } // ReadHeadFastBlockHash retrieves the hash of the current canonical head block during // fast synchronization. The difference between this and GetHeadBlockHash is that // whereas the last block hash is only updated upon a full block import, the last // fast hash is updated when importing pre-processed blocks. func ReadHeadFastBlockHash(db ethdb.KeyValueReader) common.Hash { data, _ := db.Get(headFastBlockKey) if len(data) == 0 { return common.Hash{} } return common.BytesToHash(data) } // WriteHeadFastBlockHash stores the hash of the current fast-sync head block. func WriteHeadFastBlockHash(db ethdb.KeyValueWriter, hash common.Hash) { if err := db.Put(headFastBlockKey, hash.Bytes()); err != nil { log.Crit("Failed to store last fast block's hash", "err", err) } } // ReadFastTrieProgress retrieves the number of tries nodes fast synced to allow // reportinc correct numbers across restarts. func ReadFastTrieProgress(db ethdb.KeyValueReader) uint64 { data, _ := db.Get(fastTrieProgressKey) if len(data) == 0 { return 0 } return new(big.Int).SetBytes(data).Uint64() } // WriteFastTrieProgress stores the fast sync trie process counter to support // retrieving it across restarts. func WriteFastTrieProgress(db ethdb.KeyValueWriter, count uint64) error { if err := db.Put(fastTrieProgressKey, new(big.Int).SetUint64(count).Bytes()); err != nil { log.Crit("Failed to store fast sync trie progress", "err", err) } return nil } // ReadHeaderRLP retrieves a block header in its raw RLP database encoding. func ReadHeaderRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue { // First try to look up the data in ancient database. Extra hash // comparison is necessary since ancient database only maintains // the canonical data. data, _ := db.Ancient(freezerHeaderTable, number) if len(data) > 0 && crypto.Keccak256Hash(data) == hash { return data } // Then try to look up the data in leveldb. data, _ = db.Get(headerKey(number, hash)) if len(data) > 0 { return data } // In the background freezer is moving data from leveldb to flatten files. // So during the first check for ancient db, the data is not yet in there, // but when we reach into leveldb, the data was already moved. That would // result in a not found error. data, _ = db.Ancient(freezerHeaderTable, number) if len(data) > 0 && crypto.Keccak256Hash(data) == hash { return data } return nil // Can't find the data anywhere. } // HasHeader verifies the existence of a block header corresponding to the hash. func HasHeader(db ethdb.Reader, hash common.Hash, number uint64) bool { if has, err := db.Ancient(freezerHashTable, number); err == nil && common.BytesToHash(has) == hash { return true } if has, err := db.Has(headerKey(number, hash)); !has || err != nil { return false } return true } // ReadHeader retrieves the block header corresponding to the hash. func ReadHeader(db ethdb.Reader, hash common.Hash, number uint64) *types.Header { data := ReadHeaderRLP(db, hash, number) if len(data) == 0 { return nil } header := new(types.Header) if err := rlp.Decode(bytes.NewReader(data), header); err != nil { log.Error("Invalid block header RLP", "hash", hash, "err", err) return nil } return header } // WriteHeader stores a block header into the database and also stores the hash- // to-number mapping. func WriteHeader(db ethdb.KeyValueWriter, header *types.Header) { var ( hash = header.Hash() number = header.Number.Uint64() ) // Write the hash -> number mapping WriteHeaderNumber(db, hash, number) // Write the encoded header data, err := rlp.EncodeToBytes(header) if err != nil { log.Crit("Failed to RLP encode header", "err", err) } key := headerKey(number, hash) if err := db.Put(key, data); err != nil { log.Crit("Failed to store header", "err", err) } } // DeleteHeader removes all block header data associated with a hash. func DeleteHeader(db ethdb.KeyValueWriter, hash common.Hash, number uint64) { deleteHeaderWithoutNumber(db, hash, number) if err := db.Delete(headerNumberKey(hash)); err != nil { log.Crit("Failed to delete hash to number mapping", "err", err) } } // deleteHeaderWithoutNumber removes only the block header but does not remove // the hash to number mapping. func deleteHeaderWithoutNumber(db ethdb.KeyValueWriter, hash common.Hash, number uint64) { if err := db.Delete(headerKey(number, hash)); err != nil { log.Crit("Failed to delete header", "err", err) } } // ReadBodyRLP retrieves the block body (transactions and uncles) in RLP encoding. func ReadBodyRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue { // First try to look up the data in ancient database. Extra hash // comparison is necessary since ancient database only maintains // the canonical data. data, _ := db.Ancient(freezerBodiesTable, number) if len(data) > 0 { h, _ := db.Ancient(freezerHashTable, number) if common.BytesToHash(h) == hash { return data } } // Then try to look up the data in leveldb. data, _ = db.Get(blockBodyKey(number, hash)) if len(data) > 0 { return data } // In the background freezer is moving data from leveldb to flatten files. // So during the first check for ancient db, the data is not yet in there, // but when we reach into leveldb, the data was already moved. That would // result in a not found error. data, _ = db.Ancient(freezerBodiesTable, number) if len(data) > 0 { h, _ := db.Ancient(freezerHashTable, number) if common.BytesToHash(h) == hash { return data } } return nil // Can't find the data anywhere. } // WriteBodyRLP stores an RLP encoded block body into the database. func WriteBodyRLP(db ethdb.KeyValueWriter, hash common.Hash, number uint64, rlp rlp.RawValue) { if err := db.Put(blockBodyKey(number, hash), rlp); err != nil { log.Crit("Failed to store block body", "err", err) } } // HasBody verifies the existence of a block body corresponding to the hash. func HasBody(db ethdb.Reader, hash common.Hash, number uint64) bool { //TODO: need to add isCanon check // if isCanon(db, number, hash) { // return true // } if has, err := db.Has(blockBodyKey(number, hash)); !has || err != nil { return false } return true } // ReadBody retrieves the block body corresponding to the hash. func ReadBody(db ethdb.Reader, hash common.Hash, number uint64) *types.Body { data := ReadBodyRLP(db, hash, number) if len(data) == 0 { return nil } body := new(types.Body) if err := rlp.Decode(bytes.NewReader(data), body); err != nil { log.Error("Invalid block body RLP", "hash", hash, "err", err) return nil } return body } // WriteBody stores a block body into the database. func WriteBody(db ethdb.KeyValueWriter, hash common.Hash, number uint64, body *types.Body) { data, err := rlp.EncodeToBytes(body) if err != nil { log.Crit("Failed to RLP encode body", "err", err) } WriteBodyRLP(db, hash, number, data) } // DeleteBody removes all block body data associated with a hash. func DeleteBody(db ethdb.KeyValueWriter, hash common.Hash, number uint64) { if err := db.Delete(blockBodyKey(number, hash)); err != nil { log.Crit("Failed to delete block body", "err", err) } } // ReadTdRLP retrieves a block's total difficulty corresponding to the hash in RLP encoding. func ReadTdRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue { data, _ := db.Get(headerTDKey(number, hash)) return data } // ReadTd retrieves a block's total difficulty corresponding to the hash, nil if // none found. func ReadTd(db ethdb.Reader, hash common.Hash, number uint64) *big.Int { data := ReadTdRLP(db, hash, number) if len(data) == 0 { return nil } td := new(big.Int) if err := rlp.Decode(bytes.NewReader(data), td); err != nil { log.Error("Invalid block total difficulty RLP", "hash", hash, "err", err) return nil } return td } // WriteTd stores the total difficulty of a block into the database. func WriteTd(db ethdb.KeyValueWriter, hash common.Hash, number uint64, td *big.Int) { data, err := rlp.EncodeToBytes(td) if err != nil { log.Crit("Failed to RLP encode block total difficulty", "err", err) } if err := db.Put(headerTDKey(number, hash), data); err != nil { log.Crit("Failed to store block total difficulty", "err", err) } } // DeleteTd removes all block total difficulty data associated with a hash. func DeleteTd(db ethdb.KeyValueWriter, hash common.Hash, number uint64) { if err := db.Delete(headerTDKey(number, hash)); err != nil { log.Crit("Failed to delete block total difficulty", "err", err) } } // HasReceipts verifies the existence of all the transaction receipts belonging // to a block. func HasReceipts(db ethdb.Reader, hash common.Hash, number uint64) bool { if has, err := db.Ancient(freezerHashTable, number); err == nil && common.BytesToHash(has) == hash { return true } if has, err := db.Has(blockReceiptsKey(number, hash)); !has || err != nil { return false } return true } // ReadReceiptsRLP retrieves all the transaction receipts belonging to a block in RLP encoding. func ReadReceiptsRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue { // First try to look up the data in ancient database. Extra hash // comparison is necessary since ancient database only maintains // the canonical data. data, _ := db.Ancient(freezerReceiptTable, number) if len(data) > 0 { h, _ := db.Ancient(freezerHashTable, number) if common.BytesToHash(h) == hash { return data } } // Then try to look up the data in leveldb. data, _ = db.Get(blockReceiptsKey(number, hash)) if len(data) > 0 { return data } // In the background freezer is moving data from leveldb to flatten files. // So during the first check for ancient db, the data is not yet in there, // but when we reach into leveldb, the data was already moved. That would // result in a not found error. data, _ = db.Ancient(freezerReceiptTable, number) if len(data) > 0 { h, _ := db.Ancient(freezerHashTable, number) if common.BytesToHash(h) == hash { return data } } return nil // Can't find the data anywhere. } // ReadRawReceipts retrieves all the transaction receipts belonging to a block. // The receipt metadata fields are not guaranteed to be populated, so they // should not be used. Use ReadReceipts instead if the metadata is needed. func ReadRawReceipts(db ethdb.Reader, hash common.Hash, number uint64) types.Receipts { // Retrieve the flattened receipt slice data := ReadReceiptsRLP(db, hash, number) if len(data) == 0 { return nil } // Convert the receipts from their storage form to their internal representation storageReceipts := []*types.ReceiptForStorage{} if err := rlp.DecodeBytes(data, &storageReceipts); err != nil { log.Error("Invalid receipt array RLP", "hash", hash, "err", err) return nil } receipts := make(types.Receipts, len(storageReceipts)) for i, storageReceipt := range storageReceipts { receipts[i] = (*types.Receipt)(storageReceipt) receipts[i].BlockHash = hash receipts[i].BlockNumber = new(big.Int).SetUint64(number) receipts[i].TransactionIndex = uint(i) } return receipts } // ReadReceipts retrieves all the transaction receipts belonging to a block, including // its correspoinding metadata fields. If it is unable to populate these metadata // fields then nil is returned. // // The current implementation populates these metadata fields by reading the receipts' // corresponding block body, so if the block body is not found it will return nil even // if the receipt itself is stored. func ReadReceipts(db ethdb.Reader, hash common.Hash, number uint64, config *params.ChainConfig) types.Receipts { // We're deriving many fields from the block body, retrieve beside the receipt receipts := ReadRawReceipts(db, hash, number) if receipts == nil { return nil } body := ReadBody(db, hash, number) if body == nil { log.Error("Missing body but have receipt", "hash", hash, "number", number) return nil } header := ReadHeader(db, hash, number) var baseFee *big.Int if header == nil { baseFee = big.NewInt(0) } else { baseFee = header.BaseFee } if err := receipts.DeriveFields(config, hash, number, baseFee, body.Transactions); err != nil { log.Error("Failed to derive block receipts fields", "hash", hash, "number", number, "err", err) return nil } return receipts } // WriteReceipts stores all the transaction receipts belonging to a block. func WriteReceipts(db ethdb.KeyValueWriter, hash common.Hash, number uint64, receipts types.Receipts) { // Convert the receipts into their storage form and serialize them storageReceipts := make([]*types.ReceiptForStorage, len(receipts)) for i, receipt := range receipts { storageReceipts[i] = (*types.ReceiptForStorage)(receipt) } bytes, err := rlp.EncodeToBytes(storageReceipts) if err != nil { log.Crit("Failed to encode block receipts", "err", err) } // Store the flattened receipt slice if err := db.Put(blockReceiptsKey(number, hash), bytes); err != nil { log.Crit("Failed to store block receipts", "err", err) } } // DeleteReceipts removes all receipt data associated with a block hash. func DeleteReceipts(db ethdb.KeyValueWriter, hash common.Hash, number uint64) { if err := db.Delete(blockReceiptsKey(number, hash)); err != nil { log.Crit("Failed to delete block receipts", "err", err) } } // ReceiptLogs is a barebone version of ReceiptForStorage which only keeps // the list of logs. When decoding a stored receipt into this object we // avoid creating the bloom filter. type receiptLogs struct { Logs []*types.Log } // DecodeRLP implements rlp.Decoder. func (r *receiptLogs) DecodeRLP(s *rlp.Stream) error { var stored types.ReceiptForStorage if err := s.Decode(&stored); err != nil { return err } r.Logs = make([]*types.Log, len(stored.Logs)) for i, log := range stored.Logs { r.Logs[i] = log } return nil } // DeriveLogFields fills the logs in receiptLogs with information such as block number, txhash, etc. func deriveLogFields(receipts []*receiptLogs, hash common.Hash, number uint64, txs types.Transactions) error { logIndex := uint(0) if len(txs) != len(receipts) { return errors.New("transaction and receipt count mismatch") } for i := 0; i < len(receipts); i++ { txHash := txs[i].Hash() // The derived log fields can simply be set from the block and transaction for j := 0; j < len(receipts[i].Logs); j++ { receipts[i].Logs[j].BlockNumber = number receipts[i].Logs[j].BlockHash = hash receipts[i].Logs[j].TxHash = txHash receipts[i].Logs[j].TxIndex = uint(i) receipts[i].Logs[j].Index = logIndex logIndex++ } } return nil } // ReadLogs retrieves the logs for all transactions in a block. In case // receipts is not found, a nil is returned. // Note: ReadLogs does not derive unstored log fields. func ReadLogs(db ethdb.Reader, hash common.Hash, number uint64) [][]*types.Log { // Retrieve the flattened receipt slice data := ReadReceiptsRLP(db, hash, number) if len(data) == 0 { return nil } receipts := []*receiptLogs{} if err := rlp.DecodeBytes(data, &receipts); err != nil { log.Error("Invalid receipt array RLP", "hash", hash, "err", err) return nil } logs := make([][]*types.Log, len(receipts)) for i, receipt := range receipts { logs[i] = receipt.Logs } return logs } // ReadBlock retrieves an entire block corresponding to the hash, assembling it // back from the stored header and body. If either the header or body could not // be retrieved nil is returned. // // Note, due to concurrent download of header and block body the header and thus // canonical hash can be stored in the database but the body data not (yet). func ReadBlock(db ethdb.Reader, hash common.Hash, number uint64) *types.Block { // Retrieve the block header and body contents header := ReadHeader(db, hash, number) if header == nil { return nil } body := ReadBody(db, hash, number) if body == nil { return nil } // Reassemble the block and return return types.NewBlockWithHeader(header).WithBody(*body) } // WriteBlock serializes a block into the database, header and body separately. func WriteBlock(db ethdb.KeyValueWriter, block *types.Block) { WriteBody(db, block.Hash(), block.NumberU64(), block.Body()) WriteHeader(db, block.Header()) } // DeleteBlock removes all block data associated with a hash. func DeleteBlock(db ethdb.KeyValueWriter, hash common.Hash, number uint64) { DeleteReceipts(db, hash, number) DeleteHeader(db, hash, number) DeleteBody(db, hash, number) DeleteTd(db, hash, number) } // deleteBlockWithoutNumber removes all block data associated with a hash, except // the hash to number mapping. func deleteBlockWithoutNumber(db ethdb.KeyValueWriter, hash common.Hash, number uint64) { DeleteReceipts(db, hash, number) deleteHeaderWithoutNumber(db, hash, number) DeleteBody(db, hash, number) DeleteTd(db, hash, number) } const badBlockToKeep = 10 type badBlock struct { Header *types.Header Body *types.Body } // ReadBadBlock retrieves the bad block with the corresponding block hash. func ReadBadBlock(db ethdb.Reader, hash common.Hash) *types.Block { blob, err := db.Get(badBlockKey) if err != nil { return nil } var badBlocks []*badBlock if err := rlp.DecodeBytes(blob, &badBlocks); err != nil { return nil } for _, bad := range badBlocks { if bad.Header.Hash() == hash { block := types.NewBlockWithHeader(bad.Header) if bad.Body != nil { block = block.WithBody(*bad.Body) } return block } } return nil } // ReadAllBadBlocks retrieves all the bad blocks in the database. // All returned blocks are sorted in reverse order by number. func ReadAllBadBlocks(db ethdb.Reader) []*types.Block { blob, err := db.Get(badBlockKey) if err != nil { return nil } var badBlocks []*badBlock if err := rlp.DecodeBytes(blob, &badBlocks); err != nil { return nil } var blocks []*types.Block for _, bad := range badBlocks { block := types.NewBlockWithHeader(bad.Header) if bad.Body != nil { block = block.WithBody(*bad.Body) } blocks = append(blocks, block) } return blocks } // WriteBadBlock serializes the bad block into the database. If the cumulated // bad blocks exceeds the limitation, the oldest will be dropped. func WriteBadBlock(db ethdb.KeyValueStore, block *types.Block) { blob, err := db.Get(badBlockKey) if err != nil { log.Warn("Failed to load old bad blocks", "error", err) } var badBlocks []*badBlock if len(blob) > 0 { if err := rlp.DecodeBytes(blob, &badBlocks); err != nil { log.Crit("Failed to decode old bad blocks", "error", err) } } for _, b := range badBlocks { if b.Header.Number.Uint64() == block.NumberU64() && b.Header.Hash() == block.Hash() { log.Info("Skip duplicated bad block", "number", block.NumberU64(), "hash", block.Hash()) return } } badBlocks = append(badBlocks, &badBlock{ Header: block.Header(), Body: block.Body(), }) slices.SortFunc(badBlocks, func(a, b *badBlock) int { // NOTE: sorting in descending number order. return b.Header.Number.Cmp(a.Header.Number) }) if len(badBlocks) > badBlockToKeep { badBlocks = badBlocks[:badBlockToKeep] } data, err := rlp.EncodeToBytes(badBlocks) if err != nil { log.Crit("Failed to encode bad blocks", "err", err) } if err := db.Put(badBlockKey, data); err != nil { log.Crit("Failed to write bad blocks", "err", err) } } // DeleteBadBlocks deletes all the bad blocks from the database func DeleteBadBlocks(db ethdb.KeyValueWriter) { if err := db.Delete(badBlockKey); err != nil { log.Crit("Failed to delete bad blocks", "err", err) } } // ReadHeadHeader returns the current canonical head header. func ReadHeadHeader(db ethdb.Reader) *types.Header { headHeaderHash := ReadHeadHeaderHash(db) if headHeaderHash == (common.Hash{}) { return nil } headHeaderNumber := ReadHeaderNumber(db, headHeaderHash) if headHeaderNumber == nil { return nil } return ReadHeader(db, headHeaderHash, *headHeaderNumber) } // ReadHeadHeader returns the current canonical head block. func ReadHeadBlock(db ethdb.Reader) *types.Block { headBlockHash := ReadHeadBlockHash(db) if headBlockHash == (common.Hash{}) { return nil } headBlockNumber := ReadHeaderNumber(db, headBlockHash) if headBlockNumber == nil { return nil } return ReadBlock(db, headBlockHash, *headBlockNumber) }