core: remove unnecessary fields in log (#17106 #19182)

This commit is contained in:
Daniel Liu 2025-01-22 14:30:29 +08:00
parent a843f94524
commit 92f0d07e6d
13 changed files with 223 additions and 133 deletions

View file

@ -100,7 +100,13 @@ const (
triesInMemory = 128
// BlockChainVersion ensures that an incompatible database forces a resync from scratch.
BlockChainVersion = 3
//
// During the process of upgrading the database version from 3 to 4,
// the following incompatible database changes were added.
// * the `BlockNumber`, `TxHash`, `TxIndex`, `BlockHash` and `Index` fields of log are deleted
// * the `Bloom` field of receipt is deleted
// * the `BlockIndex` and `TxIndex` fields of txlookup are deleted
BlockChainVersion uint64 = 4
// Maximum length of chain to cache by block's number
blocksHashCacheLimit = 900

View file

@ -476,18 +476,6 @@ func DeleteReceipts(db ethdb.KeyValueWriter, hash common.Hash, number uint64) {
}
}
// storedReceiptRLP is the storage encoding of a receipt.
// Re-definition in core/types/receipt.go.
type storedReceiptRLP struct {
PostStateOrStatus []byte
CumulativeGasUsed uint64
Bloom types.Bloom
TxHash common.Hash
ContractAddress common.Address
Logs []*types.LogForStorage
GasUsed uint64
}
// ReceiptLogs is a barebone version of ReceiptForStorage which only keeps
// the list of logs. When decoding a stored receipt into this object we
// avoid creating the bloom filter.
@ -497,7 +485,7 @@ type receiptLogs struct {
// DecodeRLP implements rlp.Decoder.
func (r *receiptLogs) DecodeRLP(s *rlp.Stream) error {
var stored storedReceiptRLP
var stored types.ReceiptForStorage
if err := s.Decode(&stored); err != nil {
return err
}

View file

@ -25,45 +25,31 @@ import (
"github.com/XinFinOrg/XDPoSChain/rlp"
)
type TxLookupEntry struct {
BlockHash common.Hash
BlockIndex uint64
Index uint64
}
// ReadTxLookupEntry retrieves the positional metadata associated with a transaction
// hash to allow retrieving the transaction or receipt by hash.
func ReadTxLookupEntry(db ethdb.Reader, hash common.Hash) (common.Hash, uint64, uint64) {
// Load the positional metadata from disk and bail if it fails
func ReadTxLookupEntry(db ethdb.Reader, hash common.Hash) common.Hash {
data, _ := db.Get(txLookupKey(hash))
if len(data) == 0 {
return common.Hash{}, 0, 0
return common.Hash{}
}
// Parse and return the contents of the lookup entry
var entry TxLookupEntry
if len(data) == common.HashLength {
return common.BytesToHash(data)
}
// Probably it's legacy txlookup entry data, try to decode it.
var entry LegacyTxLookupEntry
if err := rlp.DecodeBytes(data, &entry); err != nil {
log.Error("Invalid lookup entry RLP", "hash", hash, "err", err)
return common.Hash{}, 0, 0
log.Error("Invalid transaction lookup entry RLP", "hash", hash, "blob", data, "err", err)
return common.Hash{}
}
return entry.BlockHash, entry.BlockIndex, entry.Index
return entry.BlockHash
}
// WriteTxLookupEntriesByBlock stores a positional metadata for every transaction from
// a block, enabling hash based transaction and receipt lookups.
func WriteTxLookupEntriesByBlock(db ethdb.KeyValueWriter, block *types.Block) {
// Iterate over each transaction and encode its metadata
for i, tx := range block.Transactions() {
entry := TxLookupEntry{
BlockHash: block.Hash(),
BlockIndex: block.NumberU64(),
Index: uint64(i),
}
data, err := rlp.EncodeToBytes(entry)
if err != nil {
log.Crit("Failed to RLP encode TxLookupEntry", "err", err)
}
if err := db.Put(txLookupKey(tx.Hash()), data); err != nil {
log.Crit("Failed to store tx lookup entry", "err", err)
for _, tx := range block.Transactions() {
if err := db.Put(txLookupKey(tx.Hash()), block.Hash().Bytes()); err != nil {
log.Crit("Failed to store transaction lookup entry", "err", err)
}
}
}
@ -76,63 +62,80 @@ func DeleteTxLookupEntry(db ethdb.KeyValueWriter, hash common.Hash) {
// ReadTransaction retrieves a specific transaction from the database, along with
// its added positional metadata.
func ReadTransaction(db ethdb.Reader, hash common.Hash) (*types.Transaction, common.Hash, uint64, uint64) {
// Retrieve the lookup metadata and resolve the transaction from the body
blockHash, blockNumber, txIndex := ReadTxLookupEntry(db, hash)
if blockHash != (common.Hash{}) {
body := ReadBody(db, blockHash, blockNumber)
if body == nil || len(body.Transactions) <= int(txIndex) {
log.Error("Transaction referenced missing", "number", blockNumber, "hash", blockHash, "index", txIndex)
blockHash := ReadTxLookupEntry(db, hash)
if blockHash == (common.Hash{}) {
// return nil, common.Hash{}, 0, 0
// TODO(daniel): delete the following old codes
// Old transaction representation, load the transaction and it's metadata separately
data, _ := db.Get(hash.Bytes())
if len(data) == 0 {
return nil, common.Hash{}, 0, 0
}
return body.Transactions[txIndex], blockHash, blockNumber, txIndex
var tx types.Transaction
if err := rlp.DecodeBytes(data, &tx); err != nil {
return nil, common.Hash{}, 0, 0
}
// Retrieve the blockchain positional metadata
data, _ = db.Get(append(hash.Bytes(), oldTxMetaSuffix...))
if len(data) == 0 {
return nil, common.Hash{}, 0, 0
}
var entry LegacyTxLookupEntry
if err := rlp.DecodeBytes(data, &entry); err != nil {
return nil, common.Hash{}, 0, 0
}
return &tx, entry.BlockHash, entry.BlockIndex, entry.Index
}
// Old transaction representation, load the transaction and it's metadata separately
data, _ := db.Get(hash.Bytes())
if len(data) == 0 {
blockNumber := ReadHeaderNumber(db, blockHash)
if blockNumber == nil {
return nil, common.Hash{}, 0, 0
}
var tx types.Transaction
if err := rlp.DecodeBytes(data, &tx); err != nil {
body := ReadBody(db, blockHash, *blockNumber)
if body == nil {
log.Error("Transaction referenced missing", "number", blockNumber, "hash", blockHash)
return nil, common.Hash{}, 0, 0
}
// Retrieve the blockchain positional metadata
data, _ = db.Get(append(hash.Bytes(), oldTxMetaSuffix...))
if len(data) == 0 {
return nil, common.Hash{}, 0, 0
for txIndex, tx := range body.Transactions {
if tx.Hash() == hash {
return tx, blockHash, *blockNumber, uint64(txIndex)
}
}
var entry TxLookupEntry
if err := rlp.DecodeBytes(data, &entry); err != nil {
return nil, common.Hash{}, 0, 0
}
return &tx, entry.BlockHash, entry.BlockIndex, entry.Index
log.Error("Transaction not found", "number", blockNumber, "hash", blockHash, "txhash", hash)
return nil, common.Hash{}, 0, 0
}
// ReadReceipt retrieves a specific transaction receipt from the database, along with
// its added positional metadata.
func ReadReceipt(db ethdb.Reader, hash common.Hash, config *params.ChainConfig) (*types.Receipt, common.Hash, uint64, uint64) {
// Retrieve the lookup metadata and resolve the receipt from the receipts
blockHash, blockNumber, receiptIndex := ReadTxLookupEntry(db, hash)
if blockHash != (common.Hash{}) {
receipts := ReadReceipts(db, blockHash, blockNumber, config)
if len(receipts) <= int(receiptIndex) {
log.Error("Receipt refereced missing", "number", blockNumber, "hash", blockHash, "index", receiptIndex)
blockHash := ReadTxLookupEntry(db, hash)
if blockHash == (common.Hash{}) {
// return nil, common.Hash{}, 0, 0
// TODO(daniel): delete the following old codes
// Old receipt representation, load the receipt and set an unknown metadata
data, _ := db.Get(append(oldReceiptsPrefix, hash[:]...))
if len(data) == 0 {
return nil, common.Hash{}, 0, 0
}
return receipts[receiptIndex], blockHash, blockNumber, receiptIndex
var receipt types.ReceiptForStorage
err := rlp.DecodeBytes(data, &receipt)
if err != nil {
log.Error("Invalid receipt RLP", "hash", hash, "err", err)
return nil, common.Hash{}, 0, 0
}
return (*types.Receipt)(&receipt), common.Hash{}, 0, 0
}
// Old receipt representation, load the receipt and set an unknown metadata
data, _ := db.Get(append(oldReceiptsPrefix, hash[:]...))
if len(data) == 0 {
blockNumber := ReadHeaderNumber(db, blockHash)
if blockNumber == nil {
return nil, common.Hash{}, 0, 0
}
var receipt types.ReceiptForStorage
err := rlp.DecodeBytes(data, &receipt)
if err != nil {
log.Error("Invalid receipt RLP", "hash", hash, "err", err)
receipts := ReadReceipts(db, blockHash, *blockNumber, config)
for receiptIndex, receipt := range receipts {
if receipt.TxHash == hash {
return receipt, blockHash, *blockNumber, uint64(receiptIndex)
}
}
return (*types.Receipt)(&receipt), common.Hash{}, 0, 0
log.Error("Receipt not found", "number", blockNumber, "hash", blockHash, "txhash", hash)
return nil, common.Hash{}, 0, 0
}
// ReadBloomBits retrieves the compressed bloom bit vector belonging to the given

View file

@ -1,4 +1,4 @@
// Copyright 2015 The go-ethereum Authors
// Copyright 2018 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
@ -22,6 +22,7 @@ import (
"github.com/XinFinOrg/XDPoSChain/common"
"github.com/XinFinOrg/XDPoSChain/core/types"
"github.com/XinFinOrg/XDPoSChain/rlp"
)
// Tests that positional lookup metadata can be stored and retrieved.
@ -52,7 +53,7 @@ func TestLookupStorage(t *testing.T) {
if hash != block.Hash() || number != block.NumberU64() || index != uint64(i) {
t.Fatalf("tx #%d [%x]: positional metadata mismatch: have %x/%d/%d, want %x/%v/%v", i, tx.Hash(), hash, number, index, block.Hash(), block.NumberU64(), i)
}
if tx.String() != txn.String() {
if tx.Hash() != txn.Hash() {
t.Fatalf("tx #%d [%x]: transaction mismatch: have %v, want %v", i, tx.Hash(), txn, tx)
}
}
@ -64,4 +65,26 @@ func TestLookupStorage(t *testing.T) {
t.Fatalf("tx #%d [%x]: deleted transaction returned: %v", i, tx.Hash(), txn)
}
}
// Insert legacy txlookup and verify the data retrieval
for index, tx := range block.Transactions() {
entry := LegacyTxLookupEntry{
BlockHash: block.Hash(),
BlockIndex: block.NumberU64(),
Index: uint64(index),
}
data, _ := rlp.EncodeToBytes(entry)
db.Put(txLookupKey(tx.Hash()), data)
}
for i, tx := range txs {
if txn, hash, number, index := ReadTransaction(db, tx.Hash()); txn == nil {
t.Fatalf("tx #%d [%x]: transaction not found", i, tx.Hash())
} else {
if hash != block.Hash() || number != block.NumberU64() || index != uint64(i) {
t.Fatalf("tx #%d [%x]: positional metadata mismatch: have %x/%d/%d, want %x/%v/%v", i, tx.Hash(), hash, number, index, block.Hash(), block.NumberU64(), i)
}
if tx.Hash() != txn.Hash() {
t.Fatalf("tx #%d [%x]: transaction mismatch: have %v, want %v", i, tx.Hash(), txn, tx)
}
}
}
}

View file

@ -27,18 +27,30 @@ import (
"github.com/XinFinOrg/XDPoSChain/rlp"
)
// ReadDatabaseVersion reads the version number from db.
func ReadDatabaseVersion(db ethdb.KeyValueReader) int {
var vsn uint
// ReadDatabaseVersion retrieves the version number of the database.
func ReadDatabaseVersion(db ethdb.KeyValueReader) *uint64 {
var version uint64
enc, _ := db.Get(databaseVersionKey)
rlp.DecodeBytes(enc, &vsn)
return int(vsn)
if len(enc) == 0 {
return nil
}
if err := rlp.DecodeBytes(enc, &version); err != nil {
return nil
}
return &version
}
// WriteDatabaseVersion writes vsn as the version number to db.
func WriteDatabaseVersion(db ethdb.KeyValueWriter, vsn int) {
enc, _ := rlp.EncodeToBytes(uint(vsn))
db.Put(databaseVersionKey, enc)
// WriteDatabaseVersion stores the version number of the database
func WriteDatabaseVersion(db ethdb.KeyValueWriter, version uint64) {
enc, err := rlp.EncodeToBytes(version)
if err != nil {
log.Crit("Failed to encode database version", "err", err)
}
if err = db.Put(databaseVersionKey, enc); err != nil {
log.Crit("Failed to store the database version", "err", err)
}
}
// ReadChainConfig will fetch the network settings based on the given hash.

View file

@ -82,6 +82,14 @@ const (
freezerReceiptTable = "receipts"
)
// LegacyTxLookupEntry is the legacy TxLookupEntry definition with some unnecessary
// fields.
type LegacyTxLookupEntry struct {
BlockHash common.Hash
BlockIndex uint64
Index uint64
}
// encodeBlockNumber encodes a block number as big endian uint64
func encodeBlockNumber(number uint64) []byte {
enc := make([]byte, 8)

View file

@ -69,7 +69,8 @@ type rlpLog struct {
Data []byte
}
type rlpStorageLog struct {
// legacyRlpStorageLog is the previous storage encoding of a log including some redundant fields.
type legacyRlpStorageLog struct {
Address common.Address
Topics []common.Hash
Data []byte
@ -82,7 +83,8 @@ type rlpStorageLog struct {
// EncodeRLP implements rlp.Encoder.
func (l *Log) EncodeRLP(w io.Writer) error {
return rlp.Encode(w, rlpLog{Address: l.Address, Topics: l.Topics, Data: l.Data})
rl := rlpLog{Address: l.Address, Topics: l.Topics, Data: l.Data}
return rlp.Encode(w, &rl)
}
// DecodeRLP implements rlp.Decoder.
@ -105,32 +107,36 @@ type LogForStorage Log
// EncodeRLP implements rlp.Encoder.
func (l *LogForStorage) EncodeRLP(w io.Writer) error {
return rlp.Encode(w, rlpStorageLog{
Address: l.Address,
Topics: l.Topics,
Data: l.Data,
BlockNumber: l.BlockNumber,
TxHash: l.TxHash,
TxIndex: l.TxIndex,
BlockHash: l.BlockHash,
Index: l.Index,
})
rl := rlpLog{Address: l.Address, Topics: l.Topics, Data: l.Data}
return rlp.Encode(w, &rl)
}
// DecodeRLP implements rlp.Decoder.
//
// Note some redundant fields(e.g. block number, tx hash etc) will be assembled later.
func (l *LogForStorage) DecodeRLP(s *rlp.Stream) error {
var dec rlpStorageLog
err := s.Decode(&dec)
blob, err := s.Raw()
if err != nil {
return err
}
var dec rlpLog
err = rlp.DecodeBytes(blob, &dec)
if err == nil {
*l = LogForStorage{
Address: dec.Address,
Topics: dec.Topics,
Data: dec.Data,
BlockNumber: dec.BlockNumber,
TxHash: dec.TxHash,
TxIndex: dec.TxIndex,
BlockHash: dec.BlockHash,
Index: dec.Index,
Address: dec.Address,
Topics: dec.Topics,
Data: dec.Data,
}
} else {
// Try to decode log with previous definition.
var dec legacyRlpStorageLog
err = rlp.DecodeBytes(blob, &dec)
if err == nil {
*l = LogForStorage{
Address: dec.Address,
Topics: dec.Topics,
Data: dec.Data,
}
}
}
return err

View file

@ -90,8 +90,18 @@ type receiptRLP struct {
Logs []*Log
}
// receiptStorageRLP is the original storage encoding of a receipt including some unnecessary fields.
type receiptStorageRLP struct {
// v4StoredReceiptRLP is the storage encoding of a receipt used in database version 4.
type v4StoredReceiptRLP struct {
PostStateOrStatus []byte
CumulativeGasUsed uint64
TxHash common.Hash
ContractAddress common.Address
Logs []*LogForStorage
GasUsed uint64
}
// v3StoredReceiptRLP is the previous storage encoding of a receipt including some unnecessary fields.
type v3StoredReceiptRLP struct {
PostStateOrStatus []byte
CumulativeGasUsed uint64
Bloom Bloom
@ -266,10 +276,9 @@ type ReceiptForStorage Receipt
// EncodeRLP implements rlp.Encoder, and flattens all content fields of a receipt
// into an RLP stream.
func (r *ReceiptForStorage) EncodeRLP(w io.Writer) error {
enc := &receiptStorageRLP{
enc := &v4StoredReceiptRLP{
PostStateOrStatus: (*Receipt)(r).statusEncoding(),
CumulativeGasUsed: r.CumulativeGasUsed,
Bloom: r.Bloom,
TxHash: r.TxHash,
ContractAddress: r.ContractAddress,
Logs: make([]*LogForStorage, len(r.Logs)),
@ -289,28 +298,53 @@ func (r *ReceiptForStorage) DecodeRLP(s *rlp.Stream) error {
if err != nil {
return err
}
return decodeStoredReceiptRLP(r, blob)
// Try decoding from the newest format for future proofness, then the older one
// for old nodes that just upgraded. V4 was an intermediate unreleased format so
// we do need to decode it, but it's not common (try last).
if err := decodeV3StoredReceiptRLP(r, blob); err == nil {
return nil
}
return decodeV4StoredReceiptRLP(r, blob)
}
func decodeStoredReceiptRLP(r *ReceiptForStorage, blob []byte) error {
var stored receiptStorageRLP
func decodeV4StoredReceiptRLP(r *ReceiptForStorage, blob []byte) error {
var stored v4StoredReceiptRLP
if err := rlp.DecodeBytes(blob, &stored); err != nil {
return err
}
if err := (*Receipt)(r).setStatus(stored.PostStateOrStatus); err != nil {
return err
}
// Assign the consensus fields
r.CumulativeGasUsed = stored.CumulativeGasUsed
r.Bloom = stored.Bloom
r.TxHash = stored.TxHash
r.ContractAddress = stored.ContractAddress
r.GasUsed = stored.GasUsed
r.Logs = make([]*Log, len(stored.Logs))
for i, log := range stored.Logs {
r.Logs[i] = (*Log)(log)
}
// Assign the implementation fields
r.Bloom = CreateBloom(Receipts{(*Receipt)(r)})
return nil
}
func decodeV3StoredReceiptRLP(r *ReceiptForStorage, blob []byte) error {
var stored v3StoredReceiptRLP
if err := rlp.DecodeBytes(blob, &stored); err != nil {
return err
}
if err := (*Receipt)(r).setStatus(stored.PostStateOrStatus); err != nil {
return err
}
r.CumulativeGasUsed = stored.CumulativeGasUsed
r.Bloom = stored.Bloom
r.TxHash = stored.TxHash
r.ContractAddress = stored.ContractAddress
r.GasUsed = stored.GasUsed
r.Logs = make([]*Log, len(stored.Logs))
for i, log := range stored.Logs {
r.Logs[i] = (*Log)(log)
}
return nil
}

View file

@ -357,7 +357,7 @@ func TestLegacyReceiptDecoding(t *testing.T) {
}
func encodeAsV3StoredReceiptRLP(want *Receipt) ([]byte, error) {
stored := &receiptStorageRLP{
stored := &v3StoredReceiptRLP{
PostStateOrStatus: want.statusEncoding(),
CumulativeGasUsed: want.CumulativeGasUsed,
Bloom: want.Bloom,

View file

@ -151,15 +151,25 @@ func New(ctx *node.ServiceContext, config *ethconfig.Config, XDCXServ *XDCx.XDCX
if lendingServ != nil {
eth.Lending = lendingServ
}
log.Info("Initialising Ethereum protocol", "versions", ProtocolVersions, "network", config.NetworkId)
bcVersion := rawdb.ReadDatabaseVersion(chainDb)
var dbVer = "<nil>"
if bcVersion != nil {
dbVer = fmt.Sprintf("%d", *bcVersion)
}
log.Info("Initialising Ethereum protocol", "versions", ProtocolVersions, "network", config.NetworkId, "dbversion", dbVer)
if !config.SkipBcVersionCheck {
bcVersion := rawdb.ReadDatabaseVersion(chainDb)
if bcVersion != core.BlockChainVersion && bcVersion != 0 {
return nil, fmt.Errorf("blockchain DB version mismatch (%d / %d). Run geth upgradedb", bcVersion, core.BlockChainVersion)
if bcVersion != nil && *bcVersion > core.BlockChainVersion {
return nil, fmt.Errorf("database version is v%d, not supports v%d", *bcVersion, core.BlockChainVersion)
} else if bcVersion == nil || *bcVersion < core.BlockChainVersion {
if bcVersion != nil { // only print warning on upgrade, not on init
log.Warn("Upgrade blockchain database version", "from", dbVer, "to", core.BlockChainVersion)
}
rawdb.WriteDatabaseVersion(chainDb, core.BlockChainVersion)
}
rawdb.WriteDatabaseVersion(chainDb, core.BlockChainVersion)
}
var (
vmConfig = vm.Config{EnablePreimageRecording: config.EnablePreimageRecording}
cacheConfig = &core.CacheConfig{Disabled: config.NoPruning, TrieNodeLimit: config.TrieCache, TrieTimeLimit: config.TrieTimeout}

View file

@ -1172,9 +1172,9 @@ func (pm *ProtocolManager) txStatus(hashes []common.Hash) []txStatus {
// If the transaction is unknown to the pool, try looking it up locally
if stat == txpool.TxStatusUnknown {
if block, number, index := rawdb.ReadTxLookupEntry(pm.chainDb, hashes[i]); block != (common.Hash{}) {
if tx, blockHash, blockNumber, txIndex := rawdb.ReadTransaction(pm.chainDb, hashes[i]); tx != nil {
stats[i].Status = txpool.TxStatusIncluded
stats[i].Lookup = &rawdb.TxLookupEntry{BlockHash: block, BlockIndex: number, Index: index}
stats[i].Lookup = &rawdb.LegacyTxLookupEntry{BlockHash: blockHash, BlockIndex: blockNumber, Index: txIndex}
}
}
}

View file

@ -557,8 +557,8 @@ func TestTransactionStatusLes2(t *testing.T) {
// check if their status is included now
block1hash := rawdb.ReadCanonicalHash(db, 1)
test(tx1, false, txStatus{Status: txpool.TxStatusIncluded, Lookup: &rawdb.TxLookupEntry{BlockHash: block1hash, BlockIndex: 1, Index: 0}})
test(tx2, false, txStatus{Status: txpool.TxStatusIncluded, Lookup: &rawdb.TxLookupEntry{BlockHash: block1hash, BlockIndex: 1, Index: 1}})
test(tx1, false, txStatus{Status: txpool.TxStatusIncluded, Lookup: &rawdb.LegacyTxLookupEntry{BlockHash: block1hash, BlockIndex: 1, Index: 0}})
test(tx2, false, txStatus{Status: txpool.TxStatusIncluded, Lookup: &rawdb.LegacyTxLookupEntry{BlockHash: block1hash, BlockIndex: 1, Index: 1}})
// create a reorg that rolls them back
gchain, _ = core.GenerateChain(params.TestChainConfig, chain.GetBlockByNumber(0), ethash.NewFaker(), db, 2, func(i int, block *core.BlockGen) {})

View file

@ -225,6 +225,6 @@ type proofsData [][]rlp.RawValue
type txStatus struct {
Status txpool.TxStatus
Lookup *rawdb.TxLookupEntry `rlp:"nil"`
Lookup *rawdb.LegacyTxLookupEntry `rlp:"nil"`
Error string
}