mirror of
https://github.com/ethereum/go-ethereum.git
synced 2026-02-26 15:47:21 +00:00
core/rawdb, triedb/pathdb: introduce trienode history (#32596)
It's a pull request based on the #32523 , implementing the structure of trienode history.
This commit is contained in:
parent
ed264a1f19
commit
de24450dbf
13 changed files with 1850 additions and 71 deletions
|
|
@ -46,6 +46,27 @@ func DeleteStateHistoryIndexMetadata(db ethdb.KeyValueWriter) {
|
|||
}
|
||||
}
|
||||
|
||||
// ReadTrienodeHistoryIndexMetadata retrieves the metadata of trienode history index.
|
||||
func ReadTrienodeHistoryIndexMetadata(db ethdb.KeyValueReader) []byte {
|
||||
data, _ := db.Get(headTrienodeHistoryIndexKey)
|
||||
return data
|
||||
}
|
||||
|
||||
// WriteTrienodeHistoryIndexMetadata stores the metadata of trienode history index
|
||||
// into database.
|
||||
func WriteTrienodeHistoryIndexMetadata(db ethdb.KeyValueWriter, blob []byte) {
|
||||
if err := db.Put(headTrienodeHistoryIndexKey, blob); err != nil {
|
||||
log.Crit("Failed to store the metadata of trienode history index", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// DeleteTrienodeHistoryIndexMetadata removes the metadata of trienode history index.
|
||||
func DeleteTrienodeHistoryIndexMetadata(db ethdb.KeyValueWriter) {
|
||||
if err := db.Delete(headTrienodeHistoryIndexKey); err != nil {
|
||||
log.Crit("Failed to delete the metadata of trienode history index", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// ReadAccountHistoryIndex retrieves the account history index with the provided
|
||||
// account address.
|
||||
func ReadAccountHistoryIndex(db ethdb.KeyValueReader, addressHash common.Hash) []byte {
|
||||
|
|
@ -95,6 +116,30 @@ func DeleteStorageHistoryIndex(db ethdb.KeyValueWriter, addressHash common.Hash,
|
|||
}
|
||||
}
|
||||
|
||||
// ReadTrienodeHistoryIndex retrieves the trienode history index with the provided
|
||||
// account address and storage key hash.
|
||||
func ReadTrienodeHistoryIndex(db ethdb.KeyValueReader, addressHash common.Hash, path []byte) []byte {
|
||||
data, err := db.Get(trienodeHistoryIndexKey(addressHash, path))
|
||||
if err != nil || len(data) == 0 {
|
||||
return nil
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
// WriteTrienodeHistoryIndex writes the provided trienode history index into database.
|
||||
func WriteTrienodeHistoryIndex(db ethdb.KeyValueWriter, addressHash common.Hash, path []byte, data []byte) {
|
||||
if err := db.Put(trienodeHistoryIndexKey(addressHash, path), data); err != nil {
|
||||
log.Crit("Failed to store trienode history index", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// DeleteTrienodeHistoryIndex deletes the specified trienode index from the database.
|
||||
func DeleteTrienodeHistoryIndex(db ethdb.KeyValueWriter, addressHash common.Hash, path []byte) {
|
||||
if err := db.Delete(trienodeHistoryIndexKey(addressHash, path)); err != nil {
|
||||
log.Crit("Failed to delete trienode history index", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// ReadAccountHistoryIndexBlock retrieves the index block with the provided
|
||||
// account address along with the block id.
|
||||
func ReadAccountHistoryIndexBlock(db ethdb.KeyValueReader, addressHash common.Hash, blockID uint32) []byte {
|
||||
|
|
@ -143,6 +188,30 @@ func DeleteStorageHistoryIndexBlock(db ethdb.KeyValueWriter, addressHash common.
|
|||
}
|
||||
}
|
||||
|
||||
// ReadTrienodeHistoryIndexBlock retrieves the index block with the provided state
|
||||
// identifier along with the block id.
|
||||
func ReadTrienodeHistoryIndexBlock(db ethdb.KeyValueReader, addressHash common.Hash, path []byte, blockID uint32) []byte {
|
||||
data, err := db.Get(trienodeHistoryIndexBlockKey(addressHash, path, blockID))
|
||||
if err != nil || len(data) == 0 {
|
||||
return nil
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
// WriteTrienodeHistoryIndexBlock writes the provided index block into database.
|
||||
func WriteTrienodeHistoryIndexBlock(db ethdb.KeyValueWriter, addressHash common.Hash, path []byte, id uint32, data []byte) {
|
||||
if err := db.Put(trienodeHistoryIndexBlockKey(addressHash, path, id), data); err != nil {
|
||||
log.Crit("Failed to store trienode index block", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// DeleteTrienodeHistoryIndexBlock deletes the specified index block from the database.
|
||||
func DeleteTrienodeHistoryIndexBlock(db ethdb.KeyValueWriter, addressHash common.Hash, path []byte, id uint32) {
|
||||
if err := db.Delete(trienodeHistoryIndexBlockKey(addressHash, path, id)); err != nil {
|
||||
log.Crit("Failed to delete trienode index block", "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// increaseKey increase the input key by one bit. Return nil if the entire
|
||||
// addition operation overflows.
|
||||
func increaseKey(key []byte) []byte {
|
||||
|
|
@ -155,14 +224,26 @@ func increaseKey(key []byte) []byte {
|
|||
return nil
|
||||
}
|
||||
|
||||
// DeleteStateHistoryIndex completely removes all history indexing data, including
|
||||
// DeleteStateHistoryIndexes completely removes all history indexing data, including
|
||||
// indexes for accounts and storages.
|
||||
//
|
||||
// Note, this method assumes the storage space with prefix `StateHistoryIndexPrefix`
|
||||
// is exclusively occupied by the history indexing data!
|
||||
func DeleteStateHistoryIndex(db ethdb.KeyValueRangeDeleter) {
|
||||
start := StateHistoryIndexPrefix
|
||||
limit := increaseKey(bytes.Clone(StateHistoryIndexPrefix))
|
||||
func DeleteStateHistoryIndexes(db ethdb.KeyValueRangeDeleter) {
|
||||
DeleteHistoryByRange(db, StateHistoryAccountMetadataPrefix)
|
||||
DeleteHistoryByRange(db, StateHistoryStorageMetadataPrefix)
|
||||
DeleteHistoryByRange(db, StateHistoryAccountBlockPrefix)
|
||||
DeleteHistoryByRange(db, StateHistoryStorageBlockPrefix)
|
||||
}
|
||||
|
||||
// DeleteTrienodeHistoryIndexes completely removes all trienode history indexing data.
|
||||
func DeleteTrienodeHistoryIndexes(db ethdb.KeyValueRangeDeleter) {
|
||||
DeleteHistoryByRange(db, TrienodeHistoryMetadataPrefix)
|
||||
DeleteHistoryByRange(db, TrienodeHistoryBlockPrefix)
|
||||
}
|
||||
|
||||
// DeleteHistoryByRange completely removes all database entries with the specific prefix.
|
||||
// Note, this method assumes the space with the given prefix is exclusively occupied!
|
||||
func DeleteHistoryByRange(db ethdb.KeyValueRangeDeleter, prefix []byte) {
|
||||
start := prefix
|
||||
limit := increaseKey(bytes.Clone(prefix))
|
||||
|
||||
// Try to remove the data in the range by a loop, as the leveldb
|
||||
// doesn't support the native range deletion.
|
||||
|
|
|
|||
|
|
@ -299,3 +299,76 @@ func WriteStateHistory(db ethdb.AncientWriter, id uint64, meta []byte, accountIn
|
|||
})
|
||||
return err
|
||||
}
|
||||
|
||||
// ReadTrienodeHistory retrieves the trienode history corresponding to the specified id.
|
||||
// Compute the position of trienode history in freezer by minus one since the id of first
|
||||
// trienode history starts from one(zero for initial state).
|
||||
func ReadTrienodeHistory(db ethdb.AncientReaderOp, id uint64) ([]byte, []byte, []byte, error) {
|
||||
header, err := db.Ancient(trienodeHistoryHeaderTable, id-1)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
keySection, err := db.Ancient(trienodeHistoryKeySectionTable, id-1)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
valueSection, err := db.Ancient(trienodeHistoryValueSectionTable, id-1)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
return header, keySection, valueSection, nil
|
||||
}
|
||||
|
||||
// ReadTrienodeHistoryHeader retrieves the header section of trienode history.
|
||||
func ReadTrienodeHistoryHeader(db ethdb.AncientReaderOp, id uint64) ([]byte, error) {
|
||||
return db.Ancient(trienodeHistoryHeaderTable, id-1)
|
||||
}
|
||||
|
||||
// ReadTrienodeHistoryKeySection retrieves the key section of trienode history.
|
||||
func ReadTrienodeHistoryKeySection(db ethdb.AncientReaderOp, id uint64) ([]byte, error) {
|
||||
return db.Ancient(trienodeHistoryKeySectionTable, id-1)
|
||||
}
|
||||
|
||||
// ReadTrienodeHistoryValueSection retrieves the value section of trienode history.
|
||||
func ReadTrienodeHistoryValueSection(db ethdb.AncientReaderOp, id uint64) ([]byte, error) {
|
||||
return db.Ancient(trienodeHistoryValueSectionTable, id-1)
|
||||
}
|
||||
|
||||
// ReadTrienodeHistoryList retrieves the a list of trienode history corresponding
|
||||
// to the specified range.
|
||||
// Compute the position of trienode history in freezer by minus one since the id
|
||||
// of first trienode history starts from one(zero for initial state).
|
||||
func ReadTrienodeHistoryList(db ethdb.AncientReaderOp, start uint64, count uint64) ([][]byte, [][]byte, [][]byte, error) {
|
||||
header, err := db.AncientRange(trienodeHistoryHeaderTable, start-1, count, 0)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
keySection, err := db.AncientRange(trienodeHistoryKeySectionTable, start-1, count, 0)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
valueSection, err := db.AncientRange(trienodeHistoryValueSectionTable, start-1, count, 0)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
if len(header) != len(keySection) || len(header) != len(valueSection) {
|
||||
return nil, nil, nil, errors.New("trienode history is corrupted")
|
||||
}
|
||||
return header, keySection, valueSection, nil
|
||||
}
|
||||
|
||||
// WriteTrienodeHistory writes the provided trienode history to database.
|
||||
// Compute the position of trienode history in freezer by minus one since
|
||||
// the id of first state history starts from one(zero for initial state).
|
||||
func WriteTrienodeHistory(db ethdb.AncientWriter, id uint64, header []byte, keySection []byte, valueSection []byte) error {
|
||||
_, err := db.ModifyAncients(func(op ethdb.AncientWriteOp) error {
|
||||
if err := op.AppendRaw(trienodeHistoryHeaderTable, id-1, header); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := op.AppendRaw(trienodeHistoryKeySectionTable, id-1, keySection); err != nil {
|
||||
return err
|
||||
}
|
||||
return op.AppendRaw(trienodeHistoryValueSectionTable, id-1, valueSection)
|
||||
})
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -75,15 +75,38 @@ var stateFreezerTableConfigs = map[string]freezerTableConfig{
|
|||
stateHistoryStorageData: {noSnappy: false, prunable: true},
|
||||
}
|
||||
|
||||
const (
|
||||
trienodeHistoryHeaderTable = "trienode.header"
|
||||
trienodeHistoryKeySectionTable = "trienode.key"
|
||||
trienodeHistoryValueSectionTable = "trienode.value"
|
||||
)
|
||||
|
||||
// trienodeFreezerTableConfigs configures the settings for tables in the trienode freezer.
|
||||
var trienodeFreezerTableConfigs = map[string]freezerTableConfig{
|
||||
trienodeHistoryHeaderTable: {noSnappy: false, prunable: true},
|
||||
|
||||
// Disable snappy compression to allow efficient partial read.
|
||||
trienodeHistoryKeySectionTable: {noSnappy: true, prunable: true},
|
||||
|
||||
// Disable snappy compression to allow efficient partial read.
|
||||
trienodeHistoryValueSectionTable: {noSnappy: true, prunable: true},
|
||||
}
|
||||
|
||||
// The list of identifiers of ancient stores.
|
||||
var (
|
||||
ChainFreezerName = "chain" // the folder name of chain segment ancient store.
|
||||
MerkleStateFreezerName = "state" // the folder name of state history ancient store.
|
||||
VerkleStateFreezerName = "state_verkle" // the folder name of state history ancient store.
|
||||
ChainFreezerName = "chain" // the folder name of chain segment ancient store.
|
||||
MerkleStateFreezerName = "state" // the folder name of state history ancient store.
|
||||
VerkleStateFreezerName = "state_verkle" // the folder name of state history ancient store.
|
||||
MerkleTrienodeFreezerName = "trienode" // the folder name of trienode history ancient store.
|
||||
VerkleTrienodeFreezerName = "trienode_verkle" // the folder name of trienode history ancient store.
|
||||
)
|
||||
|
||||
// freezers the collections of all builtin freezers.
|
||||
var freezers = []string{ChainFreezerName, MerkleStateFreezerName, VerkleStateFreezerName}
|
||||
var freezers = []string{
|
||||
ChainFreezerName,
|
||||
MerkleStateFreezerName, VerkleStateFreezerName,
|
||||
MerkleTrienodeFreezerName, VerkleTrienodeFreezerName,
|
||||
}
|
||||
|
||||
// NewStateFreezer initializes the ancient store for state history.
|
||||
//
|
||||
|
|
@ -103,3 +126,22 @@ func NewStateFreezer(ancientDir string, verkle bool, readOnly bool) (ethdb.Reset
|
|||
}
|
||||
return newResettableFreezer(name, "eth/db/state", readOnly, stateHistoryTableSize, stateFreezerTableConfigs)
|
||||
}
|
||||
|
||||
// NewTrienodeFreezer initializes the ancient store for trienode history.
|
||||
//
|
||||
// - if the empty directory is given, initializes the pure in-memory
|
||||
// trienode freezer (e.g. dev mode).
|
||||
// - if non-empty directory is given, initializes the regular file-based
|
||||
// trienode freezer.
|
||||
func NewTrienodeFreezer(ancientDir string, verkle bool, readOnly bool) (ethdb.ResettableAncientStore, error) {
|
||||
if ancientDir == "" {
|
||||
return NewMemoryFreezer(readOnly, trienodeFreezerTableConfigs), nil
|
||||
}
|
||||
var name string
|
||||
if verkle {
|
||||
name = filepath.Join(ancientDir, VerkleTrienodeFreezerName)
|
||||
} else {
|
||||
name = filepath.Join(ancientDir, MerkleTrienodeFreezerName)
|
||||
}
|
||||
return newResettableFreezer(name, "eth/db/trienode", readOnly, stateHistoryTableSize, trienodeFreezerTableConfigs)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -80,6 +80,10 @@ var (
|
|||
// been indexed.
|
||||
headStateHistoryIndexKey = []byte("LastStateHistoryIndex")
|
||||
|
||||
// headTrienodeHistoryIndexKey tracks the ID of the latest state history that has
|
||||
// been indexed.
|
||||
headTrienodeHistoryIndexKey = []byte("LastTrienodeHistoryIndex")
|
||||
|
||||
// txIndexTailKey tracks the oldest block whose transactions have been indexed.
|
||||
txIndexTailKey = []byte("TransactionIndexTail")
|
||||
|
||||
|
|
@ -125,8 +129,10 @@ var (
|
|||
StateHistoryIndexPrefix = []byte("m") // The global prefix of state history index data
|
||||
StateHistoryAccountMetadataPrefix = []byte("ma") // StateHistoryAccountMetadataPrefix + account address hash => account metadata
|
||||
StateHistoryStorageMetadataPrefix = []byte("ms") // StateHistoryStorageMetadataPrefix + account address hash + storage slot hash => slot metadata
|
||||
TrienodeHistoryMetadataPrefix = []byte("mt") // TrienodeHistoryMetadataPrefix + account address hash + trienode path => trienode metadata
|
||||
StateHistoryAccountBlockPrefix = []byte("mba") // StateHistoryAccountBlockPrefix + account address hash + blockID => account block
|
||||
StateHistoryStorageBlockPrefix = []byte("mbs") // StateHistoryStorageBlockPrefix + account address hash + storage slot hash + blockID => slot block
|
||||
TrienodeHistoryBlockPrefix = []byte("mbt") // TrienodeHistoryBlockPrefix + account address hash + trienode path + blockID => trienode block
|
||||
|
||||
// VerklePrefix is the database prefix for Verkle trie data, which includes:
|
||||
// (a) Trie nodes
|
||||
|
|
@ -395,27 +401,34 @@ func storageHistoryIndexKey(addressHash common.Hash, storageHash common.Hash) []
|
|||
return out
|
||||
}
|
||||
|
||||
// trienodeHistoryIndexKey = TrienodeHistoryMetadataPrefix + addressHash + trienode path
|
||||
func trienodeHistoryIndexKey(addressHash common.Hash, path []byte) []byte {
|
||||
totalLen := len(TrienodeHistoryMetadataPrefix) + common.HashLength + len(path)
|
||||
out := make([]byte, totalLen)
|
||||
|
||||
off := 0
|
||||
off += copy(out[off:], TrienodeHistoryMetadataPrefix)
|
||||
off += copy(out[off:], addressHash.Bytes())
|
||||
copy(out[off:], path)
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// accountHistoryIndexBlockKey = StateHistoryAccountBlockPrefix + addressHash + blockID
|
||||
func accountHistoryIndexBlockKey(addressHash common.Hash, blockID uint32) []byte {
|
||||
var buf4 [4]byte
|
||||
binary.BigEndian.PutUint32(buf4[:], blockID)
|
||||
|
||||
totalLen := len(StateHistoryAccountBlockPrefix) + common.HashLength + 4
|
||||
out := make([]byte, totalLen)
|
||||
|
||||
off := 0
|
||||
off += copy(out[off:], StateHistoryAccountBlockPrefix)
|
||||
off += copy(out[off:], addressHash.Bytes())
|
||||
copy(out[off:], buf4[:])
|
||||
binary.BigEndian.PutUint32(out[off:], blockID)
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// storageHistoryIndexBlockKey = StateHistoryStorageBlockPrefix + addressHash + storageHash + blockID
|
||||
func storageHistoryIndexBlockKey(addressHash common.Hash, storageHash common.Hash, blockID uint32) []byte {
|
||||
var buf4 [4]byte
|
||||
binary.BigEndian.PutUint32(buf4[:], blockID)
|
||||
|
||||
totalLen := len(StateHistoryStorageBlockPrefix) + 2*common.HashLength + 4
|
||||
out := make([]byte, totalLen)
|
||||
|
||||
|
|
@ -423,7 +436,21 @@ func storageHistoryIndexBlockKey(addressHash common.Hash, storageHash common.Has
|
|||
off += copy(out[off:], StateHistoryStorageBlockPrefix)
|
||||
off += copy(out[off:], addressHash.Bytes())
|
||||
off += copy(out[off:], storageHash.Bytes())
|
||||
copy(out[off:], buf4[:])
|
||||
binary.BigEndian.PutUint32(out[off:], blockID)
|
||||
|
||||
return out
|
||||
}
|
||||
|
||||
// trienodeHistoryIndexBlockKey = TrienodeHistoryBlockPrefix + addressHash + trienode path + blockID
|
||||
func trienodeHistoryIndexBlockKey(addressHash common.Hash, path []byte, blockID uint32) []byte {
|
||||
totalLen := len(TrienodeHistoryBlockPrefix) + common.HashLength + len(path) + 4
|
||||
out := make([]byte, totalLen)
|
||||
|
||||
off := 0
|
||||
off += copy(out[off:], TrienodeHistoryBlockPrefix)
|
||||
off += copy(out[off:], addressHash.Bytes())
|
||||
off += copy(out[off:], path)
|
||||
binary.BigEndian.PutUint32(out[off:], blockID)
|
||||
|
||||
return out
|
||||
}
|
||||
|
|
|
|||
|
|
@ -232,7 +232,7 @@ func (db *Database) repairHistory() error {
|
|||
// Purge all state history indexing data first
|
||||
batch := db.diskdb.NewBatch()
|
||||
rawdb.DeleteStateHistoryIndexMetadata(batch)
|
||||
rawdb.DeleteStateHistoryIndex(batch)
|
||||
rawdb.DeleteStateHistoryIndexes(batch)
|
||||
if err := batch.Write(); err != nil {
|
||||
log.Crit("Failed to purge state history index", "err", err)
|
||||
}
|
||||
|
|
@ -426,7 +426,7 @@ func (db *Database) Enable(root common.Hash) error {
|
|||
// Purge all state history indexing data first
|
||||
batch.Reset()
|
||||
rawdb.DeleteStateHistoryIndexMetadata(batch)
|
||||
rawdb.DeleteStateHistoryIndex(batch)
|
||||
rawdb.DeleteStateHistoryIndexes(batch)
|
||||
if err := batch.Write(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -32,6 +32,9 @@ type historyType uint8
|
|||
const (
|
||||
// typeStateHistory indicates history data related to account or storage changes.
|
||||
typeStateHistory historyType = 0
|
||||
|
||||
// typeTrienodeHistory indicates history data related to trie node changes.
|
||||
typeTrienodeHistory historyType = 1
|
||||
)
|
||||
|
||||
// String returns the string format representation.
|
||||
|
|
@ -39,6 +42,8 @@ func (h historyType) String() string {
|
|||
switch h {
|
||||
case typeStateHistory:
|
||||
return "state"
|
||||
case typeTrienodeHistory:
|
||||
return "trienode"
|
||||
default:
|
||||
return fmt.Sprintf("unknown type: %d", h)
|
||||
}
|
||||
|
|
@ -48,8 +53,9 @@ func (h historyType) String() string {
|
|||
type elementType uint8
|
||||
|
||||
const (
|
||||
typeAccount elementType = 0 // represents the account data
|
||||
typeStorage elementType = 1 // represents the storage slot data
|
||||
typeAccount elementType = 0 // represents the account data
|
||||
typeStorage elementType = 1 // represents the storage slot data
|
||||
typeTrienode elementType = 2 // represents the trie node data
|
||||
)
|
||||
|
||||
// String returns the string format representation.
|
||||
|
|
@ -59,6 +65,8 @@ func (e elementType) String() string {
|
|||
return "account"
|
||||
case typeStorage:
|
||||
return "storage"
|
||||
case typeTrienode:
|
||||
return "trienode"
|
||||
default:
|
||||
return fmt.Sprintf("unknown element type: %d", e)
|
||||
}
|
||||
|
|
@ -69,11 +77,14 @@ func toHistoryType(typ elementType) historyType {
|
|||
if typ == typeAccount || typ == typeStorage {
|
||||
return typeStateHistory
|
||||
}
|
||||
if typ == typeTrienode {
|
||||
return typeTrienodeHistory
|
||||
}
|
||||
panic(fmt.Sprintf("unknown element type %v", typ))
|
||||
}
|
||||
|
||||
// stateIdent represents the identifier of a state element, which can be
|
||||
// an account or a storage slot.
|
||||
// an account, a storage slot or a trienode.
|
||||
type stateIdent struct {
|
||||
typ elementType
|
||||
|
||||
|
|
@ -91,6 +102,12 @@ type stateIdent struct {
|
|||
//
|
||||
// This field is null if the identifier refers to an account or a trie node.
|
||||
storageHash common.Hash
|
||||
|
||||
// The trie node path within the trie.
|
||||
//
|
||||
// This field is null if the identifier refers to an account or a storage slot.
|
||||
// String type is chosen to make stateIdent comparable.
|
||||
path string
|
||||
}
|
||||
|
||||
// String returns the string format state identifier.
|
||||
|
|
@ -98,7 +115,10 @@ func (ident stateIdent) String() string {
|
|||
if ident.typ == typeAccount {
|
||||
return ident.addressHash.Hex()
|
||||
}
|
||||
return ident.addressHash.Hex() + ident.storageHash.Hex()
|
||||
if ident.typ == typeStorage {
|
||||
return ident.addressHash.Hex() + ident.storageHash.Hex()
|
||||
}
|
||||
return ident.addressHash.Hex() + ident.path
|
||||
}
|
||||
|
||||
// newAccountIdent constructs a state identifier for an account.
|
||||
|
|
@ -120,8 +140,18 @@ func newStorageIdent(addressHash common.Hash, storageHash common.Hash) stateIden
|
|||
}
|
||||
}
|
||||
|
||||
// stateIdentQuery is the extension of stateIdent by adding the account address
|
||||
// and raw storage key.
|
||||
// newTrienodeIdent constructs a state identifier for a trie node.
|
||||
// The address denotes the address hash of the associated account;
|
||||
// the path denotes the path of the node within the trie;
|
||||
func newTrienodeIdent(addressHash common.Hash, path string) stateIdent {
|
||||
return stateIdent{
|
||||
typ: typeTrienode,
|
||||
addressHash: addressHash,
|
||||
path: path,
|
||||
}
|
||||
}
|
||||
|
||||
// stateIdentQuery is the extension of stateIdent by adding the raw storage key.
|
||||
type stateIdentQuery struct {
|
||||
stateIdent
|
||||
|
||||
|
|
@ -150,8 +180,19 @@ func newStorageIdentQuery(address common.Address, addressHash common.Hash, stora
|
|||
}
|
||||
}
|
||||
|
||||
// history defines the interface of historical data, implemented by stateHistory
|
||||
// and trienodeHistory (in the near future).
|
||||
// newTrienodeIdentQuery constructs a state identifier for a trie node.
|
||||
// the addressHash denotes the address hash of the associated account;
|
||||
// the path denotes the path of the node within the trie;
|
||||
//
|
||||
// nolint:unused
|
||||
func newTrienodeIdentQuery(addrHash common.Hash, path []byte) stateIdentQuery {
|
||||
return stateIdentQuery{
|
||||
stateIdent: newTrienodeIdent(addrHash, string(path)),
|
||||
}
|
||||
}
|
||||
|
||||
// history defines the interface of historical data, shared by stateHistory
|
||||
// and trienodeHistory.
|
||||
type history interface {
|
||||
// typ returns the historical data type held in the history.
|
||||
typ() historyType
|
||||
|
|
|
|||
|
|
@ -376,6 +376,8 @@ func readStateIndex(ident stateIdent, db ethdb.KeyValueReader) []byte {
|
|||
return rawdb.ReadAccountHistoryIndex(db, ident.addressHash)
|
||||
case typeStorage:
|
||||
return rawdb.ReadStorageHistoryIndex(db, ident.addressHash, ident.storageHash)
|
||||
case typeTrienode:
|
||||
return rawdb.ReadTrienodeHistoryIndex(db, ident.addressHash, []byte(ident.path))
|
||||
default:
|
||||
panic(fmt.Errorf("unknown type: %v", ident.typ))
|
||||
}
|
||||
|
|
@ -389,6 +391,8 @@ func writeStateIndex(ident stateIdent, db ethdb.KeyValueWriter, data []byte) {
|
|||
rawdb.WriteAccountHistoryIndex(db, ident.addressHash, data)
|
||||
case typeStorage:
|
||||
rawdb.WriteStorageHistoryIndex(db, ident.addressHash, ident.storageHash, data)
|
||||
case typeTrienode:
|
||||
rawdb.WriteTrienodeHistoryIndex(db, ident.addressHash, []byte(ident.path), data)
|
||||
default:
|
||||
panic(fmt.Errorf("unknown type: %v", ident.typ))
|
||||
}
|
||||
|
|
@ -402,6 +406,8 @@ func deleteStateIndex(ident stateIdent, db ethdb.KeyValueWriter) {
|
|||
rawdb.DeleteAccountHistoryIndex(db, ident.addressHash)
|
||||
case typeStorage:
|
||||
rawdb.DeleteStorageHistoryIndex(db, ident.addressHash, ident.storageHash)
|
||||
case typeTrienode:
|
||||
rawdb.DeleteTrienodeHistoryIndex(db, ident.addressHash, []byte(ident.path))
|
||||
default:
|
||||
panic(fmt.Errorf("unknown type: %v", ident.typ))
|
||||
}
|
||||
|
|
@ -415,6 +421,8 @@ func readStateIndexBlock(ident stateIdent, db ethdb.KeyValueReader, id uint32) [
|
|||
return rawdb.ReadAccountHistoryIndexBlock(db, ident.addressHash, id)
|
||||
case typeStorage:
|
||||
return rawdb.ReadStorageHistoryIndexBlock(db, ident.addressHash, ident.storageHash, id)
|
||||
case typeTrienode:
|
||||
return rawdb.ReadTrienodeHistoryIndexBlock(db, ident.addressHash, []byte(ident.path), id)
|
||||
default:
|
||||
panic(fmt.Errorf("unknown type: %v", ident.typ))
|
||||
}
|
||||
|
|
@ -428,6 +436,8 @@ func writeStateIndexBlock(ident stateIdent, db ethdb.KeyValueWriter, id uint32,
|
|||
rawdb.WriteAccountHistoryIndexBlock(db, ident.addressHash, id, data)
|
||||
case typeStorage:
|
||||
rawdb.WriteStorageHistoryIndexBlock(db, ident.addressHash, ident.storageHash, id, data)
|
||||
case typeTrienode:
|
||||
rawdb.WriteTrienodeHistoryIndexBlock(db, ident.addressHash, []byte(ident.path), id, data)
|
||||
default:
|
||||
panic(fmt.Errorf("unknown type: %v", ident.typ))
|
||||
}
|
||||
|
|
@ -441,6 +451,8 @@ func deleteStateIndexBlock(ident stateIdent, db ethdb.KeyValueWriter, id uint32)
|
|||
rawdb.DeleteAccountHistoryIndexBlock(db, ident.addressHash, id)
|
||||
case typeStorage:
|
||||
rawdb.DeleteStorageHistoryIndexBlock(db, ident.addressHash, ident.storageHash, id)
|
||||
case typeTrienode:
|
||||
rawdb.DeleteTrienodeHistoryIndexBlock(db, ident.addressHash, []byte(ident.path), id)
|
||||
default:
|
||||
panic(fmt.Errorf("unknown type: %v", ident.typ))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -36,8 +36,10 @@ const (
|
|||
// The batch size for reading state histories
|
||||
historyReadBatch = 1000
|
||||
|
||||
stateIndexV0 = uint8(0) // initial version of state index structure
|
||||
stateIndexVersion = stateIndexV0 // the current state index version
|
||||
stateHistoryIndexV0 = uint8(0) // initial version of state index structure
|
||||
stateHistoryIndexVersion = stateHistoryIndexV0 // the current state index version
|
||||
trienodeHistoryIndexV0 = uint8(0) // initial version of trienode index structure
|
||||
trienodeHistoryIndexVersion = trienodeHistoryIndexV0 // the current trienode index version
|
||||
)
|
||||
|
||||
// indexVersion returns the latest index version for the given history type.
|
||||
|
|
@ -45,7 +47,9 @@ const (
|
|||
func indexVersion(typ historyType) uint8 {
|
||||
switch typ {
|
||||
case typeStateHistory:
|
||||
return stateIndexVersion
|
||||
return stateHistoryIndexVersion
|
||||
case typeTrienodeHistory:
|
||||
return trienodeHistoryIndexVersion
|
||||
default:
|
||||
panic(fmt.Errorf("unknown history type: %d", typ))
|
||||
}
|
||||
|
|
@ -63,6 +67,8 @@ func loadIndexMetadata(db ethdb.KeyValueReader, typ historyType) *indexMetadata
|
|||
switch typ {
|
||||
case typeStateHistory:
|
||||
blob = rawdb.ReadStateHistoryIndexMetadata(db)
|
||||
case typeTrienodeHistory:
|
||||
blob = rawdb.ReadTrienodeHistoryIndexMetadata(db)
|
||||
default:
|
||||
panic(fmt.Errorf("unknown history type %d", typ))
|
||||
}
|
||||
|
|
@ -90,6 +96,8 @@ func storeIndexMetadata(db ethdb.KeyValueWriter, typ historyType, last uint64) {
|
|||
switch typ {
|
||||
case typeStateHistory:
|
||||
rawdb.WriteStateHistoryIndexMetadata(db, blob)
|
||||
case typeTrienodeHistory:
|
||||
rawdb.WriteTrienodeHistoryIndexMetadata(db, blob)
|
||||
default:
|
||||
panic(fmt.Errorf("unknown history type %d", typ))
|
||||
}
|
||||
|
|
@ -101,6 +109,8 @@ func deleteIndexMetadata(db ethdb.KeyValueWriter, typ historyType) {
|
|||
switch typ {
|
||||
case typeStateHistory:
|
||||
rawdb.DeleteStateHistoryIndexMetadata(db)
|
||||
case typeTrienodeHistory:
|
||||
rawdb.DeleteTrienodeHistoryIndexMetadata(db)
|
||||
default:
|
||||
panic(fmt.Errorf("unknown history type %d", typ))
|
||||
}
|
||||
|
|
@ -215,7 +225,11 @@ func (b *batchIndexer) finish(force bool) error {
|
|||
func indexSingle(historyID uint64, db ethdb.KeyValueStore, freezer ethdb.AncientReader, typ historyType) error {
|
||||
start := time.Now()
|
||||
defer func() {
|
||||
indexHistoryTimer.UpdateSince(start)
|
||||
if typ == typeStateHistory {
|
||||
stateIndexHistoryTimer.UpdateSince(start)
|
||||
} else if typ == typeTrienodeHistory {
|
||||
trienodeIndexHistoryTimer.UpdateSince(start)
|
||||
}
|
||||
}()
|
||||
|
||||
metadata := loadIndexMetadata(db, typ)
|
||||
|
|
@ -234,7 +248,7 @@ func indexSingle(historyID uint64, db ethdb.KeyValueStore, freezer ethdb.Ancient
|
|||
if typ == typeStateHistory {
|
||||
h, err = readStateHistory(freezer, historyID)
|
||||
} else {
|
||||
// h, err = readTrienodeHistory(freezer, historyID)
|
||||
h, err = readTrienodeHistory(freezer, historyID)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -253,7 +267,11 @@ func indexSingle(historyID uint64, db ethdb.KeyValueStore, freezer ethdb.Ancient
|
|||
func unindexSingle(historyID uint64, db ethdb.KeyValueStore, freezer ethdb.AncientReader, typ historyType) error {
|
||||
start := time.Now()
|
||||
defer func() {
|
||||
unindexHistoryTimer.UpdateSince(start)
|
||||
if typ == typeStateHistory {
|
||||
stateUnindexHistoryTimer.UpdateSince(start)
|
||||
} else if typ == typeTrienodeHistory {
|
||||
trienodeUnindexHistoryTimer.UpdateSince(start)
|
||||
}
|
||||
}()
|
||||
|
||||
metadata := loadIndexMetadata(db, typ)
|
||||
|
|
@ -272,7 +290,7 @@ func unindexSingle(historyID uint64, db ethdb.KeyValueStore, freezer ethdb.Ancie
|
|||
if typ == typeStateHistory {
|
||||
h, err = readStateHistory(freezer, historyID)
|
||||
} else {
|
||||
// h, err = readTrienodeHistory(freezer, historyID)
|
||||
h, err = readTrienodeHistory(freezer, historyID)
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -546,13 +564,13 @@ func (i *indexIniter) index(done chan struct{}, interrupt *atomic.Int32, lastID
|
|||
return
|
||||
}
|
||||
} else {
|
||||
// histories, err = readTrienodeHistories(i.freezer, current, count)
|
||||
// if err != nil {
|
||||
// // The history read might fall if the history is truncated from
|
||||
// // head due to revert operation.
|
||||
// i.log.Error("Failed to read history for indexing", "current", current, "count", count, "err", err)
|
||||
// return
|
||||
// }
|
||||
histories, err = readTrienodeHistories(i.freezer, current, count)
|
||||
if err != nil {
|
||||
// The history read might fall if the history is truncated from
|
||||
// head due to revert operation.
|
||||
i.log.Error("Failed to read history for indexing", "current", current, "count", count, "err", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
for _, h := range histories {
|
||||
if err := batch.process(h, current); err != nil {
|
||||
|
|
@ -570,7 +588,7 @@ func (i *indexIniter) index(done chan struct{}, interrupt *atomic.Int32, lastID
|
|||
done = current - beginID
|
||||
)
|
||||
eta := common.CalculateETA(done, left, time.Since(start))
|
||||
i.log.Info("Indexing state history", "processed", done, "left", left, "elapsed", common.PrettyDuration(time.Since(start)), "eta", common.PrettyDuration(eta))
|
||||
i.log.Info("Indexing history", "processed", done, "left", left, "elapsed", common.PrettyDuration(time.Since(start)), "eta", common.PrettyDuration(eta))
|
||||
}
|
||||
}
|
||||
i.indexed.Store(current - 1) // update indexing progress
|
||||
|
|
@ -657,6 +675,8 @@ func checkVersion(disk ethdb.KeyValueStore, typ historyType) {
|
|||
var blob []byte
|
||||
if typ == typeStateHistory {
|
||||
blob = rawdb.ReadStateHistoryIndexMetadata(disk)
|
||||
} else if typ == typeTrienodeHistory {
|
||||
blob = rawdb.ReadTrienodeHistoryIndexMetadata(disk)
|
||||
} else {
|
||||
panic(fmt.Errorf("unknown history type: %v", typ))
|
||||
}
|
||||
|
|
@ -666,24 +686,32 @@ func checkVersion(disk ethdb.KeyValueStore, typ historyType) {
|
|||
return
|
||||
}
|
||||
// Short circuit if the metadata is found and the version is matched
|
||||
ver := stateHistoryIndexVersion
|
||||
if typ == typeTrienodeHistory {
|
||||
ver = trienodeHistoryIndexVersion
|
||||
}
|
||||
var m indexMetadata
|
||||
err := rlp.DecodeBytes(blob, &m)
|
||||
if err == nil && m.Version == stateIndexVersion {
|
||||
if err == nil && m.Version == ver {
|
||||
return
|
||||
}
|
||||
// Version is not matched, prune the existing data and re-index from scratch
|
||||
batch := disk.NewBatch()
|
||||
if typ == typeStateHistory {
|
||||
rawdb.DeleteStateHistoryIndexMetadata(batch)
|
||||
rawdb.DeleteStateHistoryIndexes(batch)
|
||||
} else {
|
||||
rawdb.DeleteTrienodeHistoryIndexMetadata(batch)
|
||||
rawdb.DeleteTrienodeHistoryIndexes(batch)
|
||||
}
|
||||
if err := batch.Write(); err != nil {
|
||||
log.Crit("Failed to purge history index", "type", typ, "err", err)
|
||||
}
|
||||
version := "unknown"
|
||||
if err == nil {
|
||||
version = fmt.Sprintf("%d", m.Version)
|
||||
}
|
||||
|
||||
batch := disk.NewBatch()
|
||||
rawdb.DeleteStateHistoryIndexMetadata(batch)
|
||||
rawdb.DeleteStateHistoryIndex(batch)
|
||||
if err := batch.Write(); err != nil {
|
||||
log.Crit("Failed to purge state history index", "err", err)
|
||||
}
|
||||
log.Info("Cleaned up obsolete state history index", "version", version, "want", stateIndexVersion)
|
||||
log.Info("Cleaned up obsolete history index", "type", typ, "version", version, "want", version)
|
||||
}
|
||||
|
||||
// newHistoryIndexer constructs the history indexer and launches the background
|
||||
|
|
|
|||
|
|
@ -605,9 +605,9 @@ func writeStateHistory(writer ethdb.AncientWriter, dl *diffLayer) error {
|
|||
if err := rawdb.WriteStateHistory(writer, dl.stateID(), history.meta.encode(), accountIndex, storageIndex, accountData, storageData); err != nil {
|
||||
return err
|
||||
}
|
||||
historyDataBytesMeter.Mark(int64(dataSize))
|
||||
historyIndexBytesMeter.Mark(int64(indexSize))
|
||||
historyBuildTimeMeter.UpdateSince(start)
|
||||
stateHistoryDataBytesMeter.Mark(int64(dataSize))
|
||||
stateHistoryIndexBytesMeter.Mark(int64(indexSize))
|
||||
stateHistoryBuildTimeMeter.UpdateSince(start)
|
||||
log.Debug("Stored state history", "id", dl.stateID(), "block", dl.block, "data", dataSize, "index", indexSize, "elapsed", common.PrettyDuration(time.Since(start)))
|
||||
|
||||
return nil
|
||||
|
|
|
|||
|
|
@ -98,13 +98,13 @@ func testEncodeDecodeStateHistory(t *testing.T, rawStorageKey bool) {
|
|||
if !compareSet(dec.accounts, obj.accounts) {
|
||||
t.Fatal("account data is mismatched")
|
||||
}
|
||||
if !compareStorages(dec.storages, obj.storages) {
|
||||
if !compareMapSet(dec.storages, obj.storages) {
|
||||
t.Fatal("storage data is mismatched")
|
||||
}
|
||||
if !compareList(dec.accountList, obj.accountList) {
|
||||
t.Fatal("account list is mismatched")
|
||||
}
|
||||
if !compareStorageList(dec.storageList, obj.storageList) {
|
||||
if !compareMapList(dec.storageList, obj.storageList) {
|
||||
t.Fatal("storage list is mismatched")
|
||||
}
|
||||
}
|
||||
|
|
@ -292,32 +292,32 @@ func compareList[k comparable](a, b []k) bool {
|
|||
return true
|
||||
}
|
||||
|
||||
func compareStorages(a, b map[common.Address]map[common.Hash][]byte) bool {
|
||||
func compareMapSet[K1 comparable, K2 comparable](a, b map[K1]map[K2][]byte) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for h, subA := range a {
|
||||
subB, ok := b[h]
|
||||
for key, subsetA := range a {
|
||||
subsetB, ok := b[key]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
if !compareSet(subA, subB) {
|
||||
if !compareSet(subsetA, subsetB) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func compareStorageList(a, b map[common.Address][]common.Hash) bool {
|
||||
func compareMapList[K comparable, V comparable](a, b map[K][]V) bool {
|
||||
if len(a) != len(b) {
|
||||
return false
|
||||
}
|
||||
for h, la := range a {
|
||||
lb, ok := b[h]
|
||||
for key, listA := range a {
|
||||
listB, ok := b[key]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
if !compareList(la, lb) {
|
||||
if !compareList(listA, listB) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
|
|
|||
730
triedb/pathdb/history_trienode.go
Normal file
730
triedb/pathdb/history_trienode.go
Normal file
|
|
@ -0,0 +1,730 @@
|
|||
// Copyright 2025 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package pathdb
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"iter"
|
||||
"maps"
|
||||
"math"
|
||||
"slices"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/rawdb"
|
||||
"github.com/ethereum/go-ethereum/ethdb"
|
||||
"github.com/ethereum/go-ethereum/log"
|
||||
)
|
||||
|
||||
// Each trie node history entry consists of three parts (stored in three freezer
|
||||
// tables according):
|
||||
//
|
||||
// # Header
|
||||
// The header records metadata, including:
|
||||
//
|
||||
// - the history version (1 byte)
|
||||
// - the parent state root (32 bytes)
|
||||
// - the current state root (32 bytes)
|
||||
// - block number (8 bytes)
|
||||
//
|
||||
// - a lexicographically sorted list of trie IDs
|
||||
// - the corresponding offsets into the key and value sections for each trie data chunk
|
||||
//
|
||||
// Although some fields (e.g., parent state root, block number) are duplicated
|
||||
// between the state history and the trienode history, these two histories
|
||||
// operate independently. To ensure each remains self-contained and self-descriptive,
|
||||
// we have chosen to maintain these duplicate fields.
|
||||
//
|
||||
// # Key section
|
||||
// The key section stores trie node keys (paths) in a compressed format.
|
||||
// It also contains relative offsets into the value section for resolving
|
||||
// the corresponding trie node data. Note that these offsets are relative
|
||||
// to the data chunk for the trie; the chunk offset must be added to obtain
|
||||
// the absolute position.
|
||||
//
|
||||
// # Value section
|
||||
// The value section is a concatenated byte stream of all trie node data.
|
||||
// Each trie node can be retrieved using the offset and length specified
|
||||
// by its index entry.
|
||||
//
|
||||
// The header and key sections are sufficient for locating a trie node,
|
||||
// while a partial read of the value section is enough to retrieve its data.
|
||||
|
||||
// Header section:
|
||||
//
|
||||
// +----------+------------------+---------------------+---------------------+-------+------------------+---------------------+---------------------|
|
||||
// | metadata | TrieID(32 bytes) | key offset(4 bytes) | val offset(4 bytes) | ... | TrieID(32 bytes) | key offset(4 bytes) | val offset(4 bytes) |
|
||||
// +----------+------------------+---------------------+---------------------+-------+------------------+---------------------+---------------------|
|
||||
//
|
||||
//
|
||||
// Key section:
|
||||
//
|
||||
// + restart point + restart point (depends on restart interval)
|
||||
// / /
|
||||
// +---------------+---------------+---------------+---------------+---------+
|
||||
// | node entry 1 | node entry 2 | ... | node entry n | trailer |
|
||||
// +---------------+---------------+---------------+---------------+---------+
|
||||
// \ /
|
||||
// +---- restart block ------+
|
||||
//
|
||||
// node entry:
|
||||
//
|
||||
// +---- key len ----+
|
||||
// / \
|
||||
// +-------+---------+-----------+---------+-----------------------+-----------------+
|
||||
// | shared (varint) | not shared (varint) | value length (varlen) | key (varlen) |
|
||||
// +-----------------+---------------------+-----------------------+-----------------+
|
||||
//
|
||||
// trailer:
|
||||
//
|
||||
// +---- 4-bytes ----+ +---- 4-bytes ----+
|
||||
// / \ / \
|
||||
// +----------------------+------------------------+-----+--------------------------+
|
||||
// | restart_1 key offset | restart_1 value offset | ... | restart number (4-bytes) |
|
||||
// +----------------------+------------------------+-----+--------------------------+
|
||||
//
|
||||
// Note: Both the key offset and the value offset are relative to the start of
|
||||
// the trie data chunk. To obtain the absolute offset, add the offset of the
|
||||
// trie data chunk itself.
|
||||
//
|
||||
// Value section:
|
||||
//
|
||||
// +--------------+--------------+-------+---------------+
|
||||
// | node data 1 | node data 2 | ... | node data n |
|
||||
// +--------------+--------------+-------+---------------+
|
||||
//
|
||||
// NOTE: All fixed-length integer are big-endian.
|
||||
|
||||
const (
|
||||
trienodeHistoryV0 = uint8(0) // initial version of node history structure
|
||||
trienodeHistoryVersion = trienodeHistoryV0 // the default node history version
|
||||
trienodeMetadataSize = 1 + 2*common.HashLength + 8 // the size of metadata in the history
|
||||
trienodeTrieHeaderSize = 8 + common.HashLength // the size of a single trie header in history
|
||||
trienodeDataBlockRestartLen = 16 // The restart interval length of trie node block
|
||||
)
|
||||
|
||||
// trienodeMetadata describes the meta data of trienode history.
|
||||
type trienodeMetadata struct {
|
||||
version uint8 // version tag of history object
|
||||
parent common.Hash // prev-state root before the state transition
|
||||
root common.Hash // post-state root after the state transition
|
||||
block uint64 // associated block number
|
||||
}
|
||||
|
||||
// trienodeHistory represents a set of trie node changes resulting from a state
|
||||
// transition across the main account trie and all associated storage tries.
|
||||
type trienodeHistory struct {
|
||||
meta *trienodeMetadata // Metadata of the history
|
||||
owners []common.Hash // List of trie identifier sorted lexicographically
|
||||
nodeList map[common.Hash][]string // Set of node paths sorted lexicographically
|
||||
nodes map[common.Hash]map[string][]byte // Set of original value of trie nodes before state transition
|
||||
}
|
||||
|
||||
// newTrienodeHistory constructs a trienode history with the provided trie nodes.
|
||||
func newTrienodeHistory(root common.Hash, parent common.Hash, block uint64, nodes map[common.Hash]map[string][]byte) *trienodeHistory {
|
||||
nodeList := make(map[common.Hash][]string)
|
||||
for owner, subset := range nodes {
|
||||
keys := sort.StringSlice(slices.Collect(maps.Keys(subset)))
|
||||
keys.Sort()
|
||||
nodeList[owner] = keys
|
||||
}
|
||||
return &trienodeHistory{
|
||||
meta: &trienodeMetadata{
|
||||
version: trienodeHistoryVersion,
|
||||
parent: parent,
|
||||
root: root,
|
||||
block: block,
|
||||
},
|
||||
owners: slices.SortedFunc(maps.Keys(nodes), common.Hash.Cmp),
|
||||
nodeList: nodeList,
|
||||
nodes: nodes,
|
||||
}
|
||||
}
|
||||
|
||||
// sharedLen returns the length of the common prefix shared by a and b.
|
||||
func sharedLen(a, b []byte) int {
|
||||
n := min(len(a), len(b))
|
||||
for i := 0; i < n; i++ {
|
||||
if a[i] != b[i] {
|
||||
return i
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// typ implements the history interface, returning the historical data type held.
|
||||
func (h *trienodeHistory) typ() historyType {
|
||||
return typeTrienodeHistory
|
||||
}
|
||||
|
||||
// forEach implements the history interface, returning an iterator to traverse the
|
||||
// state entries in the history.
|
||||
func (h *trienodeHistory) forEach() iter.Seq[stateIdent] {
|
||||
return func(yield func(stateIdent) bool) {
|
||||
for _, owner := range h.owners {
|
||||
for _, path := range h.nodeList[owner] {
|
||||
if !yield(newTrienodeIdent(owner, path)) {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// encode serializes the contained trie nodes into bytes.
|
||||
func (h *trienodeHistory) encode() ([]byte, []byte, []byte, error) {
|
||||
var (
|
||||
buf = make([]byte, 64)
|
||||
headerSection bytes.Buffer
|
||||
keySection bytes.Buffer
|
||||
valueSection bytes.Buffer
|
||||
)
|
||||
binary.Write(&headerSection, binary.BigEndian, h.meta.version) // 1 byte
|
||||
headerSection.Write(h.meta.parent.Bytes()) // 32 bytes
|
||||
headerSection.Write(h.meta.root.Bytes()) // 32 bytes
|
||||
binary.Write(&headerSection, binary.BigEndian, h.meta.block) // 8 byte
|
||||
|
||||
for _, owner := range h.owners {
|
||||
// Fill the header section with offsets at key and value section
|
||||
headerSection.Write(owner.Bytes()) // 32 bytes
|
||||
binary.Write(&headerSection, binary.BigEndian, uint32(keySection.Len())) // 4 bytes
|
||||
|
||||
// The offset to the value section is theoretically unnecessary, since the
|
||||
// individual value offset is already tracked in the key section. However,
|
||||
// we still keep it here for two reasons:
|
||||
// - It's cheap to store (only 4 bytes for each trie).
|
||||
// - It can be useful for decoding the trie data when key is not required (e.g., in hash mode).
|
||||
binary.Write(&headerSection, binary.BigEndian, uint32(valueSection.Len())) // 4 bytes
|
||||
|
||||
// Fill the key section with node index
|
||||
var (
|
||||
prevKey []byte
|
||||
restarts []uint32
|
||||
prefixLen int
|
||||
|
||||
internalKeyOffset uint32 // key offset for the trie internally
|
||||
internalValOffset uint32 // value offset for the trie internally
|
||||
)
|
||||
for i, path := range h.nodeList[owner] {
|
||||
key := []byte(path)
|
||||
if i%trienodeDataBlockRestartLen == 0 {
|
||||
restarts = append(restarts, internalKeyOffset)
|
||||
restarts = append(restarts, internalValOffset)
|
||||
prefixLen = 0
|
||||
} else {
|
||||
prefixLen = sharedLen(prevKey, key)
|
||||
}
|
||||
value := h.nodes[owner][path]
|
||||
|
||||
// key section
|
||||
n := binary.PutUvarint(buf[0:], uint64(prefixLen)) // key length shared (varint)
|
||||
n += binary.PutUvarint(buf[n:], uint64(len(key)-prefixLen)) // key length not shared (varint)
|
||||
n += binary.PutUvarint(buf[n:], uint64(len(value))) // value length (varint)
|
||||
|
||||
if _, err := keySection.Write(buf[:n]); err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
// unshared key
|
||||
if _, err := keySection.Write(key[prefixLen:]); err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
n += len(key) - prefixLen
|
||||
prevKey = key
|
||||
|
||||
// value section
|
||||
if _, err := valueSection.Write(value); err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
internalKeyOffset += uint32(n)
|
||||
internalValOffset += uint32(len(value))
|
||||
}
|
||||
|
||||
// Encode trailer, the number of restart sections is len(restarts))/2,
|
||||
// as we track the offsets of both key and value sections.
|
||||
var trailer []byte
|
||||
for _, number := range append(restarts, uint32(len(restarts))/2) {
|
||||
binary.BigEndian.PutUint32(buf[:4], number)
|
||||
trailer = append(trailer, buf[:4]...)
|
||||
}
|
||||
if _, err := keySection.Write(trailer); err != nil {
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
}
|
||||
return headerSection.Bytes(), keySection.Bytes(), valueSection.Bytes(), nil
|
||||
}
|
||||
|
||||
// decodeHeader resolves the metadata from the header section. An error
|
||||
// should be returned if the header section is corrupted.
|
||||
func decodeHeader(data []byte) (*trienodeMetadata, []common.Hash, []uint32, []uint32, error) {
|
||||
if len(data) < trienodeMetadataSize {
|
||||
return nil, nil, nil, nil, fmt.Errorf("trienode history is too small, index size: %d", len(data))
|
||||
}
|
||||
version := data[0]
|
||||
if version != trienodeHistoryVersion {
|
||||
return nil, nil, nil, nil, fmt.Errorf("unregonized trienode history version: %d", version)
|
||||
}
|
||||
parent := common.BytesToHash(data[1 : common.HashLength+1]) // 32 bytes
|
||||
root := common.BytesToHash(data[common.HashLength+1 : common.HashLength*2+1]) // 32 bytes
|
||||
block := binary.BigEndian.Uint64(data[common.HashLength*2+1 : trienodeMetadataSize]) // 8 bytes
|
||||
|
||||
size := len(data) - trienodeMetadataSize
|
||||
if size%trienodeTrieHeaderSize != 0 {
|
||||
return nil, nil, nil, nil, fmt.Errorf("truncated trienode history data, size %d", len(data))
|
||||
}
|
||||
count := size / trienodeTrieHeaderSize
|
||||
|
||||
var (
|
||||
owners = make([]common.Hash, 0, count)
|
||||
keyOffsets = make([]uint32, 0, count)
|
||||
valOffsets = make([]uint32, 0, count)
|
||||
)
|
||||
for i := 0; i < count; i++ {
|
||||
n := trienodeMetadataSize + trienodeTrieHeaderSize*i
|
||||
owner := common.BytesToHash(data[n : n+common.HashLength])
|
||||
if i != 0 && bytes.Compare(owner.Bytes(), owners[i-1].Bytes()) <= 0 {
|
||||
return nil, nil, nil, nil, fmt.Errorf("trienode owners are out of order, prev: %v, cur: %v", owners[i-1], owner)
|
||||
}
|
||||
owners = append(owners, owner)
|
||||
|
||||
// Decode the offset to the key section
|
||||
keyOffset := binary.BigEndian.Uint32(data[n+common.HashLength : n+common.HashLength+4])
|
||||
if i != 0 && keyOffset <= keyOffsets[i-1] {
|
||||
return nil, nil, nil, nil, fmt.Errorf("key offset is out of order, prev: %v, cur: %v", keyOffsets[i-1], keyOffset)
|
||||
}
|
||||
keyOffsets = append(keyOffsets, keyOffset)
|
||||
|
||||
// Decode the offset into the value section. Note that identical value offsets
|
||||
// are valid if the node values in the last trie chunk are all zero (e.g., after
|
||||
// a trie deletion).
|
||||
valOffset := binary.BigEndian.Uint32(data[n+common.HashLength+4 : n+common.HashLength+8])
|
||||
if i != 0 && valOffset < valOffsets[i-1] {
|
||||
return nil, nil, nil, nil, fmt.Errorf("value offset is out of order, prev: %v, cur: %v", valOffsets[i-1], valOffset)
|
||||
}
|
||||
valOffsets = append(valOffsets, valOffset)
|
||||
}
|
||||
return &trienodeMetadata{
|
||||
version: version,
|
||||
parent: parent,
|
||||
root: root,
|
||||
block: block,
|
||||
}, owners, keyOffsets, valOffsets, nil
|
||||
}
|
||||
|
||||
func decodeSingle(keySection []byte, onValue func([]byte, int, int) error) ([]string, error) {
|
||||
var (
|
||||
prevKey []byte
|
||||
items int
|
||||
keyOffsets []uint32
|
||||
valOffsets []uint32
|
||||
|
||||
keyOff int // the key offset within the single trie data
|
||||
valOff int // the value offset within the single trie data
|
||||
|
||||
keys []string
|
||||
)
|
||||
// Decode the number of restart section
|
||||
if len(keySection) < 4 {
|
||||
return nil, fmt.Errorf("key section too short, size: %d", len(keySection))
|
||||
}
|
||||
nRestarts := binary.BigEndian.Uint32(keySection[len(keySection)-4:])
|
||||
|
||||
if len(keySection) < int(8*nRestarts)+4 {
|
||||
return nil, fmt.Errorf("key section too short, restarts: %d, size: %d", nRestarts, len(keySection))
|
||||
}
|
||||
for i := 0; i < int(nRestarts); i++ {
|
||||
o := len(keySection) - 4 - (int(nRestarts)-i)*8
|
||||
keyOffset := binary.BigEndian.Uint32(keySection[o : o+4])
|
||||
if i != 0 && keyOffset <= keyOffsets[i-1] {
|
||||
return nil, fmt.Errorf("key offset is out of order, prev: %v, cur: %v", keyOffsets[i-1], keyOffset)
|
||||
}
|
||||
keyOffsets = append(keyOffsets, keyOffset)
|
||||
|
||||
// Same value offset is allowed just in case all the trie nodes in the last
|
||||
// section have zero-size value.
|
||||
valOffset := binary.BigEndian.Uint32(keySection[o+4 : o+8])
|
||||
if i != 0 && valOffset < valOffsets[i-1] {
|
||||
return nil, fmt.Errorf("value offset is out of order, prev: %v, cur: %v", valOffsets[i-1], valOffset)
|
||||
}
|
||||
valOffsets = append(valOffsets, valOffset)
|
||||
}
|
||||
keyLimit := len(keySection) - 4 - int(nRestarts)*8
|
||||
|
||||
// Decode data
|
||||
for keyOff < keyLimit {
|
||||
// Validate the key and value offsets within the single trie data chunk
|
||||
if items%trienodeDataBlockRestartLen == 0 {
|
||||
if keyOff != int(keyOffsets[items/trienodeDataBlockRestartLen]) {
|
||||
return nil, fmt.Errorf("key offset is not matched, recorded: %d, want: %d", keyOffsets[items/trienodeDataBlockRestartLen], keyOff)
|
||||
}
|
||||
if valOff != int(valOffsets[items/trienodeDataBlockRestartLen]) {
|
||||
return nil, fmt.Errorf("value offset is not matched, recorded: %d, want: %d", valOffsets[items/trienodeDataBlockRestartLen], valOff)
|
||||
}
|
||||
}
|
||||
// Resolve the entry from key section
|
||||
nShared, nn := binary.Uvarint(keySection[keyOff:]) // key length shared (varint)
|
||||
keyOff += nn
|
||||
nUnshared, nn := binary.Uvarint(keySection[keyOff:]) // key length not shared (varint)
|
||||
keyOff += nn
|
||||
nValue, nn := binary.Uvarint(keySection[keyOff:]) // value length (varint)
|
||||
keyOff += nn
|
||||
|
||||
// Resolve unshared key
|
||||
if keyOff+int(nUnshared) > len(keySection) {
|
||||
return nil, fmt.Errorf("key length too long, unshared key length: %d, off: %d, section size: %d", nUnshared, keyOff, len(keySection))
|
||||
}
|
||||
unsharedKey := keySection[keyOff : keyOff+int(nUnshared)]
|
||||
keyOff += int(nUnshared)
|
||||
|
||||
// Assemble the full key
|
||||
var key []byte
|
||||
if items%trienodeDataBlockRestartLen == 0 {
|
||||
if nShared != 0 {
|
||||
return nil, fmt.Errorf("unexpected non-zero shared key prefix: %d", nShared)
|
||||
}
|
||||
key = unsharedKey
|
||||
} else {
|
||||
if int(nShared) > len(prevKey) {
|
||||
return nil, fmt.Errorf("unexpected shared key prefix: %d, prefix key length: %d", nShared, len(prevKey))
|
||||
}
|
||||
key = append([]byte{}, prevKey[:nShared]...)
|
||||
key = append(key, unsharedKey...)
|
||||
}
|
||||
if items != 0 && bytes.Compare(prevKey, key) >= 0 {
|
||||
return nil, fmt.Errorf("trienode paths are out of order, prev: %v, cur: %v", prevKey, key)
|
||||
}
|
||||
prevKey = key
|
||||
|
||||
// Resolve value
|
||||
if onValue != nil {
|
||||
if err := onValue(key, valOff, valOff+int(nValue)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
valOff += int(nValue)
|
||||
|
||||
items++
|
||||
keys = append(keys, string(key))
|
||||
}
|
||||
if keyOff != keyLimit {
|
||||
return nil, fmt.Errorf("excessive key data after decoding, offset: %d, size: %d", keyOff, keyLimit)
|
||||
}
|
||||
return keys, nil
|
||||
}
|
||||
|
||||
func decodeSingleWithValue(keySection []byte, valueSection []byte) ([]string, map[string][]byte, error) {
|
||||
var (
|
||||
offset int
|
||||
nodes = make(map[string][]byte)
|
||||
)
|
||||
paths, err := decodeSingle(keySection, func(key []byte, start int, limit int) error {
|
||||
if start != offset {
|
||||
return fmt.Errorf("gapped value section offset: %d, want: %d", start, offset)
|
||||
}
|
||||
// start == limit is allowed for zero-value trie node (e.g., non-existent node)
|
||||
if start > limit {
|
||||
return fmt.Errorf("invalid value offsets, start: %d, limit: %d", start, limit)
|
||||
}
|
||||
if start > len(valueSection) || limit > len(valueSection) {
|
||||
return fmt.Errorf("value section out of range: start: %d, limit: %d, size: %d", start, limit, len(valueSection))
|
||||
}
|
||||
nodes[string(key)] = valueSection[start:limit]
|
||||
|
||||
offset = limit
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if offset != len(valueSection) {
|
||||
return nil, nil, fmt.Errorf("excessive value data after decoding, offset: %d, size: %d", offset, len(valueSection))
|
||||
}
|
||||
return paths, nodes, nil
|
||||
}
|
||||
|
||||
// decode deserializes the contained trie nodes from the provided bytes.
|
||||
func (h *trienodeHistory) decode(header []byte, keySection []byte, valueSection []byte) error {
|
||||
metadata, owners, keyOffsets, valueOffsets, err := decodeHeader(header)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
h.meta = metadata
|
||||
h.owners = owners
|
||||
h.nodeList = make(map[common.Hash][]string)
|
||||
h.nodes = make(map[common.Hash]map[string][]byte)
|
||||
|
||||
for i := 0; i < len(owners); i++ {
|
||||
// Resolve the boundary of key section
|
||||
keyStart := keyOffsets[i]
|
||||
keyLimit := len(keySection)
|
||||
if i != len(owners)-1 {
|
||||
keyLimit = int(keyOffsets[i+1])
|
||||
}
|
||||
if int(keyStart) > len(keySection) || keyLimit > len(keySection) {
|
||||
return fmt.Errorf("invalid key offsets: keyStart: %d, keyLimit: %d, size: %d", keyStart, keyLimit, len(keySection))
|
||||
}
|
||||
|
||||
// Resolve the boundary of value section
|
||||
valStart := valueOffsets[i]
|
||||
valLimit := len(valueSection)
|
||||
if i != len(owners)-1 {
|
||||
valLimit = int(valueOffsets[i+1])
|
||||
}
|
||||
if int(valStart) > len(valueSection) || valLimit > len(valueSection) {
|
||||
return fmt.Errorf("invalid value offsets: valueStart: %d, valueLimit: %d, size: %d", valStart, valLimit, len(valueSection))
|
||||
}
|
||||
|
||||
// Decode the key and values for this specific trie
|
||||
paths, nodes, err := decodeSingleWithValue(keySection[keyStart:keyLimit], valueSection[valStart:valLimit])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
h.nodeList[owners[i]] = paths
|
||||
h.nodes[owners[i]] = nodes
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type iRange struct {
|
||||
start uint32
|
||||
limit uint32
|
||||
}
|
||||
|
||||
// singleTrienodeHistoryReader provides read access to a single trie within the
|
||||
// trienode history. It stores an offset to the trie's position in the history,
|
||||
// along with a set of per-node offsets that can be resolved on demand.
|
||||
type singleTrienodeHistoryReader struct {
|
||||
id uint64
|
||||
reader ethdb.AncientReader
|
||||
valueRange iRange // value range within the total value section
|
||||
valueInternalOffsets map[string]iRange // value offset within the single trie data
|
||||
}
|
||||
|
||||
func newSingleTrienodeHistoryReader(id uint64, reader ethdb.AncientReader, keyRange iRange, valueRange iRange) (*singleTrienodeHistoryReader, error) {
|
||||
// TODO(rjl493456442) partial freezer read should be supported
|
||||
keyData, err := rawdb.ReadTrienodeHistoryKeySection(reader, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
keyStart := int(keyRange.start)
|
||||
keyLimit := int(keyRange.limit)
|
||||
if keyLimit == math.MaxUint32 {
|
||||
keyLimit = len(keyData)
|
||||
}
|
||||
if len(keyData) < keyStart || len(keyData) < keyLimit {
|
||||
return nil, fmt.Errorf("key section too short, start: %d, limit: %d, size: %d", keyStart, keyLimit, len(keyData))
|
||||
}
|
||||
|
||||
valueOffsets := make(map[string]iRange)
|
||||
_, err = decodeSingle(keyData[keyStart:keyLimit], func(key []byte, start int, limit int) error {
|
||||
valueOffsets[string(key)] = iRange{
|
||||
start: uint32(start),
|
||||
limit: uint32(limit),
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &singleTrienodeHistoryReader{
|
||||
id: id,
|
||||
reader: reader,
|
||||
valueRange: valueRange,
|
||||
valueInternalOffsets: valueOffsets,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// read retrieves the trie node data with the provided node path.
|
||||
func (sr *singleTrienodeHistoryReader) read(path string) ([]byte, error) {
|
||||
offset, exists := sr.valueInternalOffsets[path]
|
||||
if !exists {
|
||||
return nil, fmt.Errorf("trienode %v not found", []byte(path))
|
||||
}
|
||||
// TODO(rjl493456442) partial freezer read should be supported
|
||||
valueData, err := rawdb.ReadTrienodeHistoryValueSection(sr.reader, sr.id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(valueData) < int(sr.valueRange.start) {
|
||||
return nil, fmt.Errorf("value section too short, start: %d, size: %d", sr.valueRange.start, len(valueData))
|
||||
}
|
||||
entryStart := sr.valueRange.start + offset.start
|
||||
entryLimit := sr.valueRange.start + offset.limit
|
||||
if len(valueData) < int(entryStart) || len(valueData) < int(entryLimit) {
|
||||
return nil, fmt.Errorf("value section too short, start: %d, limit: %d, size: %d", entryStart, entryLimit, len(valueData))
|
||||
}
|
||||
return valueData[int(entryStart):int(entryLimit)], nil
|
||||
}
|
||||
|
||||
// trienodeHistoryReader provides read access to node data in the trie node history.
|
||||
// It resolves data from the underlying ancient store only when needed, minimizing
|
||||
// I/O overhead.
|
||||
type trienodeHistoryReader struct {
|
||||
id uint64 // ID of the associated trienode history
|
||||
reader ethdb.AncientReader // Database reader of ancient store
|
||||
keyRanges map[common.Hash]iRange // Key ranges identifying trie chunks
|
||||
valRanges map[common.Hash]iRange // Value ranges identifying trie chunks
|
||||
iReaders map[common.Hash]*singleTrienodeHistoryReader // readers for each individual trie chunk
|
||||
}
|
||||
|
||||
// newTrienodeHistoryReader constructs the reader for specific trienode history.
|
||||
func newTrienodeHistoryReader(id uint64, reader ethdb.AncientReader) (*trienodeHistoryReader, error) {
|
||||
r := &trienodeHistoryReader{
|
||||
id: id,
|
||||
reader: reader,
|
||||
keyRanges: make(map[common.Hash]iRange),
|
||||
valRanges: make(map[common.Hash]iRange),
|
||||
iReaders: make(map[common.Hash]*singleTrienodeHistoryReader),
|
||||
}
|
||||
if err := r.decodeHeader(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// decodeHeader decodes the header section of trienode history.
|
||||
func (r *trienodeHistoryReader) decodeHeader() error {
|
||||
header, err := rawdb.ReadTrienodeHistoryHeader(r.reader, r.id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, owners, keyOffsets, valOffsets, err := decodeHeader(header)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for i, owner := range owners {
|
||||
// Decode the key range for this trie chunk
|
||||
var keyLimit uint32
|
||||
if i == len(owners)-1 {
|
||||
keyLimit = math.MaxUint32
|
||||
} else {
|
||||
keyLimit = keyOffsets[i+1]
|
||||
}
|
||||
r.keyRanges[owner] = iRange{
|
||||
start: keyOffsets[i],
|
||||
limit: keyLimit,
|
||||
}
|
||||
|
||||
// Decode the value range for this trie chunk
|
||||
var valLimit uint32
|
||||
if i == len(owners)-1 {
|
||||
valLimit = math.MaxUint32
|
||||
} else {
|
||||
valLimit = valOffsets[i+1]
|
||||
}
|
||||
r.valRanges[owner] = iRange{
|
||||
start: valOffsets[i],
|
||||
limit: valLimit,
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// read retrieves the trie node data with the provided TrieID and node path.
|
||||
func (r *trienodeHistoryReader) read(owner common.Hash, path string) ([]byte, error) {
|
||||
ir, ok := r.iReaders[owner]
|
||||
if !ok {
|
||||
keyRange, exists := r.keyRanges[owner]
|
||||
if !exists {
|
||||
return nil, fmt.Errorf("trie %x is unknown", owner)
|
||||
}
|
||||
valRange, exists := r.valRanges[owner]
|
||||
if !exists {
|
||||
return nil, fmt.Errorf("trie %x is unknown", owner)
|
||||
}
|
||||
var err error
|
||||
ir, err = newSingleTrienodeHistoryReader(r.id, r.reader, keyRange, valRange)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r.iReaders[owner] = ir
|
||||
}
|
||||
return ir.read(path)
|
||||
}
|
||||
|
||||
// writeTrienodeHistory persists the trienode history associated with the given diff layer.
|
||||
// nolint:unused
|
||||
func writeTrienodeHistory(writer ethdb.AncientWriter, dl *diffLayer) error {
|
||||
start := time.Now()
|
||||
h := newTrienodeHistory(dl.rootHash(), dl.parent.rootHash(), dl.block, dl.nodes.nodeOrigin)
|
||||
header, keySection, valueSection, err := h.encode()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Write history data into five freezer table respectively.
|
||||
if err := rawdb.WriteTrienodeHistory(writer, dl.stateID(), header, keySection, valueSection); err != nil {
|
||||
return err
|
||||
}
|
||||
trienodeHistoryDataBytesMeter.Mark(int64(len(valueSection)))
|
||||
trienodeHistoryIndexBytesMeter.Mark(int64(len(header) + len(keySection)))
|
||||
trienodeHistoryBuildTimeMeter.UpdateSince(start)
|
||||
|
||||
log.Debug(
|
||||
"Stored trienode history", "id", dl.stateID(), "block", dl.block,
|
||||
"header", common.StorageSize(len(header)),
|
||||
"keySection", common.StorageSize(len(keySection)),
|
||||
"valueSection", common.StorageSize(len(valueSection)),
|
||||
"elapsed", common.PrettyDuration(time.Since(start)),
|
||||
)
|
||||
return nil
|
||||
}
|
||||
|
||||
// readTrienodeMetadata resolves the metadata of the specified trienode history.
|
||||
// nolint:unused
|
||||
func readTrienodeMetadata(reader ethdb.AncientReader, id uint64) (*trienodeMetadata, error) {
|
||||
header, err := rawdb.ReadTrienodeHistoryHeader(reader, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
metadata, _, _, _, err := decodeHeader(header)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return metadata, nil
|
||||
}
|
||||
|
||||
// readTrienodeHistory resolves a single trienode history object with specific id.
|
||||
func readTrienodeHistory(reader ethdb.AncientReader, id uint64) (*trienodeHistory, error) {
|
||||
header, keySection, valueSection, err := rawdb.ReadTrienodeHistory(reader, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var h trienodeHistory
|
||||
if err := h.decode(header, keySection, valueSection); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &h, nil
|
||||
}
|
||||
|
||||
// readTrienodeHistories resolves a list of trienode histories with the specific range.
|
||||
func readTrienodeHistories(reader ethdb.AncientReader, start uint64, count uint64) ([]history, error) {
|
||||
headers, keySections, valueSections, err := rawdb.ReadTrienodeHistoryList(reader, start, count)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var res []history
|
||||
for i, header := range headers {
|
||||
var h trienodeHistory
|
||||
if err := h.decode(header, keySections[i], valueSections[i]); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res = append(res, &h)
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
736
triedb/pathdb/history_trienode_test.go
Normal file
736
triedb/pathdb/history_trienode_test.go
Normal file
|
|
@ -0,0 +1,736 @@
|
|||
// Copyright 2025 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package pathdb
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"math/rand"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/rawdb"
|
||||
"github.com/ethereum/go-ethereum/crypto"
|
||||
"github.com/ethereum/go-ethereum/internal/testrand"
|
||||
)
|
||||
|
||||
// randomTrienodes generates a random trienode set.
|
||||
func randomTrienodes(n int) (map[common.Hash]map[string][]byte, common.Hash) {
|
||||
var (
|
||||
root common.Hash
|
||||
nodes = make(map[common.Hash]map[string][]byte)
|
||||
)
|
||||
for i := 0; i < n; i++ {
|
||||
owner := testrand.Hash()
|
||||
if i == 0 {
|
||||
owner = common.Hash{}
|
||||
}
|
||||
nodes[owner] = make(map[string][]byte)
|
||||
|
||||
for j := 0; j < 10; j++ {
|
||||
path := testrand.Bytes(rand.Intn(10))
|
||||
for z := 0; z < len(path); z++ {
|
||||
nodes[owner][string(path[:z])] = testrand.Bytes(rand.Intn(128))
|
||||
}
|
||||
}
|
||||
// zero-size trie node, representing it was non-existent before
|
||||
for j := 0; j < 10; j++ {
|
||||
path := testrand.Bytes(32)
|
||||
nodes[owner][string(path)] = nil
|
||||
}
|
||||
// root node with zero-size path
|
||||
rnode := testrand.Bytes(256)
|
||||
nodes[owner][""] = rnode
|
||||
if owner == (common.Hash{}) {
|
||||
root = crypto.Keccak256Hash(rnode)
|
||||
}
|
||||
}
|
||||
return nodes, root
|
||||
}
|
||||
|
||||
func makeTrienodeHistory() *trienodeHistory {
|
||||
nodes, root := randomTrienodes(10)
|
||||
return newTrienodeHistory(root, common.Hash{}, 1, nodes)
|
||||
}
|
||||
|
||||
func makeTrienodeHistories(n int) []*trienodeHistory {
|
||||
var (
|
||||
parent common.Hash
|
||||
result []*trienodeHistory
|
||||
)
|
||||
for i := 0; i < n; i++ {
|
||||
nodes, root := randomTrienodes(10)
|
||||
result = append(result, newTrienodeHistory(root, parent, uint64(i+1), nodes))
|
||||
parent = root
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func TestEncodeDecodeTrienodeHistory(t *testing.T) {
|
||||
var (
|
||||
dec trienodeHistory
|
||||
obj = makeTrienodeHistory()
|
||||
)
|
||||
header, keySection, valueSection, err := obj.encode()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to encode trienode history: %v", err)
|
||||
}
|
||||
if err := dec.decode(header, keySection, valueSection); err != nil {
|
||||
t.Fatalf("Failed to decode trienode history: %v", err)
|
||||
}
|
||||
|
||||
if !reflect.DeepEqual(obj.meta, dec.meta) {
|
||||
t.Fatal("trienode metadata is mismatched")
|
||||
}
|
||||
if !compareList(dec.owners, obj.owners) {
|
||||
t.Fatal("trie owner list is mismatched")
|
||||
}
|
||||
if !compareMapList(dec.nodeList, obj.nodeList) {
|
||||
t.Fatal("trienode list is mismatched")
|
||||
}
|
||||
if !compareMapSet(dec.nodes, obj.nodes) {
|
||||
t.Fatal("trienode content is mismatched")
|
||||
}
|
||||
|
||||
// Re-encode again, ensuring the encoded blob still match
|
||||
header2, keySection2, valueSection2, err := dec.encode()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to encode trienode history: %v", err)
|
||||
}
|
||||
if !bytes.Equal(header, header2) {
|
||||
t.Fatal("re-encoded header is mismatched")
|
||||
}
|
||||
if !bytes.Equal(keySection, keySection2) {
|
||||
t.Fatal("re-encoded key section is mismatched")
|
||||
}
|
||||
if !bytes.Equal(valueSection, valueSection2) {
|
||||
t.Fatal("re-encoded value section is mismatched")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrienodeHistoryReader(t *testing.T) {
|
||||
var (
|
||||
hs = makeTrienodeHistories(10)
|
||||
freezer, _ = rawdb.NewTrienodeFreezer(t.TempDir(), false, false)
|
||||
)
|
||||
defer freezer.Close()
|
||||
|
||||
for i, h := range hs {
|
||||
header, keySection, valueSection, _ := h.encode()
|
||||
if err := rawdb.WriteTrienodeHistory(freezer, uint64(i+1), header, keySection, valueSection); err != nil {
|
||||
t.Fatalf("Failed to write trienode history: %v", err)
|
||||
}
|
||||
}
|
||||
for i, h := range hs {
|
||||
tr, err := newTrienodeHistoryReader(uint64(i+1), freezer)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to construct the history reader: %v", err)
|
||||
}
|
||||
for _, owner := range h.owners {
|
||||
nodes := h.nodes[owner]
|
||||
for key, value := range nodes {
|
||||
blob, err := tr.read(owner, key)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to read trienode history: %v", err)
|
||||
}
|
||||
if !bytes.Equal(blob, value) {
|
||||
t.Fatalf("Unexpected trie node data, want: %v, got: %v", value, blob)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for i, h := range hs {
|
||||
metadata, err := readTrienodeMetadata(freezer, uint64(i+1))
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to read trienode history metadata: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(h.meta, metadata) {
|
||||
t.Fatalf("Unexpected trienode metadata, want: %v, got: %v", h.meta, metadata)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestEmptyTrienodeHistory tests encoding/decoding of empty trienode history
|
||||
func TestEmptyTrienodeHistory(t *testing.T) {
|
||||
h := newTrienodeHistory(common.Hash{}, common.Hash{}, 1, make(map[common.Hash]map[string][]byte))
|
||||
|
||||
// Test encoding empty history
|
||||
header, keySection, valueSection, err := h.encode()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to encode empty trienode history: %v", err)
|
||||
}
|
||||
|
||||
// Verify sections are minimal but valid
|
||||
if len(header) == 0 {
|
||||
t.Fatal("Header should not be empty")
|
||||
}
|
||||
if len(keySection) != 0 {
|
||||
t.Fatal("Key section should be empty for empty history")
|
||||
}
|
||||
if len(valueSection) != 0 {
|
||||
t.Fatal("Value section should be empty for empty history")
|
||||
}
|
||||
|
||||
// Test decoding empty history
|
||||
var decoded trienodeHistory
|
||||
if err := decoded.decode(header, keySection, valueSection); err != nil {
|
||||
t.Fatalf("Failed to decode empty trienode history: %v", err)
|
||||
}
|
||||
|
||||
if len(decoded.owners) != 0 {
|
||||
t.Fatal("Decoded history should have no owners")
|
||||
}
|
||||
if len(decoded.nodeList) != 0 {
|
||||
t.Fatal("Decoded history should have no node lists")
|
||||
}
|
||||
if len(decoded.nodes) != 0 {
|
||||
t.Fatal("Decoded history should have no nodes")
|
||||
}
|
||||
}
|
||||
|
||||
// TestSingleTrieHistory tests encoding/decoding of history with single trie
|
||||
func TestSingleTrieHistory(t *testing.T) {
|
||||
nodes := make(map[common.Hash]map[string][]byte)
|
||||
owner := testrand.Hash()
|
||||
nodes[owner] = make(map[string][]byte)
|
||||
|
||||
// Add some nodes with various sizes
|
||||
nodes[owner][""] = testrand.Bytes(32) // empty key
|
||||
nodes[owner]["a"] = testrand.Bytes(1) // small value
|
||||
nodes[owner]["bb"] = testrand.Bytes(100) // medium value
|
||||
nodes[owner]["ccc"] = testrand.Bytes(1000) // large value
|
||||
nodes[owner]["dddd"] = testrand.Bytes(0) // empty value
|
||||
|
||||
h := newTrienodeHistory(common.Hash{}, common.Hash{}, 1, nodes)
|
||||
testEncodeDecode(t, h)
|
||||
}
|
||||
|
||||
// TestMultipleTries tests multiple tries with different node counts
|
||||
func TestMultipleTries(t *testing.T) {
|
||||
nodes := make(map[common.Hash]map[string][]byte)
|
||||
|
||||
// First trie with many small nodes
|
||||
owner1 := testrand.Hash()
|
||||
nodes[owner1] = make(map[string][]byte)
|
||||
for i := 0; i < 100; i++ {
|
||||
key := string(testrand.Bytes(rand.Intn(10)))
|
||||
nodes[owner1][key] = testrand.Bytes(rand.Intn(50))
|
||||
}
|
||||
|
||||
// Second trie with few large nodes
|
||||
owner2 := testrand.Hash()
|
||||
nodes[owner2] = make(map[string][]byte)
|
||||
for i := 0; i < 5; i++ {
|
||||
key := string(testrand.Bytes(rand.Intn(20)))
|
||||
nodes[owner2][key] = testrand.Bytes(1000 + rand.Intn(1000))
|
||||
}
|
||||
|
||||
// Third trie with nil values (zero-size nodes)
|
||||
owner3 := testrand.Hash()
|
||||
nodes[owner3] = make(map[string][]byte)
|
||||
for i := 0; i < 10; i++ {
|
||||
key := string(testrand.Bytes(rand.Intn(15)))
|
||||
nodes[owner3][key] = nil
|
||||
}
|
||||
|
||||
h := newTrienodeHistory(common.Hash{}, common.Hash{}, 1, nodes)
|
||||
testEncodeDecode(t, h)
|
||||
}
|
||||
|
||||
// TestLargeNodeValues tests encoding/decoding with very large node values
|
||||
func TestLargeNodeValues(t *testing.T) {
|
||||
nodes := make(map[common.Hash]map[string][]byte)
|
||||
owner := testrand.Hash()
|
||||
nodes[owner] = make(map[string][]byte)
|
||||
|
||||
// Test with progressively larger values
|
||||
sizes := []int{1024, 10 * 1024, 100 * 1024, 1024 * 1024} // 1KB, 10KB, 100KB, 1MB
|
||||
for _, size := range sizes {
|
||||
key := string(testrand.Bytes(10))
|
||||
nodes[owner][key] = testrand.Bytes(size)
|
||||
|
||||
h := newTrienodeHistory(common.Hash{}, common.Hash{}, 1, nodes)
|
||||
testEncodeDecode(t, h)
|
||||
t.Logf("Successfully tested encoding/decoding with %dKB value", size/1024)
|
||||
}
|
||||
}
|
||||
|
||||
// TestNilNodeValues tests encoding/decoding with nil (zero-length) node values
|
||||
func TestNilNodeValues(t *testing.T) {
|
||||
nodes := make(map[common.Hash]map[string][]byte)
|
||||
owner := testrand.Hash()
|
||||
nodes[owner] = make(map[string][]byte)
|
||||
|
||||
// Mix of nil and non-nil values
|
||||
nodes[owner]["nil"] = nil
|
||||
nodes[owner]["data1"] = []byte("some data")
|
||||
nodes[owner]["data2"] = []byte("more data")
|
||||
|
||||
h := newTrienodeHistory(common.Hash{}, common.Hash{}, 1, nodes)
|
||||
testEncodeDecode(t, h)
|
||||
|
||||
// Verify nil values are preserved
|
||||
_, ok := h.nodes[owner]["nil"]
|
||||
if !ok {
|
||||
t.Fatal("Nil value should be preserved")
|
||||
}
|
||||
}
|
||||
|
||||
// TestCorruptedHeader tests error handling for corrupted header data
|
||||
func TestCorruptedHeader(t *testing.T) {
|
||||
h := makeTrienodeHistory()
|
||||
header, keySection, valueSection, _ := h.encode()
|
||||
|
||||
// Test corrupted version
|
||||
corruptedHeader := make([]byte, len(header))
|
||||
copy(corruptedHeader, header)
|
||||
corruptedHeader[0] = 0xFF // Invalid version
|
||||
|
||||
var decoded trienodeHistory
|
||||
if err := decoded.decode(corruptedHeader, keySection, valueSection); err == nil {
|
||||
t.Fatal("Expected error for corrupted version")
|
||||
}
|
||||
|
||||
// Test truncated header
|
||||
truncatedHeader := header[:len(header)-5]
|
||||
if err := decoded.decode(truncatedHeader, keySection, valueSection); err == nil {
|
||||
t.Fatal("Expected error for truncated header")
|
||||
}
|
||||
|
||||
// Test header with invalid trie header size
|
||||
invalidHeader := make([]byte, len(header))
|
||||
copy(invalidHeader, header)
|
||||
invalidHeader = invalidHeader[:trienodeMetadataSize+5] // Not divisible by trie header size
|
||||
|
||||
if err := decoded.decode(invalidHeader, keySection, valueSection); err == nil {
|
||||
t.Fatal("Expected error for invalid header size")
|
||||
}
|
||||
}
|
||||
|
||||
// TestCorruptedKeySection tests error handling for corrupted key section data
|
||||
func TestCorruptedKeySection(t *testing.T) {
|
||||
h := makeTrienodeHistory()
|
||||
header, keySection, valueSection, _ := h.encode()
|
||||
|
||||
// Test empty key section when header indicates data
|
||||
if len(keySection) > 0 {
|
||||
var decoded trienodeHistory
|
||||
if err := decoded.decode(header, []byte{}, valueSection); err == nil {
|
||||
t.Fatal("Expected error for empty key section with non-empty header")
|
||||
}
|
||||
}
|
||||
|
||||
// Test truncated key section
|
||||
if len(keySection) > 10 {
|
||||
truncatedKeySection := keySection[:len(keySection)-10]
|
||||
var decoded trienodeHistory
|
||||
if err := decoded.decode(header, truncatedKeySection, valueSection); err == nil {
|
||||
t.Fatal("Expected error for truncated key section")
|
||||
}
|
||||
}
|
||||
|
||||
// Test corrupted key section with invalid varint
|
||||
corruptedKeySection := make([]byte, len(keySection))
|
||||
copy(corruptedKeySection, keySection)
|
||||
if len(corruptedKeySection) > 5 {
|
||||
corruptedKeySection[5] = 0xFF // Corrupt varint encoding
|
||||
var decoded trienodeHistory
|
||||
if err := decoded.decode(header, corruptedKeySection, valueSection); err == nil {
|
||||
t.Fatal("Expected error for corrupted varint in key section")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestCorruptedValueSection tests error handling for corrupted value section data
|
||||
func TestCorruptedValueSection(t *testing.T) {
|
||||
h := makeTrienodeHistory()
|
||||
header, keySection, valueSection, _ := h.encode()
|
||||
|
||||
// Test truncated value section
|
||||
if len(valueSection) > 10 {
|
||||
truncatedValueSection := valueSection[:len(valueSection)-10]
|
||||
var decoded trienodeHistory
|
||||
if err := decoded.decode(header, keySection, truncatedValueSection); err == nil {
|
||||
t.Fatal("Expected error for truncated value section")
|
||||
}
|
||||
}
|
||||
|
||||
// Test empty value section when key section indicates data exists
|
||||
if len(valueSection) > 0 {
|
||||
var decoded trienodeHistory
|
||||
if err := decoded.decode(header, keySection, []byte{}); err == nil {
|
||||
t.Fatal("Expected error for empty value section with non-empty key section")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestInvalidOffsets tests error handling for invalid offsets in encoded data
|
||||
func TestInvalidOffsets(t *testing.T) {
|
||||
h := makeTrienodeHistory()
|
||||
header, keySection, valueSection, _ := h.encode()
|
||||
|
||||
// Corrupt key offset in header (make it larger than key section)
|
||||
corruptedHeader := make([]byte, len(header))
|
||||
copy(corruptedHeader, header)
|
||||
corruptedHeader[trienodeMetadataSize+common.HashLength] = 0xff
|
||||
|
||||
var dec1 trienodeHistory
|
||||
if err := dec1.decode(corruptedHeader, keySection, valueSection); err == nil {
|
||||
t.Fatal("Expected error for invalid key offset")
|
||||
}
|
||||
|
||||
// Corrupt value offset in header (make it larger than value section)
|
||||
corruptedHeader = make([]byte, len(header))
|
||||
copy(corruptedHeader, header)
|
||||
corruptedHeader[trienodeMetadataSize+common.HashLength+4] = 0xff
|
||||
|
||||
var dec2 trienodeHistory
|
||||
if err := dec2.decode(corruptedHeader, keySection, valueSection); err == nil {
|
||||
t.Fatal("Expected error for invalid value offset")
|
||||
}
|
||||
}
|
||||
|
||||
// TestTrienodeHistoryReaderNonExistentPath tests reading non-existent paths
|
||||
func TestTrienodeHistoryReaderNonExistentPath(t *testing.T) {
|
||||
var (
|
||||
h = makeTrienodeHistory()
|
||||
freezer, _ = rawdb.NewTrienodeFreezer(t.TempDir(), false, false)
|
||||
)
|
||||
defer freezer.Close()
|
||||
|
||||
header, keySection, valueSection, _ := h.encode()
|
||||
if err := rawdb.WriteTrienodeHistory(freezer, 1, header, keySection, valueSection); err != nil {
|
||||
t.Fatalf("Failed to write trienode history: %v", err)
|
||||
}
|
||||
|
||||
tr, err := newTrienodeHistoryReader(1, freezer)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to construct history reader: %v", err)
|
||||
}
|
||||
|
||||
// Try to read a non-existent path
|
||||
_, err = tr.read(testrand.Hash(), "nonexistent")
|
||||
if err == nil {
|
||||
t.Fatal("Expected error for non-existent trie owner")
|
||||
}
|
||||
|
||||
// Try to read from existing owner but non-existent path
|
||||
owner := h.owners[0]
|
||||
_, err = tr.read(owner, "nonexistent-path")
|
||||
if err == nil {
|
||||
t.Fatal("Expected error for non-existent path")
|
||||
}
|
||||
}
|
||||
|
||||
// TestTrienodeHistoryReaderNilValues tests reading nil (zero-length) values
|
||||
func TestTrienodeHistoryReaderNilValues(t *testing.T) {
|
||||
nodes := make(map[common.Hash]map[string][]byte)
|
||||
owner := testrand.Hash()
|
||||
nodes[owner] = make(map[string][]byte)
|
||||
|
||||
// Add some nil values
|
||||
nodes[owner]["nil1"] = nil
|
||||
nodes[owner]["nil2"] = nil
|
||||
nodes[owner]["data1"] = []byte("some data")
|
||||
|
||||
h := newTrienodeHistory(common.Hash{}, common.Hash{}, 1, nodes)
|
||||
|
||||
var freezer, _ = rawdb.NewTrienodeFreezer(t.TempDir(), false, false)
|
||||
defer freezer.Close()
|
||||
|
||||
header, keySection, valueSection, _ := h.encode()
|
||||
if err := rawdb.WriteTrienodeHistory(freezer, 1, header, keySection, valueSection); err != nil {
|
||||
t.Fatalf("Failed to write trienode history: %v", err)
|
||||
}
|
||||
|
||||
tr, err := newTrienodeHistoryReader(1, freezer)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to construct history reader: %v", err)
|
||||
}
|
||||
|
||||
// Test reading nil values
|
||||
data1, err := tr.read(owner, "nil1")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to read nil value: %v", err)
|
||||
}
|
||||
if len(data1) != 0 {
|
||||
t.Fatal("Expected nil data for nil value")
|
||||
}
|
||||
|
||||
data2, err := tr.read(owner, "nil2")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to read nil value: %v", err)
|
||||
}
|
||||
if len(data2) != 0 {
|
||||
t.Fatal("Expected nil data for nil value")
|
||||
}
|
||||
|
||||
// Test reading non-nil value
|
||||
data3, err := tr.read(owner, "data1")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to read non-nil value: %v", err)
|
||||
}
|
||||
if !bytes.Equal(data3, []byte("some data")) {
|
||||
t.Fatal("Data mismatch for non-nil value")
|
||||
}
|
||||
}
|
||||
|
||||
// TestTrienodeHistoryReaderNilKey tests reading nil (zero-length) key
|
||||
func TestTrienodeHistoryReaderNilKey(t *testing.T) {
|
||||
nodes := make(map[common.Hash]map[string][]byte)
|
||||
owner := testrand.Hash()
|
||||
nodes[owner] = make(map[string][]byte)
|
||||
|
||||
// Add some nil values
|
||||
nodes[owner][""] = []byte("some data")
|
||||
nodes[owner]["data1"] = []byte("some data")
|
||||
|
||||
h := newTrienodeHistory(common.Hash{}, common.Hash{}, 1, nodes)
|
||||
|
||||
var freezer, _ = rawdb.NewTrienodeFreezer(t.TempDir(), false, false)
|
||||
defer freezer.Close()
|
||||
|
||||
header, keySection, valueSection, _ := h.encode()
|
||||
if err := rawdb.WriteTrienodeHistory(freezer, 1, header, keySection, valueSection); err != nil {
|
||||
t.Fatalf("Failed to write trienode history: %v", err)
|
||||
}
|
||||
|
||||
tr, err := newTrienodeHistoryReader(1, freezer)
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to construct history reader: %v", err)
|
||||
}
|
||||
|
||||
// Test reading nil values
|
||||
data1, err := tr.read(owner, "")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to read nil value: %v", err)
|
||||
}
|
||||
if !bytes.Equal(data1, []byte("some data")) {
|
||||
t.Fatal("Data mismatch for nil key")
|
||||
}
|
||||
|
||||
// Test reading non-nil value
|
||||
data2, err := tr.read(owner, "data1")
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to read non-nil value: %v", err)
|
||||
}
|
||||
if !bytes.Equal(data2, []byte("some data")) {
|
||||
t.Fatal("Data mismatch for non-nil key")
|
||||
}
|
||||
}
|
||||
|
||||
// TestTrienodeHistoryReaderIterator tests the iterator functionality
|
||||
func TestTrienodeHistoryReaderIterator(t *testing.T) {
|
||||
h := makeTrienodeHistory()
|
||||
|
||||
// Count expected entries
|
||||
expectedCount := 0
|
||||
expectedNodes := make(map[stateIdent]bool)
|
||||
for owner, nodeList := range h.nodeList {
|
||||
expectedCount += len(nodeList)
|
||||
for _, node := range nodeList {
|
||||
expectedNodes[stateIdent{
|
||||
typ: typeTrienode,
|
||||
addressHash: owner,
|
||||
path: node,
|
||||
}] = true
|
||||
}
|
||||
}
|
||||
|
||||
// Test the iterator
|
||||
actualCount := 0
|
||||
for x := range h.forEach() {
|
||||
_ = x
|
||||
actualCount++
|
||||
}
|
||||
if actualCount != expectedCount {
|
||||
t.Fatalf("Iterator count mismatch: expected %d, got %d", expectedCount, actualCount)
|
||||
}
|
||||
|
||||
// Test that iterator yields expected state identifiers
|
||||
seen := make(map[stateIdent]bool)
|
||||
for ident := range h.forEach() {
|
||||
if ident.typ != typeTrienode {
|
||||
t.Fatal("Iterator should only yield trienode history identifiers")
|
||||
}
|
||||
key := stateIdent{typ: ident.typ, addressHash: ident.addressHash, path: ident.path}
|
||||
if seen[key] {
|
||||
t.Fatal("Iterator yielded duplicate identifier")
|
||||
}
|
||||
seen[key] = true
|
||||
|
||||
if !expectedNodes[key] {
|
||||
t.Fatalf("Unexpected yielded identifier %v", key)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestSharedLen tests the sharedLen helper function
|
||||
func TestSharedLen(t *testing.T) {
|
||||
tests := []struct {
|
||||
a, b []byte
|
||||
expected int
|
||||
}{
|
||||
// Empty strings
|
||||
{[]byte(""), []byte(""), 0},
|
||||
// One empty string
|
||||
{[]byte(""), []byte("abc"), 0},
|
||||
{[]byte("abc"), []byte(""), 0},
|
||||
// No common prefix
|
||||
{[]byte("abc"), []byte("def"), 0},
|
||||
// Partial common prefix
|
||||
{[]byte("abc"), []byte("abx"), 2},
|
||||
{[]byte("prefix"), []byte("pref"), 4},
|
||||
// Complete common prefix (shorter first)
|
||||
{[]byte("ab"), []byte("abcd"), 2},
|
||||
// Complete common prefix (longer first)
|
||||
{[]byte("abcd"), []byte("ab"), 2},
|
||||
// Identical strings
|
||||
{[]byte("identical"), []byte("identical"), 9},
|
||||
// Binary data
|
||||
{[]byte{0x00, 0x01, 0x02}, []byte{0x00, 0x01, 0x03}, 2},
|
||||
// Large strings
|
||||
{bytes.Repeat([]byte("a"), 1000), bytes.Repeat([]byte("a"), 1000), 1000},
|
||||
{bytes.Repeat([]byte("a"), 1000), append(bytes.Repeat([]byte("a"), 999), []byte("b")...), 999},
|
||||
}
|
||||
|
||||
for i, test := range tests {
|
||||
result := sharedLen(test.a, test.b)
|
||||
if result != test.expected {
|
||||
t.Errorf("Test %d: sharedLen(%q, %q) = %d, expected %d",
|
||||
i, test.a, test.b, result, test.expected)
|
||||
}
|
||||
// Test commutativity
|
||||
resultReverse := sharedLen(test.b, test.a)
|
||||
if result != resultReverse {
|
||||
t.Errorf("Test %d: sharedLen is not commutative: sharedLen(a,b)=%d, sharedLen(b,a)=%d",
|
||||
i, result, resultReverse)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// TestDecodeHeaderCorruptedData tests decodeHeader with corrupted data
|
||||
func TestDecodeHeaderCorruptedData(t *testing.T) {
|
||||
// Create valid header data first
|
||||
h := makeTrienodeHistory()
|
||||
header, _, _, _ := h.encode()
|
||||
|
||||
// Test with empty header
|
||||
_, _, _, _, err := decodeHeader([]byte{})
|
||||
if err == nil {
|
||||
t.Fatal("Expected error for empty header")
|
||||
}
|
||||
|
||||
// Test with invalid version
|
||||
corruptedVersion := make([]byte, len(header))
|
||||
copy(corruptedVersion, header)
|
||||
corruptedVersion[0] = 0xFF
|
||||
_, _, _, _, err = decodeHeader(corruptedVersion)
|
||||
if err == nil {
|
||||
t.Fatal("Expected error for invalid version")
|
||||
}
|
||||
|
||||
// Test with truncated header (not divisible by trie header size)
|
||||
truncated := header[:trienodeMetadataSize+5]
|
||||
_, _, _, _, err = decodeHeader(truncated)
|
||||
if err == nil {
|
||||
t.Fatal("Expected error for truncated header")
|
||||
}
|
||||
|
||||
// Test with unordered trie owners
|
||||
unordered := make([]byte, len(header))
|
||||
copy(unordered, header)
|
||||
|
||||
// Swap two owner hashes to make them unordered
|
||||
hash1Start := trienodeMetadataSize
|
||||
hash2Start := trienodeMetadataSize + trienodeTrieHeaderSize
|
||||
hash1 := unordered[hash1Start : hash1Start+common.HashLength]
|
||||
hash2 := unordered[hash2Start : hash2Start+common.HashLength]
|
||||
|
||||
// Only swap if they would be out of order
|
||||
copy(unordered[hash1Start:hash1Start+common.HashLength], hash2)
|
||||
copy(unordered[hash2Start:hash2Start+common.HashLength], hash1)
|
||||
|
||||
_, _, _, _, err = decodeHeader(unordered)
|
||||
if err == nil {
|
||||
t.Fatal("Expected error for unordered trie owners")
|
||||
}
|
||||
}
|
||||
|
||||
// TestDecodeSingleCorruptedData tests decodeSingle with corrupted data
|
||||
func TestDecodeSingleCorruptedData(t *testing.T) {
|
||||
h := makeTrienodeHistory()
|
||||
_, keySection, _, _ := h.encode()
|
||||
|
||||
// Test with empty key section
|
||||
_, err := decodeSingle([]byte{}, nil)
|
||||
if err == nil {
|
||||
t.Fatal("Expected error for empty key section")
|
||||
}
|
||||
|
||||
// Test with key section too small for trailer
|
||||
if len(keySection) > 0 {
|
||||
_, err := decodeSingle(keySection[:3], nil) // Less than 4 bytes for trailer
|
||||
if err == nil {
|
||||
t.Fatal("Expected error for key section too small for trailer")
|
||||
}
|
||||
}
|
||||
|
||||
// Test with corrupted varint in key section
|
||||
corrupted := make([]byte, len(keySection))
|
||||
copy(corrupted, keySection)
|
||||
corrupted[5] = 0xFF // Corrupt varint
|
||||
_, err = decodeSingle(corrupted, nil)
|
||||
if err == nil {
|
||||
t.Fatal("Expected error for corrupted varint")
|
||||
}
|
||||
|
||||
// Test with corrupted trailer (invalid restart count)
|
||||
corrupted = make([]byte, len(keySection))
|
||||
copy(corrupted, keySection)
|
||||
// Set restart count to something too large
|
||||
binary.BigEndian.PutUint32(corrupted[len(corrupted)-4:], 10000)
|
||||
_, err = decodeSingle(corrupted, nil)
|
||||
if err == nil {
|
||||
t.Fatal("Expected error for invalid restart count")
|
||||
}
|
||||
}
|
||||
|
||||
// Helper function to test encode/decode cycle
|
||||
func testEncodeDecode(t *testing.T, h *trienodeHistory) {
|
||||
header, keySection, valueSection, err := h.encode()
|
||||
if err != nil {
|
||||
t.Fatalf("Failed to encode trienode history: %v", err)
|
||||
}
|
||||
|
||||
var decoded trienodeHistory
|
||||
if err := decoded.decode(header, keySection, valueSection); err != nil {
|
||||
t.Fatalf("Failed to decode trienode history: %v", err)
|
||||
}
|
||||
|
||||
// Compare the decoded history with original
|
||||
if !compareList(decoded.owners, h.owners) {
|
||||
t.Fatal("Trie owner list mismatch")
|
||||
}
|
||||
if !compareMapList(decoded.nodeList, h.nodeList) {
|
||||
t.Fatal("Trienode list mismatch")
|
||||
}
|
||||
if !compareMapSet(decoded.nodes, h.nodes) {
|
||||
t.Fatal("Trienode content mismatch")
|
||||
}
|
||||
}
|
||||
|
|
@ -69,12 +69,21 @@ var (
|
|||
gcStorageMeter = metrics.NewRegisteredMeter("pathdb/gc/storage/count", nil)
|
||||
gcStorageBytesMeter = metrics.NewRegisteredMeter("pathdb/gc/storage/bytes", nil)
|
||||
|
||||
historyBuildTimeMeter = metrics.NewRegisteredResettingTimer("pathdb/history/time", nil)
|
||||
historyDataBytesMeter = metrics.NewRegisteredMeter("pathdb/history/bytes/data", nil)
|
||||
historyIndexBytesMeter = metrics.NewRegisteredMeter("pathdb/history/bytes/index", nil)
|
||||
stateHistoryBuildTimeMeter = metrics.NewRegisteredResettingTimer("pathdb/history/state/time", nil)
|
||||
stateHistoryDataBytesMeter = metrics.NewRegisteredMeter("pathdb/history/state/bytes/data", nil)
|
||||
stateHistoryIndexBytesMeter = metrics.NewRegisteredMeter("pathdb/history/state/bytes/index", nil)
|
||||
|
||||
indexHistoryTimer = metrics.NewRegisteredResettingTimer("pathdb/history/index/time", nil)
|
||||
unindexHistoryTimer = metrics.NewRegisteredResettingTimer("pathdb/history/unindex/time", nil)
|
||||
//nolint:unused
|
||||
trienodeHistoryBuildTimeMeter = metrics.NewRegisteredResettingTimer("pathdb/history/trienode/time", nil)
|
||||
//nolint:unused
|
||||
trienodeHistoryDataBytesMeter = metrics.NewRegisteredMeter("pathdb/history/trienode/bytes/data", nil)
|
||||
//nolint:unused
|
||||
trienodeHistoryIndexBytesMeter = metrics.NewRegisteredMeter("pathdb/history/trienode/bytes/index", nil)
|
||||
|
||||
stateIndexHistoryTimer = metrics.NewRegisteredResettingTimer("pathdb/history/state/index/time", nil)
|
||||
stateUnindexHistoryTimer = metrics.NewRegisteredResettingTimer("pathdb/history/state/unindex/time", nil)
|
||||
trienodeIndexHistoryTimer = metrics.NewRegisteredResettingTimer("pathdb/history/trienode/index/time", nil)
|
||||
trienodeUnindexHistoryTimer = metrics.NewRegisteredResettingTimer("pathdb/history/trienode/unindex/time", nil)
|
||||
|
||||
lookupAddLayerTimer = metrics.NewRegisteredResettingTimer("pathdb/lookup/add/time", nil)
|
||||
lookupRemoveLayerTimer = metrics.NewRegisteredResettingTimer("pathdb/lookup/remove/time", nil)
|
||||
|
|
|
|||
Loading…
Reference in a new issue