core/rawdb, triedb/pathdb: introduce trienode history (#32596)

It's a pull request based on the #32523 , implementing the structure of
trienode history.
This commit is contained in:
rjl493456442 2025-10-10 14:51:27 +08:00 committed by GitHub
parent ed264a1f19
commit de24450dbf
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
13 changed files with 1850 additions and 71 deletions

View file

@ -46,6 +46,27 @@ func DeleteStateHistoryIndexMetadata(db ethdb.KeyValueWriter) {
} }
} }
// ReadTrienodeHistoryIndexMetadata retrieves the metadata of trienode history index.
func ReadTrienodeHistoryIndexMetadata(db ethdb.KeyValueReader) []byte {
data, _ := db.Get(headTrienodeHistoryIndexKey)
return data
}
// WriteTrienodeHistoryIndexMetadata stores the metadata of trienode history index
// into database.
func WriteTrienodeHistoryIndexMetadata(db ethdb.KeyValueWriter, blob []byte) {
if err := db.Put(headTrienodeHistoryIndexKey, blob); err != nil {
log.Crit("Failed to store the metadata of trienode history index", "err", err)
}
}
// DeleteTrienodeHistoryIndexMetadata removes the metadata of trienode history index.
func DeleteTrienodeHistoryIndexMetadata(db ethdb.KeyValueWriter) {
if err := db.Delete(headTrienodeHistoryIndexKey); err != nil {
log.Crit("Failed to delete the metadata of trienode history index", "err", err)
}
}
// ReadAccountHistoryIndex retrieves the account history index with the provided // ReadAccountHistoryIndex retrieves the account history index with the provided
// account address. // account address.
func ReadAccountHistoryIndex(db ethdb.KeyValueReader, addressHash common.Hash) []byte { func ReadAccountHistoryIndex(db ethdb.KeyValueReader, addressHash common.Hash) []byte {
@ -95,6 +116,30 @@ func DeleteStorageHistoryIndex(db ethdb.KeyValueWriter, addressHash common.Hash,
} }
} }
// ReadTrienodeHistoryIndex retrieves the trienode history index with the provided
// account address and storage key hash.
func ReadTrienodeHistoryIndex(db ethdb.KeyValueReader, addressHash common.Hash, path []byte) []byte {
data, err := db.Get(trienodeHistoryIndexKey(addressHash, path))
if err != nil || len(data) == 0 {
return nil
}
return data
}
// WriteTrienodeHistoryIndex writes the provided trienode history index into database.
func WriteTrienodeHistoryIndex(db ethdb.KeyValueWriter, addressHash common.Hash, path []byte, data []byte) {
if err := db.Put(trienodeHistoryIndexKey(addressHash, path), data); err != nil {
log.Crit("Failed to store trienode history index", "err", err)
}
}
// DeleteTrienodeHistoryIndex deletes the specified trienode index from the database.
func DeleteTrienodeHistoryIndex(db ethdb.KeyValueWriter, addressHash common.Hash, path []byte) {
if err := db.Delete(trienodeHistoryIndexKey(addressHash, path)); err != nil {
log.Crit("Failed to delete trienode history index", "err", err)
}
}
// ReadAccountHistoryIndexBlock retrieves the index block with the provided // ReadAccountHistoryIndexBlock retrieves the index block with the provided
// account address along with the block id. // account address along with the block id.
func ReadAccountHistoryIndexBlock(db ethdb.KeyValueReader, addressHash common.Hash, blockID uint32) []byte { func ReadAccountHistoryIndexBlock(db ethdb.KeyValueReader, addressHash common.Hash, blockID uint32) []byte {
@ -143,6 +188,30 @@ func DeleteStorageHistoryIndexBlock(db ethdb.KeyValueWriter, addressHash common.
} }
} }
// ReadTrienodeHistoryIndexBlock retrieves the index block with the provided state
// identifier along with the block id.
func ReadTrienodeHistoryIndexBlock(db ethdb.KeyValueReader, addressHash common.Hash, path []byte, blockID uint32) []byte {
data, err := db.Get(trienodeHistoryIndexBlockKey(addressHash, path, blockID))
if err != nil || len(data) == 0 {
return nil
}
return data
}
// WriteTrienodeHistoryIndexBlock writes the provided index block into database.
func WriteTrienodeHistoryIndexBlock(db ethdb.KeyValueWriter, addressHash common.Hash, path []byte, id uint32, data []byte) {
if err := db.Put(trienodeHistoryIndexBlockKey(addressHash, path, id), data); err != nil {
log.Crit("Failed to store trienode index block", "err", err)
}
}
// DeleteTrienodeHistoryIndexBlock deletes the specified index block from the database.
func DeleteTrienodeHistoryIndexBlock(db ethdb.KeyValueWriter, addressHash common.Hash, path []byte, id uint32) {
if err := db.Delete(trienodeHistoryIndexBlockKey(addressHash, path, id)); err != nil {
log.Crit("Failed to delete trienode index block", "err", err)
}
}
// increaseKey increase the input key by one bit. Return nil if the entire // increaseKey increase the input key by one bit. Return nil if the entire
// addition operation overflows. // addition operation overflows.
func increaseKey(key []byte) []byte { func increaseKey(key []byte) []byte {
@ -155,14 +224,26 @@ func increaseKey(key []byte) []byte {
return nil return nil
} }
// DeleteStateHistoryIndex completely removes all history indexing data, including // DeleteStateHistoryIndexes completely removes all history indexing data, including
// indexes for accounts and storages. // indexes for accounts and storages.
// func DeleteStateHistoryIndexes(db ethdb.KeyValueRangeDeleter) {
// Note, this method assumes the storage space with prefix `StateHistoryIndexPrefix` DeleteHistoryByRange(db, StateHistoryAccountMetadataPrefix)
// is exclusively occupied by the history indexing data! DeleteHistoryByRange(db, StateHistoryStorageMetadataPrefix)
func DeleteStateHistoryIndex(db ethdb.KeyValueRangeDeleter) { DeleteHistoryByRange(db, StateHistoryAccountBlockPrefix)
start := StateHistoryIndexPrefix DeleteHistoryByRange(db, StateHistoryStorageBlockPrefix)
limit := increaseKey(bytes.Clone(StateHistoryIndexPrefix)) }
// DeleteTrienodeHistoryIndexes completely removes all trienode history indexing data.
func DeleteTrienodeHistoryIndexes(db ethdb.KeyValueRangeDeleter) {
DeleteHistoryByRange(db, TrienodeHistoryMetadataPrefix)
DeleteHistoryByRange(db, TrienodeHistoryBlockPrefix)
}
// DeleteHistoryByRange completely removes all database entries with the specific prefix.
// Note, this method assumes the space with the given prefix is exclusively occupied!
func DeleteHistoryByRange(db ethdb.KeyValueRangeDeleter, prefix []byte) {
start := prefix
limit := increaseKey(bytes.Clone(prefix))
// Try to remove the data in the range by a loop, as the leveldb // Try to remove the data in the range by a loop, as the leveldb
// doesn't support the native range deletion. // doesn't support the native range deletion.

View file

@ -299,3 +299,76 @@ func WriteStateHistory(db ethdb.AncientWriter, id uint64, meta []byte, accountIn
}) })
return err return err
} }
// ReadTrienodeHistory retrieves the trienode history corresponding to the specified id.
// Compute the position of trienode history in freezer by minus one since the id of first
// trienode history starts from one(zero for initial state).
func ReadTrienodeHistory(db ethdb.AncientReaderOp, id uint64) ([]byte, []byte, []byte, error) {
header, err := db.Ancient(trienodeHistoryHeaderTable, id-1)
if err != nil {
return nil, nil, nil, err
}
keySection, err := db.Ancient(trienodeHistoryKeySectionTable, id-1)
if err != nil {
return nil, nil, nil, err
}
valueSection, err := db.Ancient(trienodeHistoryValueSectionTable, id-1)
if err != nil {
return nil, nil, nil, err
}
return header, keySection, valueSection, nil
}
// ReadTrienodeHistoryHeader retrieves the header section of trienode history.
func ReadTrienodeHistoryHeader(db ethdb.AncientReaderOp, id uint64) ([]byte, error) {
return db.Ancient(trienodeHistoryHeaderTable, id-1)
}
// ReadTrienodeHistoryKeySection retrieves the key section of trienode history.
func ReadTrienodeHistoryKeySection(db ethdb.AncientReaderOp, id uint64) ([]byte, error) {
return db.Ancient(trienodeHistoryKeySectionTable, id-1)
}
// ReadTrienodeHistoryValueSection retrieves the value section of trienode history.
func ReadTrienodeHistoryValueSection(db ethdb.AncientReaderOp, id uint64) ([]byte, error) {
return db.Ancient(trienodeHistoryValueSectionTable, id-1)
}
// ReadTrienodeHistoryList retrieves the a list of trienode history corresponding
// to the specified range.
// Compute the position of trienode history in freezer by minus one since the id
// of first trienode history starts from one(zero for initial state).
func ReadTrienodeHistoryList(db ethdb.AncientReaderOp, start uint64, count uint64) ([][]byte, [][]byte, [][]byte, error) {
header, err := db.AncientRange(trienodeHistoryHeaderTable, start-1, count, 0)
if err != nil {
return nil, nil, nil, err
}
keySection, err := db.AncientRange(trienodeHistoryKeySectionTable, start-1, count, 0)
if err != nil {
return nil, nil, nil, err
}
valueSection, err := db.AncientRange(trienodeHistoryValueSectionTable, start-1, count, 0)
if err != nil {
return nil, nil, nil, err
}
if len(header) != len(keySection) || len(header) != len(valueSection) {
return nil, nil, nil, errors.New("trienode history is corrupted")
}
return header, keySection, valueSection, nil
}
// WriteTrienodeHistory writes the provided trienode history to database.
// Compute the position of trienode history in freezer by minus one since
// the id of first state history starts from one(zero for initial state).
func WriteTrienodeHistory(db ethdb.AncientWriter, id uint64, header []byte, keySection []byte, valueSection []byte) error {
_, err := db.ModifyAncients(func(op ethdb.AncientWriteOp) error {
if err := op.AppendRaw(trienodeHistoryHeaderTable, id-1, header); err != nil {
return err
}
if err := op.AppendRaw(trienodeHistoryKeySectionTable, id-1, keySection); err != nil {
return err
}
return op.AppendRaw(trienodeHistoryValueSectionTable, id-1, valueSection)
})
return err
}

View file

@ -75,15 +75,38 @@ var stateFreezerTableConfigs = map[string]freezerTableConfig{
stateHistoryStorageData: {noSnappy: false, prunable: true}, stateHistoryStorageData: {noSnappy: false, prunable: true},
} }
const (
trienodeHistoryHeaderTable = "trienode.header"
trienodeHistoryKeySectionTable = "trienode.key"
trienodeHistoryValueSectionTable = "trienode.value"
)
// trienodeFreezerTableConfigs configures the settings for tables in the trienode freezer.
var trienodeFreezerTableConfigs = map[string]freezerTableConfig{
trienodeHistoryHeaderTable: {noSnappy: false, prunable: true},
// Disable snappy compression to allow efficient partial read.
trienodeHistoryKeySectionTable: {noSnappy: true, prunable: true},
// Disable snappy compression to allow efficient partial read.
trienodeHistoryValueSectionTable: {noSnappy: true, prunable: true},
}
// The list of identifiers of ancient stores. // The list of identifiers of ancient stores.
var ( var (
ChainFreezerName = "chain" // the folder name of chain segment ancient store. ChainFreezerName = "chain" // the folder name of chain segment ancient store.
MerkleStateFreezerName = "state" // the folder name of state history ancient store. MerkleStateFreezerName = "state" // the folder name of state history ancient store.
VerkleStateFreezerName = "state_verkle" // the folder name of state history ancient store. VerkleStateFreezerName = "state_verkle" // the folder name of state history ancient store.
MerkleTrienodeFreezerName = "trienode" // the folder name of trienode history ancient store.
VerkleTrienodeFreezerName = "trienode_verkle" // the folder name of trienode history ancient store.
) )
// freezers the collections of all builtin freezers. // freezers the collections of all builtin freezers.
var freezers = []string{ChainFreezerName, MerkleStateFreezerName, VerkleStateFreezerName} var freezers = []string{
ChainFreezerName,
MerkleStateFreezerName, VerkleStateFreezerName,
MerkleTrienodeFreezerName, VerkleTrienodeFreezerName,
}
// NewStateFreezer initializes the ancient store for state history. // NewStateFreezer initializes the ancient store for state history.
// //
@ -103,3 +126,22 @@ func NewStateFreezer(ancientDir string, verkle bool, readOnly bool) (ethdb.Reset
} }
return newResettableFreezer(name, "eth/db/state", readOnly, stateHistoryTableSize, stateFreezerTableConfigs) return newResettableFreezer(name, "eth/db/state", readOnly, stateHistoryTableSize, stateFreezerTableConfigs)
} }
// NewTrienodeFreezer initializes the ancient store for trienode history.
//
// - if the empty directory is given, initializes the pure in-memory
// trienode freezer (e.g. dev mode).
// - if non-empty directory is given, initializes the regular file-based
// trienode freezer.
func NewTrienodeFreezer(ancientDir string, verkle bool, readOnly bool) (ethdb.ResettableAncientStore, error) {
if ancientDir == "" {
return NewMemoryFreezer(readOnly, trienodeFreezerTableConfigs), nil
}
var name string
if verkle {
name = filepath.Join(ancientDir, VerkleTrienodeFreezerName)
} else {
name = filepath.Join(ancientDir, MerkleTrienodeFreezerName)
}
return newResettableFreezer(name, "eth/db/trienode", readOnly, stateHistoryTableSize, trienodeFreezerTableConfigs)
}

View file

@ -80,6 +80,10 @@ var (
// been indexed. // been indexed.
headStateHistoryIndexKey = []byte("LastStateHistoryIndex") headStateHistoryIndexKey = []byte("LastStateHistoryIndex")
// headTrienodeHistoryIndexKey tracks the ID of the latest state history that has
// been indexed.
headTrienodeHistoryIndexKey = []byte("LastTrienodeHistoryIndex")
// txIndexTailKey tracks the oldest block whose transactions have been indexed. // txIndexTailKey tracks the oldest block whose transactions have been indexed.
txIndexTailKey = []byte("TransactionIndexTail") txIndexTailKey = []byte("TransactionIndexTail")
@ -125,8 +129,10 @@ var (
StateHistoryIndexPrefix = []byte("m") // The global prefix of state history index data StateHistoryIndexPrefix = []byte("m") // The global prefix of state history index data
StateHistoryAccountMetadataPrefix = []byte("ma") // StateHistoryAccountMetadataPrefix + account address hash => account metadata StateHistoryAccountMetadataPrefix = []byte("ma") // StateHistoryAccountMetadataPrefix + account address hash => account metadata
StateHistoryStorageMetadataPrefix = []byte("ms") // StateHistoryStorageMetadataPrefix + account address hash + storage slot hash => slot metadata StateHistoryStorageMetadataPrefix = []byte("ms") // StateHistoryStorageMetadataPrefix + account address hash + storage slot hash => slot metadata
TrienodeHistoryMetadataPrefix = []byte("mt") // TrienodeHistoryMetadataPrefix + account address hash + trienode path => trienode metadata
StateHistoryAccountBlockPrefix = []byte("mba") // StateHistoryAccountBlockPrefix + account address hash + blockID => account block StateHistoryAccountBlockPrefix = []byte("mba") // StateHistoryAccountBlockPrefix + account address hash + blockID => account block
StateHistoryStorageBlockPrefix = []byte("mbs") // StateHistoryStorageBlockPrefix + account address hash + storage slot hash + blockID => slot block StateHistoryStorageBlockPrefix = []byte("mbs") // StateHistoryStorageBlockPrefix + account address hash + storage slot hash + blockID => slot block
TrienodeHistoryBlockPrefix = []byte("mbt") // TrienodeHistoryBlockPrefix + account address hash + trienode path + blockID => trienode block
// VerklePrefix is the database prefix for Verkle trie data, which includes: // VerklePrefix is the database prefix for Verkle trie data, which includes:
// (a) Trie nodes // (a) Trie nodes
@ -395,27 +401,34 @@ func storageHistoryIndexKey(addressHash common.Hash, storageHash common.Hash) []
return out return out
} }
// trienodeHistoryIndexKey = TrienodeHistoryMetadataPrefix + addressHash + trienode path
func trienodeHistoryIndexKey(addressHash common.Hash, path []byte) []byte {
totalLen := len(TrienodeHistoryMetadataPrefix) + common.HashLength + len(path)
out := make([]byte, totalLen)
off := 0
off += copy(out[off:], TrienodeHistoryMetadataPrefix)
off += copy(out[off:], addressHash.Bytes())
copy(out[off:], path)
return out
}
// accountHistoryIndexBlockKey = StateHistoryAccountBlockPrefix + addressHash + blockID // accountHistoryIndexBlockKey = StateHistoryAccountBlockPrefix + addressHash + blockID
func accountHistoryIndexBlockKey(addressHash common.Hash, blockID uint32) []byte { func accountHistoryIndexBlockKey(addressHash common.Hash, blockID uint32) []byte {
var buf4 [4]byte
binary.BigEndian.PutUint32(buf4[:], blockID)
totalLen := len(StateHistoryAccountBlockPrefix) + common.HashLength + 4 totalLen := len(StateHistoryAccountBlockPrefix) + common.HashLength + 4
out := make([]byte, totalLen) out := make([]byte, totalLen)
off := 0 off := 0
off += copy(out[off:], StateHistoryAccountBlockPrefix) off += copy(out[off:], StateHistoryAccountBlockPrefix)
off += copy(out[off:], addressHash.Bytes()) off += copy(out[off:], addressHash.Bytes())
copy(out[off:], buf4[:]) binary.BigEndian.PutUint32(out[off:], blockID)
return out return out
} }
// storageHistoryIndexBlockKey = StateHistoryStorageBlockPrefix + addressHash + storageHash + blockID // storageHistoryIndexBlockKey = StateHistoryStorageBlockPrefix + addressHash + storageHash + blockID
func storageHistoryIndexBlockKey(addressHash common.Hash, storageHash common.Hash, blockID uint32) []byte { func storageHistoryIndexBlockKey(addressHash common.Hash, storageHash common.Hash, blockID uint32) []byte {
var buf4 [4]byte
binary.BigEndian.PutUint32(buf4[:], blockID)
totalLen := len(StateHistoryStorageBlockPrefix) + 2*common.HashLength + 4 totalLen := len(StateHistoryStorageBlockPrefix) + 2*common.HashLength + 4
out := make([]byte, totalLen) out := make([]byte, totalLen)
@ -423,7 +436,21 @@ func storageHistoryIndexBlockKey(addressHash common.Hash, storageHash common.Has
off += copy(out[off:], StateHistoryStorageBlockPrefix) off += copy(out[off:], StateHistoryStorageBlockPrefix)
off += copy(out[off:], addressHash.Bytes()) off += copy(out[off:], addressHash.Bytes())
off += copy(out[off:], storageHash.Bytes()) off += copy(out[off:], storageHash.Bytes())
copy(out[off:], buf4[:]) binary.BigEndian.PutUint32(out[off:], blockID)
return out
}
// trienodeHistoryIndexBlockKey = TrienodeHistoryBlockPrefix + addressHash + trienode path + blockID
func trienodeHistoryIndexBlockKey(addressHash common.Hash, path []byte, blockID uint32) []byte {
totalLen := len(TrienodeHistoryBlockPrefix) + common.HashLength + len(path) + 4
out := make([]byte, totalLen)
off := 0
off += copy(out[off:], TrienodeHistoryBlockPrefix)
off += copy(out[off:], addressHash.Bytes())
off += copy(out[off:], path)
binary.BigEndian.PutUint32(out[off:], blockID)
return out return out
} }

View file

@ -232,7 +232,7 @@ func (db *Database) repairHistory() error {
// Purge all state history indexing data first // Purge all state history indexing data first
batch := db.diskdb.NewBatch() batch := db.diskdb.NewBatch()
rawdb.DeleteStateHistoryIndexMetadata(batch) rawdb.DeleteStateHistoryIndexMetadata(batch)
rawdb.DeleteStateHistoryIndex(batch) rawdb.DeleteStateHistoryIndexes(batch)
if err := batch.Write(); err != nil { if err := batch.Write(); err != nil {
log.Crit("Failed to purge state history index", "err", err) log.Crit("Failed to purge state history index", "err", err)
} }
@ -426,7 +426,7 @@ func (db *Database) Enable(root common.Hash) error {
// Purge all state history indexing data first // Purge all state history indexing data first
batch.Reset() batch.Reset()
rawdb.DeleteStateHistoryIndexMetadata(batch) rawdb.DeleteStateHistoryIndexMetadata(batch)
rawdb.DeleteStateHistoryIndex(batch) rawdb.DeleteStateHistoryIndexes(batch)
if err := batch.Write(); err != nil { if err := batch.Write(); err != nil {
return err return err
} }

View file

@ -32,6 +32,9 @@ type historyType uint8
const ( const (
// typeStateHistory indicates history data related to account or storage changes. // typeStateHistory indicates history data related to account or storage changes.
typeStateHistory historyType = 0 typeStateHistory historyType = 0
// typeTrienodeHistory indicates history data related to trie node changes.
typeTrienodeHistory historyType = 1
) )
// String returns the string format representation. // String returns the string format representation.
@ -39,6 +42,8 @@ func (h historyType) String() string {
switch h { switch h {
case typeStateHistory: case typeStateHistory:
return "state" return "state"
case typeTrienodeHistory:
return "trienode"
default: default:
return fmt.Sprintf("unknown type: %d", h) return fmt.Sprintf("unknown type: %d", h)
} }
@ -48,8 +53,9 @@ func (h historyType) String() string {
type elementType uint8 type elementType uint8
const ( const (
typeAccount elementType = 0 // represents the account data typeAccount elementType = 0 // represents the account data
typeStorage elementType = 1 // represents the storage slot data typeStorage elementType = 1 // represents the storage slot data
typeTrienode elementType = 2 // represents the trie node data
) )
// String returns the string format representation. // String returns the string format representation.
@ -59,6 +65,8 @@ func (e elementType) String() string {
return "account" return "account"
case typeStorage: case typeStorage:
return "storage" return "storage"
case typeTrienode:
return "trienode"
default: default:
return fmt.Sprintf("unknown element type: %d", e) return fmt.Sprintf("unknown element type: %d", e)
} }
@ -69,11 +77,14 @@ func toHistoryType(typ elementType) historyType {
if typ == typeAccount || typ == typeStorage { if typ == typeAccount || typ == typeStorage {
return typeStateHistory return typeStateHistory
} }
if typ == typeTrienode {
return typeTrienodeHistory
}
panic(fmt.Sprintf("unknown element type %v", typ)) panic(fmt.Sprintf("unknown element type %v", typ))
} }
// stateIdent represents the identifier of a state element, which can be // stateIdent represents the identifier of a state element, which can be
// an account or a storage slot. // an account, a storage slot or a trienode.
type stateIdent struct { type stateIdent struct {
typ elementType typ elementType
@ -91,6 +102,12 @@ type stateIdent struct {
// //
// This field is null if the identifier refers to an account or a trie node. // This field is null if the identifier refers to an account or a trie node.
storageHash common.Hash storageHash common.Hash
// The trie node path within the trie.
//
// This field is null if the identifier refers to an account or a storage slot.
// String type is chosen to make stateIdent comparable.
path string
} }
// String returns the string format state identifier. // String returns the string format state identifier.
@ -98,7 +115,10 @@ func (ident stateIdent) String() string {
if ident.typ == typeAccount { if ident.typ == typeAccount {
return ident.addressHash.Hex() return ident.addressHash.Hex()
} }
return ident.addressHash.Hex() + ident.storageHash.Hex() if ident.typ == typeStorage {
return ident.addressHash.Hex() + ident.storageHash.Hex()
}
return ident.addressHash.Hex() + ident.path
} }
// newAccountIdent constructs a state identifier for an account. // newAccountIdent constructs a state identifier for an account.
@ -120,8 +140,18 @@ func newStorageIdent(addressHash common.Hash, storageHash common.Hash) stateIden
} }
} }
// stateIdentQuery is the extension of stateIdent by adding the account address // newTrienodeIdent constructs a state identifier for a trie node.
// and raw storage key. // The address denotes the address hash of the associated account;
// the path denotes the path of the node within the trie;
func newTrienodeIdent(addressHash common.Hash, path string) stateIdent {
return stateIdent{
typ: typeTrienode,
addressHash: addressHash,
path: path,
}
}
// stateIdentQuery is the extension of stateIdent by adding the raw storage key.
type stateIdentQuery struct { type stateIdentQuery struct {
stateIdent stateIdent
@ -150,8 +180,19 @@ func newStorageIdentQuery(address common.Address, addressHash common.Hash, stora
} }
} }
// history defines the interface of historical data, implemented by stateHistory // newTrienodeIdentQuery constructs a state identifier for a trie node.
// and trienodeHistory (in the near future). // the addressHash denotes the address hash of the associated account;
// the path denotes the path of the node within the trie;
//
// nolint:unused
func newTrienodeIdentQuery(addrHash common.Hash, path []byte) stateIdentQuery {
return stateIdentQuery{
stateIdent: newTrienodeIdent(addrHash, string(path)),
}
}
// history defines the interface of historical data, shared by stateHistory
// and trienodeHistory.
type history interface { type history interface {
// typ returns the historical data type held in the history. // typ returns the historical data type held in the history.
typ() historyType typ() historyType

View file

@ -376,6 +376,8 @@ func readStateIndex(ident stateIdent, db ethdb.KeyValueReader) []byte {
return rawdb.ReadAccountHistoryIndex(db, ident.addressHash) return rawdb.ReadAccountHistoryIndex(db, ident.addressHash)
case typeStorage: case typeStorage:
return rawdb.ReadStorageHistoryIndex(db, ident.addressHash, ident.storageHash) return rawdb.ReadStorageHistoryIndex(db, ident.addressHash, ident.storageHash)
case typeTrienode:
return rawdb.ReadTrienodeHistoryIndex(db, ident.addressHash, []byte(ident.path))
default: default:
panic(fmt.Errorf("unknown type: %v", ident.typ)) panic(fmt.Errorf("unknown type: %v", ident.typ))
} }
@ -389,6 +391,8 @@ func writeStateIndex(ident stateIdent, db ethdb.KeyValueWriter, data []byte) {
rawdb.WriteAccountHistoryIndex(db, ident.addressHash, data) rawdb.WriteAccountHistoryIndex(db, ident.addressHash, data)
case typeStorage: case typeStorage:
rawdb.WriteStorageHistoryIndex(db, ident.addressHash, ident.storageHash, data) rawdb.WriteStorageHistoryIndex(db, ident.addressHash, ident.storageHash, data)
case typeTrienode:
rawdb.WriteTrienodeHistoryIndex(db, ident.addressHash, []byte(ident.path), data)
default: default:
panic(fmt.Errorf("unknown type: %v", ident.typ)) panic(fmt.Errorf("unknown type: %v", ident.typ))
} }
@ -402,6 +406,8 @@ func deleteStateIndex(ident stateIdent, db ethdb.KeyValueWriter) {
rawdb.DeleteAccountHistoryIndex(db, ident.addressHash) rawdb.DeleteAccountHistoryIndex(db, ident.addressHash)
case typeStorage: case typeStorage:
rawdb.DeleteStorageHistoryIndex(db, ident.addressHash, ident.storageHash) rawdb.DeleteStorageHistoryIndex(db, ident.addressHash, ident.storageHash)
case typeTrienode:
rawdb.DeleteTrienodeHistoryIndex(db, ident.addressHash, []byte(ident.path))
default: default:
panic(fmt.Errorf("unknown type: %v", ident.typ)) panic(fmt.Errorf("unknown type: %v", ident.typ))
} }
@ -415,6 +421,8 @@ func readStateIndexBlock(ident stateIdent, db ethdb.KeyValueReader, id uint32) [
return rawdb.ReadAccountHistoryIndexBlock(db, ident.addressHash, id) return rawdb.ReadAccountHistoryIndexBlock(db, ident.addressHash, id)
case typeStorage: case typeStorage:
return rawdb.ReadStorageHistoryIndexBlock(db, ident.addressHash, ident.storageHash, id) return rawdb.ReadStorageHistoryIndexBlock(db, ident.addressHash, ident.storageHash, id)
case typeTrienode:
return rawdb.ReadTrienodeHistoryIndexBlock(db, ident.addressHash, []byte(ident.path), id)
default: default:
panic(fmt.Errorf("unknown type: %v", ident.typ)) panic(fmt.Errorf("unknown type: %v", ident.typ))
} }
@ -428,6 +436,8 @@ func writeStateIndexBlock(ident stateIdent, db ethdb.KeyValueWriter, id uint32,
rawdb.WriteAccountHistoryIndexBlock(db, ident.addressHash, id, data) rawdb.WriteAccountHistoryIndexBlock(db, ident.addressHash, id, data)
case typeStorage: case typeStorage:
rawdb.WriteStorageHistoryIndexBlock(db, ident.addressHash, ident.storageHash, id, data) rawdb.WriteStorageHistoryIndexBlock(db, ident.addressHash, ident.storageHash, id, data)
case typeTrienode:
rawdb.WriteTrienodeHistoryIndexBlock(db, ident.addressHash, []byte(ident.path), id, data)
default: default:
panic(fmt.Errorf("unknown type: %v", ident.typ)) panic(fmt.Errorf("unknown type: %v", ident.typ))
} }
@ -441,6 +451,8 @@ func deleteStateIndexBlock(ident stateIdent, db ethdb.KeyValueWriter, id uint32)
rawdb.DeleteAccountHistoryIndexBlock(db, ident.addressHash, id) rawdb.DeleteAccountHistoryIndexBlock(db, ident.addressHash, id)
case typeStorage: case typeStorage:
rawdb.DeleteStorageHistoryIndexBlock(db, ident.addressHash, ident.storageHash, id) rawdb.DeleteStorageHistoryIndexBlock(db, ident.addressHash, ident.storageHash, id)
case typeTrienode:
rawdb.DeleteTrienodeHistoryIndexBlock(db, ident.addressHash, []byte(ident.path), id)
default: default:
panic(fmt.Errorf("unknown type: %v", ident.typ)) panic(fmt.Errorf("unknown type: %v", ident.typ))
} }

View file

@ -36,8 +36,10 @@ const (
// The batch size for reading state histories // The batch size for reading state histories
historyReadBatch = 1000 historyReadBatch = 1000
stateIndexV0 = uint8(0) // initial version of state index structure stateHistoryIndexV0 = uint8(0) // initial version of state index structure
stateIndexVersion = stateIndexV0 // the current state index version stateHistoryIndexVersion = stateHistoryIndexV0 // the current state index version
trienodeHistoryIndexV0 = uint8(0) // initial version of trienode index structure
trienodeHistoryIndexVersion = trienodeHistoryIndexV0 // the current trienode index version
) )
// indexVersion returns the latest index version for the given history type. // indexVersion returns the latest index version for the given history type.
@ -45,7 +47,9 @@ const (
func indexVersion(typ historyType) uint8 { func indexVersion(typ historyType) uint8 {
switch typ { switch typ {
case typeStateHistory: case typeStateHistory:
return stateIndexVersion return stateHistoryIndexVersion
case typeTrienodeHistory:
return trienodeHistoryIndexVersion
default: default:
panic(fmt.Errorf("unknown history type: %d", typ)) panic(fmt.Errorf("unknown history type: %d", typ))
} }
@ -63,6 +67,8 @@ func loadIndexMetadata(db ethdb.KeyValueReader, typ historyType) *indexMetadata
switch typ { switch typ {
case typeStateHistory: case typeStateHistory:
blob = rawdb.ReadStateHistoryIndexMetadata(db) blob = rawdb.ReadStateHistoryIndexMetadata(db)
case typeTrienodeHistory:
blob = rawdb.ReadTrienodeHistoryIndexMetadata(db)
default: default:
panic(fmt.Errorf("unknown history type %d", typ)) panic(fmt.Errorf("unknown history type %d", typ))
} }
@ -90,6 +96,8 @@ func storeIndexMetadata(db ethdb.KeyValueWriter, typ historyType, last uint64) {
switch typ { switch typ {
case typeStateHistory: case typeStateHistory:
rawdb.WriteStateHistoryIndexMetadata(db, blob) rawdb.WriteStateHistoryIndexMetadata(db, blob)
case typeTrienodeHistory:
rawdb.WriteTrienodeHistoryIndexMetadata(db, blob)
default: default:
panic(fmt.Errorf("unknown history type %d", typ)) panic(fmt.Errorf("unknown history type %d", typ))
} }
@ -101,6 +109,8 @@ func deleteIndexMetadata(db ethdb.KeyValueWriter, typ historyType) {
switch typ { switch typ {
case typeStateHistory: case typeStateHistory:
rawdb.DeleteStateHistoryIndexMetadata(db) rawdb.DeleteStateHistoryIndexMetadata(db)
case typeTrienodeHistory:
rawdb.DeleteTrienodeHistoryIndexMetadata(db)
default: default:
panic(fmt.Errorf("unknown history type %d", typ)) panic(fmt.Errorf("unknown history type %d", typ))
} }
@ -215,7 +225,11 @@ func (b *batchIndexer) finish(force bool) error {
func indexSingle(historyID uint64, db ethdb.KeyValueStore, freezer ethdb.AncientReader, typ historyType) error { func indexSingle(historyID uint64, db ethdb.KeyValueStore, freezer ethdb.AncientReader, typ historyType) error {
start := time.Now() start := time.Now()
defer func() { defer func() {
indexHistoryTimer.UpdateSince(start) if typ == typeStateHistory {
stateIndexHistoryTimer.UpdateSince(start)
} else if typ == typeTrienodeHistory {
trienodeIndexHistoryTimer.UpdateSince(start)
}
}() }()
metadata := loadIndexMetadata(db, typ) metadata := loadIndexMetadata(db, typ)
@ -234,7 +248,7 @@ func indexSingle(historyID uint64, db ethdb.KeyValueStore, freezer ethdb.Ancient
if typ == typeStateHistory { if typ == typeStateHistory {
h, err = readStateHistory(freezer, historyID) h, err = readStateHistory(freezer, historyID)
} else { } else {
// h, err = readTrienodeHistory(freezer, historyID) h, err = readTrienodeHistory(freezer, historyID)
} }
if err != nil { if err != nil {
return err return err
@ -253,7 +267,11 @@ func indexSingle(historyID uint64, db ethdb.KeyValueStore, freezer ethdb.Ancient
func unindexSingle(historyID uint64, db ethdb.KeyValueStore, freezer ethdb.AncientReader, typ historyType) error { func unindexSingle(historyID uint64, db ethdb.KeyValueStore, freezer ethdb.AncientReader, typ historyType) error {
start := time.Now() start := time.Now()
defer func() { defer func() {
unindexHistoryTimer.UpdateSince(start) if typ == typeStateHistory {
stateUnindexHistoryTimer.UpdateSince(start)
} else if typ == typeTrienodeHistory {
trienodeUnindexHistoryTimer.UpdateSince(start)
}
}() }()
metadata := loadIndexMetadata(db, typ) metadata := loadIndexMetadata(db, typ)
@ -272,7 +290,7 @@ func unindexSingle(historyID uint64, db ethdb.KeyValueStore, freezer ethdb.Ancie
if typ == typeStateHistory { if typ == typeStateHistory {
h, err = readStateHistory(freezer, historyID) h, err = readStateHistory(freezer, historyID)
} else { } else {
// h, err = readTrienodeHistory(freezer, historyID) h, err = readTrienodeHistory(freezer, historyID)
} }
if err != nil { if err != nil {
return err return err
@ -546,13 +564,13 @@ func (i *indexIniter) index(done chan struct{}, interrupt *atomic.Int32, lastID
return return
} }
} else { } else {
// histories, err = readTrienodeHistories(i.freezer, current, count) histories, err = readTrienodeHistories(i.freezer, current, count)
// if err != nil { if err != nil {
// // The history read might fall if the history is truncated from // The history read might fall if the history is truncated from
// // head due to revert operation. // head due to revert operation.
// i.log.Error("Failed to read history for indexing", "current", current, "count", count, "err", err) i.log.Error("Failed to read history for indexing", "current", current, "count", count, "err", err)
// return return
// } }
} }
for _, h := range histories { for _, h := range histories {
if err := batch.process(h, current); err != nil { if err := batch.process(h, current); err != nil {
@ -570,7 +588,7 @@ func (i *indexIniter) index(done chan struct{}, interrupt *atomic.Int32, lastID
done = current - beginID done = current - beginID
) )
eta := common.CalculateETA(done, left, time.Since(start)) eta := common.CalculateETA(done, left, time.Since(start))
i.log.Info("Indexing state history", "processed", done, "left", left, "elapsed", common.PrettyDuration(time.Since(start)), "eta", common.PrettyDuration(eta)) i.log.Info("Indexing history", "processed", done, "left", left, "elapsed", common.PrettyDuration(time.Since(start)), "eta", common.PrettyDuration(eta))
} }
} }
i.indexed.Store(current - 1) // update indexing progress i.indexed.Store(current - 1) // update indexing progress
@ -657,6 +675,8 @@ func checkVersion(disk ethdb.KeyValueStore, typ historyType) {
var blob []byte var blob []byte
if typ == typeStateHistory { if typ == typeStateHistory {
blob = rawdb.ReadStateHistoryIndexMetadata(disk) blob = rawdb.ReadStateHistoryIndexMetadata(disk)
} else if typ == typeTrienodeHistory {
blob = rawdb.ReadTrienodeHistoryIndexMetadata(disk)
} else { } else {
panic(fmt.Errorf("unknown history type: %v", typ)) panic(fmt.Errorf("unknown history type: %v", typ))
} }
@ -666,24 +686,32 @@ func checkVersion(disk ethdb.KeyValueStore, typ historyType) {
return return
} }
// Short circuit if the metadata is found and the version is matched // Short circuit if the metadata is found and the version is matched
ver := stateHistoryIndexVersion
if typ == typeTrienodeHistory {
ver = trienodeHistoryIndexVersion
}
var m indexMetadata var m indexMetadata
err := rlp.DecodeBytes(blob, &m) err := rlp.DecodeBytes(blob, &m)
if err == nil && m.Version == stateIndexVersion { if err == nil && m.Version == ver {
return return
} }
// Version is not matched, prune the existing data and re-index from scratch // Version is not matched, prune the existing data and re-index from scratch
batch := disk.NewBatch()
if typ == typeStateHistory {
rawdb.DeleteStateHistoryIndexMetadata(batch)
rawdb.DeleteStateHistoryIndexes(batch)
} else {
rawdb.DeleteTrienodeHistoryIndexMetadata(batch)
rawdb.DeleteTrienodeHistoryIndexes(batch)
}
if err := batch.Write(); err != nil {
log.Crit("Failed to purge history index", "type", typ, "err", err)
}
version := "unknown" version := "unknown"
if err == nil { if err == nil {
version = fmt.Sprintf("%d", m.Version) version = fmt.Sprintf("%d", m.Version)
} }
log.Info("Cleaned up obsolete history index", "type", typ, "version", version, "want", version)
batch := disk.NewBatch()
rawdb.DeleteStateHistoryIndexMetadata(batch)
rawdb.DeleteStateHistoryIndex(batch)
if err := batch.Write(); err != nil {
log.Crit("Failed to purge state history index", "err", err)
}
log.Info("Cleaned up obsolete state history index", "version", version, "want", stateIndexVersion)
} }
// newHistoryIndexer constructs the history indexer and launches the background // newHistoryIndexer constructs the history indexer and launches the background

View file

@ -605,9 +605,9 @@ func writeStateHistory(writer ethdb.AncientWriter, dl *diffLayer) error {
if err := rawdb.WriteStateHistory(writer, dl.stateID(), history.meta.encode(), accountIndex, storageIndex, accountData, storageData); err != nil { if err := rawdb.WriteStateHistory(writer, dl.stateID(), history.meta.encode(), accountIndex, storageIndex, accountData, storageData); err != nil {
return err return err
} }
historyDataBytesMeter.Mark(int64(dataSize)) stateHistoryDataBytesMeter.Mark(int64(dataSize))
historyIndexBytesMeter.Mark(int64(indexSize)) stateHistoryIndexBytesMeter.Mark(int64(indexSize))
historyBuildTimeMeter.UpdateSince(start) stateHistoryBuildTimeMeter.UpdateSince(start)
log.Debug("Stored state history", "id", dl.stateID(), "block", dl.block, "data", dataSize, "index", indexSize, "elapsed", common.PrettyDuration(time.Since(start))) log.Debug("Stored state history", "id", dl.stateID(), "block", dl.block, "data", dataSize, "index", indexSize, "elapsed", common.PrettyDuration(time.Since(start)))
return nil return nil

View file

@ -98,13 +98,13 @@ func testEncodeDecodeStateHistory(t *testing.T, rawStorageKey bool) {
if !compareSet(dec.accounts, obj.accounts) { if !compareSet(dec.accounts, obj.accounts) {
t.Fatal("account data is mismatched") t.Fatal("account data is mismatched")
} }
if !compareStorages(dec.storages, obj.storages) { if !compareMapSet(dec.storages, obj.storages) {
t.Fatal("storage data is mismatched") t.Fatal("storage data is mismatched")
} }
if !compareList(dec.accountList, obj.accountList) { if !compareList(dec.accountList, obj.accountList) {
t.Fatal("account list is mismatched") t.Fatal("account list is mismatched")
} }
if !compareStorageList(dec.storageList, obj.storageList) { if !compareMapList(dec.storageList, obj.storageList) {
t.Fatal("storage list is mismatched") t.Fatal("storage list is mismatched")
} }
} }
@ -292,32 +292,32 @@ func compareList[k comparable](a, b []k) bool {
return true return true
} }
func compareStorages(a, b map[common.Address]map[common.Hash][]byte) bool { func compareMapSet[K1 comparable, K2 comparable](a, b map[K1]map[K2][]byte) bool {
if len(a) != len(b) { if len(a) != len(b) {
return false return false
} }
for h, subA := range a { for key, subsetA := range a {
subB, ok := b[h] subsetB, ok := b[key]
if !ok { if !ok {
return false return false
} }
if !compareSet(subA, subB) { if !compareSet(subsetA, subsetB) {
return false return false
} }
} }
return true return true
} }
func compareStorageList(a, b map[common.Address][]common.Hash) bool { func compareMapList[K comparable, V comparable](a, b map[K][]V) bool {
if len(a) != len(b) { if len(a) != len(b) {
return false return false
} }
for h, la := range a { for key, listA := range a {
lb, ok := b[h] listB, ok := b[key]
if !ok { if !ok {
return false return false
} }
if !compareList(la, lb) { if !compareList(listA, listB) {
return false return false
} }
} }

View file

@ -0,0 +1,730 @@
// Copyright 2025 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package pathdb
import (
"bytes"
"encoding/binary"
"fmt"
"iter"
"maps"
"math"
"slices"
"sort"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/log"
)
// Each trie node history entry consists of three parts (stored in three freezer
// tables according):
//
// # Header
// The header records metadata, including:
//
// - the history version (1 byte)
// - the parent state root (32 bytes)
// - the current state root (32 bytes)
// - block number (8 bytes)
//
// - a lexicographically sorted list of trie IDs
// - the corresponding offsets into the key and value sections for each trie data chunk
//
// Although some fields (e.g., parent state root, block number) are duplicated
// between the state history and the trienode history, these two histories
// operate independently. To ensure each remains self-contained and self-descriptive,
// we have chosen to maintain these duplicate fields.
//
// # Key section
// The key section stores trie node keys (paths) in a compressed format.
// It also contains relative offsets into the value section for resolving
// the corresponding trie node data. Note that these offsets are relative
// to the data chunk for the trie; the chunk offset must be added to obtain
// the absolute position.
//
// # Value section
// The value section is a concatenated byte stream of all trie node data.
// Each trie node can be retrieved using the offset and length specified
// by its index entry.
//
// The header and key sections are sufficient for locating a trie node,
// while a partial read of the value section is enough to retrieve its data.
// Header section:
//
// +----------+------------------+---------------------+---------------------+-------+------------------+---------------------+---------------------|
// | metadata | TrieID(32 bytes) | key offset(4 bytes) | val offset(4 bytes) | ... | TrieID(32 bytes) | key offset(4 bytes) | val offset(4 bytes) |
// +----------+------------------+---------------------+---------------------+-------+------------------+---------------------+---------------------|
//
//
// Key section:
//
// + restart point + restart point (depends on restart interval)
// / /
// +---------------+---------------+---------------+---------------+---------+
// | node entry 1 | node entry 2 | ... | node entry n | trailer |
// +---------------+---------------+---------------+---------------+---------+
// \ /
// +---- restart block ------+
//
// node entry:
//
// +---- key len ----+
// / \
// +-------+---------+-----------+---------+-----------------------+-----------------+
// | shared (varint) | not shared (varint) | value length (varlen) | key (varlen) |
// +-----------------+---------------------+-----------------------+-----------------+
//
// trailer:
//
// +---- 4-bytes ----+ +---- 4-bytes ----+
// / \ / \
// +----------------------+------------------------+-----+--------------------------+
// | restart_1 key offset | restart_1 value offset | ... | restart number (4-bytes) |
// +----------------------+------------------------+-----+--------------------------+
//
// Note: Both the key offset and the value offset are relative to the start of
// the trie data chunk. To obtain the absolute offset, add the offset of the
// trie data chunk itself.
//
// Value section:
//
// +--------------+--------------+-------+---------------+
// | node data 1 | node data 2 | ... | node data n |
// +--------------+--------------+-------+---------------+
//
// NOTE: All fixed-length integer are big-endian.
const (
trienodeHistoryV0 = uint8(0) // initial version of node history structure
trienodeHistoryVersion = trienodeHistoryV0 // the default node history version
trienodeMetadataSize = 1 + 2*common.HashLength + 8 // the size of metadata in the history
trienodeTrieHeaderSize = 8 + common.HashLength // the size of a single trie header in history
trienodeDataBlockRestartLen = 16 // The restart interval length of trie node block
)
// trienodeMetadata describes the meta data of trienode history.
type trienodeMetadata struct {
version uint8 // version tag of history object
parent common.Hash // prev-state root before the state transition
root common.Hash // post-state root after the state transition
block uint64 // associated block number
}
// trienodeHistory represents a set of trie node changes resulting from a state
// transition across the main account trie and all associated storage tries.
type trienodeHistory struct {
meta *trienodeMetadata // Metadata of the history
owners []common.Hash // List of trie identifier sorted lexicographically
nodeList map[common.Hash][]string // Set of node paths sorted lexicographically
nodes map[common.Hash]map[string][]byte // Set of original value of trie nodes before state transition
}
// newTrienodeHistory constructs a trienode history with the provided trie nodes.
func newTrienodeHistory(root common.Hash, parent common.Hash, block uint64, nodes map[common.Hash]map[string][]byte) *trienodeHistory {
nodeList := make(map[common.Hash][]string)
for owner, subset := range nodes {
keys := sort.StringSlice(slices.Collect(maps.Keys(subset)))
keys.Sort()
nodeList[owner] = keys
}
return &trienodeHistory{
meta: &trienodeMetadata{
version: trienodeHistoryVersion,
parent: parent,
root: root,
block: block,
},
owners: slices.SortedFunc(maps.Keys(nodes), common.Hash.Cmp),
nodeList: nodeList,
nodes: nodes,
}
}
// sharedLen returns the length of the common prefix shared by a and b.
func sharedLen(a, b []byte) int {
n := min(len(a), len(b))
for i := 0; i < n; i++ {
if a[i] != b[i] {
return i
}
}
return n
}
// typ implements the history interface, returning the historical data type held.
func (h *trienodeHistory) typ() historyType {
return typeTrienodeHistory
}
// forEach implements the history interface, returning an iterator to traverse the
// state entries in the history.
func (h *trienodeHistory) forEach() iter.Seq[stateIdent] {
return func(yield func(stateIdent) bool) {
for _, owner := range h.owners {
for _, path := range h.nodeList[owner] {
if !yield(newTrienodeIdent(owner, path)) {
return
}
}
}
}
}
// encode serializes the contained trie nodes into bytes.
func (h *trienodeHistory) encode() ([]byte, []byte, []byte, error) {
var (
buf = make([]byte, 64)
headerSection bytes.Buffer
keySection bytes.Buffer
valueSection bytes.Buffer
)
binary.Write(&headerSection, binary.BigEndian, h.meta.version) // 1 byte
headerSection.Write(h.meta.parent.Bytes()) // 32 bytes
headerSection.Write(h.meta.root.Bytes()) // 32 bytes
binary.Write(&headerSection, binary.BigEndian, h.meta.block) // 8 byte
for _, owner := range h.owners {
// Fill the header section with offsets at key and value section
headerSection.Write(owner.Bytes()) // 32 bytes
binary.Write(&headerSection, binary.BigEndian, uint32(keySection.Len())) // 4 bytes
// The offset to the value section is theoretically unnecessary, since the
// individual value offset is already tracked in the key section. However,
// we still keep it here for two reasons:
// - It's cheap to store (only 4 bytes for each trie).
// - It can be useful for decoding the trie data when key is not required (e.g., in hash mode).
binary.Write(&headerSection, binary.BigEndian, uint32(valueSection.Len())) // 4 bytes
// Fill the key section with node index
var (
prevKey []byte
restarts []uint32
prefixLen int
internalKeyOffset uint32 // key offset for the trie internally
internalValOffset uint32 // value offset for the trie internally
)
for i, path := range h.nodeList[owner] {
key := []byte(path)
if i%trienodeDataBlockRestartLen == 0 {
restarts = append(restarts, internalKeyOffset)
restarts = append(restarts, internalValOffset)
prefixLen = 0
} else {
prefixLen = sharedLen(prevKey, key)
}
value := h.nodes[owner][path]
// key section
n := binary.PutUvarint(buf[0:], uint64(prefixLen)) // key length shared (varint)
n += binary.PutUvarint(buf[n:], uint64(len(key)-prefixLen)) // key length not shared (varint)
n += binary.PutUvarint(buf[n:], uint64(len(value))) // value length (varint)
if _, err := keySection.Write(buf[:n]); err != nil {
return nil, nil, nil, err
}
// unshared key
if _, err := keySection.Write(key[prefixLen:]); err != nil {
return nil, nil, nil, err
}
n += len(key) - prefixLen
prevKey = key
// value section
if _, err := valueSection.Write(value); err != nil {
return nil, nil, nil, err
}
internalKeyOffset += uint32(n)
internalValOffset += uint32(len(value))
}
// Encode trailer, the number of restart sections is len(restarts))/2,
// as we track the offsets of both key and value sections.
var trailer []byte
for _, number := range append(restarts, uint32(len(restarts))/2) {
binary.BigEndian.PutUint32(buf[:4], number)
trailer = append(trailer, buf[:4]...)
}
if _, err := keySection.Write(trailer); err != nil {
return nil, nil, nil, err
}
}
return headerSection.Bytes(), keySection.Bytes(), valueSection.Bytes(), nil
}
// decodeHeader resolves the metadata from the header section. An error
// should be returned if the header section is corrupted.
func decodeHeader(data []byte) (*trienodeMetadata, []common.Hash, []uint32, []uint32, error) {
if len(data) < trienodeMetadataSize {
return nil, nil, nil, nil, fmt.Errorf("trienode history is too small, index size: %d", len(data))
}
version := data[0]
if version != trienodeHistoryVersion {
return nil, nil, nil, nil, fmt.Errorf("unregonized trienode history version: %d", version)
}
parent := common.BytesToHash(data[1 : common.HashLength+1]) // 32 bytes
root := common.BytesToHash(data[common.HashLength+1 : common.HashLength*2+1]) // 32 bytes
block := binary.BigEndian.Uint64(data[common.HashLength*2+1 : trienodeMetadataSize]) // 8 bytes
size := len(data) - trienodeMetadataSize
if size%trienodeTrieHeaderSize != 0 {
return nil, nil, nil, nil, fmt.Errorf("truncated trienode history data, size %d", len(data))
}
count := size / trienodeTrieHeaderSize
var (
owners = make([]common.Hash, 0, count)
keyOffsets = make([]uint32, 0, count)
valOffsets = make([]uint32, 0, count)
)
for i := 0; i < count; i++ {
n := trienodeMetadataSize + trienodeTrieHeaderSize*i
owner := common.BytesToHash(data[n : n+common.HashLength])
if i != 0 && bytes.Compare(owner.Bytes(), owners[i-1].Bytes()) <= 0 {
return nil, nil, nil, nil, fmt.Errorf("trienode owners are out of order, prev: %v, cur: %v", owners[i-1], owner)
}
owners = append(owners, owner)
// Decode the offset to the key section
keyOffset := binary.BigEndian.Uint32(data[n+common.HashLength : n+common.HashLength+4])
if i != 0 && keyOffset <= keyOffsets[i-1] {
return nil, nil, nil, nil, fmt.Errorf("key offset is out of order, prev: %v, cur: %v", keyOffsets[i-1], keyOffset)
}
keyOffsets = append(keyOffsets, keyOffset)
// Decode the offset into the value section. Note that identical value offsets
// are valid if the node values in the last trie chunk are all zero (e.g., after
// a trie deletion).
valOffset := binary.BigEndian.Uint32(data[n+common.HashLength+4 : n+common.HashLength+8])
if i != 0 && valOffset < valOffsets[i-1] {
return nil, nil, nil, nil, fmt.Errorf("value offset is out of order, prev: %v, cur: %v", valOffsets[i-1], valOffset)
}
valOffsets = append(valOffsets, valOffset)
}
return &trienodeMetadata{
version: version,
parent: parent,
root: root,
block: block,
}, owners, keyOffsets, valOffsets, nil
}
func decodeSingle(keySection []byte, onValue func([]byte, int, int) error) ([]string, error) {
var (
prevKey []byte
items int
keyOffsets []uint32
valOffsets []uint32
keyOff int // the key offset within the single trie data
valOff int // the value offset within the single trie data
keys []string
)
// Decode the number of restart section
if len(keySection) < 4 {
return nil, fmt.Errorf("key section too short, size: %d", len(keySection))
}
nRestarts := binary.BigEndian.Uint32(keySection[len(keySection)-4:])
if len(keySection) < int(8*nRestarts)+4 {
return nil, fmt.Errorf("key section too short, restarts: %d, size: %d", nRestarts, len(keySection))
}
for i := 0; i < int(nRestarts); i++ {
o := len(keySection) - 4 - (int(nRestarts)-i)*8
keyOffset := binary.BigEndian.Uint32(keySection[o : o+4])
if i != 0 && keyOffset <= keyOffsets[i-1] {
return nil, fmt.Errorf("key offset is out of order, prev: %v, cur: %v", keyOffsets[i-1], keyOffset)
}
keyOffsets = append(keyOffsets, keyOffset)
// Same value offset is allowed just in case all the trie nodes in the last
// section have zero-size value.
valOffset := binary.BigEndian.Uint32(keySection[o+4 : o+8])
if i != 0 && valOffset < valOffsets[i-1] {
return nil, fmt.Errorf("value offset is out of order, prev: %v, cur: %v", valOffsets[i-1], valOffset)
}
valOffsets = append(valOffsets, valOffset)
}
keyLimit := len(keySection) - 4 - int(nRestarts)*8
// Decode data
for keyOff < keyLimit {
// Validate the key and value offsets within the single trie data chunk
if items%trienodeDataBlockRestartLen == 0 {
if keyOff != int(keyOffsets[items/trienodeDataBlockRestartLen]) {
return nil, fmt.Errorf("key offset is not matched, recorded: %d, want: %d", keyOffsets[items/trienodeDataBlockRestartLen], keyOff)
}
if valOff != int(valOffsets[items/trienodeDataBlockRestartLen]) {
return nil, fmt.Errorf("value offset is not matched, recorded: %d, want: %d", valOffsets[items/trienodeDataBlockRestartLen], valOff)
}
}
// Resolve the entry from key section
nShared, nn := binary.Uvarint(keySection[keyOff:]) // key length shared (varint)
keyOff += nn
nUnshared, nn := binary.Uvarint(keySection[keyOff:]) // key length not shared (varint)
keyOff += nn
nValue, nn := binary.Uvarint(keySection[keyOff:]) // value length (varint)
keyOff += nn
// Resolve unshared key
if keyOff+int(nUnshared) > len(keySection) {
return nil, fmt.Errorf("key length too long, unshared key length: %d, off: %d, section size: %d", nUnshared, keyOff, len(keySection))
}
unsharedKey := keySection[keyOff : keyOff+int(nUnshared)]
keyOff += int(nUnshared)
// Assemble the full key
var key []byte
if items%trienodeDataBlockRestartLen == 0 {
if nShared != 0 {
return nil, fmt.Errorf("unexpected non-zero shared key prefix: %d", nShared)
}
key = unsharedKey
} else {
if int(nShared) > len(prevKey) {
return nil, fmt.Errorf("unexpected shared key prefix: %d, prefix key length: %d", nShared, len(prevKey))
}
key = append([]byte{}, prevKey[:nShared]...)
key = append(key, unsharedKey...)
}
if items != 0 && bytes.Compare(prevKey, key) >= 0 {
return nil, fmt.Errorf("trienode paths are out of order, prev: %v, cur: %v", prevKey, key)
}
prevKey = key
// Resolve value
if onValue != nil {
if err := onValue(key, valOff, valOff+int(nValue)); err != nil {
return nil, err
}
}
valOff += int(nValue)
items++
keys = append(keys, string(key))
}
if keyOff != keyLimit {
return nil, fmt.Errorf("excessive key data after decoding, offset: %d, size: %d", keyOff, keyLimit)
}
return keys, nil
}
func decodeSingleWithValue(keySection []byte, valueSection []byte) ([]string, map[string][]byte, error) {
var (
offset int
nodes = make(map[string][]byte)
)
paths, err := decodeSingle(keySection, func(key []byte, start int, limit int) error {
if start != offset {
return fmt.Errorf("gapped value section offset: %d, want: %d", start, offset)
}
// start == limit is allowed for zero-value trie node (e.g., non-existent node)
if start > limit {
return fmt.Errorf("invalid value offsets, start: %d, limit: %d", start, limit)
}
if start > len(valueSection) || limit > len(valueSection) {
return fmt.Errorf("value section out of range: start: %d, limit: %d, size: %d", start, limit, len(valueSection))
}
nodes[string(key)] = valueSection[start:limit]
offset = limit
return nil
})
if err != nil {
return nil, nil, err
}
if offset != len(valueSection) {
return nil, nil, fmt.Errorf("excessive value data after decoding, offset: %d, size: %d", offset, len(valueSection))
}
return paths, nodes, nil
}
// decode deserializes the contained trie nodes from the provided bytes.
func (h *trienodeHistory) decode(header []byte, keySection []byte, valueSection []byte) error {
metadata, owners, keyOffsets, valueOffsets, err := decodeHeader(header)
if err != nil {
return err
}
h.meta = metadata
h.owners = owners
h.nodeList = make(map[common.Hash][]string)
h.nodes = make(map[common.Hash]map[string][]byte)
for i := 0; i < len(owners); i++ {
// Resolve the boundary of key section
keyStart := keyOffsets[i]
keyLimit := len(keySection)
if i != len(owners)-1 {
keyLimit = int(keyOffsets[i+1])
}
if int(keyStart) > len(keySection) || keyLimit > len(keySection) {
return fmt.Errorf("invalid key offsets: keyStart: %d, keyLimit: %d, size: %d", keyStart, keyLimit, len(keySection))
}
// Resolve the boundary of value section
valStart := valueOffsets[i]
valLimit := len(valueSection)
if i != len(owners)-1 {
valLimit = int(valueOffsets[i+1])
}
if int(valStart) > len(valueSection) || valLimit > len(valueSection) {
return fmt.Errorf("invalid value offsets: valueStart: %d, valueLimit: %d, size: %d", valStart, valLimit, len(valueSection))
}
// Decode the key and values for this specific trie
paths, nodes, err := decodeSingleWithValue(keySection[keyStart:keyLimit], valueSection[valStart:valLimit])
if err != nil {
return err
}
h.nodeList[owners[i]] = paths
h.nodes[owners[i]] = nodes
}
return nil
}
type iRange struct {
start uint32
limit uint32
}
// singleTrienodeHistoryReader provides read access to a single trie within the
// trienode history. It stores an offset to the trie's position in the history,
// along with a set of per-node offsets that can be resolved on demand.
type singleTrienodeHistoryReader struct {
id uint64
reader ethdb.AncientReader
valueRange iRange // value range within the total value section
valueInternalOffsets map[string]iRange // value offset within the single trie data
}
func newSingleTrienodeHistoryReader(id uint64, reader ethdb.AncientReader, keyRange iRange, valueRange iRange) (*singleTrienodeHistoryReader, error) {
// TODO(rjl493456442) partial freezer read should be supported
keyData, err := rawdb.ReadTrienodeHistoryKeySection(reader, id)
if err != nil {
return nil, err
}
keyStart := int(keyRange.start)
keyLimit := int(keyRange.limit)
if keyLimit == math.MaxUint32 {
keyLimit = len(keyData)
}
if len(keyData) < keyStart || len(keyData) < keyLimit {
return nil, fmt.Errorf("key section too short, start: %d, limit: %d, size: %d", keyStart, keyLimit, len(keyData))
}
valueOffsets := make(map[string]iRange)
_, err = decodeSingle(keyData[keyStart:keyLimit], func(key []byte, start int, limit int) error {
valueOffsets[string(key)] = iRange{
start: uint32(start),
limit: uint32(limit),
}
return nil
})
if err != nil {
return nil, err
}
return &singleTrienodeHistoryReader{
id: id,
reader: reader,
valueRange: valueRange,
valueInternalOffsets: valueOffsets,
}, nil
}
// read retrieves the trie node data with the provided node path.
func (sr *singleTrienodeHistoryReader) read(path string) ([]byte, error) {
offset, exists := sr.valueInternalOffsets[path]
if !exists {
return nil, fmt.Errorf("trienode %v not found", []byte(path))
}
// TODO(rjl493456442) partial freezer read should be supported
valueData, err := rawdb.ReadTrienodeHistoryValueSection(sr.reader, sr.id)
if err != nil {
return nil, err
}
if len(valueData) < int(sr.valueRange.start) {
return nil, fmt.Errorf("value section too short, start: %d, size: %d", sr.valueRange.start, len(valueData))
}
entryStart := sr.valueRange.start + offset.start
entryLimit := sr.valueRange.start + offset.limit
if len(valueData) < int(entryStart) || len(valueData) < int(entryLimit) {
return nil, fmt.Errorf("value section too short, start: %d, limit: %d, size: %d", entryStart, entryLimit, len(valueData))
}
return valueData[int(entryStart):int(entryLimit)], nil
}
// trienodeHistoryReader provides read access to node data in the trie node history.
// It resolves data from the underlying ancient store only when needed, minimizing
// I/O overhead.
type trienodeHistoryReader struct {
id uint64 // ID of the associated trienode history
reader ethdb.AncientReader // Database reader of ancient store
keyRanges map[common.Hash]iRange // Key ranges identifying trie chunks
valRanges map[common.Hash]iRange // Value ranges identifying trie chunks
iReaders map[common.Hash]*singleTrienodeHistoryReader // readers for each individual trie chunk
}
// newTrienodeHistoryReader constructs the reader for specific trienode history.
func newTrienodeHistoryReader(id uint64, reader ethdb.AncientReader) (*trienodeHistoryReader, error) {
r := &trienodeHistoryReader{
id: id,
reader: reader,
keyRanges: make(map[common.Hash]iRange),
valRanges: make(map[common.Hash]iRange),
iReaders: make(map[common.Hash]*singleTrienodeHistoryReader),
}
if err := r.decodeHeader(); err != nil {
return nil, err
}
return r, nil
}
// decodeHeader decodes the header section of trienode history.
func (r *trienodeHistoryReader) decodeHeader() error {
header, err := rawdb.ReadTrienodeHistoryHeader(r.reader, r.id)
if err != nil {
return err
}
_, owners, keyOffsets, valOffsets, err := decodeHeader(header)
if err != nil {
return err
}
for i, owner := range owners {
// Decode the key range for this trie chunk
var keyLimit uint32
if i == len(owners)-1 {
keyLimit = math.MaxUint32
} else {
keyLimit = keyOffsets[i+1]
}
r.keyRanges[owner] = iRange{
start: keyOffsets[i],
limit: keyLimit,
}
// Decode the value range for this trie chunk
var valLimit uint32
if i == len(owners)-1 {
valLimit = math.MaxUint32
} else {
valLimit = valOffsets[i+1]
}
r.valRanges[owner] = iRange{
start: valOffsets[i],
limit: valLimit,
}
}
return nil
}
// read retrieves the trie node data with the provided TrieID and node path.
func (r *trienodeHistoryReader) read(owner common.Hash, path string) ([]byte, error) {
ir, ok := r.iReaders[owner]
if !ok {
keyRange, exists := r.keyRanges[owner]
if !exists {
return nil, fmt.Errorf("trie %x is unknown", owner)
}
valRange, exists := r.valRanges[owner]
if !exists {
return nil, fmt.Errorf("trie %x is unknown", owner)
}
var err error
ir, err = newSingleTrienodeHistoryReader(r.id, r.reader, keyRange, valRange)
if err != nil {
return nil, err
}
r.iReaders[owner] = ir
}
return ir.read(path)
}
// writeTrienodeHistory persists the trienode history associated with the given diff layer.
// nolint:unused
func writeTrienodeHistory(writer ethdb.AncientWriter, dl *diffLayer) error {
start := time.Now()
h := newTrienodeHistory(dl.rootHash(), dl.parent.rootHash(), dl.block, dl.nodes.nodeOrigin)
header, keySection, valueSection, err := h.encode()
if err != nil {
return err
}
// Write history data into five freezer table respectively.
if err := rawdb.WriteTrienodeHistory(writer, dl.stateID(), header, keySection, valueSection); err != nil {
return err
}
trienodeHistoryDataBytesMeter.Mark(int64(len(valueSection)))
trienodeHistoryIndexBytesMeter.Mark(int64(len(header) + len(keySection)))
trienodeHistoryBuildTimeMeter.UpdateSince(start)
log.Debug(
"Stored trienode history", "id", dl.stateID(), "block", dl.block,
"header", common.StorageSize(len(header)),
"keySection", common.StorageSize(len(keySection)),
"valueSection", common.StorageSize(len(valueSection)),
"elapsed", common.PrettyDuration(time.Since(start)),
)
return nil
}
// readTrienodeMetadata resolves the metadata of the specified trienode history.
// nolint:unused
func readTrienodeMetadata(reader ethdb.AncientReader, id uint64) (*trienodeMetadata, error) {
header, err := rawdb.ReadTrienodeHistoryHeader(reader, id)
if err != nil {
return nil, err
}
metadata, _, _, _, err := decodeHeader(header)
if err != nil {
return nil, err
}
return metadata, nil
}
// readTrienodeHistory resolves a single trienode history object with specific id.
func readTrienodeHistory(reader ethdb.AncientReader, id uint64) (*trienodeHistory, error) {
header, keySection, valueSection, err := rawdb.ReadTrienodeHistory(reader, id)
if err != nil {
return nil, err
}
var h trienodeHistory
if err := h.decode(header, keySection, valueSection); err != nil {
return nil, err
}
return &h, nil
}
// readTrienodeHistories resolves a list of trienode histories with the specific range.
func readTrienodeHistories(reader ethdb.AncientReader, start uint64, count uint64) ([]history, error) {
headers, keySections, valueSections, err := rawdb.ReadTrienodeHistoryList(reader, start, count)
if err != nil {
return nil, err
}
var res []history
for i, header := range headers {
var h trienodeHistory
if err := h.decode(header, keySections[i], valueSections[i]); err != nil {
return nil, err
}
res = append(res, &h)
}
return res, nil
}

View file

@ -0,0 +1,736 @@
// Copyright 2025 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package pathdb
import (
"bytes"
"encoding/binary"
"math/rand"
"reflect"
"testing"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/internal/testrand"
)
// randomTrienodes generates a random trienode set.
func randomTrienodes(n int) (map[common.Hash]map[string][]byte, common.Hash) {
var (
root common.Hash
nodes = make(map[common.Hash]map[string][]byte)
)
for i := 0; i < n; i++ {
owner := testrand.Hash()
if i == 0 {
owner = common.Hash{}
}
nodes[owner] = make(map[string][]byte)
for j := 0; j < 10; j++ {
path := testrand.Bytes(rand.Intn(10))
for z := 0; z < len(path); z++ {
nodes[owner][string(path[:z])] = testrand.Bytes(rand.Intn(128))
}
}
// zero-size trie node, representing it was non-existent before
for j := 0; j < 10; j++ {
path := testrand.Bytes(32)
nodes[owner][string(path)] = nil
}
// root node with zero-size path
rnode := testrand.Bytes(256)
nodes[owner][""] = rnode
if owner == (common.Hash{}) {
root = crypto.Keccak256Hash(rnode)
}
}
return nodes, root
}
func makeTrienodeHistory() *trienodeHistory {
nodes, root := randomTrienodes(10)
return newTrienodeHistory(root, common.Hash{}, 1, nodes)
}
func makeTrienodeHistories(n int) []*trienodeHistory {
var (
parent common.Hash
result []*trienodeHistory
)
for i := 0; i < n; i++ {
nodes, root := randomTrienodes(10)
result = append(result, newTrienodeHistory(root, parent, uint64(i+1), nodes))
parent = root
}
return result
}
func TestEncodeDecodeTrienodeHistory(t *testing.T) {
var (
dec trienodeHistory
obj = makeTrienodeHistory()
)
header, keySection, valueSection, err := obj.encode()
if err != nil {
t.Fatalf("Failed to encode trienode history: %v", err)
}
if err := dec.decode(header, keySection, valueSection); err != nil {
t.Fatalf("Failed to decode trienode history: %v", err)
}
if !reflect.DeepEqual(obj.meta, dec.meta) {
t.Fatal("trienode metadata is mismatched")
}
if !compareList(dec.owners, obj.owners) {
t.Fatal("trie owner list is mismatched")
}
if !compareMapList(dec.nodeList, obj.nodeList) {
t.Fatal("trienode list is mismatched")
}
if !compareMapSet(dec.nodes, obj.nodes) {
t.Fatal("trienode content is mismatched")
}
// Re-encode again, ensuring the encoded blob still match
header2, keySection2, valueSection2, err := dec.encode()
if err != nil {
t.Fatalf("Failed to encode trienode history: %v", err)
}
if !bytes.Equal(header, header2) {
t.Fatal("re-encoded header is mismatched")
}
if !bytes.Equal(keySection, keySection2) {
t.Fatal("re-encoded key section is mismatched")
}
if !bytes.Equal(valueSection, valueSection2) {
t.Fatal("re-encoded value section is mismatched")
}
}
func TestTrienodeHistoryReader(t *testing.T) {
var (
hs = makeTrienodeHistories(10)
freezer, _ = rawdb.NewTrienodeFreezer(t.TempDir(), false, false)
)
defer freezer.Close()
for i, h := range hs {
header, keySection, valueSection, _ := h.encode()
if err := rawdb.WriteTrienodeHistory(freezer, uint64(i+1), header, keySection, valueSection); err != nil {
t.Fatalf("Failed to write trienode history: %v", err)
}
}
for i, h := range hs {
tr, err := newTrienodeHistoryReader(uint64(i+1), freezer)
if err != nil {
t.Fatalf("Failed to construct the history reader: %v", err)
}
for _, owner := range h.owners {
nodes := h.nodes[owner]
for key, value := range nodes {
blob, err := tr.read(owner, key)
if err != nil {
t.Fatalf("Failed to read trienode history: %v", err)
}
if !bytes.Equal(blob, value) {
t.Fatalf("Unexpected trie node data, want: %v, got: %v", value, blob)
}
}
}
}
for i, h := range hs {
metadata, err := readTrienodeMetadata(freezer, uint64(i+1))
if err != nil {
t.Fatalf("Failed to read trienode history metadata: %v", err)
}
if !reflect.DeepEqual(h.meta, metadata) {
t.Fatalf("Unexpected trienode metadata, want: %v, got: %v", h.meta, metadata)
}
}
}
// TestEmptyTrienodeHistory tests encoding/decoding of empty trienode history
func TestEmptyTrienodeHistory(t *testing.T) {
h := newTrienodeHistory(common.Hash{}, common.Hash{}, 1, make(map[common.Hash]map[string][]byte))
// Test encoding empty history
header, keySection, valueSection, err := h.encode()
if err != nil {
t.Fatalf("Failed to encode empty trienode history: %v", err)
}
// Verify sections are minimal but valid
if len(header) == 0 {
t.Fatal("Header should not be empty")
}
if len(keySection) != 0 {
t.Fatal("Key section should be empty for empty history")
}
if len(valueSection) != 0 {
t.Fatal("Value section should be empty for empty history")
}
// Test decoding empty history
var decoded trienodeHistory
if err := decoded.decode(header, keySection, valueSection); err != nil {
t.Fatalf("Failed to decode empty trienode history: %v", err)
}
if len(decoded.owners) != 0 {
t.Fatal("Decoded history should have no owners")
}
if len(decoded.nodeList) != 0 {
t.Fatal("Decoded history should have no node lists")
}
if len(decoded.nodes) != 0 {
t.Fatal("Decoded history should have no nodes")
}
}
// TestSingleTrieHistory tests encoding/decoding of history with single trie
func TestSingleTrieHistory(t *testing.T) {
nodes := make(map[common.Hash]map[string][]byte)
owner := testrand.Hash()
nodes[owner] = make(map[string][]byte)
// Add some nodes with various sizes
nodes[owner][""] = testrand.Bytes(32) // empty key
nodes[owner]["a"] = testrand.Bytes(1) // small value
nodes[owner]["bb"] = testrand.Bytes(100) // medium value
nodes[owner]["ccc"] = testrand.Bytes(1000) // large value
nodes[owner]["dddd"] = testrand.Bytes(0) // empty value
h := newTrienodeHistory(common.Hash{}, common.Hash{}, 1, nodes)
testEncodeDecode(t, h)
}
// TestMultipleTries tests multiple tries with different node counts
func TestMultipleTries(t *testing.T) {
nodes := make(map[common.Hash]map[string][]byte)
// First trie with many small nodes
owner1 := testrand.Hash()
nodes[owner1] = make(map[string][]byte)
for i := 0; i < 100; i++ {
key := string(testrand.Bytes(rand.Intn(10)))
nodes[owner1][key] = testrand.Bytes(rand.Intn(50))
}
// Second trie with few large nodes
owner2 := testrand.Hash()
nodes[owner2] = make(map[string][]byte)
for i := 0; i < 5; i++ {
key := string(testrand.Bytes(rand.Intn(20)))
nodes[owner2][key] = testrand.Bytes(1000 + rand.Intn(1000))
}
// Third trie with nil values (zero-size nodes)
owner3 := testrand.Hash()
nodes[owner3] = make(map[string][]byte)
for i := 0; i < 10; i++ {
key := string(testrand.Bytes(rand.Intn(15)))
nodes[owner3][key] = nil
}
h := newTrienodeHistory(common.Hash{}, common.Hash{}, 1, nodes)
testEncodeDecode(t, h)
}
// TestLargeNodeValues tests encoding/decoding with very large node values
func TestLargeNodeValues(t *testing.T) {
nodes := make(map[common.Hash]map[string][]byte)
owner := testrand.Hash()
nodes[owner] = make(map[string][]byte)
// Test with progressively larger values
sizes := []int{1024, 10 * 1024, 100 * 1024, 1024 * 1024} // 1KB, 10KB, 100KB, 1MB
for _, size := range sizes {
key := string(testrand.Bytes(10))
nodes[owner][key] = testrand.Bytes(size)
h := newTrienodeHistory(common.Hash{}, common.Hash{}, 1, nodes)
testEncodeDecode(t, h)
t.Logf("Successfully tested encoding/decoding with %dKB value", size/1024)
}
}
// TestNilNodeValues tests encoding/decoding with nil (zero-length) node values
func TestNilNodeValues(t *testing.T) {
nodes := make(map[common.Hash]map[string][]byte)
owner := testrand.Hash()
nodes[owner] = make(map[string][]byte)
// Mix of nil and non-nil values
nodes[owner]["nil"] = nil
nodes[owner]["data1"] = []byte("some data")
nodes[owner]["data2"] = []byte("more data")
h := newTrienodeHistory(common.Hash{}, common.Hash{}, 1, nodes)
testEncodeDecode(t, h)
// Verify nil values are preserved
_, ok := h.nodes[owner]["nil"]
if !ok {
t.Fatal("Nil value should be preserved")
}
}
// TestCorruptedHeader tests error handling for corrupted header data
func TestCorruptedHeader(t *testing.T) {
h := makeTrienodeHistory()
header, keySection, valueSection, _ := h.encode()
// Test corrupted version
corruptedHeader := make([]byte, len(header))
copy(corruptedHeader, header)
corruptedHeader[0] = 0xFF // Invalid version
var decoded trienodeHistory
if err := decoded.decode(corruptedHeader, keySection, valueSection); err == nil {
t.Fatal("Expected error for corrupted version")
}
// Test truncated header
truncatedHeader := header[:len(header)-5]
if err := decoded.decode(truncatedHeader, keySection, valueSection); err == nil {
t.Fatal("Expected error for truncated header")
}
// Test header with invalid trie header size
invalidHeader := make([]byte, len(header))
copy(invalidHeader, header)
invalidHeader = invalidHeader[:trienodeMetadataSize+5] // Not divisible by trie header size
if err := decoded.decode(invalidHeader, keySection, valueSection); err == nil {
t.Fatal("Expected error for invalid header size")
}
}
// TestCorruptedKeySection tests error handling for corrupted key section data
func TestCorruptedKeySection(t *testing.T) {
h := makeTrienodeHistory()
header, keySection, valueSection, _ := h.encode()
// Test empty key section when header indicates data
if len(keySection) > 0 {
var decoded trienodeHistory
if err := decoded.decode(header, []byte{}, valueSection); err == nil {
t.Fatal("Expected error for empty key section with non-empty header")
}
}
// Test truncated key section
if len(keySection) > 10 {
truncatedKeySection := keySection[:len(keySection)-10]
var decoded trienodeHistory
if err := decoded.decode(header, truncatedKeySection, valueSection); err == nil {
t.Fatal("Expected error for truncated key section")
}
}
// Test corrupted key section with invalid varint
corruptedKeySection := make([]byte, len(keySection))
copy(corruptedKeySection, keySection)
if len(corruptedKeySection) > 5 {
corruptedKeySection[5] = 0xFF // Corrupt varint encoding
var decoded trienodeHistory
if err := decoded.decode(header, corruptedKeySection, valueSection); err == nil {
t.Fatal("Expected error for corrupted varint in key section")
}
}
}
// TestCorruptedValueSection tests error handling for corrupted value section data
func TestCorruptedValueSection(t *testing.T) {
h := makeTrienodeHistory()
header, keySection, valueSection, _ := h.encode()
// Test truncated value section
if len(valueSection) > 10 {
truncatedValueSection := valueSection[:len(valueSection)-10]
var decoded trienodeHistory
if err := decoded.decode(header, keySection, truncatedValueSection); err == nil {
t.Fatal("Expected error for truncated value section")
}
}
// Test empty value section when key section indicates data exists
if len(valueSection) > 0 {
var decoded trienodeHistory
if err := decoded.decode(header, keySection, []byte{}); err == nil {
t.Fatal("Expected error for empty value section with non-empty key section")
}
}
}
// TestInvalidOffsets tests error handling for invalid offsets in encoded data
func TestInvalidOffsets(t *testing.T) {
h := makeTrienodeHistory()
header, keySection, valueSection, _ := h.encode()
// Corrupt key offset in header (make it larger than key section)
corruptedHeader := make([]byte, len(header))
copy(corruptedHeader, header)
corruptedHeader[trienodeMetadataSize+common.HashLength] = 0xff
var dec1 trienodeHistory
if err := dec1.decode(corruptedHeader, keySection, valueSection); err == nil {
t.Fatal("Expected error for invalid key offset")
}
// Corrupt value offset in header (make it larger than value section)
corruptedHeader = make([]byte, len(header))
copy(corruptedHeader, header)
corruptedHeader[trienodeMetadataSize+common.HashLength+4] = 0xff
var dec2 trienodeHistory
if err := dec2.decode(corruptedHeader, keySection, valueSection); err == nil {
t.Fatal("Expected error for invalid value offset")
}
}
// TestTrienodeHistoryReaderNonExistentPath tests reading non-existent paths
func TestTrienodeHistoryReaderNonExistentPath(t *testing.T) {
var (
h = makeTrienodeHistory()
freezer, _ = rawdb.NewTrienodeFreezer(t.TempDir(), false, false)
)
defer freezer.Close()
header, keySection, valueSection, _ := h.encode()
if err := rawdb.WriteTrienodeHistory(freezer, 1, header, keySection, valueSection); err != nil {
t.Fatalf("Failed to write trienode history: %v", err)
}
tr, err := newTrienodeHistoryReader(1, freezer)
if err != nil {
t.Fatalf("Failed to construct history reader: %v", err)
}
// Try to read a non-existent path
_, err = tr.read(testrand.Hash(), "nonexistent")
if err == nil {
t.Fatal("Expected error for non-existent trie owner")
}
// Try to read from existing owner but non-existent path
owner := h.owners[0]
_, err = tr.read(owner, "nonexistent-path")
if err == nil {
t.Fatal("Expected error for non-existent path")
}
}
// TestTrienodeHistoryReaderNilValues tests reading nil (zero-length) values
func TestTrienodeHistoryReaderNilValues(t *testing.T) {
nodes := make(map[common.Hash]map[string][]byte)
owner := testrand.Hash()
nodes[owner] = make(map[string][]byte)
// Add some nil values
nodes[owner]["nil1"] = nil
nodes[owner]["nil2"] = nil
nodes[owner]["data1"] = []byte("some data")
h := newTrienodeHistory(common.Hash{}, common.Hash{}, 1, nodes)
var freezer, _ = rawdb.NewTrienodeFreezer(t.TempDir(), false, false)
defer freezer.Close()
header, keySection, valueSection, _ := h.encode()
if err := rawdb.WriteTrienodeHistory(freezer, 1, header, keySection, valueSection); err != nil {
t.Fatalf("Failed to write trienode history: %v", err)
}
tr, err := newTrienodeHistoryReader(1, freezer)
if err != nil {
t.Fatalf("Failed to construct history reader: %v", err)
}
// Test reading nil values
data1, err := tr.read(owner, "nil1")
if err != nil {
t.Fatalf("Failed to read nil value: %v", err)
}
if len(data1) != 0 {
t.Fatal("Expected nil data for nil value")
}
data2, err := tr.read(owner, "nil2")
if err != nil {
t.Fatalf("Failed to read nil value: %v", err)
}
if len(data2) != 0 {
t.Fatal("Expected nil data for nil value")
}
// Test reading non-nil value
data3, err := tr.read(owner, "data1")
if err != nil {
t.Fatalf("Failed to read non-nil value: %v", err)
}
if !bytes.Equal(data3, []byte("some data")) {
t.Fatal("Data mismatch for non-nil value")
}
}
// TestTrienodeHistoryReaderNilKey tests reading nil (zero-length) key
func TestTrienodeHistoryReaderNilKey(t *testing.T) {
nodes := make(map[common.Hash]map[string][]byte)
owner := testrand.Hash()
nodes[owner] = make(map[string][]byte)
// Add some nil values
nodes[owner][""] = []byte("some data")
nodes[owner]["data1"] = []byte("some data")
h := newTrienodeHistory(common.Hash{}, common.Hash{}, 1, nodes)
var freezer, _ = rawdb.NewTrienodeFreezer(t.TempDir(), false, false)
defer freezer.Close()
header, keySection, valueSection, _ := h.encode()
if err := rawdb.WriteTrienodeHistory(freezer, 1, header, keySection, valueSection); err != nil {
t.Fatalf("Failed to write trienode history: %v", err)
}
tr, err := newTrienodeHistoryReader(1, freezer)
if err != nil {
t.Fatalf("Failed to construct history reader: %v", err)
}
// Test reading nil values
data1, err := tr.read(owner, "")
if err != nil {
t.Fatalf("Failed to read nil value: %v", err)
}
if !bytes.Equal(data1, []byte("some data")) {
t.Fatal("Data mismatch for nil key")
}
// Test reading non-nil value
data2, err := tr.read(owner, "data1")
if err != nil {
t.Fatalf("Failed to read non-nil value: %v", err)
}
if !bytes.Equal(data2, []byte("some data")) {
t.Fatal("Data mismatch for non-nil key")
}
}
// TestTrienodeHistoryReaderIterator tests the iterator functionality
func TestTrienodeHistoryReaderIterator(t *testing.T) {
h := makeTrienodeHistory()
// Count expected entries
expectedCount := 0
expectedNodes := make(map[stateIdent]bool)
for owner, nodeList := range h.nodeList {
expectedCount += len(nodeList)
for _, node := range nodeList {
expectedNodes[stateIdent{
typ: typeTrienode,
addressHash: owner,
path: node,
}] = true
}
}
// Test the iterator
actualCount := 0
for x := range h.forEach() {
_ = x
actualCount++
}
if actualCount != expectedCount {
t.Fatalf("Iterator count mismatch: expected %d, got %d", expectedCount, actualCount)
}
// Test that iterator yields expected state identifiers
seen := make(map[stateIdent]bool)
for ident := range h.forEach() {
if ident.typ != typeTrienode {
t.Fatal("Iterator should only yield trienode history identifiers")
}
key := stateIdent{typ: ident.typ, addressHash: ident.addressHash, path: ident.path}
if seen[key] {
t.Fatal("Iterator yielded duplicate identifier")
}
seen[key] = true
if !expectedNodes[key] {
t.Fatalf("Unexpected yielded identifier %v", key)
}
}
}
// TestSharedLen tests the sharedLen helper function
func TestSharedLen(t *testing.T) {
tests := []struct {
a, b []byte
expected int
}{
// Empty strings
{[]byte(""), []byte(""), 0},
// One empty string
{[]byte(""), []byte("abc"), 0},
{[]byte("abc"), []byte(""), 0},
// No common prefix
{[]byte("abc"), []byte("def"), 0},
// Partial common prefix
{[]byte("abc"), []byte("abx"), 2},
{[]byte("prefix"), []byte("pref"), 4},
// Complete common prefix (shorter first)
{[]byte("ab"), []byte("abcd"), 2},
// Complete common prefix (longer first)
{[]byte("abcd"), []byte("ab"), 2},
// Identical strings
{[]byte("identical"), []byte("identical"), 9},
// Binary data
{[]byte{0x00, 0x01, 0x02}, []byte{0x00, 0x01, 0x03}, 2},
// Large strings
{bytes.Repeat([]byte("a"), 1000), bytes.Repeat([]byte("a"), 1000), 1000},
{bytes.Repeat([]byte("a"), 1000), append(bytes.Repeat([]byte("a"), 999), []byte("b")...), 999},
}
for i, test := range tests {
result := sharedLen(test.a, test.b)
if result != test.expected {
t.Errorf("Test %d: sharedLen(%q, %q) = %d, expected %d",
i, test.a, test.b, result, test.expected)
}
// Test commutativity
resultReverse := sharedLen(test.b, test.a)
if result != resultReverse {
t.Errorf("Test %d: sharedLen is not commutative: sharedLen(a,b)=%d, sharedLen(b,a)=%d",
i, result, resultReverse)
}
}
}
// TestDecodeHeaderCorruptedData tests decodeHeader with corrupted data
func TestDecodeHeaderCorruptedData(t *testing.T) {
// Create valid header data first
h := makeTrienodeHistory()
header, _, _, _ := h.encode()
// Test with empty header
_, _, _, _, err := decodeHeader([]byte{})
if err == nil {
t.Fatal("Expected error for empty header")
}
// Test with invalid version
corruptedVersion := make([]byte, len(header))
copy(corruptedVersion, header)
corruptedVersion[0] = 0xFF
_, _, _, _, err = decodeHeader(corruptedVersion)
if err == nil {
t.Fatal("Expected error for invalid version")
}
// Test with truncated header (not divisible by trie header size)
truncated := header[:trienodeMetadataSize+5]
_, _, _, _, err = decodeHeader(truncated)
if err == nil {
t.Fatal("Expected error for truncated header")
}
// Test with unordered trie owners
unordered := make([]byte, len(header))
copy(unordered, header)
// Swap two owner hashes to make them unordered
hash1Start := trienodeMetadataSize
hash2Start := trienodeMetadataSize + trienodeTrieHeaderSize
hash1 := unordered[hash1Start : hash1Start+common.HashLength]
hash2 := unordered[hash2Start : hash2Start+common.HashLength]
// Only swap if they would be out of order
copy(unordered[hash1Start:hash1Start+common.HashLength], hash2)
copy(unordered[hash2Start:hash2Start+common.HashLength], hash1)
_, _, _, _, err = decodeHeader(unordered)
if err == nil {
t.Fatal("Expected error for unordered trie owners")
}
}
// TestDecodeSingleCorruptedData tests decodeSingle with corrupted data
func TestDecodeSingleCorruptedData(t *testing.T) {
h := makeTrienodeHistory()
_, keySection, _, _ := h.encode()
// Test with empty key section
_, err := decodeSingle([]byte{}, nil)
if err == nil {
t.Fatal("Expected error for empty key section")
}
// Test with key section too small for trailer
if len(keySection) > 0 {
_, err := decodeSingle(keySection[:3], nil) // Less than 4 bytes for trailer
if err == nil {
t.Fatal("Expected error for key section too small for trailer")
}
}
// Test with corrupted varint in key section
corrupted := make([]byte, len(keySection))
copy(corrupted, keySection)
corrupted[5] = 0xFF // Corrupt varint
_, err = decodeSingle(corrupted, nil)
if err == nil {
t.Fatal("Expected error for corrupted varint")
}
// Test with corrupted trailer (invalid restart count)
corrupted = make([]byte, len(keySection))
copy(corrupted, keySection)
// Set restart count to something too large
binary.BigEndian.PutUint32(corrupted[len(corrupted)-4:], 10000)
_, err = decodeSingle(corrupted, nil)
if err == nil {
t.Fatal("Expected error for invalid restart count")
}
}
// Helper function to test encode/decode cycle
func testEncodeDecode(t *testing.T, h *trienodeHistory) {
header, keySection, valueSection, err := h.encode()
if err != nil {
t.Fatalf("Failed to encode trienode history: %v", err)
}
var decoded trienodeHistory
if err := decoded.decode(header, keySection, valueSection); err != nil {
t.Fatalf("Failed to decode trienode history: %v", err)
}
// Compare the decoded history with original
if !compareList(decoded.owners, h.owners) {
t.Fatal("Trie owner list mismatch")
}
if !compareMapList(decoded.nodeList, h.nodeList) {
t.Fatal("Trienode list mismatch")
}
if !compareMapSet(decoded.nodes, h.nodes) {
t.Fatal("Trienode content mismatch")
}
}

View file

@ -69,12 +69,21 @@ var (
gcStorageMeter = metrics.NewRegisteredMeter("pathdb/gc/storage/count", nil) gcStorageMeter = metrics.NewRegisteredMeter("pathdb/gc/storage/count", nil)
gcStorageBytesMeter = metrics.NewRegisteredMeter("pathdb/gc/storage/bytes", nil) gcStorageBytesMeter = metrics.NewRegisteredMeter("pathdb/gc/storage/bytes", nil)
historyBuildTimeMeter = metrics.NewRegisteredResettingTimer("pathdb/history/time", nil) stateHistoryBuildTimeMeter = metrics.NewRegisteredResettingTimer("pathdb/history/state/time", nil)
historyDataBytesMeter = metrics.NewRegisteredMeter("pathdb/history/bytes/data", nil) stateHistoryDataBytesMeter = metrics.NewRegisteredMeter("pathdb/history/state/bytes/data", nil)
historyIndexBytesMeter = metrics.NewRegisteredMeter("pathdb/history/bytes/index", nil) stateHistoryIndexBytesMeter = metrics.NewRegisteredMeter("pathdb/history/state/bytes/index", nil)
indexHistoryTimer = metrics.NewRegisteredResettingTimer("pathdb/history/index/time", nil) //nolint:unused
unindexHistoryTimer = metrics.NewRegisteredResettingTimer("pathdb/history/unindex/time", nil) trienodeHistoryBuildTimeMeter = metrics.NewRegisteredResettingTimer("pathdb/history/trienode/time", nil)
//nolint:unused
trienodeHistoryDataBytesMeter = metrics.NewRegisteredMeter("pathdb/history/trienode/bytes/data", nil)
//nolint:unused
trienodeHistoryIndexBytesMeter = metrics.NewRegisteredMeter("pathdb/history/trienode/bytes/index", nil)
stateIndexHistoryTimer = metrics.NewRegisteredResettingTimer("pathdb/history/state/index/time", nil)
stateUnindexHistoryTimer = metrics.NewRegisteredResettingTimer("pathdb/history/state/unindex/time", nil)
trienodeIndexHistoryTimer = metrics.NewRegisteredResettingTimer("pathdb/history/trienode/index/time", nil)
trienodeUnindexHistoryTimer = metrics.NewRegisteredResettingTimer("pathdb/history/trienode/unindex/time", nil)
lookupAddLayerTimer = metrics.NewRegisteredResettingTimer("pathdb/lookup/add/time", nil) lookupAddLayerTimer = metrics.NewRegisteredResettingTimer("pathdb/lookup/add/time", nil)
lookupRemoveLayerTimer = metrics.NewRegisteredResettingTimer("pathdb/lookup/remove/time", nil) lookupRemoveLayerTimer = metrics.NewRegisteredResettingTimer("pathdb/lookup/remove/time", nil)