From 507c50d572ce5e096a16f93d5f3d59b4418ec36c Mon Sep 17 00:00:00 2001 From: Gary Rong Date: Mon, 22 Sep 2025 14:19:06 +0800 Subject: [PATCH] triedb/pathdb: include metadata in header section --- triedb/pathdb/history_trienode.go | 100 +++++++++++++++------ triedb/pathdb/history_trienode_test.go | 115 ++++++++++++++++--------- 2 files changed, 151 insertions(+), 64 deletions(-) diff --git a/triedb/pathdb/history_trienode.go b/triedb/pathdb/history_trienode.go index defee8d3cd..28d3027a1a 100644 --- a/triedb/pathdb/history_trienode.go +++ b/triedb/pathdb/history_trienode.go @@ -38,10 +38,20 @@ import ( // // # Header // The header records metadata, including: -// - the history version +// +// - the history version (1 byte) +// - the parent state root (32 bytes) +// - the current state root (32 bytes) +// - block number (8 bytes) +// // - a lexicographically sorted list of trie IDs // - the corresponding offsets into the key and value sections for each trie data chunk // +// Although some fields (e.g., parent state root, block number) are duplicated +// between the state history and the trienode history, these two histories +// operate independently. To ensure each remains self-contained and self-descriptive, +// we have chosen to maintain these duplicate fields. +// // # Key section // The key section stores trie node keys (paths) in a compressed format. // It also contains relative offsets into the value section for resolving @@ -60,7 +70,7 @@ import ( // Header section: // // +----------+------------------+---------------------+---------------------+-------+------------------+---------------------+---------------------| -// | ver (1B) | TrieID(32 bytes) | key offset(4 bytes) | val offset(4 bytes) | ... | TrieID(32 bytes) | key offset(4 bytes) | val offset(4 bytes) | +// | metadata | TrieID(32 bytes) | key offset(4 bytes) | val offset(4 bytes) | ... | TrieID(32 bytes) | key offset(4 bytes) | val offset(4 bytes) | // +----------+------------------+---------------------+---------------------+-------+------------------+---------------------+---------------------| // // @@ -103,23 +113,32 @@ import ( // NOTE: All fixed-length integer are big-endian. const ( - trienodeHistoryV0 = uint8(0) // initial version of node history structure - trienodeHistoryVersion = trienodeHistoryV0 // the default node history version - trienodeVersionSize = 1 // the size of version tag in the history - trienodeTrieHeaderSize = 8 + common.HashLength // the size of a single trie header in history - trienodeDataBlockRestartLen = 16 // The restart interval length of trie node block + trienodeHistoryV0 = uint8(0) // initial version of node history structure + trienodeHistoryVersion = trienodeHistoryV0 // the default node history version + trienodeMetadataSize = 1 + 2*common.HashLength + 8 // the size of metadata in the history + trienodeTrieHeaderSize = 8 + common.HashLength // the size of a single trie header in history + trienodeDataBlockRestartLen = 16 // The restart interval length of trie node block ) +// trienodeMetadata describes the meta data of trienode history. +type trienodeMetadata struct { + version uint8 // version tag of history object + parent common.Hash // prev-state root before the state transition + root common.Hash // post-state root after the state transition + block uint64 // associated block number +} + // trienodeHistory represents a set of trie node changes resulting from a state // transition across the main account trie and all associated storage tries. type trienodeHistory struct { + meta *trienodeMetadata // Metadata of the history owners []common.Hash // List of trie identifier sorted lexicographically nodeList map[common.Hash][]string // Set of node paths sorted lexicographically nodes map[common.Hash]map[string][]byte // Set of original value of trie nodes before state transition } // newTrienodeHistory constructs a trienode history with the provided trie nodes. -func newTrienodeHistory(nodes map[common.Hash]map[string][]byte) *trienodeHistory { +func newTrienodeHistory(root common.Hash, parent common.Hash, block uint64, nodes map[common.Hash]map[string][]byte) *trienodeHistory { nodeList := make(map[common.Hash][]string) for owner, subset := range nodes { keys := sort.StringSlice(slices.Collect(maps.Keys(subset))) @@ -127,6 +146,12 @@ func newTrienodeHistory(nodes map[common.Hash]map[string][]byte) *trienodeHistor nodeList[owner] = keys } return &trienodeHistory{ + meta: &trienodeMetadata{ + version: trienodeHistoryVersion, + parent: parent, + root: root, + block: block, + }, owners: slices.SortedFunc(maps.Keys(nodes), common.Hash.Cmp), nodeList: nodeList, nodes: nodes, @@ -174,7 +199,10 @@ func (h *trienodeHistory) encode() ([]byte, []byte, []byte, error) { keySection bytes.Buffer valueSection bytes.Buffer ) - binary.Write(&headerSection, binary.BigEndian, trienodeHistoryVersion) // 1 byte + binary.Write(&headerSection, binary.BigEndian, h.meta.version) // 1 byte + headerSection.Write(h.meta.parent.Bytes()) // 32 bytes + headerSection.Write(h.meta.root.Bytes()) // 32 bytes + binary.Write(&headerSection, binary.BigEndian, h.meta.block) // 8 byte for _, owner := range h.owners { // Fill the header section with offsets at key and value section @@ -246,17 +274,21 @@ func (h *trienodeHistory) encode() ([]byte, []byte, []byte, error) { // decodeHeader resolves the metadata from the header section. An error // should be returned if the header section is corrupted. -func decodeHeader(data []byte) ([]common.Hash, []uint32, []uint32, error) { - if len(data) < trienodeVersionSize { - return nil, nil, nil, fmt.Errorf("trienode history is too small, index size: %d", len(data)) +func decodeHeader(data []byte) (*trienodeMetadata, []common.Hash, []uint32, []uint32, error) { + if len(data) < trienodeMetadataSize { + return nil, nil, nil, nil, fmt.Errorf("trienode history is too small, index size: %d", len(data)) } version := data[0] if version != trienodeHistoryVersion { - return nil, nil, nil, fmt.Errorf("unregonized trienode history version: %d", version) + return nil, nil, nil, nil, fmt.Errorf("unregonized trienode history version: %d", version) } - size := len(data) - trienodeVersionSize + parent := common.BytesToHash(data[1 : common.HashLength+1]) // 32 bytes + root := common.BytesToHash(data[common.HashLength+1 : common.HashLength*2+1]) // 32 bytes + block := binary.BigEndian.Uint64(data[common.HashLength*2+1 : trienodeMetadataSize]) // 8 bytes + + size := len(data) - trienodeMetadataSize if size%trienodeTrieHeaderSize != 0 { - return nil, nil, nil, fmt.Errorf("truncated trienode history data, size %d", len(data)) + return nil, nil, nil, nil, fmt.Errorf("truncated trienode history data, size %d", len(data)) } count := size / trienodeTrieHeaderSize @@ -266,17 +298,17 @@ func decodeHeader(data []byte) ([]common.Hash, []uint32, []uint32, error) { valOffsets = make([]uint32, 0, count) ) for i := 0; i < count; i++ { - n := trienodeVersionSize + trienodeTrieHeaderSize*i + n := trienodeMetadataSize + trienodeTrieHeaderSize*i owner := common.BytesToHash(data[n : n+common.HashLength]) if i != 0 && bytes.Compare(owner.Bytes(), owners[i-1].Bytes()) <= 0 { - return nil, nil, nil, fmt.Errorf("trienode owners are out of order, prev: %v, cur: %v", owners[i-1], owner) + return nil, nil, nil, nil, fmt.Errorf("trienode owners are out of order, prev: %v, cur: %v", owners[i-1], owner) } owners = append(owners, owner) // Decode the offset to the key section keyOffset := binary.BigEndian.Uint32(data[n+common.HashLength : n+common.HashLength+4]) if i != 0 && keyOffset <= keyOffsets[i-1] { - return nil, nil, nil, fmt.Errorf("key offset is out of order, prev: %v, cur: %v", keyOffsets[i-1], keyOffset) + return nil, nil, nil, nil, fmt.Errorf("key offset is out of order, prev: %v, cur: %v", keyOffsets[i-1], keyOffset) } keyOffsets = append(keyOffsets, keyOffset) @@ -285,11 +317,16 @@ func decodeHeader(data []byte) ([]common.Hash, []uint32, []uint32, error) { // a trie deletion). valOffset := binary.BigEndian.Uint32(data[n+common.HashLength+4 : n+common.HashLength+8]) if i != 0 && valOffset < valOffsets[i-1] { - return nil, nil, nil, fmt.Errorf("value offset is out of order, prev: %v, cur: %v", valOffsets[i-1], valOffset) + return nil, nil, nil, nil, fmt.Errorf("value offset is out of order, prev: %v, cur: %v", valOffsets[i-1], valOffset) } valOffsets = append(valOffsets, valOffset) } - return owners, keyOffsets, valOffsets, nil + return &trienodeMetadata{ + version: version, + parent: parent, + root: root, + block: block, + }, owners, keyOffsets, valOffsets, nil } func decodeSingle(keySection []byte, onValue func([]byte, int, int) error) ([]string, error) { @@ -425,10 +462,11 @@ func decodeSingleWithValue(keySection []byte, valueSection []byte) ([]string, ma // decode deserializes the contained trie nodes from the provided bytes. func (h *trienodeHistory) decode(header []byte, keySection []byte, valueSection []byte) error { - owners, keyOffsets, valueOffsets, err := decodeHeader(header) + metadata, owners, keyOffsets, valueOffsets, err := decodeHeader(header) if err != nil { return err } + h.meta = metadata h.owners = owners h.nodeList = make(map[common.Hash][]string) h.nodes = make(map[common.Hash]map[string][]byte) @@ -562,13 +600,13 @@ func newTrienodeHistoryReader(id uint64, reader ethdb.AncientReader) (*trienodeH return r, nil } -// decodeHeader decodes the metadata of trienode history from the header section. +// decodeHeader decodes the header section of trienode history. func (r *trienodeHistoryReader) decodeHeader() error { header, err := rawdb.ReadTrienodeHistoryHeader(r.reader, r.id) if err != nil { return err } - owners, keyOffsets, valOffsets, err := decodeHeader(header) + _, owners, keyOffsets, valOffsets, err := decodeHeader(header) if err != nil { return err } @@ -626,7 +664,7 @@ func (r *trienodeHistoryReader) read(owner common.Hash, path string) ([]byte, er // nolint:unused func writeTrienodeHistory(writer ethdb.AncientWriter, dl *diffLayer) error { start := time.Now() - h := newTrienodeHistory(dl.nodes.nodeOrigin) + h := newTrienodeHistory(dl.rootHash(), dl.parent.rootHash(), dl.block, dl.nodes.nodeOrigin) header, keySection, valueSection, err := h.encode() if err != nil { return err @@ -649,6 +687,20 @@ func writeTrienodeHistory(writer ethdb.AncientWriter, dl *diffLayer) error { return nil } +// readTrienodeMetadata resolves the metadata of the specified trienode history. +// nolint:unused +func readTrienodeMetadata(reader ethdb.AncientReader, id uint64) (*trienodeMetadata, error) { + header, err := rawdb.ReadTrienodeHistoryHeader(reader, id) + if err != nil { + return nil, err + } + metadata, _, _, _, err := decodeHeader(header) + if err != nil { + return nil, err + } + return metadata, nil +} + // readTrienodeHistory resolves a single trienode history object with specific id. func readTrienodeHistory(reader ethdb.AncientReader, id uint64) (*trienodeHistory, error) { header, keySection, valueSection, err := rawdb.ReadTrienodeHistory(reader, id) diff --git a/triedb/pathdb/history_trienode_test.go b/triedb/pathdb/history_trienode_test.go index 69d13fdbb4..b02c0d5380 100644 --- a/triedb/pathdb/history_trienode_test.go +++ b/triedb/pathdb/history_trienode_test.go @@ -20,18 +20,26 @@ import ( "bytes" "encoding/binary" "math/rand" + "reflect" "testing" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/internal/testrand" ) // randomTrienodes generates a random trienode set. -func randomTrienodes(n int) map[common.Hash]map[string][]byte { - nodes := make(map[common.Hash]map[string][]byte) +func randomTrienodes(n int) (map[common.Hash]map[string][]byte, common.Hash) { + var ( + root common.Hash + nodes = make(map[common.Hash]map[string][]byte) + ) for i := 0; i < n; i++ { owner := testrand.Hash() + if i == 0 { + owner = common.Hash{} + } nodes[owner] = make(map[string][]byte) for j := 0; j < 10; j++ { @@ -48,20 +56,29 @@ func randomTrienodes(n int) map[common.Hash]map[string][]byte { nodes[owner][string(path)] = nil } // root node with zero-size path - nodes[owner][""] = testrand.Bytes(10) + rnode := testrand.Bytes(256) + nodes[owner][""] = rnode + if owner == (common.Hash{}) { + root = crypto.Keccak256Hash(rnode) + } } - return nodes + return nodes, root } func makeTrinodeHistory() *trienodeHistory { - return newTrienodeHistory(randomTrienodes(10)) + nodes, root := randomTrienodes(10) + return newTrienodeHistory(root, common.Hash{}, 1, nodes) } func makeTrienodeHistories(n int) []*trienodeHistory { - var result []*trienodeHistory + var ( + parent common.Hash + result []*trienodeHistory + ) for i := 0; i < n; i++ { - h := makeTrinodeHistory() - result = append(result, h) + nodes, root := randomTrienodes(10) + result = append(result, newTrienodeHistory(root, parent, uint64(i+1), nodes)) + parent = root } return result } @@ -78,6 +95,10 @@ func TestEncodeDecodeTrienodeHistory(t *testing.T) { if err := dec.decode(header, keySection, valueSection); err != nil { t.Fatalf("Failed to decode trienode history: %v", err) } + + if !reflect.DeepEqual(obj.meta, dec.meta) { + t.Fatal("trienode metadata is mismatched") + } if !compareList(dec.owners, obj.owners) { t.Fatal("trie owner list is mismatched") } @@ -120,11 +141,20 @@ func TestTrienodeHistoryReader(t *testing.T) { } } } + for i, h := range hs { + metadata, err := readTrienodeMetadata(freezer, uint64(i+1)) + if err != nil { + t.Fatalf("Failed to read trienode history metadata: %v", err) + } + if !reflect.DeepEqual(h.meta, metadata) { + t.Fatalf("Unexpected trienode metadata, want: %v, got: %v", h.meta, metadata) + } + } } // TestEmptyTrienodeHistory tests encoding/decoding of empty trienode history func TestEmptyTrienodeHistory(t *testing.T) { - h := newTrienodeHistory(make(map[common.Hash]map[string][]byte)) + h := newTrienodeHistory(common.Hash{}, common.Hash{}, 1, make(map[common.Hash]map[string][]byte)) // Test encoding empty history header, keySection, valueSection, err := h.encode() @@ -173,7 +203,7 @@ func TestSingleTrieHistory(t *testing.T) { nodes[owner]["ccc"] = testrand.Bytes(1000) // large value nodes[owner]["dddd"] = testrand.Bytes(0) // empty value - h := newTrienodeHistory(nodes) + h := newTrienodeHistory(common.Hash{}, common.Hash{}, 1, nodes) testEncodeDecode(t, h) } @@ -205,7 +235,7 @@ func TestMultipleTries(t *testing.T) { nodes[owner3][key] = nil } - h := newTrienodeHistory(nodes) + h := newTrienodeHistory(common.Hash{}, common.Hash{}, 1, nodes) testEncodeDecode(t, h) } @@ -221,7 +251,7 @@ func TestLargeNodeValues(t *testing.T) { key := string(testrand.Bytes(10)) nodes[owner][key] = testrand.Bytes(size) - h := newTrienodeHistory(nodes) + h := newTrienodeHistory(common.Hash{}, common.Hash{}, 1, nodes) testEncodeDecode(t, h) t.Logf("Successfully tested encoding/decoding with %dKB value", size/1024) } @@ -234,21 +264,16 @@ func TestNilNodeValues(t *testing.T) { nodes[owner] = make(map[string][]byte) // Mix of nil and non-nil values - nodes[owner]["nil1"] = nil - nodes[owner]["nil2"] = nil + nodes[owner]["nil"] = nil nodes[owner]["data1"] = []byte("some data") - nodes[owner]["nil3"] = nil nodes[owner]["data2"] = []byte("more data") - nodes[owner]["nil4"] = nil - h := newTrienodeHistory(nodes) + h := newTrienodeHistory(common.Hash{}, common.Hash{}, 1, nodes) testEncodeDecode(t, h) // Verify nil values are preserved - if h.nodes[owner]["nil1"] != nil { - t.Fatal("Nil value should be preserved") - } - if h.nodes[owner]["nil3"] != nil { + _, ok := h.nodes[owner]["nil"] + if !ok { t.Fatal("Nil value should be preserved") } } @@ -275,14 +300,12 @@ func TestCorruptedHeader(t *testing.T) { } // Test header with invalid trie header size - if len(header) > trienodeVersionSize { - invalidHeader := make([]byte, len(header)) - copy(invalidHeader, header) - invalidHeader = invalidHeader[:trienodeVersionSize+5] // Not divisible by trie header size + invalidHeader := make([]byte, len(header)) + copy(invalidHeader, header) + invalidHeader = invalidHeader[:trienodeMetadataSize+5] // Not divisible by trie header size - if err := decoded.decode(invalidHeader, keySection, valueSection); err == nil { - t.Fatal("Expected error for invalid header size") - } + if err := decoded.decode(invalidHeader, keySection, valueSection); err == nil { + t.Fatal("Expected error for invalid header size") } } @@ -351,7 +374,7 @@ func TestInvalidOffsets(t *testing.T) { // Corrupt key offset in header (make it larger than key section) corruptedHeader := make([]byte, len(header)) copy(corruptedHeader, header) - corruptedHeader[trienodeVersionSize+common.HashLength] = 0xff + corruptedHeader[trienodeMetadataSize+common.HashLength] = 0xff var dec1 trienodeHistory if err := dec1.decode(corruptedHeader, keySection, valueSection); err == nil { @@ -361,7 +384,7 @@ func TestInvalidOffsets(t *testing.T) { // Corrupt value offset in header (make it larger than value section) corruptedHeader = make([]byte, len(header)) copy(corruptedHeader, header) - corruptedHeader[trienodeVersionSize+common.HashLength+4] = 0xff + corruptedHeader[trienodeMetadataSize+common.HashLength+4] = 0xff var dec2 trienodeHistory if err := dec2.decode(corruptedHeader, keySection, valueSection); err == nil { @@ -412,7 +435,7 @@ func TestTrienodeHistoryReaderNilValues(t *testing.T) { nodes[owner]["nil2"] = nil nodes[owner]["data1"] = []byte("some data") - h := newTrienodeHistory(nodes) + h := newTrienodeHistory(common.Hash{}, common.Hash{}, 1, nodes) var freezer, _ = rawdb.NewTrienodeFreezer(t.TempDir(), false, false) defer freezer.Close() @@ -464,7 +487,7 @@ func TestTrienodeHistoryReaderNilKey(t *testing.T) { nodes[owner][""] = []byte("some data") nodes[owner]["data1"] = []byte("some data") - h := newTrienodeHistory(nodes) + h := newTrienodeHistory(common.Hash{}, common.Hash{}, 1, nodes) var freezer, _ = rawdb.NewTrienodeFreezer(t.TempDir(), false, false) defer freezer.Close() @@ -504,8 +527,16 @@ func TestTrienodeHistoryReaderIterator(t *testing.T) { // Count expected entries expectedCount := 0 - for _, nodeList := range h.nodeList { + expectedNodes := make(map[stateIdent]bool) + for owner, nodeList := range h.nodeList { expectedCount += len(nodeList) + for _, node := range nodeList { + expectedNodes[stateIdent{ + typ: typeTrienode, + addressHash: owner, + path: node, + }] = true + } } // Test the iterator @@ -529,6 +560,10 @@ func TestTrienodeHistoryReaderIterator(t *testing.T) { t.Fatal("Iterator yielded duplicate identifier") } seen[key] = true + + if !expectedNodes[key] { + t.Fatalf("Unexpected yielded identifier %v", key) + } } } @@ -583,7 +618,7 @@ func TestDecodeHeaderCorruptedData(t *testing.T) { header, _, _, _ := h.encode() // Test with empty header - _, _, _, err := decodeHeader([]byte{}) + _, _, _, _, err := decodeHeader([]byte{}) if err == nil { t.Fatal("Expected error for empty header") } @@ -592,14 +627,14 @@ func TestDecodeHeaderCorruptedData(t *testing.T) { corruptedVersion := make([]byte, len(header)) copy(corruptedVersion, header) corruptedVersion[0] = 0xFF - _, _, _, err = decodeHeader(corruptedVersion) + _, _, _, _, err = decodeHeader(corruptedVersion) if err == nil { t.Fatal("Expected error for invalid version") } // Test with truncated header (not divisible by trie header size) - truncated := header[:trienodeVersionSize+5] - _, _, _, err = decodeHeader(truncated) + truncated := header[:trienodeMetadataSize+5] + _, _, _, _, err = decodeHeader(truncated) if err == nil { t.Fatal("Expected error for truncated header") } @@ -609,8 +644,8 @@ func TestDecodeHeaderCorruptedData(t *testing.T) { copy(unordered, header) // Swap two owner hashes to make them unordered - hash1Start := trienodeVersionSize - hash2Start := trienodeVersionSize + trienodeTrieHeaderSize + hash1Start := trienodeMetadataSize + hash2Start := trienodeMetadataSize + trienodeTrieHeaderSize hash1 := unordered[hash1Start : hash1Start+common.HashLength] hash2 := unordered[hash2Start : hash2Start+common.HashLength] @@ -618,7 +653,7 @@ func TestDecodeHeaderCorruptedData(t *testing.T) { copy(unordered[hash1Start:hash1Start+common.HashLength], hash2) copy(unordered[hash2Start:hash2Start+common.HashLength], hash1) - _, _, _, err = decodeHeader(unordered) + _, _, _, _, err = decodeHeader(unordered) if err == nil { t.Fatal("Expected error for unordered trie owners") }