triedb/pathdb: include metadata in header section

triedb/pathdb/history_trienode.go

@@ -38,10 +38,20 @@ import (
 //
 // # Header
 // The header records metadata, including:
-//   - the history version
+//
+//   - the history version    (1 byte)
+//   - the parent state root  (32 bytes)
+//   - the current state root (32 bytes)
+//   - block number           (8 bytes)
+//
 //   - a lexicographically sorted list of trie IDs
 //   - the corresponding offsets into the key and value sections for each trie data chunk
 //
+// Although some fields (e.g., parent state root, block number) are duplicated
+// between the state history and the trienode history, these two histories
+// operate independently. To ensure each remains self-contained and self-descriptive,
+// we have chosen to maintain these duplicate fields.
+//
 // # Key section
 // The key section stores trie node keys (paths) in a compressed format.
 // It also contains relative offsets into the value section for resolving
@@ -60,7 +70,7 @@ import (
 // Header section:
 //
 //    +----------+------------------+---------------------+---------------------+-------+------------------+---------------------+---------------------|
-//    | ver (1B) | TrieID(32 bytes) | key offset(4 bytes) | val offset(4 bytes) |  ...  | TrieID(32 bytes) | key offset(4 bytes) | val offset(4 bytes) |
+//    | metadata | TrieID(32 bytes) | key offset(4 bytes) | val offset(4 bytes) |  ...  | TrieID(32 bytes) | key offset(4 bytes) | val offset(4 bytes) |
 //    +----------+------------------+---------------------+---------------------+-------+------------------+---------------------+---------------------|
 //
 //
@@ -103,23 +113,32 @@ import (
 // NOTE: All fixed-length integer are big-endian.
 
 const (
-	trienodeHistoryV0           = uint8(0)              // initial version of node history structure
+	trienodeHistoryV0           = uint8(0)                    // initial version of node history structure
-	trienodeHistoryVersion      = trienodeHistoryV0     // the default node history version
+	trienodeHistoryVersion      = trienodeHistoryV0           // the default node history version
-	trienodeVersionSize         = 1                     // the size of version tag in the history
+	trienodeMetadataSize        = 1 + 2*common.HashLength + 8 // the size of metadata in the history
-	trienodeTrieHeaderSize      = 8 + common.HashLength // the size of a single trie header in history
+	trienodeTrieHeaderSize      = 8 + common.HashLength       // the size of a single trie header in history
-	trienodeDataBlockRestartLen = 16                    // The restart interval length of trie node block
+	trienodeDataBlockRestartLen = 16                          // The restart interval length of trie node block
 )
 
+// trienodeMetadata describes the meta data of trienode history.
+type trienodeMetadata struct {
+	version uint8       // version tag of history object
+	parent  common.Hash // prev-state root before the state transition
+	root    common.Hash // post-state root after the state transition
+	block   uint64      // associated block number
+}
+
 // trienodeHistory represents a set of trie node changes resulting from a state
 // transition across the main account trie and all associated storage tries.
 type trienodeHistory struct {
+	meta     *trienodeMetadata                 // Metadata of the history
 	owners   []common.Hash                     // List of trie identifier sorted lexicographically
 	nodeList map[common.Hash][]string          // Set of node paths sorted lexicographically
 	nodes    map[common.Hash]map[string][]byte // Set of original value of trie nodes before state transition
 }
 
 // newTrienodeHistory constructs a trienode history with the provided trie nodes.
-func newTrienodeHistory(nodes map[common.Hash]map[string][]byte) *trienodeHistory {
+func newTrienodeHistory(root common.Hash, parent common.Hash, block uint64, nodes map[common.Hash]map[string][]byte) *trienodeHistory {
 	nodeList := make(map[common.Hash][]string)
 	for owner, subset := range nodes {
 		keys := sort.StringSlice(slices.Collect(maps.Keys(subset)))
@@ -127,6 +146,12 @@ func newTrienodeHistory(nodes map[common.Hash]map[string][]byte) *trienodeHistor
 		nodeList[owner] = keys
 	}
 	return &trienodeHistory{
+		meta: &trienodeMetadata{
+			version: trienodeHistoryVersion,
+			parent:  parent,
+			root:    root,
+			block:   block,
+		},
 		owners:   slices.SortedFunc(maps.Keys(nodes), common.Hash.Cmp),
 		nodeList: nodeList,
 		nodes:    nodes,
@@ -174,7 +199,10 @@ func (h *trienodeHistory) encode() ([]byte, []byte, []byte, error) {
 		keySection    bytes.Buffer
 		valueSection  bytes.Buffer
 	)
-	binary.Write(&headerSection, binary.BigEndian, trienodeHistoryVersion) // 1 byte
+	binary.Write(&headerSection, binary.BigEndian, h.meta.version) // 1 byte
+	headerSection.Write(h.meta.parent.Bytes())                     // 32 bytes
+	headerSection.Write(h.meta.root.Bytes())                       // 32 bytes
+	binary.Write(&headerSection, binary.BigEndian, h.meta.block)   // 8 byte
 
 	for _, owner := range h.owners {
 		// Fill the header section with offsets at key and value section
@@ -246,17 +274,21 @@ func (h *trienodeHistory) encode() ([]byte, []byte, []byte, error) {
 
 // decodeHeader resolves the metadata from the header section. An error
 // should be returned if the header section is corrupted.
-func decodeHeader(data []byte) ([]common.Hash, []uint32, []uint32, error) {
+func decodeHeader(data []byte) (*trienodeMetadata, []common.Hash, []uint32, []uint32, error) {
-	if len(data) < trienodeVersionSize {
+	if len(data) < trienodeMetadataSize {
-		return nil, nil, nil, fmt.Errorf("trienode history is too small, index size: %d", len(data))
+		return nil, nil, nil, nil, fmt.Errorf("trienode history is too small, index size: %d", len(data))
 	}
 	version := data[0]
 	if version != trienodeHistoryVersion {
-		return nil, nil, nil, fmt.Errorf("unregonized trienode history version: %d", version)
+		return nil, nil, nil, nil, fmt.Errorf("unregonized trienode history version: %d", version)
 	}
-	size := len(data) - trienodeVersionSize
+	parent := common.BytesToHash(data[1 : common.HashLength+1])                          // 32 bytes
+	root := common.BytesToHash(data[common.HashLength+1 : common.HashLength*2+1])        // 32 bytes
+	block := binary.BigEndian.Uint64(data[common.HashLength*2+1 : trienodeMetadataSize]) // 8 bytes
+
+	size := len(data) - trienodeMetadataSize
 	if size%trienodeTrieHeaderSize != 0 {
-		return nil, nil, nil, fmt.Errorf("truncated trienode history data, size %d", len(data))
+		return nil, nil, nil, nil, fmt.Errorf("truncated trienode history data, size %d", len(data))
 	}
 	count := size / trienodeTrieHeaderSize
 
@@ -266,17 +298,17 @@ func decodeHeader(data []byte) ([]common.Hash, []uint32, []uint32, error) {
 		valOffsets = make([]uint32, 0, count)
 	)
 	for i := 0; i < count; i++ {
-		n := trienodeVersionSize + trienodeTrieHeaderSize*i
+		n := trienodeMetadataSize + trienodeTrieHeaderSize*i
 		owner := common.BytesToHash(data[n : n+common.HashLength])
 		if i != 0 && bytes.Compare(owner.Bytes(), owners[i-1].Bytes()) <= 0 {
-			return nil, nil, nil, fmt.Errorf("trienode owners are out of order, prev: %v, cur: %v", owners[i-1], owner)
+			return nil, nil, nil, nil, fmt.Errorf("trienode owners are out of order, prev: %v, cur: %v", owners[i-1], owner)
 		}
 		owners = append(owners, owner)
 
 		// Decode the offset to the key section
 		keyOffset := binary.BigEndian.Uint32(data[n+common.HashLength : n+common.HashLength+4])
 		if i != 0 && keyOffset <= keyOffsets[i-1] {
-			return nil, nil, nil, fmt.Errorf("key offset is out of order, prev: %v, cur: %v", keyOffsets[i-1], keyOffset)
+			return nil, nil, nil, nil, fmt.Errorf("key offset is out of order, prev: %v, cur: %v", keyOffsets[i-1], keyOffset)
 		}
 		keyOffsets = append(keyOffsets, keyOffset)
 
@@ -285,11 +317,16 @@ func decodeHeader(data []byte) ([]common.Hash, []uint32, []uint32, error) {
 		// a trie deletion).
 		valOffset := binary.BigEndian.Uint32(data[n+common.HashLength+4 : n+common.HashLength+8])
 		if i != 0 && valOffset < valOffsets[i-1] {
-			return nil, nil, nil, fmt.Errorf("value offset is out of order, prev: %v, cur: %v", valOffsets[i-1], valOffset)
+			return nil, nil, nil, nil, fmt.Errorf("value offset is out of order, prev: %v, cur: %v", valOffsets[i-1], valOffset)
 		}
 		valOffsets = append(valOffsets, valOffset)
 	}
-	return owners, keyOffsets, valOffsets, nil
+	return &trienodeMetadata{
+		version: version,
+		parent:  parent,
+		root:    root,
+		block:   block,
+	}, owners, keyOffsets, valOffsets, nil
 }
 
 func decodeSingle(keySection []byte, onValue func([]byte, int, int) error) ([]string, error) {
@@ -425,10 +462,11 @@ func decodeSingleWithValue(keySection []byte, valueSection []byte) ([]string, ma
 
 // decode deserializes the contained trie nodes from the provided bytes.
 func (h *trienodeHistory) decode(header []byte, keySection []byte, valueSection []byte) error {
-	owners, keyOffsets, valueOffsets, err := decodeHeader(header)
+	metadata, owners, keyOffsets, valueOffsets, err := decodeHeader(header)
 	if err != nil {
 		return err
 	}
+	h.meta = metadata
 	h.owners = owners
 	h.nodeList = make(map[common.Hash][]string)
 	h.nodes = make(map[common.Hash]map[string][]byte)
@@ -562,13 +600,13 @@ func newTrienodeHistoryReader(id uint64, reader ethdb.AncientReader) (*trienodeH
 	return r, nil
 }
 
-// decodeHeader decodes the metadata of trienode history from the header section.
+// decodeHeader decodes the header section of trienode history.
 func (r *trienodeHistoryReader) decodeHeader() error {
 	header, err := rawdb.ReadTrienodeHistoryHeader(r.reader, r.id)
 	if err != nil {
 		return err
 	}
-	owners, keyOffsets, valOffsets, err := decodeHeader(header)
+	_, owners, keyOffsets, valOffsets, err := decodeHeader(header)
 	if err != nil {
 		return err
 	}
@@ -626,7 +664,7 @@ func (r *trienodeHistoryReader) read(owner common.Hash, path string) ([]byte, er
 // nolint:unused
 func writeTrienodeHistory(writer ethdb.AncientWriter, dl *diffLayer) error {
 	start := time.Now()
-	h := newTrienodeHistory(dl.nodes.nodeOrigin)
+	h := newTrienodeHistory(dl.rootHash(), dl.parent.rootHash(), dl.block, dl.nodes.nodeOrigin)
 	header, keySection, valueSection, err := h.encode()
 	if err != nil {
 		return err
@@ -649,6 +687,20 @@ func writeTrienodeHistory(writer ethdb.AncientWriter, dl *diffLayer) error {
 	return nil
 }
 
+// readTrienodeMetadata resolves the metadata of the specified trienode history.
+// nolint:unused
+func readTrienodeMetadata(reader ethdb.AncientReader, id uint64) (*trienodeMetadata, error) {
+	header, err := rawdb.ReadTrienodeHistoryHeader(reader, id)
+	if err != nil {
+		return nil, err
+	}
+	metadata, _, _, _, err := decodeHeader(header)
+	if err != nil {
+		return nil, err
+	}
+	return metadata, nil
+}
+
 // readTrienodeHistory resolves a single trienode history object with specific id.
 func readTrienodeHistory(reader ethdb.AncientReader, id uint64) (*trienodeHistory, error) {
 	header, keySection, valueSection, err := rawdb.ReadTrienodeHistory(reader, id)

triedb/pathdb/history_trienode_test.go

@@ -20,18 +20,26 @@ import (
 	"bytes"
 	"encoding/binary"
 	"math/rand"
+	"reflect"
 	"testing"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/rawdb"
+	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/internal/testrand"
 )
 
 // randomTrienodes generates a random trienode set.
-func randomTrienodes(n int) map[common.Hash]map[string][]byte {
+func randomTrienodes(n int) (map[common.Hash]map[string][]byte, common.Hash) {
-	nodes := make(map[common.Hash]map[string][]byte)
+	var (
+		root  common.Hash
+		nodes = make(map[common.Hash]map[string][]byte)
+	)
 	for i := 0; i < n; i++ {
 		owner := testrand.Hash()
+		if i == 0 {
+			owner = common.Hash{}
+		}
 		nodes[owner] = make(map[string][]byte)
 
 		for j := 0; j < 10; j++ {
@@ -48,20 +56,29 @@ func randomTrienodes(n int) map[common.Hash]map[string][]byte {
 			nodes[owner][string(path)] = nil
 		}
 		// root node with zero-size path
-		nodes[owner][""] = testrand.Bytes(10)
+		rnode := testrand.Bytes(256)
+		nodes[owner][""] = rnode
+		if owner == (common.Hash{}) {
+			root = crypto.Keccak256Hash(rnode)
+		}
 	}
-	return nodes
+	return nodes, root
 }
 
 func makeTrinodeHistory() *trienodeHistory {
-	return newTrienodeHistory(randomTrienodes(10))
+	nodes, root := randomTrienodes(10)
+	return newTrienodeHistory(root, common.Hash{}, 1, nodes)
 }
 
 func makeTrienodeHistories(n int) []*trienodeHistory {
-	var result []*trienodeHistory
+	var (
+		parent common.Hash
+		result []*trienodeHistory
+	)
 	for i := 0; i < n; i++ {
-		h := makeTrinodeHistory()
+		nodes, root := randomTrienodes(10)
-		result = append(result, h)
+		result = append(result, newTrienodeHistory(root, parent, uint64(i+1), nodes))
+		parent = root
 	}
 	return result
 }
@@ -78,6 +95,10 @@ func TestEncodeDecodeTrienodeHistory(t *testing.T) {
 	if err := dec.decode(header, keySection, valueSection); err != nil {
 		t.Fatalf("Failed to decode trienode history: %v", err)
 	}
+
+	if !reflect.DeepEqual(obj.meta, dec.meta) {
+		t.Fatal("trienode metadata is mismatched")
+	}
 	if !compareList(dec.owners, obj.owners) {
 		t.Fatal("trie owner list is mismatched")
 	}
@@ -120,11 +141,20 @@ func TestTrienodeHistoryReader(t *testing.T) {
 			}
 		}
 	}
+	for i, h := range hs {
+		metadata, err := readTrienodeMetadata(freezer, uint64(i+1))
+		if err != nil {
+			t.Fatalf("Failed to read trienode history metadata: %v", err)
+		}
+		if !reflect.DeepEqual(h.meta, metadata) {
+			t.Fatalf("Unexpected trienode metadata, want: %v, got: %v", h.meta, metadata)
+		}
+	}
 }
 
 // TestEmptyTrienodeHistory tests encoding/decoding of empty trienode history
 func TestEmptyTrienodeHistory(t *testing.T) {
-	h := newTrienodeHistory(make(map[common.Hash]map[string][]byte))
+	h := newTrienodeHistory(common.Hash{}, common.Hash{}, 1, make(map[common.Hash]map[string][]byte))
 
 	// Test encoding empty history
 	header, keySection, valueSection, err := h.encode()
@@ -173,7 +203,7 @@ func TestSingleTrieHistory(t *testing.T) {
 	nodes[owner]["ccc"] = testrand.Bytes(1000) // large value
 	nodes[owner]["dddd"] = testrand.Bytes(0)   // empty value
 
-	h := newTrienodeHistory(nodes)
+	h := newTrienodeHistory(common.Hash{}, common.Hash{}, 1, nodes)
 	testEncodeDecode(t, h)
 }
 
@@ -205,7 +235,7 @@ func TestMultipleTries(t *testing.T) {
 		nodes[owner3][key] = nil
 	}
 
-	h := newTrienodeHistory(nodes)
+	h := newTrienodeHistory(common.Hash{}, common.Hash{}, 1, nodes)
 	testEncodeDecode(t, h)
 }
 
@@ -221,7 +251,7 @@ func TestLargeNodeValues(t *testing.T) {
 		key := string(testrand.Bytes(10))
 		nodes[owner][key] = testrand.Bytes(size)
 
-		h := newTrienodeHistory(nodes)
+		h := newTrienodeHistory(common.Hash{}, common.Hash{}, 1, nodes)
 		testEncodeDecode(t, h)
 		t.Logf("Successfully tested encoding/decoding with %dKB value", size/1024)
 	}
@@ -234,21 +264,16 @@ func TestNilNodeValues(t *testing.T) {
 	nodes[owner] = make(map[string][]byte)
 
 	// Mix of nil and non-nil values
-	nodes[owner]["nil1"] = nil
+	nodes[owner]["nil"] = nil
-	nodes[owner]["nil2"] = nil
 	nodes[owner]["data1"] = []byte("some data")
-	nodes[owner]["nil3"] = nil
 	nodes[owner]["data2"] = []byte("more data")
-	nodes[owner]["nil4"] = nil
 
-	h := newTrienodeHistory(nodes)
+	h := newTrienodeHistory(common.Hash{}, common.Hash{}, 1, nodes)
 	testEncodeDecode(t, h)
 
 	// Verify nil values are preserved
-	if h.nodes[owner]["nil1"] != nil {
+	_, ok := h.nodes[owner]["nil"]
-		t.Fatal("Nil value should be preserved")
+	if !ok {
-	}
-	if h.nodes[owner]["nil3"] != nil {
 		t.Fatal("Nil value should be preserved")
 	}
 }
@@ -275,14 +300,12 @@ func TestCorruptedHeader(t *testing.T) {
 	}
 
 	// Test header with invalid trie header size
-	if len(header) > trienodeVersionSize {
+	invalidHeader := make([]byte, len(header))
-		invalidHeader := make([]byte, len(header))
+	copy(invalidHeader, header)
-		copy(invalidHeader, header)
+	invalidHeader = invalidHeader[:trienodeMetadataSize+5] // Not divisible by trie header size
-		invalidHeader = invalidHeader[:trienodeVersionSize+5] // Not divisible by trie header size
 
-		if err := decoded.decode(invalidHeader, keySection, valueSection); err == nil {
+	if err := decoded.decode(invalidHeader, keySection, valueSection); err == nil {
-			t.Fatal("Expected error for invalid header size")
+		t.Fatal("Expected error for invalid header size")
-		}
 	}
 }
 
@@ -351,7 +374,7 @@ func TestInvalidOffsets(t *testing.T) {
 	// Corrupt key offset in header (make it larger than key section)
 	corruptedHeader := make([]byte, len(header))
 	copy(corruptedHeader, header)
-	corruptedHeader[trienodeVersionSize+common.HashLength] = 0xff
+	corruptedHeader[trienodeMetadataSize+common.HashLength] = 0xff
 
 	var dec1 trienodeHistory
 	if err := dec1.decode(corruptedHeader, keySection, valueSection); err == nil {
@@ -361,7 +384,7 @@ func TestInvalidOffsets(t *testing.T) {
 	// Corrupt value offset in header (make it larger than value section)
 	corruptedHeader = make([]byte, len(header))
 	copy(corruptedHeader, header)
-	corruptedHeader[trienodeVersionSize+common.HashLength+4] = 0xff
+	corruptedHeader[trienodeMetadataSize+common.HashLength+4] = 0xff
 
 	var dec2 trienodeHistory
 	if err := dec2.decode(corruptedHeader, keySection, valueSection); err == nil {
@@ -412,7 +435,7 @@ func TestTrienodeHistoryReaderNilValues(t *testing.T) {
 	nodes[owner]["nil2"] = nil
 	nodes[owner]["data1"] = []byte("some data")
 
-	h := newTrienodeHistory(nodes)
+	h := newTrienodeHistory(common.Hash{}, common.Hash{}, 1, nodes)
 
 	var freezer, _ = rawdb.NewTrienodeFreezer(t.TempDir(), false, false)
 	defer freezer.Close()
@@ -464,7 +487,7 @@ func TestTrienodeHistoryReaderNilKey(t *testing.T) {
 	nodes[owner][""] = []byte("some data")
 	nodes[owner]["data1"] = []byte("some data")
 
-	h := newTrienodeHistory(nodes)
+	h := newTrienodeHistory(common.Hash{}, common.Hash{}, 1, nodes)
 
 	var freezer, _ = rawdb.NewTrienodeFreezer(t.TempDir(), false, false)
 	defer freezer.Close()
@@ -504,8 +527,16 @@ func TestTrienodeHistoryReaderIterator(t *testing.T) {
 
 	// Count expected entries
 	expectedCount := 0
-	for _, nodeList := range h.nodeList {
+	expectedNodes := make(map[stateIdent]bool)
+	for owner, nodeList := range h.nodeList {
 		expectedCount += len(nodeList)
+		for _, node := range nodeList {
+			expectedNodes[stateIdent{
+				typ:         typeTrienode,
+				addressHash: owner,
+				path:        node,
+			}] = true
+		}
 	}
 
 	// Test the iterator
@@ -529,6 +560,10 @@ func TestTrienodeHistoryReaderIterator(t *testing.T) {
 			t.Fatal("Iterator yielded duplicate identifier")
 		}
 		seen[key] = true
+
+		if !expectedNodes[key] {
+			t.Fatalf("Unexpected yielded identifier %v", key)
+		}
 	}
 }
 
@@ -583,7 +618,7 @@ func TestDecodeHeaderCorruptedData(t *testing.T) {
 	header, _, _, _ := h.encode()
 
 	// Test with empty header
-	_, _, _, err := decodeHeader([]byte{})
+	_, _, _, _, err := decodeHeader([]byte{})
 	if err == nil {
 		t.Fatal("Expected error for empty header")
 	}
@@ -592,14 +627,14 @@ func TestDecodeHeaderCorruptedData(t *testing.T) {
 	corruptedVersion := make([]byte, len(header))
 	copy(corruptedVersion, header)
 	corruptedVersion[0] = 0xFF
-	_, _, _, err = decodeHeader(corruptedVersion)
+	_, _, _, _, err = decodeHeader(corruptedVersion)
 	if err == nil {
 		t.Fatal("Expected error for invalid version")
 	}
 
 	// Test with truncated header (not divisible by trie header size)
-	truncated := header[:trienodeVersionSize+5]
+	truncated := header[:trienodeMetadataSize+5]
-	_, _, _, err = decodeHeader(truncated)
+	_, _, _, _, err = decodeHeader(truncated)
 	if err == nil {
 		t.Fatal("Expected error for truncated header")
 	}
@@ -609,8 +644,8 @@ func TestDecodeHeaderCorruptedData(t *testing.T) {
 	copy(unordered, header)
 
 	// Swap two owner hashes to make them unordered
-	hash1Start := trienodeVersionSize
+	hash1Start := trienodeMetadataSize
-	hash2Start := trienodeVersionSize + trienodeTrieHeaderSize
+	hash2Start := trienodeMetadataSize + trienodeTrieHeaderSize
 	hash1 := unordered[hash1Start : hash1Start+common.HashLength]
 	hash2 := unordered[hash2Start : hash2Start+common.HashLength]
 
@@ -618,7 +653,7 @@ func TestDecodeHeaderCorruptedData(t *testing.T) {
 	copy(unordered[hash1Start:hash1Start+common.HashLength], hash2)
 	copy(unordered[hash2Start:hash2Start+common.HashLength], hash1)
 
-	_, _, _, err = decodeHeader(unordered)
+	_, _, _, _, err = decodeHeader(unordered)
 	if err == nil {
 		t.Fatal("Expected error for unordered trie owners")
 	}