diff --git a/cmd/evm/internal/t8ntool/transition.go b/cmd/evm/internal/t8ntool/transition.go index af60333cbd..1226656eb9 100644 --- a/cmd/evm/internal/t8ntool/transition.go +++ b/cmd/evm/internal/t8ntool/transition.go @@ -428,7 +428,7 @@ func BinKeys(ctx *cli.Context) error { db := triedb.NewDatabase(rawdb.NewMemoryDatabase(), triedb.VerkleDefaults) defer db.Close() - bt, err := genBinTrieFromAlloc(alloc, db) + bt, err := genBinTrieFromAlloc(alloc, db, db.BinTrieGroupDepth()) if err != nil { return fmt.Errorf("error generating bt: %w", err) } @@ -472,7 +472,7 @@ func BinTrieRoot(ctx *cli.Context) error { db := triedb.NewDatabase(rawdb.NewMemoryDatabase(), triedb.VerkleDefaults) defer db.Close() - bt, err := genBinTrieFromAlloc(alloc, db) + bt, err := genBinTrieFromAlloc(alloc, db, db.BinTrieGroupDepth()) if err != nil { return fmt.Errorf("error generating bt: %w", err) } @@ -482,8 +482,8 @@ func BinTrieRoot(ctx *cli.Context) error { } // TODO(@CPerezz): Should this go to `bintrie` module? -func genBinTrieFromAlloc(alloc core.GenesisAlloc, db database.NodeDatabase) (*bintrie.BinaryTrie, error) { - bt, err := bintrie.NewBinaryTrie(types.EmptyBinaryHash, db) +func genBinTrieFromAlloc(alloc core.GenesisAlloc, db database.NodeDatabase, groupDepth int) (*bintrie.BinaryTrie, error) { + bt, err := bintrie.NewBinaryTrie(types.EmptyBinaryHash, db, groupDepth) if err != nil { return nil, err } diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 844397b734..6775dce676 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -289,6 +289,12 @@ var ( Value: ethconfig.Defaults.EnableStateSizeTracking, Category: flags.StateCategory, } + BinTrieGroupDepthFlag = &cli.IntFlag{ + Name: "bintrie.groupdepth", + Usage: "Number of levels per serialized group in binary trie (1-8, default 8). Lower values create smaller groups with more nodes.", + Value: 8, + Category: flags.StateCategory, + } StateHistoryFlag = &cli.Uint64Flag{ Name: "history.state", Usage: "Number of recent blocks to retain state history for, only relevant in state.scheme=path (default = 90,000 blocks, 0 = entire chain)", @@ -1720,9 +1726,6 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.IsSet(TrienodeHistoryFlag.Name) { cfg.TrienodeHistory = ctx.Int64(TrienodeHistoryFlag.Name) } - if ctx.IsSet(TrienodeHistoryFullValueCheckpointFlag.Name) { - cfg.NodeFullValueCheckpoint = uint32(ctx.Uint(TrienodeHistoryFullValueCheckpointFlag.Name)) - } if ctx.IsSet(StateSchemeFlag.Name) { cfg.StateScheme = ctx.String(StateSchemeFlag.Name) } @@ -2338,6 +2341,7 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readonly bool) (*core.BlockCh StateHistory: ctx.Uint64(StateHistoryFlag.Name), TrienodeHistory: ctx.Int64(TrienodeHistoryFlag.Name), NodeFullValueCheckpoint: uint32(ctx.Uint(TrienodeHistoryFullValueCheckpointFlag.Name)), + BinTrieGroupDepth: ctx.Int(BinTrieGroupDepthFlag.Name), // Disable transaction indexing/unindexing. TxLookupLimit: -1, diff --git a/core/blockchain.go b/core/blockchain.go index 8741b8b937..b7329cad66 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -169,9 +169,10 @@ type BlockChainConfig struct { TrieNoAsyncFlush bool // Whether the asynchronous buffer flushing is disallowed TrieJournalDirectory string // Directory path to the journal used for persisting trie data across node restarts - Preimages bool // Whether to store preimage of trie key to the disk - StateScheme string // Scheme used to store ethereum states and merkle tree nodes on top - ArchiveMode bool // Whether to enable the archive mode + Preimages bool // Whether to store preimage of trie key to the disk + StateScheme string // Scheme used to store ethereum states and merkle tree nodes on top + ArchiveMode bool // Whether to enable the archive mode + BinTrieGroupDepth int // Number of levels per serialized group in binary trie (1-8) // Number of blocks from the chain head for which state histories are retained. // If set to 0, all state histories across the entire chain will be retained; @@ -255,8 +256,9 @@ func (cfg BlockChainConfig) WithNoAsyncFlush(on bool) *BlockChainConfig { // triedbConfig derives the configures for trie database. func (cfg *BlockChainConfig) triedbConfig(isVerkle bool) *triedb.Config { config := &triedb.Config{ - Preimages: cfg.Preimages, - IsVerkle: isVerkle, + Preimages: cfg.Preimages, + IsVerkle: isVerkle, + BinTrieGroupDepth: cfg.BinTrieGroupDepth, } if cfg.StateScheme == rawdb.HashScheme { config.HashDB = &hashdb.Config{ diff --git a/core/state/database.go b/core/state/database.go index 4a5547d075..82e585ff57 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -246,7 +246,7 @@ func (db *CachingDB) OpenTrie(root common.Hash) (Trie, error) { if ts.Transitioned() { // Use BinaryTrie instead of VerkleTrie when IsVerkle is set // (IsVerkle actually means Binary Trie mode in this codebase) - return bintrie.NewBinaryTrie(root, db.triedb) + return bintrie.NewBinaryTrie(root, db.triedb, db.triedb.BinTrieGroupDepth()) } } tr, err := trie.NewStateTrie(trie.StateTrieID(root), db.triedb) diff --git a/core/state/reader.go b/core/state/reader.go index 2db9d1f9b4..0d094030d9 100644 --- a/core/state/reader.go +++ b/core/state/reader.go @@ -302,7 +302,7 @@ func newTrieReader(root common.Hash, db *triedb.Database) (*trieReader, error) { tr, err = trie.NewStateTrie(trie.StateTrieID(root), db) } else { // When IsVerkle() is true, create a BinaryTrie wrapped in TransitionTrie - binTrie, binErr := bintrie.NewBinaryTrie(root, db) + binTrie, binErr := bintrie.NewBinaryTrie(root, db, db.BinTrieGroupDepth()) if binErr != nil { return nil, binErr } diff --git a/eth/backend.go b/eth/backend.go index aed1542aeb..d765a663e9 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -232,6 +232,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { StateHistory: config.StateHistory, TrienodeHistory: config.TrienodeHistory, NodeFullValueCheckpoint: config.NodeFullValueCheckpoint, + BinTrieGroupDepth: config.BinTrieGroupDepth, StateScheme: scheme, ChainHistoryMode: config.HistoryMode, TxLookupLimit: int64(min(config.TransactionHistory, math.MaxInt64)), diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index e58c4b884a..1505ce6524 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -59,6 +59,7 @@ var Defaults = Config{ StateHistory: pathdb.Defaults.StateHistory, TrienodeHistory: pathdb.Defaults.TrienodeHistory, NodeFullValueCheckpoint: pathdb.Defaults.FullValueCheckpoint, + BinTrieGroupDepth: 8, // byte-aligned groups by default DatabaseCache: 512, TrieCleanCache: 154, TrieDirtyCache: 256, @@ -125,6 +126,11 @@ type Config struct { // consistent with persistent state. StateScheme string `toml:",omitempty"` + // BinTrieGroupDepth is the number of levels per serialized group in binary trie. + // Valid values are 1-8, with 8 being the default (byte-aligned groups). + // Lower values create smaller groups with more nodes. + BinTrieGroupDepth int `toml:",omitempty"` + // RequiredBlocks is a set of block number -> hash mappings which must be in the // canonical chain of all remote peers. Setting the option makes geth verify the // presence of these blocks for every new peer connection. diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index 6f94a409e5..09e1ee1e78 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -34,6 +34,7 @@ func (c Config) MarshalTOML() (interface{}, error) { TrienodeHistory int64 `toml:",omitempty"` NodeFullValueCheckpoint uint32 `toml:",omitempty"` StateScheme string `toml:",omitempty"` + BinTrieGroupDepth int `toml:",omitempty"` RequiredBlocks map[uint64]common.Hash `toml:"-"` SlowBlockThreshold time.Duration `toml:",omitempty"` SkipBcVersionCheck bool `toml:"-"` @@ -87,6 +88,7 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.TrienodeHistory = c.TrienodeHistory enc.NodeFullValueCheckpoint = c.NodeFullValueCheckpoint enc.StateScheme = c.StateScheme + enc.BinTrieGroupDepth = c.BinTrieGroupDepth enc.RequiredBlocks = c.RequiredBlocks enc.SlowBlockThreshold = c.SlowBlockThreshold enc.SkipBcVersionCheck = c.SkipBcVersionCheck @@ -144,6 +146,7 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { TrienodeHistory *int64 `toml:",omitempty"` NodeFullValueCheckpoint *uint32 `toml:",omitempty"` StateScheme *string `toml:",omitempty"` + BinTrieGroupDepth *int `toml:",omitempty"` RequiredBlocks map[uint64]common.Hash `toml:"-"` SlowBlockThreshold *time.Duration `toml:",omitempty"` SkipBcVersionCheck *bool `toml:"-"` @@ -234,6 +237,9 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.StateScheme != nil { c.StateScheme = *dec.StateScheme } + if dec.BinTrieGroupDepth != nil { + c.BinTrieGroupDepth = *dec.BinTrieGroupDepth + } if dec.RequiredBlocks != nil { c.RequiredBlocks = dec.RequiredBlocks } diff --git a/trie/bintrie/binary_node.go b/trie/bintrie/binary_node.go index e1eee7a7bd..07a9107642 100644 --- a/trie/bintrie/binary_node.go +++ b/trie/bintrie/binary_node.go @@ -31,21 +31,30 @@ type ( var zero [32]byte const ( - StemNodeWidth = 256 // Number of child per leaf node - StemSize = 31 // Number of bytes to travel before reaching a group of leaves - NodeTypeBytes = 1 // Size of node type prefix in serialization - HashSize = 32 // Size of a hash in bytes - BitmapSize = 32 // Size of the bitmap in a stem node + StemNodeWidth = 256 // Number of child per leaf node + StemSize = 31 // Number of bytes to travel before reaching a group of leaves + NodeTypeBytes = 1 // Size of node type prefix in serialization + HashSize = 32 // Size of a hash in bytes + StemBitmapSize = 32 // Size of the bitmap in a stem node (256 values = 32 bytes) - // GroupDepth is the number of levels in a grouped subtree serialization. - // Groups are byte-aligned (depth % 8 == 0). This may become configurable later. + // MaxGroupDepth is the maximum allowed group depth for InternalNode serialization. + // Valid group depths are 1-8, where depth N means 2^N bottom-layer positions. // Serialization format for InternalNode groups: - // [1 byte type] [1 byte group depth (1-8)] [32 byte bitmap] [N × 32 byte hashes] - // The bitmap has 2^groupDepth bits, indicating which bottom-layer children are present. - // Only present children's hashes are stored, in order. - GroupDepth = 8 + // [1 byte type] [1 byte group depth (1-8)] [variable bitmap] [N × 32 byte hashes] + // The bitmap has 2^groupDepth bits (BitmapSizeForDepth bytes), indicating which + // bottom-layer children are present. Only present children's hashes are stored. + MaxGroupDepth = 8 ) +// BitmapSizeForDepth returns the bitmap size in bytes for a given group depth. +// For depths 1-3, returns 1 byte. For depths 4-8, returns 2^(depth-3) bytes. +func BitmapSizeForDepth(groupDepth int) int { + if groupDepth <= 3 { + return 1 + } + return 1 << (groupDepth - 3) +} + const ( nodeTypeStem = iota + 1 // Stem node, contains a stem and a bitmap of values nodeTypeInternal @@ -59,7 +68,7 @@ type BinaryNode interface { Hash() common.Hash GetValuesAtStem([]byte, NodeResolverFn) ([][]byte, error) InsertValuesAtStem([]byte, [][]byte, NodeResolverFn, int) (BinaryNode, error) - CollectNodes([]byte, NodeFlushFn) error + CollectNodes([]byte, NodeFlushFn, int) error // groupDepth parameter for serialization toDot(parent, path string) string GetHeight() int @@ -106,25 +115,28 @@ func serializeSubtree(node BinaryNode, remainingDepth int, position int, bitmap } // SerializeNode serializes a binary trie node into a byte slice. -func SerializeNode(node BinaryNode) []byte { +// groupDepth specifies how many levels to include in an InternalNode group (1-8). +func SerializeNode(node BinaryNode, groupDepth int) []byte { + if groupDepth < 1 || groupDepth > MaxGroupDepth { + panic("groupDepth must be between 1 and 8") + } switch n := (node).(type) { case *InternalNode: - // InternalNode group: 1 byte type + 1 byte group depth + 32 byte bitmap + N×32 byte hashes - groupDepth := GroupDepth - - var bitmap [BitmapSize]byte + // InternalNode group: 1 byte type + 1 byte group depth + variable bitmap + N×32 byte hashes + bitmapSize := BitmapSizeForDepth(groupDepth) + bitmap := make([]byte, bitmapSize) var hashes []common.Hash - serializeSubtree(n, groupDepth, 0, bitmap[:], &hashes) + serializeSubtree(n, groupDepth, 0, bitmap, &hashes) // Build serialized output - serializedLen := NodeTypeBytes + 1 + BitmapSize + len(hashes)*HashSize + serializedLen := NodeTypeBytes + 1 + bitmapSize + len(hashes)*HashSize serialized := make([]byte, serializedLen) serialized[0] = nodeTypeInternal serialized[1] = byte(groupDepth) - copy(serialized[2:2+BitmapSize], bitmap[:]) + copy(serialized[2:2+bitmapSize], bitmap) - offset := NodeTypeBytes + 1 + BitmapSize + offset := NodeTypeBytes + 1 + bitmapSize for _, h := range hashes { copy(serialized[offset:offset+HashSize], h.Bytes()) offset += HashSize @@ -207,16 +219,20 @@ func DeserializeNode(serialized []byte, depth int) (BinaryNode, error) { switch serialized[0] { case nodeTypeInternal: - // Grouped format: 1 byte type + 1 byte group depth + 32 byte bitmap + N×32 byte hashes - if len(serialized) < NodeTypeBytes+1+BitmapSize { + // Grouped format: 1 byte type + 1 byte group depth + variable bitmap + N×32 byte hashes + if len(serialized) < NodeTypeBytes+1 { return nil, invalidSerializedLength } groupDepth := int(serialized[1]) - if groupDepth < 1 || groupDepth > GroupDepth { + if groupDepth < 1 || groupDepth > MaxGroupDepth { return nil, errors.New("invalid group depth") } - bitmap := serialized[2 : 2+BitmapSize] - hashData := serialized[2+BitmapSize:] + bitmapSize := BitmapSizeForDepth(groupDepth) + if len(serialized) < NodeTypeBytes+1+bitmapSize { + return nil, invalidSerializedLength + } + bitmap := serialized[2 : 2+bitmapSize] + hashData := serialized[2+bitmapSize:] // Count present children from bitmap hashIdx := 0 diff --git a/trie/bintrie/binary_node_test.go b/trie/bintrie/binary_node_test.go index a04f08c9b3..2fe068d869 100644 --- a/trie/bintrie/binary_node_test.go +++ b/trie/bintrie/binary_node_test.go @@ -37,27 +37,28 @@ func TestSerializeDeserializeInternalNode(t *testing.T) { right: HashedNode(rightHash), } - // Serialize the node - serialized := SerializeNode(node) + // Serialize the node with default group depth of 8 + serialized := SerializeNode(node, MaxGroupDepth) // Check the serialized format: type byte + group depth byte + 32 byte bitmap + N*32 byte hashes if serialized[0] != nodeTypeInternal { t.Errorf("Expected type byte to be %d, got %d", nodeTypeInternal, serialized[0]) } - if serialized[1] != GroupDepth { - t.Errorf("Expected group depth to be %d, got %d", GroupDepth, serialized[1]) + if serialized[1] != MaxGroupDepth { + t.Errorf("Expected group depth to be %d, got %d", MaxGroupDepth, serialized[1]) } // Expected length: 1 (type) + 1 (group depth) + 32 (bitmap) + 2*32 (two hashes) = 98 bytes - expectedLen := NodeTypeBytes + 1 + BitmapSize + 2*HashSize + bitmapSize := BitmapSizeForDepth(MaxGroupDepth) + expectedLen := NodeTypeBytes + 1 + bitmapSize + 2*HashSize if len(serialized) != expectedLen { t.Errorf("Expected serialized length to be %d, got %d", expectedLen, len(serialized)) } // The left child (HashedNode) terminates at remainingDepth=7, so it's placed at position 0<<7 = 0 // The right child (HashedNode) terminates at remainingDepth=7, so it's placed at position 1<<7 = 128 - bitmap := serialized[2 : 2+BitmapSize] + bitmap := serialized[2 : 2+bitmapSize] if bitmap[0]&0x80 == 0 { // bit 0 (MSB of byte 0) t.Error("Expected bit 0 to be set in bitmap (left child)") } @@ -135,8 +136,8 @@ func TestSerializeDeserializeStemNode(t *testing.T) { depth: 10, } - // Serialize the node - serialized := SerializeNode(node) + // Serialize the node (groupDepth doesn't affect StemNode serialization) + serialized := SerializeNode(node, MaxGroupDepth) // Check the serialized format if serialized[0] != nodeTypeStem { @@ -214,8 +215,8 @@ func TestDeserializeInvalidType(t *testing.T) { // TestDeserializeInvalidLength tests deserialization with invalid data length func TestDeserializeInvalidLength(t *testing.T) { - // InternalNode with type byte 1 but wrong length - invalidData := []byte{nodeTypeInternal, 0, 0} // Too short for internal node + // InternalNode with valid type byte and group depth but too short for bitmap + invalidData := []byte{nodeTypeInternal, 8, 0, 0} // Too short for bitmap (needs 32 bytes) _, err := DeserializeNode(invalidData, 0) if err == nil { diff --git a/trie/bintrie/empty.go b/trie/bintrie/empty.go index 7cfe373b35..7d426e15bf 100644 --- a/trie/bintrie/empty.go +++ b/trie/bintrie/empty.go @@ -59,7 +59,7 @@ func (e Empty) InsertValuesAtStem(key []byte, values [][]byte, _ NodeResolverFn, }, nil } -func (e Empty) CollectNodes(_ []byte, _ NodeFlushFn) error { +func (e Empty) CollectNodes(_ []byte, _ NodeFlushFn, _ int) error { return nil } diff --git a/trie/bintrie/empty_test.go b/trie/bintrie/empty_test.go index 574ae1830b..07dcefcaff 100644 --- a/trie/bintrie/empty_test.go +++ b/trie/bintrie/empty_test.go @@ -186,7 +186,7 @@ func TestEmptyCollectNodes(t *testing.T) { collected = append(collected, n) } - err := node.CollectNodes([]byte{0, 1, 0}, flushFn) + err := node.CollectNodes([]byte{0, 1, 0}, flushFn, MaxGroupDepth) if err != nil { t.Fatalf("Unexpected error: %v", err) } diff --git a/trie/bintrie/group_debug_test.go b/trie/bintrie/group_debug_test.go index cc54a6f68c..87a8d6ea43 100644 --- a/trie/bintrie/group_debug_test.go +++ b/trie/bintrie/group_debug_test.go @@ -18,11 +18,12 @@ func TestGroupedSerializationDebug(t *testing.T) { right: HashedNode(rightHash), } - serialized := SerializeNode(node) + serialized := SerializeNode(node, MaxGroupDepth) t.Logf("Serialized length: %d", len(serialized)) t.Logf("Type: %d, GroupDepth: %d", serialized[0], serialized[1]) - bitmap := serialized[2 : 2+BitmapSize] + bitmapSize := BitmapSizeForDepth(MaxGroupDepth) + bitmap := serialized[2 : 2+bitmapSize] t.Logf("Bitmap: %x", bitmap) // Count and show set bits @@ -69,12 +70,13 @@ func TestFullDepth8Tree(t *testing.T) { // Build a full 8-level tree root := buildFullTree(0, 8) - serialized := SerializeNode(root) + serialized := SerializeNode(root, MaxGroupDepth) t.Logf("Full tree serialized length: %d", len(serialized)) t.Logf("Expected: 1 + 1 + 32 + 256*32 = %d", 1+1+32+256*32) // Count set bits in bitmap - bitmap := serialized[2 : 2+BitmapSize] + bitmapSize := BitmapSizeForDepth(MaxGroupDepth) + bitmap := serialized[2 : 2+bitmapSize] count := 0 for i := 0; i < 256; i++ { if bitmap[i/8]>>(7-(i%8))&1 == 1 { @@ -146,7 +148,7 @@ func TestRoundTripPreservesHashes(t *testing.T) { root := buildTreeWithHashes(0, 8, 0, hashes) - serialized := SerializeNode(root) + serialized := SerializeNode(root, MaxGroupDepth) deserialized, err := DeserializeNode(serialized, 0) if err != nil { t.Fatalf("Error: %v", err) @@ -205,9 +207,9 @@ func TestCollectNodesGrouping(t *testing.T) { }{pathCopy, node}) // Serialize and store by hash - serialized := SerializeNode(node) + serialized := SerializeNode(node, MaxGroupDepth) serializedNodes[node.Hash()] = serialized - }) + }, MaxGroupDepth) if err != nil { t.Fatalf("CollectNodes failed: %v", err) } @@ -362,3 +364,64 @@ func buildDeepTreeUnique(depth, maxDepth, position int) BinaryNode { right: buildDeepTreeUnique(depth+1, maxDepth, position*2+1), } } + +// TestVariableGroupDepth tests serialization with different group depths (1-8) +func TestVariableGroupDepth(t *testing.T) { + for groupDepth := 1; groupDepth <= MaxGroupDepth; groupDepth++ { + t.Run(fmt.Sprintf("groupDepth=%d", groupDepth), func(t *testing.T) { + // Build a tree with depth equal to groupDepth * 2 (two full groups) + treeDepth := groupDepth * 2 + root := buildDeepTreeUnique(0, treeDepth, 0) + originalHash := root.Hash() + + // Serialize with this group depth + serialized := SerializeNode(root, groupDepth) + + // Verify header + if serialized[0] != nodeTypeInternal { + t.Errorf("Expected type byte %d, got %d", nodeTypeInternal, serialized[0]) + } + if int(serialized[1]) != groupDepth { + t.Errorf("Expected group depth %d, got %d", groupDepth, serialized[1]) + } + + // Verify bitmap size + expectedBitmapSize := BitmapSizeForDepth(groupDepth) + expectedMinLen := 1 + 1 + expectedBitmapSize // type + depth + bitmap + if len(serialized) < expectedMinLen { + t.Errorf("Serialized data too short: got %d, expected at least %d", len(serialized), expectedMinLen) + } + + // Deserialize and verify hash matches + deserialized, err := DeserializeNode(serialized, 0) + if err != nil { + t.Fatalf("DeserializeNode failed: %v", err) + } + + if deserialized.Hash() != originalHash { + t.Errorf("Hash mismatch after round-trip: expected %x, got %x", originalHash, deserialized.Hash()) + } + + // Collect nodes and verify correct grouping + var collectedDepths []int + err = root.CollectNodes(nil, func(path []byte, node BinaryNode) { + if in, ok := node.(*InternalNode); ok { + collectedDepths = append(collectedDepths, in.depth) + } + }, groupDepth) + if err != nil { + t.Fatalf("CollectNodes failed: %v", err) + } + + // Verify all collected nodes are at group boundaries + for _, depth := range collectedDepths { + if depth%groupDepth != 0 { + t.Errorf("Collected node at depth %d, but groupDepth is %d (not a boundary)", depth, groupDepth) + } + } + + t.Logf("groupDepth=%d: serialized=%d bytes, collected=%d nodes at depths %v", + groupDepth, len(serialized), len(collectedDepths), collectedDepths) + }) + } +} diff --git a/trie/bintrie/hashed_node.go b/trie/bintrie/hashed_node.go index e4d8c2e7ac..a3b884ecdb 100644 --- a/trie/bintrie/hashed_node.go +++ b/trie/bintrie/hashed_node.go @@ -80,7 +80,7 @@ func (h HashedNode) toDot(parent string, path string) string { return ret } -func (h HashedNode) CollectNodes([]byte, NodeFlushFn) error { +func (h HashedNode) CollectNodes([]byte, NodeFlushFn, int) error { // HashedNodes are already persisted in the database and don't need to be collected. return nil } diff --git a/trie/bintrie/hashed_node_test.go b/trie/bintrie/hashed_node_test.go index f9e6984888..9fba61bdff 100644 --- a/trie/bintrie/hashed_node_test.go +++ b/trie/bintrie/hashed_node_test.go @@ -135,7 +135,7 @@ func TestHashedNodeInsertValuesAtStem(t *testing.T) { } // Serialize the node - serialized := SerializeNode(originalNode) + serialized := SerializeNode(originalNode, MaxGroupDepth) // Create a mock resolver that returns the serialized node validResolver := func(path []byte, hash common.Hash) ([]byte, error) { diff --git a/trie/bintrie/internal_node.go b/trie/bintrie/internal_node.go index 6351731fe0..9fcb8818d8 100644 --- a/trie/bintrie/internal_node.go +++ b/trie/bintrie/internal_node.go @@ -184,15 +184,18 @@ func (bt *InternalNode) InsertValuesAtStem(stem []byte, values [][]byte, resolve return bt, err } -// CollectNodes collects all child nodes at group boundaries (every GroupDepth levels), -// and flushes them into the provided node collector. Each flush serializes an 8-level +// CollectNodes collects all child nodes at group boundaries (every groupDepth levels), +// and flushes them into the provided node collector. Each flush serializes a groupDepth-level // subtree group. Nodes within a group are not flushed individually. -func (bt *InternalNode) CollectNodes(path []byte, flushfn NodeFlushFn) error { - // Only flush at group boundaries (depth % GroupDepth == 0) - if bt.depth%GroupDepth == 0 { +func (bt *InternalNode) CollectNodes(path []byte, flushfn NodeFlushFn, groupDepth int) error { + if groupDepth < 1 || groupDepth > MaxGroupDepth { + return errors.New("groupDepth must be between 1 and 8") + } + // Only flush at group boundaries (depth % groupDepth == 0) + if bt.depth%groupDepth == 0 { // We're at a group boundary - first collect any nodes in deeper groups, // then flush this group - if err := bt.collectChildGroups(path, flushfn, GroupDepth-1); err != nil { + if err := bt.collectChildGroups(path, flushfn, groupDepth, groupDepth-1); err != nil { return err } flushfn(path, bt) @@ -200,22 +203,22 @@ func (bt *InternalNode) CollectNodes(path []byte, flushfn NodeFlushFn) error { } // Not at a group boundary - this shouldn't happen if we're called correctly from root // but handle it by continuing to traverse - return bt.collectChildGroups(path, flushfn, GroupDepth-(bt.depth%GroupDepth)-1) + return bt.collectChildGroups(path, flushfn, groupDepth, groupDepth-(bt.depth%groupDepth)-1) } // collectChildGroups traverses within a group to find and collect nodes in the next group. // remainingLevels is how many more levels below the current node until we reach the group boundary. // When remainingLevels=0, the current node's children are at the next group boundary. -func (bt *InternalNode) collectChildGroups(path []byte, flushfn NodeFlushFn, remainingLevels int) error { +func (bt *InternalNode) collectChildGroups(path []byte, flushfn NodeFlushFn, groupDepth int, remainingLevels int) error { if remainingLevels == 0 { // Current node is at depth (groupBoundary - 1), its children are at the next group boundary if bt.left != nil { - if err := bt.left.CollectNodes(appendBit(path, 0), flushfn); err != nil { + if err := bt.left.CollectNodes(appendBit(path, 0), flushfn, groupDepth); err != nil { return err } } if bt.right != nil { - if err := bt.right.CollectNodes(appendBit(path, 1), flushfn); err != nil { + if err := bt.right.CollectNodes(appendBit(path, 1), flushfn, groupDepth); err != nil { return err } } @@ -226,12 +229,12 @@ func (bt *InternalNode) collectChildGroups(path []byte, flushfn NodeFlushFn, rem if bt.left != nil { switch n := bt.left.(type) { case *InternalNode: - if err := n.collectChildGroups(appendBit(path, 0), flushfn, remainingLevels-1); err != nil { + if err := n.collectChildGroups(appendBit(path, 0), flushfn, groupDepth, remainingLevels-1); err != nil { return err } default: // StemNode, HashedNode, or Empty - they handle their own collection - if err := bt.left.CollectNodes(appendBit(path, 0), flushfn); err != nil { + if err := bt.left.CollectNodes(appendBit(path, 0), flushfn, groupDepth); err != nil { return err } } @@ -239,12 +242,12 @@ func (bt *InternalNode) collectChildGroups(path []byte, flushfn NodeFlushFn, rem if bt.right != nil { switch n := bt.right.(type) { case *InternalNode: - if err := n.collectChildGroups(appendBit(path, 1), flushfn, remainingLevels-1); err != nil { + if err := n.collectChildGroups(appendBit(path, 1), flushfn, groupDepth, remainingLevels-1); err != nil { return err } default: // StemNode, HashedNode, or Empty - they handle their own collection - if err := bt.right.CollectNodes(appendBit(path, 1), flushfn); err != nil { + if err := bt.right.CollectNodes(appendBit(path, 1), flushfn, groupDepth); err != nil { return err } } diff --git a/trie/bintrie/internal_node_test.go b/trie/bintrie/internal_node_test.go index 158d8b7147..c8173fd69e 100644 --- a/trie/bintrie/internal_node_test.go +++ b/trie/bintrie/internal_node_test.go @@ -95,7 +95,7 @@ func TestInternalNodeGetWithResolver(t *testing.T) { Values: values[:], depth: 1, } - return SerializeNode(stemNode), nil + return SerializeNode(stemNode, MaxGroupDepth), nil } return nil, errors.New("node not found") } @@ -379,7 +379,7 @@ func TestInternalNodeCollectNodes(t *testing.T) { collectedNodes = append(collectedNodes, n) } - err := node.CollectNodes([]byte{1}, flushFn) + err := node.CollectNodes([]byte{1}, flushFn, MaxGroupDepth) if err != nil { t.Fatalf("Failed to collect nodes: %v", err) } diff --git a/trie/bintrie/iterator.go b/trie/bintrie/iterator.go index 9b863ed1e3..ba900b4321 100644 --- a/trie/bintrie/iterator.go +++ b/trie/bintrie/iterator.go @@ -184,7 +184,7 @@ func (it *binaryNodeIterator) Path() []byte { // NodeBlob returns the serialized bytes of the current node. func (it *binaryNodeIterator) NodeBlob() []byte { - return SerializeNode(it.current) + return SerializeNode(it.current, it.trie.groupDepth) } // Leaf returns true iff the current node is a leaf node. diff --git a/trie/bintrie/stem_node.go b/trie/bintrie/stem_node.go index 60856b42ce..5b7fd56e27 100644 --- a/trie/bintrie/stem_node.go +++ b/trie/bintrie/stem_node.go @@ -134,8 +134,8 @@ func (bt *StemNode) Hash() common.Hash { } // CollectNodes collects all child nodes at a given path, and flushes it -// into the provided node collector. -func (bt *StemNode) CollectNodes(path []byte, flush NodeFlushFn) error { +// into the provided node collector. groupDepth is ignored for StemNodes. +func (bt *StemNode) CollectNodes(path []byte, flush NodeFlushFn, groupDepth int) error { flush(path, bt) return nil } diff --git a/trie/bintrie/stem_node_test.go b/trie/bintrie/stem_node_test.go index d8d6844427..595fb7b4f3 100644 --- a/trie/bintrie/stem_node_test.go +++ b/trie/bintrie/stem_node_test.go @@ -347,7 +347,7 @@ func TestStemNodeCollectNodes(t *testing.T) { collectedNodes = append(collectedNodes, n) } - err := node.CollectNodes([]byte{0, 1, 0}, flushFn) + err := node.CollectNodes([]byte{0, 1, 0}, flushFn, MaxGroupDepth) if err != nil { t.Fatalf("Failed to collect nodes: %v", err) } diff --git a/trie/bintrie/trie.go b/trie/bintrie/trie.go index c082d57bdf..5faec691b1 100644 --- a/trie/bintrie/trie.go +++ b/trie/bintrie/trie.go @@ -115,9 +115,10 @@ func NewBinaryNode() BinaryNode { // BinaryTrie is the implementation of https://eips.ethereum.org/EIPS/eip-7864. type BinaryTrie struct { - root BinaryNode - reader *trie.Reader - tracer *trie.PrevalueTracer + root BinaryNode + reader *trie.Reader + tracer *trie.PrevalueTracer + groupDepth int // Number of levels per serialized group (1-8, default 8) } // ToDot converts the binary trie to a DOT language representation. Useful for debugging. @@ -127,15 +128,20 @@ func (t *BinaryTrie) ToDot() string { } // NewBinaryTrie creates a new binary trie. -func NewBinaryTrie(root common.Hash, db database.NodeDatabase) (*BinaryTrie, error) { +// groupDepth specifies the number of levels per serialized group (1-8). +func NewBinaryTrie(root common.Hash, db database.NodeDatabase, groupDepth int) (*BinaryTrie, error) { + if groupDepth < 1 || groupDepth > MaxGroupDepth { + groupDepth = MaxGroupDepth // Default to 8 + } reader, err := trie.NewReader(root, common.Hash{}, db) if err != nil { return nil, err } t := &BinaryTrie{ - root: NewBinaryNode(), - reader: reader, - tracer: trie.NewPrevalueTracer(), + root: NewBinaryNode(), + reader: reader, + tracer: trie.NewPrevalueTracer(), + groupDepth: groupDepth, } // Parse the root node if it's not empty if root != types.EmptyBinaryHash && root != types.EmptyRootHash { @@ -325,9 +331,9 @@ func (t *BinaryTrie) Commit(_ bool) (common.Hash, *trienode.NodeSet) { // The root can be any type of BinaryNode (InternalNode, StemNode, etc.) err := t.root.CollectNodes(nil, func(path []byte, node BinaryNode) { - serialized := SerializeNode(node) + serialized := SerializeNode(node, t.groupDepth) nodeset.AddNode(path, trienode.NewNodeWithPrev(node.Hash(), serialized, t.tracer.Get(path))) - }) + }, t.groupDepth) if err != nil { panic(fmt.Errorf("CollectNodes failed: %v", err)) } @@ -355,9 +361,10 @@ func (t *BinaryTrie) Prove(key []byte, proofDb ethdb.KeyValueWriter) error { // Copy creates a deep copy of the trie. func (t *BinaryTrie) Copy() *BinaryTrie { return &BinaryTrie{ - root: t.root.Copy(), - reader: t.reader, - tracer: t.tracer.Copy(), + root: t.root.Copy(), + reader: t.reader, + tracer: t.tracer.Copy(), + groupDepth: t.groupDepth, } } diff --git a/triedb/database.go b/triedb/database.go index e7e47bb91a..5ea3c62e6c 100644 --- a/triedb/database.go +++ b/triedb/database.go @@ -31,10 +31,11 @@ import ( // Config defines all necessary options for database. type Config struct { - Preimages bool // Flag whether the preimage of node key is recorded - IsVerkle bool // Flag whether the db is holding a verkle tree - HashDB *hashdb.Config // Configs for hash-based scheme - PathDB *pathdb.Config // Configs for experimental path-based scheme + Preimages bool // Flag whether the preimage of node key is recorded + IsVerkle bool // Flag whether the db is holding a verkle tree + BinTrieGroupDepth int // Number of levels per serialized group in binary trie (1-8, default 8) + HashDB *hashdb.Config // Configs for hash-based scheme + PathDB *pathdb.Config // Configs for experimental path-based scheme } // HashDefaults represents a config for using hash-based scheme with @@ -48,9 +49,10 @@ var HashDefaults = &Config{ // VerkleDefaults represents a config for holding verkle trie data // using path-based scheme with default settings. var VerkleDefaults = &Config{ - Preimages: false, - IsVerkle: true, - PathDB: pathdb.Defaults, + Preimages: false, + IsVerkle: true, + BinTrieGroupDepth: 8, // Default to byte-aligned groups + PathDB: pathdb.Defaults, } // backend defines the methods needed to access/update trie nodes in different @@ -380,6 +382,15 @@ func (db *Database) IsVerkle() bool { return db.config.IsVerkle } +// BinTrieGroupDepth returns the group depth for binary trie serialization (1-8). +// Returns 8 as default if not configured. +func (db *Database) BinTrieGroupDepth() int { + if db.config.BinTrieGroupDepth < 1 || db.config.BinTrieGroupDepth > 8 { + return 8 // Default + } + return db.config.BinTrieGroupDepth +} + // Disk returns the underlying disk database. func (db *Database) Disk() ethdb.Database { return db.disk