mirror of
https://github.com/ethereum/go-ethereum.git
synced 2026-06-19 21:31:37 +00:00
nomt/core: redesign for EIP-7864 compatibility (Phase A)
Replace Keccak256+MSB tagging with SHA256. Remove leaf nodes entirely, replacing them with opaque stem hashes. Simplify NodeKind to just Terminator and Internal. Add HashStem (8-level binary SHA256 tree matching bintrie StemNode.Hash). Reduce max trie depth from 256 to 248 (31-byte stem path). Replace BuildTrie/LeafOp with BuildInternalTree/StemKeyValue. Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
weiihann
parent
cb3e13d93d
commit
2db24e85a3
7 changed files with 276 additions and 428 deletions
|
|
@ -1,46 +1,62 @@
|
|||
package core
|
||||
|
||||
import "golang.org/x/crypto/sha3"
|
||||
import "crypto/sha256"
|
||||
|
||||
// HashLeaf computes the hash of a leaf node: keccak256(keyPath || valueHash)
|
||||
// with the MSB of byte 0 set to 1.
|
||||
func HashLeaf(data *LeafData) Node {
|
||||
h := sha3.NewLegacyKeccak256()
|
||||
h.Write(data.KeyPath[:])
|
||||
h.Write(data.ValueHash[:])
|
||||
var out Node
|
||||
h.Sum(out[:0])
|
||||
setMSB(&out)
|
||||
return out
|
||||
}
|
||||
const (
|
||||
// StemSize is the number of bytes in a stem path (248 bits).
|
||||
StemSize = 31
|
||||
|
||||
// HashInternal computes the hash of an internal node: keccak256(left || right)
|
||||
// with the MSB of byte 0 cleared to 0.
|
||||
// StemNodeWidth is the number of value slots per stem node.
|
||||
StemNodeWidth = 256
|
||||
|
||||
// HashSize is the size of a SHA256 hash in bytes.
|
||||
HashSize = 32
|
||||
)
|
||||
|
||||
// HashInternal computes SHA256(left || right) matching EIP-7864's InternalNode.Hash().
|
||||
func HashInternal(data *InternalData) Node {
|
||||
h := sha3.NewLegacyKeccak256()
|
||||
h := sha256.New()
|
||||
h.Write(data.Left[:])
|
||||
h.Write(data.Right[:])
|
||||
var out Node
|
||||
h.Sum(out[:0])
|
||||
clearMSB(&out)
|
||||
return out
|
||||
}
|
||||
|
||||
// HashValue computes keccak256 of an arbitrary-length value.
|
||||
func HashValue(value []byte) ValueHash {
|
||||
h := sha3.NewLegacyKeccak256()
|
||||
h.Write(value)
|
||||
var out ValueHash
|
||||
// HashStem computes the stem node hash matching EIP-7864's StemNode.Hash().
|
||||
//
|
||||
// Algorithm:
|
||||
// 1. SHA256 each non-nil value to get 256 leaf hashes (nil → zero hash)
|
||||
// 2. Build an 8-level binary SHA256 tree (256 → 128 → ... → 1 root)
|
||||
// Skip pairs where both children are zero (produce zero parent)
|
||||
// 3. Final hash: SHA256(stem || 0x00 || subtreeRoot)
|
||||
func HashStem(stem [StemSize]byte, values [StemNodeWidth][]byte) Node {
|
||||
var data [StemNodeWidth]Node
|
||||
for i, v := range values {
|
||||
if v != nil {
|
||||
data[i] = sha256.Sum256(v)
|
||||
}
|
||||
}
|
||||
|
||||
h := sha256.New()
|
||||
for level := 1; level <= 8; level++ {
|
||||
for i := range StemNodeWidth / (1 << level) {
|
||||
if data[i*2] == (Node{}) && data[i*2+1] == (Node{}) {
|
||||
data[i] = Node{}
|
||||
continue
|
||||
}
|
||||
h.Reset()
|
||||
h.Write(data[i*2][:])
|
||||
h.Write(data[i*2+1][:])
|
||||
h.Sum(data[i][:0])
|
||||
}
|
||||
}
|
||||
|
||||
h.Reset()
|
||||
h.Write(stem[:])
|
||||
h.Write([]byte{0x00})
|
||||
h.Write(data[0][:])
|
||||
var out Node
|
||||
h.Sum(out[:0])
|
||||
return out
|
||||
}
|
||||
|
||||
// setMSB sets the most significant bit (bit 7 of byte 0) to 1.
|
||||
func setMSB(n *Node) {
|
||||
n[0] |= 0x80
|
||||
}
|
||||
|
||||
// clearMSB clears the most significant bit (bit 7 of byte 0) to 0.
|
||||
func clearMSB(n *Node) {
|
||||
n[0] &= 0x7F
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,42 +1,40 @@
|
|||
// Package core defines the fundamental data structures for a NOMT binary
|
||||
// merkle trie. All types are pure computation with no I/O dependencies.
|
||||
// merkle trie aligned with EIP-7864. All types are pure computation with
|
||||
// no I/O dependencies.
|
||||
package core
|
||||
|
||||
// Node is a 256-bit hash representing a node in the binary merkle trie.
|
||||
// The MSB of byte 0 discriminates leaves (MSB=1) from internal nodes (MSB=0).
|
||||
// The all-zeros value is reserved as the Terminator.
|
||||
// The all-zeros value is reserved as the Terminator (empty sub-trie).
|
||||
// Unlike the previous NOMT design, there is no MSB tagging — nodes are
|
||||
// either terminators (zero) or opaque hashes (non-zero).
|
||||
type Node = [32]byte
|
||||
|
||||
// KeyPath is the 256-bit lookup path for a key in the trie.
|
||||
// KeyPath is a full 256-bit key (31-byte stem + 1-byte suffix).
|
||||
// Used for flat state lookups where the full 32-byte key is needed.
|
||||
type KeyPath = [32]byte
|
||||
|
||||
// ValueHash is the 256-bit hash of a value stored at a leaf.
|
||||
type ValueHash = [32]byte
|
||||
// StemPath is the 248-bit (31-byte) stem portion of a key.
|
||||
// In the EIP-7864 trie, internal nodes traverse bits 0-247, then
|
||||
// stem nodes hold 256 value slots indexed by the last byte.
|
||||
type StemPath = [StemSize]byte
|
||||
|
||||
// Terminator is the special node value denoting an empty sub-trie.
|
||||
// When this appears at a location, no key with a matching path prefix has a value.
|
||||
var Terminator Node
|
||||
|
||||
// NodeKind discriminates the three kinds of trie nodes.
|
||||
// NodeKind discriminates the two kinds of trie nodes in the page tree.
|
||||
type NodeKind int
|
||||
|
||||
const (
|
||||
// NodeTerminator indicates an empty sub-trie (all-zero node).
|
||||
NodeTerminator NodeKind = iota
|
||||
// NodeLeaf indicates a leaf node (MSB of byte 0 is 1).
|
||||
NodeLeaf
|
||||
// NodeInternal indicates an internal (branch) node (MSB of byte 0 is 0, non-zero).
|
||||
// NodeInternal indicates a non-zero hash (internal node or stem hash).
|
||||
NodeInternal
|
||||
)
|
||||
|
||||
// NodeKindOf returns the kind of the given node using MSB discrimination.
|
||||
//
|
||||
// If the MSB of byte 0 is set, it is a leaf. If the node is all zeros,
|
||||
// it is a terminator. Otherwise it is an internal node.
|
||||
// NodeKindOf returns the kind of the given node.
|
||||
// In EIP-7864, the page tree only stores terminators and opaque hashes.
|
||||
func NodeKindOf(n *Node) NodeKind {
|
||||
if n[0]>>7 == 1 {
|
||||
return NodeLeaf
|
||||
}
|
||||
if *n == Terminator {
|
||||
return NodeTerminator
|
||||
}
|
||||
|
|
@ -48,24 +46,8 @@ func IsTerminator(n *Node) bool {
|
|||
return *n == Terminator
|
||||
}
|
||||
|
||||
// IsLeaf reports whether the node's MSB indicates a leaf.
|
||||
func IsLeaf(n *Node) bool {
|
||||
return n[0]>>7 == 1
|
||||
}
|
||||
|
||||
// IsInternal reports whether the node is a non-terminator internal node.
|
||||
func IsInternal(n *Node) bool {
|
||||
return n[0]>>7 == 0 && *n != Terminator
|
||||
}
|
||||
|
||||
// InternalData holds the preimage of an internal (branch) node.
|
||||
type InternalData struct {
|
||||
Left Node
|
||||
Right Node
|
||||
}
|
||||
|
||||
// LeafData holds the preimage of a leaf node.
|
||||
type LeafData struct {
|
||||
KeyPath KeyPath
|
||||
ValueHash ValueHash
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
package core
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
|
|
@ -24,23 +25,18 @@ func TestNodeKindOf(t *testing.T) {
|
|||
want: NodeTerminator,
|
||||
},
|
||||
{
|
||||
name: "leaf with MSB set",
|
||||
name: "non-zero hash is internal",
|
||||
node: Node{0x80, 0x01, 0x02},
|
||||
want: NodeLeaf,
|
||||
want: NodeInternal,
|
||||
},
|
||||
{
|
||||
name: "leaf with all bits set in first byte",
|
||||
node: Node{0xFF, 0x01},
|
||||
want: NodeLeaf,
|
||||
},
|
||||
{
|
||||
name: "internal node",
|
||||
name: "any non-zero is internal",
|
||||
node: Node{0x01, 0x02, 0x03},
|
||||
want: NodeInternal,
|
||||
},
|
||||
{
|
||||
name: "internal with MSB clear",
|
||||
node: Node{0x7F, 0xFF, 0xFF},
|
||||
name: "high bits set is still internal",
|
||||
node: Node{0xFF, 0xFF, 0xFF},
|
||||
want: NodeInternal,
|
||||
},
|
||||
}
|
||||
|
|
@ -60,55 +56,6 @@ func TestIsTerminator(t *testing.T) {
|
|||
assert.False(t, IsTerminator(&nonZero))
|
||||
}
|
||||
|
||||
func TestIsLeaf(t *testing.T) {
|
||||
leaf := Node{0x80}
|
||||
assert.True(t, IsLeaf(&leaf))
|
||||
|
||||
internal := Node{0x7F, 0xFF}
|
||||
assert.False(t, IsLeaf(&internal))
|
||||
}
|
||||
|
||||
func TestIsInternal(t *testing.T) {
|
||||
internal := Node{0x01}
|
||||
assert.True(t, IsInternal(&internal))
|
||||
|
||||
leaf := Node{0x80}
|
||||
assert.False(t, IsInternal(&leaf))
|
||||
|
||||
assert.False(t, IsInternal(&Terminator))
|
||||
}
|
||||
|
||||
func TestHashLeafSetsMSB(t *testing.T) {
|
||||
data := &LeafData{
|
||||
KeyPath: KeyPath{0x01, 0x02, 0x03},
|
||||
ValueHash: ValueHash{0x04, 0x05, 0x06},
|
||||
}
|
||||
result := HashLeaf(data)
|
||||
require.True(t, IsLeaf(&result), "HashLeaf must produce a leaf node")
|
||||
require.False(t, IsTerminator(&result))
|
||||
}
|
||||
|
||||
func TestHashInternalClearsMSB(t *testing.T) {
|
||||
data := &InternalData{
|
||||
Left: Node{0xFF, 0x01},
|
||||
Right: Node{0x80, 0x02},
|
||||
}
|
||||
result := HashInternal(data)
|
||||
require.True(t, IsInternal(&result),
|
||||
"HashInternal must produce an internal node")
|
||||
require.False(t, IsLeaf(&result))
|
||||
}
|
||||
|
||||
func TestHashLeafDeterministic(t *testing.T) {
|
||||
data := &LeafData{
|
||||
KeyPath: KeyPath{0xAB, 0xCD},
|
||||
ValueHash: ValueHash{0xEF, 0x01},
|
||||
}
|
||||
h1 := HashLeaf(data)
|
||||
h2 := HashLeaf(data)
|
||||
assert.Equal(t, h1, h2, "same inputs must produce same hash")
|
||||
}
|
||||
|
||||
func TestHashInternalDeterministic(t *testing.T) {
|
||||
data := &InternalData{
|
||||
Left: Node{0x11, 0x22},
|
||||
|
|
@ -119,32 +66,77 @@ func TestHashInternalDeterministic(t *testing.T) {
|
|||
assert.Equal(t, h1, h2, "same inputs must produce same hash")
|
||||
}
|
||||
|
||||
func TestHashLeafDiffersFromInternal(t *testing.T) {
|
||||
// Using the same 64-byte preimage for both should produce different
|
||||
// hashes due to MSB tagging (even if the raw keccak is the same,
|
||||
// the MSB bit will differ).
|
||||
var key KeyPath
|
||||
var val ValueHash
|
||||
for i := range key {
|
||||
key[i] = byte(i)
|
||||
}
|
||||
for i := range val {
|
||||
val[i] = byte(i + 32)
|
||||
}
|
||||
func TestHashInternalMatchesSHA256(t *testing.T) {
|
||||
left := Node{0x01, 0x02, 0x03}
|
||||
right := Node{0x04, 0x05, 0x06}
|
||||
|
||||
leaf := HashLeaf(&LeafData{KeyPath: key, ValueHash: val})
|
||||
internal := HashInternal(&InternalData{Left: Node(key), Right: Node(val)})
|
||||
data := &InternalData{Left: left, Right: right}
|
||||
got := HashInternal(data)
|
||||
|
||||
// They share the same keccak input, but MSB tagging makes them differ.
|
||||
assert.NotEqual(t, leaf, internal,
|
||||
"leaf and internal hashes must differ due to MSB tagging")
|
||||
// Manual SHA256(left || right)
|
||||
h := sha256.New()
|
||||
h.Write(left[:])
|
||||
h.Write(right[:])
|
||||
expected := h.Sum(nil)
|
||||
|
||||
assert.Equal(t, expected, got[:])
|
||||
}
|
||||
|
||||
func TestHashValue(t *testing.T) {
|
||||
v1 := HashValue([]byte("hello"))
|
||||
v2 := HashValue([]byte("hello"))
|
||||
v3 := HashValue([]byte("world"))
|
||||
|
||||
assert.Equal(t, v1, v2, "same value must produce same hash")
|
||||
assert.NotEqual(t, v1, v3, "different values must differ")
|
||||
func TestHashInternalNoMSBTagging(t *testing.T) {
|
||||
// With SHA256, the MSB is determined by the hash output, not forced.
|
||||
// Just verify it produces a non-zero, non-terminator result.
|
||||
data := &InternalData{
|
||||
Left: Node{0xFF, 0x01},
|
||||
Right: Node{0x80, 0x02},
|
||||
}
|
||||
result := HashInternal(data)
|
||||
require.False(t, IsTerminator(&result))
|
||||
}
|
||||
|
||||
func TestHashStemDeterministic(t *testing.T) {
|
||||
var stem StemPath
|
||||
stem[0] = 0xAB
|
||||
stem[1] = 0xCD
|
||||
|
||||
var values [StemNodeWidth][]byte
|
||||
values[0] = make([]byte, 32)
|
||||
values[0][0] = 0x01
|
||||
|
||||
h1 := HashStem(stem, values)
|
||||
h2 := HashStem(stem, values)
|
||||
assert.Equal(t, h1, h2, "same inputs must produce same hash")
|
||||
}
|
||||
|
||||
func TestHashStemAllNilIsNotZero(t *testing.T) {
|
||||
// Even with all nil values, the stem hash includes the stem bytes,
|
||||
// so it should NOT be the zero hash (unless stem is also zero and
|
||||
// subtree root is zero... let's check).
|
||||
var stem StemPath
|
||||
var values [StemNodeWidth][]byte
|
||||
|
||||
// With all-zero stem and all-nil values, the subtree root is zero.
|
||||
// Final = SHA256(zero_stem || 0x00 || zero_hash)
|
||||
result := HashStem(stem, values)
|
||||
assert.False(t, IsTerminator(&result),
|
||||
"stem hash should not be terminator even with empty values")
|
||||
}
|
||||
|
||||
func TestHashStemSingleValue(t *testing.T) {
|
||||
var stem StemPath
|
||||
stem[0] = 0x42
|
||||
|
||||
var values [StemNodeWidth][]byte
|
||||
val := make([]byte, 32)
|
||||
val[0] = 0xFF
|
||||
values[0] = val
|
||||
|
||||
result := HashStem(stem, values)
|
||||
assert.False(t, IsTerminator(&result))
|
||||
}
|
||||
|
||||
func TestStemPathType(t *testing.T) {
|
||||
// Verify StemPath is 31 bytes.
|
||||
var sp StemPath
|
||||
assert.Equal(t, StemSize, len(sp))
|
||||
assert.Equal(t, 31, len(sp))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,13 +13,17 @@ func NewTriePosition() TriePosition {
|
|||
return TriePosition{}
|
||||
}
|
||||
|
||||
// MaxTrieDepth is the maximum depth of the internal node tree (248 bits = 31 bytes).
|
||||
// Stem nodes exist at this depth; the last 8 bits are the stem suffix.
|
||||
const MaxTrieDepth = StemSize * 8 // 248
|
||||
|
||||
// TriePositionFromPathAndDepth creates a TriePosition at the given depth
|
||||
// within the path. Panics if depth is 0.
|
||||
func TriePositionFromPathAndDepth(path KeyPath, depth uint16) TriePosition {
|
||||
if depth == 0 {
|
||||
panic("triepos: depth must be non-zero")
|
||||
}
|
||||
if depth > 256 {
|
||||
if depth > MaxTrieDepth {
|
||||
panic("triepos: depth out of range")
|
||||
}
|
||||
pagePath := lastPagePath(path[:], depth)
|
||||
|
|
@ -57,8 +61,8 @@ func (p *TriePosition) NodeIndex() int {
|
|||
|
||||
// Down moves the position down by 1 bit (left if bit=false, right if bit=true).
|
||||
func (p *TriePosition) Down(bit bool) {
|
||||
if p.depth == 256 {
|
||||
panic("triepos: can't descend past 256 bits")
|
||||
if p.depth >= MaxTrieDepth {
|
||||
panic("triepos: can't descend past 248 bits")
|
||||
}
|
||||
if int(p.depth)%PageDepth == 0 {
|
||||
// Entering a new page: node index resets.
|
||||
|
|
|
|||
|
|
@ -158,13 +158,23 @@ func TestTriePositionChildPageIndex(t *testing.T) {
|
|||
assert.Equal(t, uint8(0), p.ChildPageIndex())
|
||||
}
|
||||
|
||||
func TestTriePositionMax255Depth(t *testing.T) {
|
||||
func TestTriePositionMax248Depth(t *testing.T) {
|
||||
p := NewTriePosition()
|
||||
for range 255 {
|
||||
for range 247 {
|
||||
p.Down(true)
|
||||
}
|
||||
assert.Equal(t, uint16(255), p.Depth())
|
||||
// One more descent should work (to 256).
|
||||
assert.Equal(t, uint16(247), p.Depth())
|
||||
// One more descent should work (to 248, the max).
|
||||
p.Down(false)
|
||||
assert.Equal(t, uint16(256), p.Depth())
|
||||
assert.Equal(t, uint16(248), p.Depth())
|
||||
}
|
||||
|
||||
func TestTriePositionPanicsBeyond248(t *testing.T) {
|
||||
p := NewTriePosition()
|
||||
for range 248 {
|
||||
p.Down(true)
|
||||
}
|
||||
assert.Equal(t, uint16(248), p.Depth())
|
||||
// Going one more should panic.
|
||||
assert.Panics(t, func() { p.Down(false) })
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,48 +1,40 @@
|
|||
package core
|
||||
|
||||
import "sort"
|
||||
|
||||
// LeafOp represents a leaf operation: set or delete.
|
||||
// A nil ValueHash pointer means delete.
|
||||
type LeafOp struct {
|
||||
Key KeyPath
|
||||
Value *ValueHash
|
||||
// StemKeyValue is a resolved (stemPath, stemHash) pair for the page tree.
|
||||
// The stem hash is precomputed by the integration layer using HashStem.
|
||||
type StemKeyValue struct {
|
||||
Stem StemPath // 31-byte stem (248 bits)
|
||||
Hash Node // precomputed SHA256 stem hash
|
||||
}
|
||||
|
||||
// KeyValue is a resolved (key, value) pair for trie building.
|
||||
type KeyValue struct {
|
||||
Key KeyPath
|
||||
Value ValueHash
|
||||
}
|
||||
|
||||
// WriteNodeKind enumerates the types of write commands from BuildTrie.
|
||||
// WriteNodeKind enumerates the types of write commands from BuildInternalTree.
|
||||
type WriteNodeKind int
|
||||
|
||||
const (
|
||||
WriteNodeLeaf WriteNodeKind = iota
|
||||
WriteNodeInternal
|
||||
WriteNodeTerminator
|
||||
WriteNodeStem WriteNodeKind = iota // opaque stem hash placed at tree bottom
|
||||
WriteNodeInternal // internal node (hash of left+right)
|
||||
WriteNodeTerminator // empty sub-trie
|
||||
)
|
||||
|
||||
// WriteNode represents a node to be written during trie building.
|
||||
type WriteNode struct {
|
||||
Kind WriteNodeKind
|
||||
Node Node
|
||||
LeafData *LeafData // set for leaf writes
|
||||
InternalData *InternalData // set for internal writes
|
||||
|
||||
// Navigation: move up 1 before writing (true for internal nodes and
|
||||
// non-first leaves).
|
||||
// non-first stem placements).
|
||||
GoUp bool
|
||||
// Navigation: bits to descend after going up (only for leaf writes).
|
||||
// Navigation: bits to descend after going up (only for stem writes).
|
||||
DownBits []bool
|
||||
}
|
||||
|
||||
// SharedBits counts the number of shared prefix bits between two key paths,
|
||||
// starting after `skip` bits.
|
||||
func SharedBits(a, b *KeyPath, skip int) int {
|
||||
// StemSharedBits counts the number of shared prefix bits between two stem
|
||||
// paths, starting after `skip` bits.
|
||||
func StemSharedBits(a, b *StemPath, skip int) int {
|
||||
count := 0
|
||||
for i := skip; i < 256; i++ {
|
||||
maxBits := StemSize * 8 // 248
|
||||
for i := skip; i < maxBits; i++ {
|
||||
aBit := (a[i/8] >> (7 - i%8)) & 1
|
||||
bBit := (b[i/8] >> (7 - i%8)) & 1
|
||||
if aBit != bBit {
|
||||
|
|
@ -53,163 +45,104 @@ func SharedBits(a, b *KeyPath, skip int) int {
|
|||
return count
|
||||
}
|
||||
|
||||
// LeafOpsSpliced creates a combined operation list from an existing leaf and
|
||||
// new operations. If the existing leaf's key is not in ops, it is spliced in.
|
||||
// Deletions (nil value) are filtered out.
|
||||
func LeafOpsSpliced(existingLeaf *LeafData, ops []LeafOp) []KeyValue {
|
||||
// Find splice position: where the existing leaf would be inserted.
|
||||
spliceIndex := -1
|
||||
if existingLeaf != nil {
|
||||
idx := sort.Search(len(ops), func(i int) bool {
|
||||
return keyPathCmp(&ops[i].Key, &existingLeaf.KeyPath) >= 0
|
||||
})
|
||||
if idx >= len(ops) || ops[idx].Key != existingLeaf.KeyPath {
|
||||
spliceIndex = idx
|
||||
}
|
||||
}
|
||||
|
||||
result := make([]KeyValue, 0, len(ops)+1)
|
||||
|
||||
if spliceIndex < 0 {
|
||||
// No splicing needed — just filter out deletes.
|
||||
for _, op := range ops {
|
||||
if op.Value != nil {
|
||||
result = append(result, KeyValue{op.Key, *op.Value})
|
||||
}
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
// Before splice point.
|
||||
for _, op := range ops[:spliceIndex] {
|
||||
if op.Value != nil {
|
||||
result = append(result, KeyValue{op.Key, *op.Value})
|
||||
}
|
||||
}
|
||||
|
||||
// The existing leaf.
|
||||
result = append(result, KeyValue{
|
||||
existingLeaf.KeyPath,
|
||||
existingLeaf.ValueHash,
|
||||
})
|
||||
|
||||
// After splice point.
|
||||
for _, op := range ops[spliceIndex:] {
|
||||
if op.Value != nil {
|
||||
result = append(result, KeyValue{op.Key, *op.Value})
|
||||
}
|
||||
}
|
||||
|
||||
return result
|
||||
}
|
||||
|
||||
// BuildTrie builds a compact sub-trie from sorted (key, value) pairs.
|
||||
// BuildInternalTree builds a compact internal-node sub-trie from sorted
|
||||
// (stem, hash) pairs.
|
||||
//
|
||||
// skip: the number of prefix bits already consumed (all ops share this prefix).
|
||||
// ops: sorted (KeyPath, ValueHash) pairs.
|
||||
// ops: sorted StemKeyValue pairs (by stem path).
|
||||
// visit: callback invoked for each computed node, bottom-up.
|
||||
//
|
||||
// Returns the root node of the built sub-trie.
|
||||
//
|
||||
// The algorithm uses a 3-pointer sliding window (a, b, c) over the sorted
|
||||
// ops to determine each leaf's depth based on shared bits with its neighbors.
|
||||
// Internal nodes are computed by hashing up a left-frontier stack.
|
||||
func BuildTrie(skip int, ops []KeyValue, visit func(WriteNode)) Node {
|
||||
// This replaces the old BuildTrie. The key difference: there are no leaf
|
||||
// nodes — stem hashes are opaque values placed at tree positions, and
|
||||
// internal nodes are always SHA256(left || right) with no MSB tagging
|
||||
// or leaf compaction.
|
||||
func BuildInternalTree(skip int, ops []StemKeyValue, visit func(WriteNode)) Node {
|
||||
if len(ops) == 0 {
|
||||
visit(WriteNode{Kind: WriteNodeTerminator, Node: Terminator})
|
||||
return Terminator
|
||||
}
|
||||
|
||||
if len(ops) == 1 {
|
||||
ld := LeafData{
|
||||
KeyPath: ops[0].Key,
|
||||
ValueHash: ops[0].Value,
|
||||
}
|
||||
h := HashLeaf(&ld)
|
||||
visit(WriteNode{
|
||||
Kind: WriteNodeLeaf,
|
||||
Node: h,
|
||||
LeafData: &ld,
|
||||
GoUp: false,
|
||||
Kind: WriteNodeStem,
|
||||
Node: ops[0].Hash,
|
||||
GoUp: false,
|
||||
})
|
||||
return h
|
||||
return ops[0].Hash
|
||||
}
|
||||
|
||||
// 3-pointer left-frontier algorithm.
|
||||
// 3-pointer left-frontier algorithm (same structure as old BuildTrie
|
||||
// but without leaf hashing or leaf compaction).
|
||||
type pendingSibling struct {
|
||||
node Node
|
||||
layer int
|
||||
}
|
||||
pendingSiblings := make([]pendingSibling, 0, 16)
|
||||
|
||||
commonAfterPrefix := func(k1, k2 *KeyPath) int {
|
||||
return SharedBits(k1, k2, skip)
|
||||
commonAfterPrefix := func(s1, s2 *StemPath) int {
|
||||
return StemSharedBits(s1, s2, skip)
|
||||
}
|
||||
|
||||
// Sliding window: a, b, c.
|
||||
var aKey *KeyPath
|
||||
var aVal *ValueHash
|
||||
var aStem *StemPath
|
||||
|
||||
for bIdx := 0; bIdx < len(ops); bIdx++ {
|
||||
thisKey := &ops[bIdx].Key
|
||||
thisVal := &ops[bIdx].Value
|
||||
thisStem := &ops[bIdx].Stem
|
||||
thisHash := ops[bIdx].Hash
|
||||
|
||||
var n1 *int
|
||||
if aKey != nil {
|
||||
v := commonAfterPrefix(aKey, thisKey)
|
||||
if aStem != nil {
|
||||
v := commonAfterPrefix(aStem, thisStem)
|
||||
n1 = &v
|
||||
}
|
||||
|
||||
var n2 *int
|
||||
if bIdx+1 < len(ops) {
|
||||
v := commonAfterPrefix(&ops[bIdx+1].Key, thisKey)
|
||||
v := commonAfterPrefix(&ops[bIdx+1].Stem, thisStem)
|
||||
n2 = &v
|
||||
}
|
||||
|
||||
ld := LeafData{KeyPath: *thisKey, ValueHash: *thisVal}
|
||||
leaf := HashLeaf(&ld)
|
||||
|
||||
var leafDepth, hashUpLayers int
|
||||
var stemDepth, hashUpLayers int
|
||||
switch {
|
||||
case n1 == nil && n2 == nil:
|
||||
leafDepth = 0
|
||||
stemDepth = 0
|
||||
hashUpLayers = 0
|
||||
case n1 == nil && n2 != nil:
|
||||
leafDepth = *n2 + 1
|
||||
stemDepth = *n2 + 1
|
||||
hashUpLayers = 0
|
||||
case n1 != nil && n2 == nil:
|
||||
leafDepth = *n1 + 1
|
||||
stemDepth = *n1 + 1
|
||||
hashUpLayers = *n1 + 1
|
||||
default:
|
||||
leafDepth = max(*n1, *n2) + 1
|
||||
stemDepth = max(*n1, *n2) + 1
|
||||
hashUpLayers = 0
|
||||
if *n1 > *n2 {
|
||||
hashUpLayers = *n1 - *n2
|
||||
}
|
||||
}
|
||||
|
||||
layer := leafDepth
|
||||
lastNode := leaf
|
||||
layer := stemDepth
|
||||
lastNode := thisHash
|
||||
|
||||
// Compute down bits for the visitor.
|
||||
downStart := skip
|
||||
if n1 != nil {
|
||||
downStart = skip + *n1
|
||||
}
|
||||
leafEndBit := skip + leafDepth
|
||||
stemEndBit := skip + stemDepth
|
||||
|
||||
var downBits []bool
|
||||
if leafEndBit > downStart {
|
||||
downBits = make([]bool, leafEndBit-downStart)
|
||||
for i := downStart; i < leafEndBit; i++ {
|
||||
downBits[i-downStart] = bitAt(thisKey, i)
|
||||
if stemEndBit > downStart {
|
||||
downBits = make([]bool, stemEndBit-downStart)
|
||||
for i := downStart; i < stemEndBit; i++ {
|
||||
downBits[i-downStart] = stemBitAt(thisStem, i)
|
||||
}
|
||||
}
|
||||
|
||||
visit(WriteNode{
|
||||
Kind: WriteNodeLeaf,
|
||||
Node: leaf,
|
||||
LeafData: &ld,
|
||||
Kind: WriteNodeStem,
|
||||
Node: thisHash,
|
||||
GoUp: n1 != nil,
|
||||
DownBits: downBits,
|
||||
})
|
||||
|
|
@ -217,10 +150,9 @@ func BuildTrie(skip int, ops []KeyValue, visit func(WriteNode)) Node {
|
|||
// Hash upward.
|
||||
for h := 0; h < hashUpLayers; h++ {
|
||||
layer--
|
||||
bitIdx := skip + layer // the bit at this layer
|
||||
bit := bitAt(thisKey, bitIdx)
|
||||
bitIdx := skip + layer
|
||||
bit := stemBitAt(thisStem, bitIdx)
|
||||
|
||||
// Pop sibling from pending if it matches.
|
||||
var sibling Node
|
||||
if len(pendingSiblings) > 0 &&
|
||||
pendingSiblings[len(pendingSiblings)-1].layer == layer+1 {
|
||||
|
|
@ -247,10 +179,8 @@ func BuildTrie(skip int, ops []KeyValue, visit func(WriteNode)) Node {
|
|||
pendingSiblings = append(pendingSiblings,
|
||||
pendingSibling{node: lastNode, layer: layer})
|
||||
|
||||
aKey = thisKey
|
||||
aVal = thisVal
|
||||
aStem = thisStem
|
||||
}
|
||||
_ = aVal // used in the loop to track state
|
||||
|
||||
if len(pendingSiblings) > 0 {
|
||||
return pendingSiblings[len(pendingSiblings)-1].node
|
||||
|
|
@ -258,11 +188,11 @@ func BuildTrie(skip int, ops []KeyValue, visit func(WriteNode)) Node {
|
|||
return Terminator
|
||||
}
|
||||
|
||||
func bitAt(key *KeyPath, idx int) bool {
|
||||
return (key[idx/8]>>(7-idx%8))&1 == 1
|
||||
func stemBitAt(stem *StemPath, idx int) bool {
|
||||
return (stem[idx/8]>>(7-idx%8))&1 == 1
|
||||
}
|
||||
|
||||
func keyPathCmp(a, b *KeyPath) int {
|
||||
func stemPathCmp(a, b *StemPath) int {
|
||||
for i := range a {
|
||||
if a[i] < b[i] {
|
||||
return -1
|
||||
|
|
|
|||
|
|
@ -7,43 +7,45 @@ import (
|
|||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestSharedBits(t *testing.T) {
|
||||
func TestStemSharedBits(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
a, b KeyPath
|
||||
a, b StemPath
|
||||
skip int
|
||||
expected int
|
||||
}{
|
||||
{"identical", KeyPath{0xFF}, KeyPath{0xFF}, 0, 256},
|
||||
{"differ at bit 0", KeyPath{0x80}, KeyPath{0x00}, 0, 0},
|
||||
{"share 4 bits", KeyPath{0xF0}, KeyPath{0xF8}, 0, 4},
|
||||
{"with skip", KeyPath{0xF0}, KeyPath{0xF8}, 4, 0},
|
||||
{"identical", StemPath{0xFF}, StemPath{0xFF}, 0, 248},
|
||||
{"differ at bit 0", StemPath{0x80}, StemPath{0x00}, 0, 0},
|
||||
{"share 4 bits", StemPath{0xF0}, StemPath{0xF8}, 0, 4},
|
||||
{"with skip", StemPath{0xF0}, StemPath{0xF8}, 4, 0},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got := SharedBits(&tt.a, &tt.b, tt.skip)
|
||||
got := StemSharedBits(&tt.a, &tt.b, tt.skip)
|
||||
assert.Equal(t, tt.expected, got)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// makeKV creates a (key, value) pair where both key and value are filled with b.
|
||||
func makeKV(b byte) KeyValue {
|
||||
var key KeyPath
|
||||
var val ValueHash
|
||||
for i := range key {
|
||||
key[i] = b
|
||||
// makeSKV creates a StemKeyValue where the stem is filled with b and
|
||||
// the hash is a deterministic non-zero value derived from b.
|
||||
func makeSKV(b byte) StemKeyValue {
|
||||
var stem StemPath
|
||||
for i := range stem {
|
||||
stem[i] = b
|
||||
}
|
||||
for i := range val {
|
||||
val[i] = b
|
||||
// Use a simple deterministic hash for testing.
|
||||
var hash Node
|
||||
for i := range hash {
|
||||
hash[i] = b ^ byte(i)
|
||||
}
|
||||
return KeyValue{Key: key, Value: val}
|
||||
return StemKeyValue{Stem: stem, Hash: hash}
|
||||
}
|
||||
|
||||
func TestBuildTrieEmpty(t *testing.T) {
|
||||
func TestBuildInternalTreeEmpty(t *testing.T) {
|
||||
var visited []WriteNode
|
||||
root := BuildTrie(0, nil, func(wn WriteNode) {
|
||||
root := BuildInternalTree(0, nil, func(wn WriteNode) {
|
||||
visited = append(visited, wn)
|
||||
})
|
||||
|
||||
|
|
@ -52,121 +54,95 @@ func TestBuildTrieEmpty(t *testing.T) {
|
|||
assert.Equal(t, Terminator, root)
|
||||
}
|
||||
|
||||
func TestBuildTrieSingleLeaf(t *testing.T) {
|
||||
kv := makeKV(0xFF)
|
||||
func TestBuildInternalTreeSingleStem(t *testing.T) {
|
||||
skv := makeSKV(0xFF)
|
||||
var visited []WriteNode
|
||||
root := BuildTrie(0, []KeyValue{kv}, func(wn WriteNode) {
|
||||
root := BuildInternalTree(0, []StemKeyValue{skv}, func(wn WriteNode) {
|
||||
visited = append(visited, wn)
|
||||
})
|
||||
|
||||
require.Len(t, visited, 1)
|
||||
assert.Equal(t, WriteNodeLeaf, visited[0].Kind)
|
||||
assert.Equal(t, WriteNodeStem, visited[0].Kind)
|
||||
assert.False(t, visited[0].GoUp)
|
||||
assert.True(t, IsLeaf(&root))
|
||||
|
||||
expected := HashLeaf(&LeafData{
|
||||
KeyPath: kv.Key,
|
||||
ValueHash: kv.Value,
|
||||
})
|
||||
assert.Equal(t, expected, root)
|
||||
assert.False(t, IsTerminator(&root))
|
||||
assert.Equal(t, skv.Hash, root)
|
||||
}
|
||||
|
||||
func TestBuildTrieTwoLeaves(t *testing.T) {
|
||||
// Keys: 0x00... and 0xFF... differ at bit 0.
|
||||
kv0 := makeKV(0x00)
|
||||
kvF := makeKV(0xFF)
|
||||
func TestBuildInternalTreeTwoStems(t *testing.T) {
|
||||
// Stems: 0x00... and 0xFF... differ at bit 0.
|
||||
skv0 := makeSKV(0x00)
|
||||
skvF := makeSKV(0xFF)
|
||||
|
||||
var visited []WriteNode
|
||||
root := BuildTrie(0, []KeyValue{kv0, kvF}, func(wn WriteNode) {
|
||||
root := BuildInternalTree(0, []StemKeyValue{skv0, skvF}, func(wn WriteNode) {
|
||||
visited = append(visited, wn)
|
||||
})
|
||||
|
||||
// Should visit: leaf_0, leaf_F, internal(leaf_0, leaf_F).
|
||||
// Should visit: stem_0, stem_F, internal(stem_0, stem_F).
|
||||
require.Len(t, visited, 3)
|
||||
assert.Equal(t, WriteNodeLeaf, visited[0].Kind)
|
||||
assert.Equal(t, WriteNodeLeaf, visited[1].Kind)
|
||||
assert.Equal(t, WriteNodeStem, visited[0].Kind)
|
||||
assert.Equal(t, WriteNodeStem, visited[1].Kind)
|
||||
assert.Equal(t, WriteNodeInternal, visited[2].Kind)
|
||||
|
||||
leaf0 := HashLeaf(&LeafData{KeyPath: kv0.Key, ValueHash: kv0.Value})
|
||||
leafF := HashLeaf(&LeafData{KeyPath: kvF.Key, ValueHash: kvF.Value})
|
||||
expected := HashInternal(&InternalData{Left: leaf0, Right: leafF})
|
||||
|
||||
expected := HashInternal(&InternalData{Left: skv0.Hash, Right: skvF.Hash})
|
||||
assert.Equal(t, expected, root)
|
||||
assert.True(t, IsInternal(&root))
|
||||
}
|
||||
|
||||
func TestBuildTrieThreeLeaves(t *testing.T) {
|
||||
// Three keys sharing common prefixes.
|
||||
// 0b00010001... = 0x11
|
||||
// 0b00010010... = 0x12
|
||||
// 0b00010100... = 0x14
|
||||
kv1 := makeKV(0x11)
|
||||
kv2 := makeKV(0x12)
|
||||
kv3 := makeKV(0x14)
|
||||
func TestBuildInternalTreeThreeStems(t *testing.T) {
|
||||
skv1 := makeSKV(0x11)
|
||||
skv2 := makeSKV(0x12)
|
||||
skv3 := makeSKV(0x14)
|
||||
|
||||
var visited []WriteNode
|
||||
root := BuildTrie(0, []KeyValue{kv1, kv2, kv3}, func(wn WriteNode) {
|
||||
root := BuildInternalTree(0, []StemKeyValue{skv1, skv2, skv3}, func(wn WriteNode) {
|
||||
visited = append(visited, wn)
|
||||
})
|
||||
|
||||
assert.True(t, IsInternal(&root) || IsLeaf(&root),
|
||||
"root should be non-terminator")
|
||||
assert.False(t, IsTerminator(&root))
|
||||
|
||||
// Verify determinism.
|
||||
var visited2 []WriteNode
|
||||
root2 := BuildTrie(0, []KeyValue{kv1, kv2, kv3}, func(wn WriteNode) {
|
||||
root2 := BuildInternalTree(0, []StemKeyValue{skv1, skv2, skv3}, func(wn WriteNode) {
|
||||
visited2 = append(visited2, wn)
|
||||
})
|
||||
assert.Equal(t, root, root2)
|
||||
assert.Equal(t, len(visited), len(visited2))
|
||||
}
|
||||
|
||||
func TestBuildTrieWithSkip(t *testing.T) {
|
||||
// Keys all share prefix 0001 (4 bits): 0x11, 0x12, 0x14.
|
||||
kv1 := makeKV(0x11)
|
||||
kv2 := makeKV(0x12)
|
||||
kv3 := makeKV(0x14)
|
||||
func TestBuildInternalTreeWithSkip(t *testing.T) {
|
||||
skv1 := makeSKV(0x11)
|
||||
skv2 := makeSKV(0x12)
|
||||
skv3 := makeSKV(0x14)
|
||||
|
||||
var visited []WriteNode
|
||||
root := BuildTrie(4, []KeyValue{kv1, kv2, kv3}, func(wn WriteNode) {
|
||||
root := BuildInternalTree(4, []StemKeyValue{skv1, skv2, skv3}, func(wn WriteNode) {
|
||||
visited = append(visited, wn)
|
||||
})
|
||||
|
||||
// Should produce a non-trivial sub-trie.
|
||||
assert.False(t, IsTerminator(&root))
|
||||
assert.True(t, len(visited) >= 3)
|
||||
}
|
||||
|
||||
func TestBuildTrieMultiValue(t *testing.T) {
|
||||
// Matches the Rust multi_value test pattern.
|
||||
// 0b00010000 = 0x10
|
||||
// 0b00100000 = 0x20
|
||||
// 0b01000000 = 0x40
|
||||
// 0b10100000 = 0xA0
|
||||
// 0b10110000 = 0xB0
|
||||
kvA := makeKV(0x10)
|
||||
kvB := makeKV(0x20)
|
||||
kvC := makeKV(0x40)
|
||||
kvD := makeKV(0xA0)
|
||||
kvE := makeKV(0xB0)
|
||||
func TestBuildInternalTreeMultiValue(t *testing.T) {
|
||||
skvA := makeSKV(0x10)
|
||||
skvB := makeSKV(0x20)
|
||||
skvC := makeSKV(0x40)
|
||||
skvD := makeSKV(0xA0)
|
||||
skvE := makeSKV(0xB0)
|
||||
|
||||
var nodes []Node
|
||||
root := BuildTrie(0, []KeyValue{kvA, kvB, kvC, kvD, kvE},
|
||||
root := BuildInternalTree(0, []StemKeyValue{skvA, skvB, skvC, skvD, skvE},
|
||||
func(wn WriteNode) {
|
||||
nodes = append(nodes, wn.Node)
|
||||
})
|
||||
|
||||
// Manually verify the trie structure.
|
||||
leafA := HashLeaf(&LeafData{KeyPath: kvA.Key, ValueHash: kvA.Value})
|
||||
leafB := HashLeaf(&LeafData{KeyPath: kvB.Key, ValueHash: kvB.Value})
|
||||
leafC := HashLeaf(&LeafData{KeyPath: kvC.Key, ValueHash: kvC.Value})
|
||||
leafD := HashLeaf(&LeafData{KeyPath: kvD.Key, ValueHash: kvD.Value})
|
||||
leafE := HashLeaf(&LeafData{KeyPath: kvE.Key, ValueHash: kvE.Value})
|
||||
// A=0x10 (0001...), B=0x20 (0010...), C=0x40 (0100...)
|
||||
// D=0xA0 (1010...), E=0xB0 (1011...)
|
||||
branchAB := HashInternal(&InternalData{Left: skvA.Hash, Right: skvB.Hash})
|
||||
branchABC := HashInternal(&InternalData{Left: branchAB, Right: skvC.Hash})
|
||||
|
||||
branchAB := HashInternal(&InternalData{Left: leafA, Right: leafB})
|
||||
branchABC := HashInternal(&InternalData{Left: branchAB, Right: leafC})
|
||||
|
||||
branchDE1 := HashInternal(&InternalData{Left: leafD, Right: leafE})
|
||||
branchDE1 := HashInternal(&InternalData{Left: skvD.Hash, Right: skvE.Hash})
|
||||
branchDE2 := HashInternal(&InternalData{Left: Terminator, Right: branchDE1})
|
||||
branchDE3 := HashInternal(&InternalData{Left: branchDE2, Right: Terminator})
|
||||
|
||||
|
|
@ -174,65 +150,3 @@ func TestBuildTrieMultiValue(t *testing.T) {
|
|||
|
||||
assert.Equal(t, expected, root)
|
||||
}
|
||||
|
||||
func TestLeafOpsSplicedNoExisting(t *testing.T) {
|
||||
val := ValueHash{0x01}
|
||||
ops := []LeafOp{
|
||||
{Key: KeyPath{0x10}, Value: &val},
|
||||
{Key: KeyPath{0x20}, Value: &val},
|
||||
}
|
||||
|
||||
result := LeafOpsSpliced(nil, ops)
|
||||
assert.Len(t, result, 2)
|
||||
}
|
||||
|
||||
func TestLeafOpsSplicedWithExistingLeaf(t *testing.T) {
|
||||
val := ValueHash{0x01}
|
||||
ops := []LeafOp{
|
||||
{Key: KeyPath{0x10}, Value: &val},
|
||||
{Key: KeyPath{0x30}, Value: &val},
|
||||
}
|
||||
|
||||
existing := &LeafData{
|
||||
KeyPath: KeyPath{0x20},
|
||||
ValueHash: ValueHash{0x02},
|
||||
}
|
||||
|
||||
result := LeafOpsSpliced(existing, ops)
|
||||
assert.Len(t, result, 3)
|
||||
assert.Equal(t, KeyPath{0x10}, result[0].Key)
|
||||
assert.Equal(t, KeyPath{0x20}, result[1].Key)
|
||||
assert.Equal(t, KeyPath{0x30}, result[2].Key)
|
||||
}
|
||||
|
||||
func TestLeafOpsSplicedDeleteFiltered(t *testing.T) {
|
||||
val := ValueHash{0x01}
|
||||
ops := []LeafOp{
|
||||
{Key: KeyPath{0x10}, Value: &val},
|
||||
{Key: KeyPath{0x20}, Value: nil}, // delete
|
||||
{Key: KeyPath{0x30}, Value: &val},
|
||||
}
|
||||
|
||||
result := LeafOpsSpliced(nil, ops)
|
||||
assert.Len(t, result, 2)
|
||||
assert.Equal(t, KeyPath{0x10}, result[0].Key)
|
||||
assert.Equal(t, KeyPath{0x30}, result[1].Key)
|
||||
}
|
||||
|
||||
func TestLeafOpsSplicedExistingKeyInOps(t *testing.T) {
|
||||
val := ValueHash{0x01}
|
||||
newVal := ValueHash{0x99}
|
||||
ops := []LeafOp{
|
||||
{Key: KeyPath{0x20}, Value: &newVal},
|
||||
}
|
||||
|
||||
existing := &LeafData{
|
||||
KeyPath: KeyPath{0x20},
|
||||
ValueHash: val,
|
||||
}
|
||||
|
||||
// The existing leaf should NOT be spliced because its key is in ops.
|
||||
result := LeafOpsSpliced(existing, ops)
|
||||
assert.Len(t, result, 1)
|
||||
assert.Equal(t, newVal, result[0].Value)
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue