forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-07-16 01:40:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

nomt/core: redesign for EIP-7864 compatibility (Phase A)

Browse source 
Replace Keccak256+MSB tagging with SHA256. Remove leaf nodes entirely,
replacing them with opaque stem hashes. Simplify NodeKind to just
Terminator and Internal. Add HashStem (8-level binary SHA256 tree
matching bintrie StemNode.Hash). Reduce max trie depth from 256 to 248
(31-byte stem path). Replace BuildTrie/LeafOp with
BuildInternalTree/StemKeyValue.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
weiihann 2026-02-12 21:59:53 +08:00
parent cb3e13d93d
commit 2db24e85a3
7 changed files with 276 additions and 428 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

78
nomt/core/hasher.go
View file

@ -1,46 +1,62 @@
package core package core
import "golang.org/x/crypto/sha3" import "crypto/sha256"
// HashLeaf computes the hash of a leaf node: keccak256(keyPath || valueHash) const (
// with the MSB of byte 0 set to 1. // StemSize is the number of bytes in a stem path (248 bits).
func HashLeaf(data *LeafData) Node { StemSize = 31
h := sha3.NewLegacyKeccak256()
h.Write(data.KeyPath[:])
h.Write(data.ValueHash[:])
var out Node
h.Sum(out[:0])
setMSB(&out)
return out
}
// HashInternal computes the hash of an internal node: keccak256(left || right) // StemNodeWidth is the number of value slots per stem node.
// with the MSB of byte 0 cleared to 0. StemNodeWidth = 256
// HashSize is the size of a SHA256 hash in bytes.
HashSize = 32
)
// HashInternal computes SHA256(left || right) matching EIP-7864's InternalNode.Hash().
func HashInternal(data *InternalData) Node { func HashInternal(data *InternalData) Node {
h := sha3.NewLegacyKeccak256() h := sha256.New()
h.Write(data.Left[:]) h.Write(data.Left[:])
h.Write(data.Right[:]) h.Write(data.Right[:])
var out Node var out Node
h.Sum(out[:0]) h.Sum(out[:0])
clearMSB(&out)
return out return out
} }
// HashValue computes keccak256 of an arbitrary-length value. // HashStem computes the stem node hash matching EIP-7864's StemNode.Hash().
func HashValue(value []byte) ValueHash { //
h := sha3.NewLegacyKeccak256() // Algorithm:
h.Write(value) // 1. SHA256 each non-nil value to get 256 leaf hashes (nil → zero hash)
var out ValueHash // 2. Build an 8-level binary SHA256 tree (256 → 128 → ... → 1 root)
// Skip pairs where both children are zero (produce zero parent)
// 3. Final hash: SHA256(stem || 0x00 || subtreeRoot)
func HashStem(stem [StemSize]byte, values [StemNodeWidth][]byte) Node {
var data [StemNodeWidth]Node
for i, v := range values {
if v != nil {
data[i] = sha256.Sum256(v)
}
}
h := sha256.New()
for level := 1; level <= 8; level++ {
for i := range StemNodeWidth / (1 << level) {
if data[i*2] == (Node{}) && data[i*2+1] == (Node{}) {
data[i] = Node{}
continue
}
h.Reset()
h.Write(data[i*2][:])
h.Write(data[i*2+1][:])
h.Sum(data[i][:0])
}
}
h.Reset()
h.Write(stem[:])
h.Write([]byte{0x00})
h.Write(data[0][:])
var out Node
h.Sum(out[:0]) h.Sum(out[:0])
return out return out
} }
// setMSB sets the most significant bit (bit 7 of byte 0) to 1.
func setMSB(n *Node) {
n[0] |= 0x80
}
// clearMSB clears the most significant bit (bit 7 of byte 0) to 0.
func clearMSB(n *Node) {
n[0] &= 0x7F
}

48
nomt/core/node.go
View file

@ -1,42 +1,40 @@
// Package core defines the fundamental data structures for a NOMT binary // Package core defines the fundamental data structures for a NOMT binary
// merkle trie. All types are pure computation with no I/O dependencies. // merkle trie aligned with EIP-7864. All types are pure computation with
// no I/O dependencies.
package core package core
// Node is a 256-bit hash representing a node in the binary merkle trie. // Node is a 256-bit hash representing a node in the binary merkle trie.
// The MSB of byte 0 discriminates leaves (MSB=1) from internal nodes (MSB=0). // The all-zeros value is reserved as the Terminator (empty sub-trie).
// The all-zeros value is reserved as the Terminator. // Unlike the previous NOMT design, there is no MSB tagging — nodes are
// either terminators (zero) or opaque hashes (non-zero).
type Node = [32]byte type Node = [32]byte
// KeyPath is the 256-bit lookup path for a key in the trie. // KeyPath is a full 256-bit key (31-byte stem + 1-byte suffix).
// Used for flat state lookups where the full 32-byte key is needed.
type KeyPath = [32]byte type KeyPath = [32]byte
// ValueHash is the 256-bit hash of a value stored at a leaf. // StemPath is the 248-bit (31-byte) stem portion of a key.
type ValueHash = [32]byte // In the EIP-7864 trie, internal nodes traverse bits 0-247, then
// stem nodes hold 256 value slots indexed by the last byte.
type StemPath = [StemSize]byte
// Terminator is the special node value denoting an empty sub-trie. // Terminator is the special node value denoting an empty sub-trie.
// When this appears at a location, no key with a matching path prefix has a value. // When this appears at a location, no key with a matching path prefix has a value.
var Terminator Node var Terminator Node
// NodeKind discriminates the three kinds of trie nodes. // NodeKind discriminates the two kinds of trie nodes in the page tree.
type NodeKind int type NodeKind int
const ( const (
// NodeTerminator indicates an empty sub-trie (all-zero node). // NodeTerminator indicates an empty sub-trie (all-zero node).
NodeTerminator NodeKind = iota NodeTerminator NodeKind = iota
// NodeLeaf indicates a leaf node (MSB of byte 0 is 1). // NodeInternal indicates a non-zero hash (internal node or stem hash).
NodeLeaf
// NodeInternal indicates an internal (branch) node (MSB of byte 0 is 0, non-zero).
NodeInternal NodeInternal
) )
// NodeKindOf returns the kind of the given node using MSB discrimination. // NodeKindOf returns the kind of the given node.
// // In EIP-7864, the page tree only stores terminators and opaque hashes.
// If the MSB of byte 0 is set, it is a leaf. If the node is all zeros,
// it is a terminator. Otherwise it is an internal node.
func NodeKindOf(n *Node) NodeKind { func NodeKindOf(n *Node) NodeKind {
if n[0]>>7 == 1 {
return NodeLeaf
}
if *n == Terminator { if *n == Terminator {
return NodeTerminator return NodeTerminator
} }
@ -48,24 +46,8 @@ func IsTerminator(n *Node) bool {
return *n == Terminator return *n == Terminator
} }
// IsLeaf reports whether the node's MSB indicates a leaf.
func IsLeaf(n *Node) bool {
return n[0]>>7 == 1
}
// IsInternal reports whether the node is a non-terminator internal node.
func IsInternal(n *Node) bool {
return n[0]>>7 == 0 && *n != Terminator
}
// InternalData holds the preimage of an internal (branch) node. // InternalData holds the preimage of an internal (branch) node.
type InternalData struct { type InternalData struct {
Left Node Left Node
Right Node Right Node
} }
// LeafData holds the preimage of a leaf node.
type LeafData struct {
KeyPath KeyPath
ValueHash ValueHash
}

158
nomt/core/node_test.go
View file

@ -1,6 +1,7 @@
package core package core
import ( import (
"crypto/sha256"
"testing" "testing"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
@ -24,23 +25,18 @@ func TestNodeKindOf(t *testing.T) {
want: NodeTerminator, want: NodeTerminator,
}, },
{ {
name: "leaf with MSB set", name: "non-zero hash is internal",
node: Node{0x80, 0x01, 0x02}, node: Node{0x80, 0x01, 0x02},
want: NodeLeaf, want: NodeInternal,
}, },
{ {
name: "leaf with all bits set in first byte", name: "any non-zero is internal",
node: Node{0xFF, 0x01},
want: NodeLeaf,
},
{
name: "internal node",
node: Node{0x01, 0x02, 0x03}, node: Node{0x01, 0x02, 0x03},
want: NodeInternal, want: NodeInternal,
}, },
{ {
name: "internal with MSB clear", name: "high bits set is still internal",
node: Node{0x7F, 0xFF, 0xFF}, node: Node{0xFF, 0xFF, 0xFF},
want: NodeInternal, want: NodeInternal,
}, },
} }
@ -60,55 +56,6 @@ func TestIsTerminator(t *testing.T) {
assert.False(t, IsTerminator(&nonZero)) assert.False(t, IsTerminator(&nonZero))
} }
func TestIsLeaf(t *testing.T) {
leaf := Node{0x80}
assert.True(t, IsLeaf(&leaf))
internal := Node{0x7F, 0xFF}
assert.False(t, IsLeaf(&internal))
}
func TestIsInternal(t *testing.T) {
internal := Node{0x01}
assert.True(t, IsInternal(&internal))
leaf := Node{0x80}
assert.False(t, IsInternal(&leaf))
assert.False(t, IsInternal(&Terminator))
}
func TestHashLeafSetsMSB(t *testing.T) {
data := &LeafData{
KeyPath: KeyPath{0x01, 0x02, 0x03},
ValueHash: ValueHash{0x04, 0x05, 0x06},
}
result := HashLeaf(data)
require.True(t, IsLeaf(&result), "HashLeaf must produce a leaf node")
require.False(t, IsTerminator(&result))
}
func TestHashInternalClearsMSB(t *testing.T) {
data := &InternalData{
Left: Node{0xFF, 0x01},
Right: Node{0x80, 0x02},
}
result := HashInternal(data)
require.True(t, IsInternal(&result),
"HashInternal must produce an internal node")
require.False(t, IsLeaf(&result))
}
func TestHashLeafDeterministic(t *testing.T) {
data := &LeafData{
KeyPath: KeyPath{0xAB, 0xCD},
ValueHash: ValueHash{0xEF, 0x01},
}
h1 := HashLeaf(data)
h2 := HashLeaf(data)
assert.Equal(t, h1, h2, "same inputs must produce same hash")
}
func TestHashInternalDeterministic(t *testing.T) { func TestHashInternalDeterministic(t *testing.T) {
data := &InternalData{ data := &InternalData{
Left: Node{0x11, 0x22}, Left: Node{0x11, 0x22},
@ -119,32 +66,77 @@ func TestHashInternalDeterministic(t *testing.T) {
assert.Equal(t, h1, h2, "same inputs must produce same hash") assert.Equal(t, h1, h2, "same inputs must produce same hash")
} }
func TestHashLeafDiffersFromInternal(t *testing.T) { func TestHashInternalMatchesSHA256(t *testing.T) {
// Using the same 64-byte preimage for both should produce different left := Node{0x01, 0x02, 0x03}
// hashes due to MSB tagging (even if the raw keccak is the same, right := Node{0x04, 0x05, 0x06}
// the MSB bit will differ).
var key KeyPath
var val ValueHash
for i := range key {
key[i] = byte(i)
}
for i := range val {
val[i] = byte(i + 32)
}
leaf := HashLeaf(&LeafData{KeyPath: key, ValueHash: val}) data := &InternalData{Left: left, Right: right}
internal := HashInternal(&InternalData{Left: Node(key), Right: Node(val)}) got := HashInternal(data)
// They share the same keccak input, but MSB tagging makes them differ. // Manual SHA256(left || right)
assert.NotEqual(t, leaf, internal, h := sha256.New()
"leaf and internal hashes must differ due to MSB tagging") h.Write(left[:])
h.Write(right[:])
expected := h.Sum(nil)
assert.Equal(t, expected, got[:])
} }
func TestHashValue(t *testing.T) { func TestHashInternalNoMSBTagging(t *testing.T) {
v1 := HashValue([]byte("hello")) // With SHA256, the MSB is determined by the hash output, not forced.
v2 := HashValue([]byte("hello")) // Just verify it produces a non-zero, non-terminator result.
v3 := HashValue([]byte("world")) data := &InternalData{
Left: Node{0xFF, 0x01},
assert.Equal(t, v1, v2, "same value must produce same hash") Right: Node{0x80, 0x02},
assert.NotEqual(t, v1, v3, "different values must differ") }
result := HashInternal(data)
require.False(t, IsTerminator(&result))
}
func TestHashStemDeterministic(t *testing.T) {
var stem StemPath
stem[0] = 0xAB
stem[1] = 0xCD
var values [StemNodeWidth][]byte
values[0] = make([]byte, 32)
values[0][0] = 0x01
h1 := HashStem(stem, values)
h2 := HashStem(stem, values)
assert.Equal(t, h1, h2, "same inputs must produce same hash")
}
func TestHashStemAllNilIsNotZero(t *testing.T) {
// Even with all nil values, the stem hash includes the stem bytes,
// so it should NOT be the zero hash (unless stem is also zero and
// subtree root is zero... let's check).
var stem StemPath
var values [StemNodeWidth][]byte
// With all-zero stem and all-nil values, the subtree root is zero.
// Final = SHA256(zero_stem || 0x00 || zero_hash)
result := HashStem(stem, values)
assert.False(t, IsTerminator(&result),
"stem hash should not be terminator even with empty values")
}
func TestHashStemSingleValue(t *testing.T) {
var stem StemPath
stem[0] = 0x42
var values [StemNodeWidth][]byte
val := make([]byte, 32)
val[0] = 0xFF
values[0] = val
result := HashStem(stem, values)
assert.False(t, IsTerminator(&result))
}
func TestStemPathType(t *testing.T) {
// Verify StemPath is 31 bytes.
var sp StemPath
assert.Equal(t, StemSize, len(sp))
assert.Equal(t, 31, len(sp))
} }

10
nomt/core/triepos.go
View file

@ -13,13 +13,17 @@ func NewTriePosition() TriePosition {
return TriePosition{} return TriePosition{}
} }
// MaxTrieDepth is the maximum depth of the internal node tree (248 bits = 31 bytes).
// Stem nodes exist at this depth; the last 8 bits are the stem suffix.
const MaxTrieDepth = StemSize * 8 // 248
// TriePositionFromPathAndDepth creates a TriePosition at the given depth // TriePositionFromPathAndDepth creates a TriePosition at the given depth
// within the path. Panics if depth is 0. // within the path. Panics if depth is 0.
func TriePositionFromPathAndDepth(path KeyPath, depth uint16) TriePosition { func TriePositionFromPathAndDepth(path KeyPath, depth uint16) TriePosition {
if depth == 0 { if depth == 0 {
panic("triepos: depth must be non-zero") panic("triepos: depth must be non-zero")
} }
if depth > 256 { if depth > MaxTrieDepth {
panic("triepos: depth out of range") panic("triepos: depth out of range")
} }
pagePath := lastPagePath(path[:], depth) pagePath := lastPagePath(path[:], depth)
@ -57,8 +61,8 @@ func (p *TriePosition) NodeIndex() int {
// Down moves the position down by 1 bit (left if bit=false, right if bit=true). // Down moves the position down by 1 bit (left if bit=false, right if bit=true).
func (p *TriePosition) Down(bit bool) { func (p *TriePosition) Down(bit bool) {
if p.depth == 256 { if p.depth >= MaxTrieDepth {
panic("triepos: can't descend past 256 bits") panic("triepos: can't descend past 248 bits")
} }
if int(p.depth)%PageDepth == 0 { if int(p.depth)%PageDepth == 0 {
// Entering a new page: node index resets. // Entering a new page: node index resets.

20
nomt/core/triepos_test.go
View file

@ -158,13 +158,23 @@ func TestTriePositionChildPageIndex(t *testing.T) {
assert.Equal(t, uint8(0), p.ChildPageIndex()) assert.Equal(t, uint8(0), p.ChildPageIndex())
} }
func TestTriePositionMax255Depth(t *testing.T) { func TestTriePositionMax248Depth(t *testing.T) {
p := NewTriePosition() p := NewTriePosition()
for range 255 { for range 247 {
p.Down(true) p.Down(true)
} }
assert.Equal(t, uint16(255), p.Depth()) assert.Equal(t, uint16(247), p.Depth())
// One more descent should work (to 256). // One more descent should work (to 248, the max).
p.Down(false) p.Down(false)
assert.Equal(t, uint16(256), p.Depth()) assert.Equal(t, uint16(248), p.Depth())
}
func TestTriePositionPanicsBeyond248(t *testing.T) {
p := NewTriePosition()
for range 248 {
p.Down(true)
}
assert.Equal(t, uint16(248), p.Depth())
// Going one more should panic.
assert.Panics(t, func() { p.Down(false) })
} }

186
nomt/core/update.go
View file

@ -1,48 +1,40 @@
package core package core
import "sort" // StemKeyValue is a resolved (stemPath, stemHash) pair for the page tree.
// The stem hash is precomputed by the integration layer using HashStem.
// LeafOp represents a leaf operation: set or delete. type StemKeyValue struct {
// A nil ValueHash pointer means delete. Stem StemPath // 31-byte stem (248 bits)
type LeafOp struct { Hash Node // precomputed SHA256 stem hash
Key KeyPath
Value *ValueHash
} }
// KeyValue is a resolved (key, value) pair for trie building. // WriteNodeKind enumerates the types of write commands from BuildInternalTree.
type KeyValue struct {
Key KeyPath
Value ValueHash
}
// WriteNodeKind enumerates the types of write commands from BuildTrie.
type WriteNodeKind int type WriteNodeKind int
const ( const (
WriteNodeLeaf WriteNodeKind = iota WriteNodeStem WriteNodeKind = iota // opaque stem hash placed at tree bottom
WriteNodeInternal WriteNodeInternal // internal node (hash of left+right)
WriteNodeTerminator WriteNodeTerminator // empty sub-trie
) )
// WriteNode represents a node to be written during trie building. // WriteNode represents a node to be written during trie building.
type WriteNode struct { type WriteNode struct {
Kind WriteNodeKind Kind WriteNodeKind
Node Node Node Node
LeafData *LeafData // set for leaf writes
InternalData *InternalData // set for internal writes InternalData *InternalData // set for internal writes
// Navigation: move up 1 before writing (true for internal nodes and // Navigation: move up 1 before writing (true for internal nodes and
// non-first leaves). // non-first stem placements).
GoUp bool GoUp bool
// Navigation: bits to descend after going up (only for leaf writes). // Navigation: bits to descend after going up (only for stem writes).
DownBits []bool DownBits []bool
} }
// SharedBits counts the number of shared prefix bits between two key paths, // StemSharedBits counts the number of shared prefix bits between two stem
// starting after `skip` bits. // paths, starting after `skip` bits.
func SharedBits(a, b *KeyPath, skip int) int { func StemSharedBits(a, b *StemPath, skip int) int {
count := 0 count := 0
for i := skip; i < 256; i++ { maxBits := StemSize * 8 // 248
for i := skip; i < maxBits; i++ {
aBit := (a[i/8] >> (7 - i%8)) & 1 aBit := (a[i/8] >> (7 - i%8)) & 1
bBit := (b[i/8] >> (7 - i%8)) & 1 bBit := (b[i/8] >> (7 - i%8)) & 1
if aBit != bBit { if aBit != bBit {
@ -53,163 +45,104 @@ func SharedBits(a, b *KeyPath, skip int) int {
return count return count
} }
// LeafOpsSpliced creates a combined operation list from an existing leaf and // BuildInternalTree builds a compact internal-node sub-trie from sorted
// new operations. If the existing leaf's key is not in ops, it is spliced in. // (stem, hash) pairs.
// Deletions (nil value) are filtered out.
func LeafOpsSpliced(existingLeaf *LeafData, ops []LeafOp) []KeyValue {
// Find splice position: where the existing leaf would be inserted.
spliceIndex := -1
if existingLeaf != nil {
idx := sort.Search(len(ops), func(i int) bool {
return keyPathCmp(&ops[i].Key, &existingLeaf.KeyPath) >= 0
})
if idx >= len(ops) || ops[idx].Key != existingLeaf.KeyPath {
spliceIndex = idx
}
}
result := make([]KeyValue, 0, len(ops)+1)
if spliceIndex < 0 {
// No splicing needed — just filter out deletes.
for _, op := range ops {
if op.Value != nil {
result = append(result, KeyValue{op.Key, *op.Value})
}
}
return result
}
// Before splice point.
for _, op := range ops[:spliceIndex] {
if op.Value != nil {
result = append(result, KeyValue{op.Key, *op.Value})
}
}
// The existing leaf.
result = append(result, KeyValue{
existingLeaf.KeyPath,
existingLeaf.ValueHash,
})
// After splice point.
for _, op := range ops[spliceIndex:] {
if op.Value != nil {
result = append(result, KeyValue{op.Key, *op.Value})
}
}
return result
}
// BuildTrie builds a compact sub-trie from sorted (key, value) pairs.
// //
// skip: the number of prefix bits already consumed (all ops share this prefix). // skip: the number of prefix bits already consumed (all ops share this prefix).
// ops: sorted (KeyPath, ValueHash) pairs. // ops: sorted StemKeyValue pairs (by stem path).
// visit: callback invoked for each computed node, bottom-up. // visit: callback invoked for each computed node, bottom-up.
// //
// Returns the root node of the built sub-trie. // Returns the root node of the built sub-trie.
// //
// The algorithm uses a 3-pointer sliding window (a, b, c) over the sorted // This replaces the old BuildTrie. The key difference: there are no leaf
// ops to determine each leaf's depth based on shared bits with its neighbors. // nodes — stem hashes are opaque values placed at tree positions, and
// Internal nodes are computed by hashing up a left-frontier stack. // internal nodes are always SHA256(left || right) with no MSB tagging
func BuildTrie(skip int, ops []KeyValue, visit func(WriteNode)) Node { // or leaf compaction.
func BuildInternalTree(skip int, ops []StemKeyValue, visit func(WriteNode)) Node {
if len(ops) == 0 { if len(ops) == 0 {
visit(WriteNode{Kind: WriteNodeTerminator, Node: Terminator}) visit(WriteNode{Kind: WriteNodeTerminator, Node: Terminator})
return Terminator return Terminator
} }
if len(ops) == 1 { if len(ops) == 1 {
ld := LeafData{
KeyPath: ops[0].Key,
ValueHash: ops[0].Value,
}
h := HashLeaf(&ld)
visit(WriteNode{ visit(WriteNode{
Kind: WriteNodeLeaf, Kind: WriteNodeStem,
Node: h, Node: ops[0].Hash,
LeafData: &ld, GoUp: false,
GoUp: false,
}) })
return h return ops[0].Hash
} }
// 3-pointer left-frontier algorithm. // 3-pointer left-frontier algorithm (same structure as old BuildTrie
// but without leaf hashing or leaf compaction).
type pendingSibling struct { type pendingSibling struct {
node Node node Node
layer int layer int
} }
pendingSiblings := make([]pendingSibling, 0, 16) pendingSiblings := make([]pendingSibling, 0, 16)
commonAfterPrefix := func(k1, k2 *KeyPath) int { commonAfterPrefix := func(s1, s2 *StemPath) int {
return SharedBits(k1, k2, skip) return StemSharedBits(s1, s2, skip)
} }
// Sliding window: a, b, c. var aStem *StemPath
var aKey *KeyPath
var aVal *ValueHash
for bIdx := 0; bIdx < len(ops); bIdx++ { for bIdx := 0; bIdx < len(ops); bIdx++ {
thisKey := &ops[bIdx].Key thisStem := &ops[bIdx].Stem
thisVal := &ops[bIdx].Value thisHash := ops[bIdx].Hash
var n1 *int var n1 *int
if aKey != nil { if aStem != nil {
v := commonAfterPrefix(aKey, thisKey) v := commonAfterPrefix(aStem, thisStem)
n1 = &v n1 = &v
} }
var n2 *int var n2 *int
if bIdx+1 < len(ops) { if bIdx+1 < len(ops) {
v := commonAfterPrefix(&ops[bIdx+1].Key, thisKey) v := commonAfterPrefix(&ops[bIdx+1].Stem, thisStem)
n2 = &v n2 = &v
} }
ld := LeafData{KeyPath: *thisKey, ValueHash: *thisVal} var stemDepth, hashUpLayers int
leaf := HashLeaf(&ld)
var leafDepth, hashUpLayers int
switch { switch {
case n1 == nil && n2 == nil: case n1 == nil && n2 == nil:
leafDepth = 0 stemDepth = 0
hashUpLayers = 0 hashUpLayers = 0
case n1 == nil && n2 != nil: case n1 == nil && n2 != nil:
leafDepth = *n2 + 1 stemDepth = *n2 + 1
hashUpLayers = 0 hashUpLayers = 0
case n1 != nil && n2 == nil: case n1 != nil && n2 == nil:
leafDepth = *n1 + 1 stemDepth = *n1 + 1
hashUpLayers = *n1 + 1 hashUpLayers = *n1 + 1
default: default:
leafDepth = max(*n1, *n2) + 1 stemDepth = max(*n1, *n2) + 1
hashUpLayers = 0 hashUpLayers = 0
if *n1 > *n2 { if *n1 > *n2 {
hashUpLayers = *n1 - *n2 hashUpLayers = *n1 - *n2
} }
} }
layer := leafDepth layer := stemDepth
lastNode := leaf lastNode := thisHash
// Compute down bits for the visitor. // Compute down bits for the visitor.
downStart := skip downStart := skip
if n1 != nil { if n1 != nil {
downStart = skip + *n1 downStart = skip + *n1
} }
leafEndBit := skip + leafDepth stemEndBit := skip + stemDepth
var downBits []bool var downBits []bool
if leafEndBit > downStart { if stemEndBit > downStart {
downBits = make([]bool, leafEndBit-downStart) downBits = make([]bool, stemEndBit-downStart)
for i := downStart; i < leafEndBit; i++ { for i := downStart; i < stemEndBit; i++ {
downBits[i-downStart] = bitAt(thisKey, i) downBits[i-downStart] = stemBitAt(thisStem, i)
} }
} }
visit(WriteNode{ visit(WriteNode{
Kind: WriteNodeLeaf, Kind: WriteNodeStem,
Node: leaf, Node: thisHash,
LeafData: &ld,
GoUp: n1 != nil, GoUp: n1 != nil,
DownBits: downBits, DownBits: downBits,
}) })
@ -217,10 +150,9 @@ func BuildTrie(skip int, ops []KeyValue, visit func(WriteNode)) Node {
// Hash upward. // Hash upward.
for h := 0; h < hashUpLayers; h++ { for h := 0; h < hashUpLayers; h++ {
layer-- layer--
bitIdx := skip + layer // the bit at this layer bitIdx := skip + layer
bit := bitAt(thisKey, bitIdx) bit := stemBitAt(thisStem, bitIdx)
// Pop sibling from pending if it matches.
var sibling Node var sibling Node
if len(pendingSiblings) > 0 && if len(pendingSiblings) > 0 &&
pendingSiblings[len(pendingSiblings)-1].layer == layer+1 { pendingSiblings[len(pendingSiblings)-1].layer == layer+1 {
@ -247,10 +179,8 @@ func BuildTrie(skip int, ops []KeyValue, visit func(WriteNode)) Node {
pendingSiblings = append(pendingSiblings, pendingSiblings = append(pendingSiblings,
pendingSibling{node: lastNode, layer: layer}) pendingSibling{node: lastNode, layer: layer})
aKey = thisKey aStem = thisStem
aVal = thisVal
} }
_ = aVal // used in the loop to track state
if len(pendingSiblings) > 0 { if len(pendingSiblings) > 0 {
return pendingSiblings[len(pendingSiblings)-1].node return pendingSiblings[len(pendingSiblings)-1].node
@ -258,11 +188,11 @@ func BuildTrie(skip int, ops []KeyValue, visit func(WriteNode)) Node {
return Terminator return Terminator
} }
func bitAt(key *KeyPath, idx int) bool { func stemBitAt(stem *StemPath, idx int) bool {
return (key[idx/8]>>(7-idx%8))&1 == 1 return (stem[idx/8]>>(7-idx%8))&1 == 1
} }
func keyPathCmp(a, b *KeyPath) int { func stemPathCmp(a, b *StemPath) int {
for i := range a { for i := range a {
if a[i] < b[i] { if a[i] < b[i] {
return -1 return -1

204
nomt/core/update_test.go
View file

@ -7,43 +7,45 @@ import (
"github.com/stretchr/testify/require" "github.com/stretchr/testify/require"
) )
func TestSharedBits(t *testing.T) { func TestStemSharedBits(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
a, b KeyPath a, b StemPath
skip int skip int
expected int expected int
}{ }{
{"identical", KeyPath{0xFF}, KeyPath{0xFF}, 0, 256}, {"identical", StemPath{0xFF}, StemPath{0xFF}, 0, 248},
{"differ at bit 0", KeyPath{0x80}, KeyPath{0x00}, 0, 0}, {"differ at bit 0", StemPath{0x80}, StemPath{0x00}, 0, 0},
{"share 4 bits", KeyPath{0xF0}, KeyPath{0xF8}, 0, 4}, {"share 4 bits", StemPath{0xF0}, StemPath{0xF8}, 0, 4},
{"with skip", KeyPath{0xF0}, KeyPath{0xF8}, 4, 0}, {"with skip", StemPath{0xF0}, StemPath{0xF8}, 4, 0},
} }
for _, tt := range tests { for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) { t.Run(tt.name, func(t *testing.T) {
got := SharedBits(&tt.a, &tt.b, tt.skip) got := StemSharedBits(&tt.a, &tt.b, tt.skip)
assert.Equal(t, tt.expected, got) assert.Equal(t, tt.expected, got)
}) })
} }
} }
// makeKV creates a (key, value) pair where both key and value are filled with b. // makeSKV creates a StemKeyValue where the stem is filled with b and
func makeKV(b byte) KeyValue { // the hash is a deterministic non-zero value derived from b.
var key KeyPath func makeSKV(b byte) StemKeyValue {
var val ValueHash var stem StemPath
for i := range key { for i := range stem {
key[i] = b stem[i] = b
} }
for i := range val { // Use a simple deterministic hash for testing.
val[i] = b var hash Node
for i := range hash {
hash[i] = b ^ byte(i)
} }
return KeyValue{Key: key, Value: val} return StemKeyValue{Stem: stem, Hash: hash}
} }
func TestBuildTrieEmpty(t *testing.T) { func TestBuildInternalTreeEmpty(t *testing.T) {
var visited []WriteNode var visited []WriteNode
root := BuildTrie(0, nil, func(wn WriteNode) { root := BuildInternalTree(0, nil, func(wn WriteNode) {
visited = append(visited, wn) visited = append(visited, wn)
}) })
@ -52,121 +54,95 @@ func TestBuildTrieEmpty(t *testing.T) {
assert.Equal(t, Terminator, root) assert.Equal(t, Terminator, root)
} }
func TestBuildTrieSingleLeaf(t *testing.T) { func TestBuildInternalTreeSingleStem(t *testing.T) {
kv := makeKV(0xFF) skv := makeSKV(0xFF)
var visited []WriteNode var visited []WriteNode
root := BuildTrie(0, []KeyValue{kv}, func(wn WriteNode) { root := BuildInternalTree(0, []StemKeyValue{skv}, func(wn WriteNode) {
visited = append(visited, wn) visited = append(visited, wn)
}) })
require.Len(t, visited, 1) require.Len(t, visited, 1)
assert.Equal(t, WriteNodeLeaf, visited[0].Kind) assert.Equal(t, WriteNodeStem, visited[0].Kind)
assert.False(t, visited[0].GoUp) assert.False(t, visited[0].GoUp)
assert.True(t, IsLeaf(&root)) assert.False(t, IsTerminator(&root))
assert.Equal(t, skv.Hash, root)
expected := HashLeaf(&LeafData{
KeyPath: kv.Key,
ValueHash: kv.Value,
})
assert.Equal(t, expected, root)
} }
func TestBuildTrieTwoLeaves(t *testing.T) { func TestBuildInternalTreeTwoStems(t *testing.T) {
// Keys: 0x00... and 0xFF... differ at bit 0. // Stems: 0x00... and 0xFF... differ at bit 0.
kv0 := makeKV(0x00) skv0 := makeSKV(0x00)
kvF := makeKV(0xFF) skvF := makeSKV(0xFF)
var visited []WriteNode var visited []WriteNode
root := BuildTrie(0, []KeyValue{kv0, kvF}, func(wn WriteNode) { root := BuildInternalTree(0, []StemKeyValue{skv0, skvF}, func(wn WriteNode) {
visited = append(visited, wn) visited = append(visited, wn)
}) })
// Should visit: leaf_0, leaf_F, internal(leaf_0, leaf_F). // Should visit: stem_0, stem_F, internal(stem_0, stem_F).
require.Len(t, visited, 3) require.Len(t, visited, 3)
assert.Equal(t, WriteNodeLeaf, visited[0].Kind) assert.Equal(t, WriteNodeStem, visited[0].Kind)
assert.Equal(t, WriteNodeLeaf, visited[1].Kind) assert.Equal(t, WriteNodeStem, visited[1].Kind)
assert.Equal(t, WriteNodeInternal, visited[2].Kind) assert.Equal(t, WriteNodeInternal, visited[2].Kind)
leaf0 := HashLeaf(&LeafData{KeyPath: kv0.Key, ValueHash: kv0.Value}) expected := HashInternal(&InternalData{Left: skv0.Hash, Right: skvF.Hash})
leafF := HashLeaf(&LeafData{KeyPath: kvF.Key, ValueHash: kvF.Value})
expected := HashInternal(&InternalData{Left: leaf0, Right: leafF})
assert.Equal(t, expected, root) assert.Equal(t, expected, root)
assert.True(t, IsInternal(&root))
} }
func TestBuildTrieThreeLeaves(t *testing.T) { func TestBuildInternalTreeThreeStems(t *testing.T) {
// Three keys sharing common prefixes. skv1 := makeSKV(0x11)
// 0b00010001... = 0x11 skv2 := makeSKV(0x12)
// 0b00010010... = 0x12 skv3 := makeSKV(0x14)
// 0b00010100... = 0x14
kv1 := makeKV(0x11)
kv2 := makeKV(0x12)
kv3 := makeKV(0x14)
var visited []WriteNode var visited []WriteNode
root := BuildTrie(0, []KeyValue{kv1, kv2, kv3}, func(wn WriteNode) { root := BuildInternalTree(0, []StemKeyValue{skv1, skv2, skv3}, func(wn WriteNode) {
visited = append(visited, wn) visited = append(visited, wn)
}) })
assert.True(t, IsInternal(&root) || IsLeaf(&root), assert.False(t, IsTerminator(&root))
"root should be non-terminator")
// Verify determinism. // Verify determinism.
var visited2 []WriteNode var visited2 []WriteNode
root2 := BuildTrie(0, []KeyValue{kv1, kv2, kv3}, func(wn WriteNode) { root2 := BuildInternalTree(0, []StemKeyValue{skv1, skv2, skv3}, func(wn WriteNode) {
visited2 = append(visited2, wn) visited2 = append(visited2, wn)
}) })
assert.Equal(t, root, root2) assert.Equal(t, root, root2)
assert.Equal(t, len(visited), len(visited2)) assert.Equal(t, len(visited), len(visited2))
} }
func TestBuildTrieWithSkip(t *testing.T) { func TestBuildInternalTreeWithSkip(t *testing.T) {
// Keys all share prefix 0001 (4 bits): 0x11, 0x12, 0x14. skv1 := makeSKV(0x11)
kv1 := makeKV(0x11) skv2 := makeSKV(0x12)
kv2 := makeKV(0x12) skv3 := makeSKV(0x14)
kv3 := makeKV(0x14)
var visited []WriteNode var visited []WriteNode
root := BuildTrie(4, []KeyValue{kv1, kv2, kv3}, func(wn WriteNode) { root := BuildInternalTree(4, []StemKeyValue{skv1, skv2, skv3}, func(wn WriteNode) {
visited = append(visited, wn) visited = append(visited, wn)
}) })
// Should produce a non-trivial sub-trie.
assert.False(t, IsTerminator(&root)) assert.False(t, IsTerminator(&root))
assert.True(t, len(visited) >= 3) assert.True(t, len(visited) >= 3)
} }
func TestBuildTrieMultiValue(t *testing.T) { func TestBuildInternalTreeMultiValue(t *testing.T) {
// Matches the Rust multi_value test pattern. skvA := makeSKV(0x10)
// 0b00010000 = 0x10 skvB := makeSKV(0x20)
// 0b00100000 = 0x20 skvC := makeSKV(0x40)
// 0b01000000 = 0x40 skvD := makeSKV(0xA0)
// 0b10100000 = 0xA0 skvE := makeSKV(0xB0)
// 0b10110000 = 0xB0
kvA := makeKV(0x10)
kvB := makeKV(0x20)
kvC := makeKV(0x40)
kvD := makeKV(0xA0)
kvE := makeKV(0xB0)
var nodes []Node var nodes []Node
root := BuildTrie(0, []KeyValue{kvA, kvB, kvC, kvD, kvE}, root := BuildInternalTree(0, []StemKeyValue{skvA, skvB, skvC, skvD, skvE},
func(wn WriteNode) { func(wn WriteNode) {
nodes = append(nodes, wn.Node) nodes = append(nodes, wn.Node)
}) })
// Manually verify the trie structure. // Manually verify the trie structure.
leafA := HashLeaf(&LeafData{KeyPath: kvA.Key, ValueHash: kvA.Value}) // A=0x10 (0001...), B=0x20 (0010...), C=0x40 (0100...)
leafB := HashLeaf(&LeafData{KeyPath: kvB.Key, ValueHash: kvB.Value}) // D=0xA0 (1010...), E=0xB0 (1011...)
leafC := HashLeaf(&LeafData{KeyPath: kvC.Key, ValueHash: kvC.Value}) branchAB := HashInternal(&InternalData{Left: skvA.Hash, Right: skvB.Hash})
leafD := HashLeaf(&LeafData{KeyPath: kvD.Key, ValueHash: kvD.Value}) branchABC := HashInternal(&InternalData{Left: branchAB, Right: skvC.Hash})
leafE := HashLeaf(&LeafData{KeyPath: kvE.Key, ValueHash: kvE.Value})
branchAB := HashInternal(&InternalData{Left: leafA, Right: leafB}) branchDE1 := HashInternal(&InternalData{Left: skvD.Hash, Right: skvE.Hash})
branchABC := HashInternal(&InternalData{Left: branchAB, Right: leafC})
branchDE1 := HashInternal(&InternalData{Left: leafD, Right: leafE})
branchDE2 := HashInternal(&InternalData{Left: Terminator, Right: branchDE1}) branchDE2 := HashInternal(&InternalData{Left: Terminator, Right: branchDE1})
branchDE3 := HashInternal(&InternalData{Left: branchDE2, Right: Terminator}) branchDE3 := HashInternal(&InternalData{Left: branchDE2, Right: Terminator})
@ -174,65 +150,3 @@ func TestBuildTrieMultiValue(t *testing.T) {
assert.Equal(t, expected, root) assert.Equal(t, expected, root)
} }
func TestLeafOpsSplicedNoExisting(t *testing.T) {
val := ValueHash{0x01}
ops := []LeafOp{
{Key: KeyPath{0x10}, Value: &val},
{Key: KeyPath{0x20}, Value: &val},
}
result := LeafOpsSpliced(nil, ops)
assert.Len(t, result, 2)
}
func TestLeafOpsSplicedWithExistingLeaf(t *testing.T) {
val := ValueHash{0x01}
ops := []LeafOp{
{Key: KeyPath{0x10}, Value: &val},
{Key: KeyPath{0x30}, Value: &val},
}
existing := &LeafData{
KeyPath: KeyPath{0x20},
ValueHash: ValueHash{0x02},
}
result := LeafOpsSpliced(existing, ops)
assert.Len(t, result, 3)
assert.Equal(t, KeyPath{0x10}, result[0].Key)
assert.Equal(t, KeyPath{0x20}, result[1].Key)
assert.Equal(t, KeyPath{0x30}, result[2].Key)
}
func TestLeafOpsSplicedDeleteFiltered(t *testing.T) {
val := ValueHash{0x01}
ops := []LeafOp{
{Key: KeyPath{0x10}, Value: &val},
{Key: KeyPath{0x20}, Value: nil}, // delete
{Key: KeyPath{0x30}, Value: &val},
}
result := LeafOpsSpliced(nil, ops)
assert.Len(t, result, 2)
assert.Equal(t, KeyPath{0x10}, result[0].Key)
assert.Equal(t, KeyPath{0x30}, result[1].Key)
}
func TestLeafOpsSplicedExistingKeyInOps(t *testing.T) {
val := ValueHash{0x01}
newVal := ValueHash{0x99}
ops := []LeafOp{
{Key: KeyPath{0x20}, Value: &newVal},
}
existing := &LeafData{
KeyPath: KeyPath{0x20},
ValueHash: val,
}
// The existing leaf should NOT be spliced because its key is in ops.
result := LeafOpsSpliced(existing, ops)
assert.Len(t, result, 1)
assert.Equal(t, newVal, result[0].Value)
}