mirror of
https://github.com/ethereum/go-ethereum.git
synced 2026-06-19 13:21:37 +00:00
use pebble instead of custom db
This commit is contained in:
weiihann
parent
43b69c4cfb
commit
a145150c39
18 changed files with 726 additions and 2443 deletions
1032
nomt/DESIGN.md
1032
nomt/DESIGN.md
File diff suppressed because it is too large
Load diff
|
|
@ -1,399 +0,0 @@
|
|||
package bitbox
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/ethereum/go-ethereum/nomt/core"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// --- HT File Layout Tests ---
|
||||
|
||||
func TestHTOffsetsMetaByteOffset(t *testing.T) {
|
||||
offsets := NewHTOffsets(8192)
|
||||
assert.Equal(t, int64(pageSize), offsets.MetaByteOffset(0))
|
||||
assert.Equal(t, int64(pageSize+1), offsets.MetaByteOffset(1))
|
||||
}
|
||||
|
||||
func TestHTOffsetsDataPageOffset(t *testing.T) {
|
||||
// capacity=4096 → 1 meta page
|
||||
offsets := NewHTOffsets(4096)
|
||||
assert.Equal(t, uint64(1), offsets.MetaPages)
|
||||
|
||||
// Data starts at: header(4096) + 1 meta page(4096) = 8192
|
||||
assert.Equal(t, int64(8192), offsets.DataPageOffset(0))
|
||||
assert.Equal(t, int64(8192+4096), offsets.DataPageOffset(1))
|
||||
}
|
||||
|
||||
func TestHTOffsetsTotalFileSize(t *testing.T) {
|
||||
offsets := NewHTOffsets(4096)
|
||||
// header(4096) + 1 meta page(4096) + 4096 data pages * 4096
|
||||
expected := int64(4096 + 4096 + 4096*4096)
|
||||
assert.Equal(t, expected, offsets.TotalFileSize())
|
||||
}
|
||||
|
||||
func TestHTOffsetsMetaPagesRoundup(t *testing.T) {
|
||||
offsets := NewHTOffsets(5000)
|
||||
assert.Equal(t, uint64(2), offsets.MetaPages)
|
||||
}
|
||||
|
||||
func TestCreateOpenHTFile(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "test.ht")
|
||||
|
||||
seed := HashSeedFromUint64(42, 99)
|
||||
f, offsets, err := CreateHTFile(path, 1024, seed)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, uint64(1024), offsets.Capacity)
|
||||
f.Close()
|
||||
|
||||
f2, offsets2, seed2, occ, err := OpenHTFile(path)
|
||||
require.NoError(t, err)
|
||||
defer f2.Close()
|
||||
|
||||
assert.Equal(t, seed, seed2)
|
||||
assert.Equal(t, uint64(1024), offsets2.Capacity)
|
||||
assert.Equal(t, uint64(0), occ)
|
||||
}
|
||||
|
||||
// --- Meta Byte Tests ---
|
||||
|
||||
func TestMetaByteEncoding(t *testing.T) {
|
||||
assert.True(t, IsEmpty(MetaEmpty))
|
||||
assert.False(t, IsOccupied(MetaEmpty))
|
||||
assert.False(t, IsTombstone(MetaEmpty))
|
||||
|
||||
assert.True(t, IsTombstone(MetaTombstone))
|
||||
assert.False(t, IsEmpty(MetaTombstone))
|
||||
assert.False(t, IsOccupied(MetaTombstone))
|
||||
|
||||
occupied := MakeOccupied(0xFFFFFFFFFFFFFFFF)
|
||||
assert.True(t, IsOccupied(occupied))
|
||||
assert.False(t, IsEmpty(occupied))
|
||||
assert.False(t, IsTombstone(occupied))
|
||||
}
|
||||
|
||||
func TestMetaByteTagMatching(t *testing.T) {
|
||||
hash := uint64(0xABCDEF1234567890)
|
||||
meta := MakeOccupied(hash)
|
||||
assert.True(t, TagMatches(meta, hash))
|
||||
|
||||
// Different high bits should not match.
|
||||
differentHash := uint64(0x1234EF1234567890)
|
||||
assert.False(t, TagMatches(meta, differentHash))
|
||||
}
|
||||
|
||||
func TestMetaMapSetGet(t *testing.T) {
|
||||
mm := NewMetaMap(8192)
|
||||
assert.Equal(t, MetaEmpty, mm.Get(0))
|
||||
|
||||
mm.Set(100, MakeOccupied(12345))
|
||||
assert.True(t, IsOccupied(mm.Get(100)))
|
||||
}
|
||||
|
||||
func TestMetaMapDirtyTracking(t *testing.T) {
|
||||
mm := NewMetaMap(8192) // 2 meta pages
|
||||
assert.Empty(t, mm.DirtyMetaPages())
|
||||
|
||||
mm.Set(0, MetaTombstone) // page 0
|
||||
mm.Set(5000, MetaTombstone) // page 1
|
||||
|
||||
dirty := mm.DirtyMetaPages()
|
||||
assert.Len(t, dirty, 2)
|
||||
assert.Contains(t, dirty, uint64(0))
|
||||
assert.Contains(t, dirty, uint64(1))
|
||||
|
||||
mm.ClearDirty()
|
||||
assert.Empty(t, mm.DirtyMetaPages())
|
||||
}
|
||||
|
||||
// --- Probe Sequence Tests ---
|
||||
|
||||
func TestProbeSequenceInitial(t *testing.T) {
|
||||
p := NewProbeSequence(42, 1024)
|
||||
assert.Equal(t, uint64(42%1024), p.Bucket())
|
||||
assert.Equal(t, uint64(42), p.Hash())
|
||||
}
|
||||
|
||||
func TestProbeSequenceTriangular(t *testing.T) {
|
||||
p := NewProbeSequence(0, 16) // initial bucket = 0
|
||||
assert.Equal(t, uint64(0), p.Bucket())
|
||||
|
||||
p.Next() // step=1 → (0+1)%16 = 1
|
||||
assert.Equal(t, uint64(1), p.Bucket())
|
||||
|
||||
p.Next() // step=2 → (1+2)%16 = 3
|
||||
assert.Equal(t, uint64(3), p.Bucket())
|
||||
|
||||
p.Next() // step=3 → (3+3)%16 = 6
|
||||
assert.Equal(t, uint64(6), p.Bucket())
|
||||
|
||||
p.Next() // step=4 → (6+4)%16 = 10
|
||||
assert.Equal(t, uint64(10), p.Bucket())
|
||||
}
|
||||
|
||||
func TestProbeSequenceVisitsAll(t *testing.T) {
|
||||
// With power-of-2 capacity, triangular probing should visit all buckets.
|
||||
capacity := uint64(16)
|
||||
p := NewProbeSequence(0, capacity)
|
||||
|
||||
visited := make(map[uint64]bool, capacity)
|
||||
for range capacity {
|
||||
visited[p.Bucket()] = true
|
||||
p.Next()
|
||||
}
|
||||
|
||||
assert.Equal(t, int(capacity), len(visited),
|
||||
"triangular probing should visit all buckets")
|
||||
}
|
||||
|
||||
func TestHashPageID(t *testing.T) {
|
||||
seed := HashSeedFromUint64(1, 2)
|
||||
root := core.RootPageID()
|
||||
h1 := HashPageID(seed, root)
|
||||
h2 := HashPageID(seed, root)
|
||||
assert.Equal(t, h1, h2, "same inputs should produce same hash")
|
||||
|
||||
// Different seed should produce different hash.
|
||||
seed2 := HashSeedFromUint64(3, 4)
|
||||
h3 := HashPageID(seed2, root)
|
||||
assert.NotEqual(t, h1, h3)
|
||||
}
|
||||
|
||||
// --- DB Integration Tests ---
|
||||
|
||||
func TestDBCreateAndOpen(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "test.bitbox")
|
||||
|
||||
seed := HashSeedFromUint64(1, 2)
|
||||
db, err := Create(path, 1024, seed)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, uint64(1024), db.Capacity())
|
||||
assert.Equal(t, int64(0), db.Occupied())
|
||||
require.NoError(t, db.Sync())
|
||||
require.NoError(t, db.Close())
|
||||
|
||||
db2, err := Open(path)
|
||||
require.NoError(t, err)
|
||||
defer db2.Close()
|
||||
assert.Equal(t, seed, db2.Seed())
|
||||
assert.Equal(t, uint64(1024), db2.Capacity())
|
||||
}
|
||||
|
||||
func TestDBStoreAndLoad(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "test.bitbox")
|
||||
|
||||
seed := HashSeedFromUint64(1, 2)
|
||||
db, err := Create(path, 1024, seed)
|
||||
require.NoError(t, err)
|
||||
defer db.Close()
|
||||
|
||||
// Store a page.
|
||||
rootID := core.RootPageID()
|
||||
page := new(core.RawPage)
|
||||
page.SetNodeAt(0, core.Node{0x42})
|
||||
|
||||
bucket, err := db.StorePage(rootID, page)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, int64(1), db.Occupied())
|
||||
|
||||
// Load it back.
|
||||
loaded, loadBucket, found, err := db.LoadPage(rootID)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, found)
|
||||
assert.Equal(t, bucket, loadBucket)
|
||||
assert.Equal(t, core.Node{0x42}, loaded.NodeAt(0))
|
||||
}
|
||||
|
||||
func TestDBStoreOverwrite(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "test.bitbox")
|
||||
|
||||
seed := HashSeedFromUint64(1, 2)
|
||||
db, err := Create(path, 1024, seed)
|
||||
require.NoError(t, err)
|
||||
defer db.Close()
|
||||
|
||||
rootID := core.RootPageID()
|
||||
page1 := new(core.RawPage)
|
||||
page1.SetNodeAt(0, core.Node{0x01})
|
||||
_, err = db.StorePage(rootID, page1)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Overwrite with new data.
|
||||
page2 := new(core.RawPage)
|
||||
page2.SetNodeAt(0, core.Node{0x02})
|
||||
_, err = db.StorePage(rootID, page2)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Should still only have 1 occupied.
|
||||
assert.Equal(t, int64(1), db.Occupied())
|
||||
|
||||
loaded, _, found, err := db.LoadPage(rootID)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, found)
|
||||
assert.Equal(t, core.Node{0x02}, loaded.NodeAt(0))
|
||||
}
|
||||
|
||||
func TestDBDelete(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "test.bitbox")
|
||||
|
||||
seed := HashSeedFromUint64(1, 2)
|
||||
db, err := Create(path, 1024, seed)
|
||||
require.NoError(t, err)
|
||||
defer db.Close()
|
||||
|
||||
rootID := core.RootPageID()
|
||||
page := new(core.RawPage)
|
||||
_, err = db.StorePage(rootID, page)
|
||||
require.NoError(t, err)
|
||||
|
||||
deleted, err := db.DeletePage(rootID)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, deleted)
|
||||
assert.Equal(t, int64(0), db.Occupied())
|
||||
|
||||
_, _, found, err := db.LoadPage(rootID)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, found)
|
||||
}
|
||||
|
||||
func TestDBDeleteNonexistent(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "test.bitbox")
|
||||
|
||||
seed := HashSeedFromUint64(1, 2)
|
||||
db, err := Create(path, 1024, seed)
|
||||
require.NoError(t, err)
|
||||
defer db.Close()
|
||||
|
||||
rootID := core.RootPageID()
|
||||
deleted, err := db.DeletePage(rootID)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, deleted)
|
||||
}
|
||||
|
||||
func TestDBLoadMiss(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "test.bitbox")
|
||||
|
||||
seed := HashSeedFromUint64(1, 2)
|
||||
db, err := Create(path, 1024, seed)
|
||||
require.NoError(t, err)
|
||||
defer db.Close()
|
||||
|
||||
rootID := core.RootPageID()
|
||||
_, _, found, err := db.LoadPage(rootID)
|
||||
require.NoError(t, err)
|
||||
assert.False(t, found)
|
||||
}
|
||||
|
||||
func TestDBMultiplePages(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "test.bitbox")
|
||||
|
||||
seed := HashSeedFromUint64(1, 2)
|
||||
db, err := Create(path, 1024, seed)
|
||||
require.NoError(t, err)
|
||||
defer db.Close()
|
||||
|
||||
rootID := core.RootPageID()
|
||||
childID, err := rootID.ChildPageID(0)
|
||||
require.NoError(t, err)
|
||||
childID2, err := rootID.ChildPageID(1)
|
||||
require.NoError(t, err)
|
||||
|
||||
// Store 3 pages.
|
||||
for i, pid := range []core.PageID{rootID, childID, childID2} {
|
||||
page := new(core.RawPage)
|
||||
page.SetNodeAt(0, core.Node{byte(i + 1)})
|
||||
_, err := db.StorePage(pid, page)
|
||||
require.NoError(t, err)
|
||||
}
|
||||
|
||||
assert.Equal(t, int64(3), db.Occupied())
|
||||
|
||||
// Load each one.
|
||||
for i, pid := range []core.PageID{rootID, childID, childID2} {
|
||||
loaded, _, found, err := db.LoadPage(pid)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, found, "page %d", i)
|
||||
assert.Equal(t, core.Node{byte(i + 1)}, loaded.NodeAt(0))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDBPersistAndReopen(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "test.bitbox")
|
||||
|
||||
seed := HashSeedFromUint64(1, 2)
|
||||
db, err := Create(path, 1024, seed)
|
||||
require.NoError(t, err)
|
||||
|
||||
rootID := core.RootPageID()
|
||||
page := new(core.RawPage)
|
||||
page.SetNodeAt(0, core.Node{0xAB})
|
||||
_, err = db.StorePage(rootID, page)
|
||||
require.NoError(t, err)
|
||||
|
||||
require.NoError(t, db.Sync())
|
||||
require.NoError(t, db.Close())
|
||||
|
||||
// Reopen and verify.
|
||||
db2, err := Open(path)
|
||||
require.NoError(t, err)
|
||||
defer db2.Close()
|
||||
|
||||
loaded, _, found, err := db2.LoadPage(rootID)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, found)
|
||||
assert.Equal(t, core.Node{0xAB}, loaded.NodeAt(0))
|
||||
}
|
||||
|
||||
func TestDBCapacityMustBePowerOf2(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "test.bitbox")
|
||||
seed := HashSeedFromUint64(1, 2)
|
||||
|
||||
_, err := Create(path, 1000, seed)
|
||||
assert.Error(t, err)
|
||||
// Cleanup any partial file.
|
||||
os.Remove(path)
|
||||
}
|
||||
|
||||
func TestDBDeleteAndReinsert(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "test.bitbox")
|
||||
|
||||
seed := HashSeedFromUint64(1, 2)
|
||||
db, err := Create(path, 1024, seed)
|
||||
require.NoError(t, err)
|
||||
defer db.Close()
|
||||
|
||||
rootID := core.RootPageID()
|
||||
|
||||
// Insert → delete → insert should work.
|
||||
page1 := new(core.RawPage)
|
||||
page1.SetNodeAt(0, core.Node{0x01})
|
||||
_, err = db.StorePage(rootID, page1)
|
||||
require.NoError(t, err)
|
||||
|
||||
_, err = db.DeletePage(rootID)
|
||||
require.NoError(t, err)
|
||||
|
||||
page2 := new(core.RawPage)
|
||||
page2.SetNodeAt(0, core.Node{0x02})
|
||||
_, err = db.StorePage(rootID, page2)
|
||||
require.NoError(t, err)
|
||||
|
||||
loaded, _, found, err := db.LoadPage(rootID)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, found)
|
||||
assert.Equal(t, core.Node{0x02}, loaded.NodeAt(0))
|
||||
}
|
||||
|
|
@ -1,284 +0,0 @@
|
|||
package bitbox
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"os"
|
||||
"sync/atomic"
|
||||
|
||||
"github.com/ethereum/go-ethereum/nomt/core"
|
||||
)
|
||||
|
||||
// DB is the Bitbox on-disk hash table for storing trie pages.
|
||||
type DB struct {
|
||||
file *os.File
|
||||
offsets HTOffsets
|
||||
metaMap *MetaMap
|
||||
seed [16]byte
|
||||
capacity uint64
|
||||
occupied atomic.Int64
|
||||
}
|
||||
|
||||
// Create creates a new Bitbox database at the given path.
|
||||
// Capacity must be a power of 2.
|
||||
func Create(path string, capacity uint64, seed [16]byte) (*DB, error) {
|
||||
if capacity == 0 || capacity&(capacity-1) != 0 {
|
||||
return nil, fmt.Errorf("bitbox: capacity must be a power of 2")
|
||||
}
|
||||
|
||||
f, offsets, err := CreateHTFile(path, capacity, seed)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
mm := NewMetaMap(capacity)
|
||||
|
||||
db := &DB{
|
||||
file: f,
|
||||
offsets: offsets,
|
||||
metaMap: mm,
|
||||
seed: seed,
|
||||
capacity: capacity,
|
||||
}
|
||||
return db, nil
|
||||
}
|
||||
|
||||
// Open opens an existing Bitbox database.
|
||||
func Open(path string) (*DB, error) {
|
||||
f, offsets, seed, occupied, err := OpenHTFile(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
mm, err := LoadMetaMap(f, offsets)
|
||||
if err != nil {
|
||||
f.Close()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
db := &DB{
|
||||
file: f,
|
||||
offsets: offsets,
|
||||
metaMap: mm,
|
||||
seed: seed,
|
||||
capacity: offsets.Capacity,
|
||||
}
|
||||
db.occupied.Store(int64(occupied))
|
||||
return db, nil
|
||||
}
|
||||
|
||||
// Close closes the database file.
|
||||
func (db *DB) Close() error {
|
||||
return db.file.Close()
|
||||
}
|
||||
|
||||
// Seed returns the hash seed.
|
||||
func (db *DB) Seed() [16]byte {
|
||||
return db.seed
|
||||
}
|
||||
|
||||
// Capacity returns the total number of buckets.
|
||||
func (db *DB) Capacity() uint64 {
|
||||
return db.capacity
|
||||
}
|
||||
|
||||
// Occupied returns the number of occupied buckets.
|
||||
func (db *DB) Occupied() int64 {
|
||||
return db.occupied.Load()
|
||||
}
|
||||
|
||||
// LoadPage reads a page from the hash table by probing for its PageID.
|
||||
// Returns the page, the bucket index where it was found, and whether it exists.
|
||||
func (db *DB) LoadPage(pageID core.PageID) (
|
||||
*core.RawPage, uint64, bool, error,
|
||||
) {
|
||||
hash := HashPageID(db.seed, pageID)
|
||||
probe := NewProbeSequence(hash, db.capacity)
|
||||
encodedID := pageID.Encode()
|
||||
|
||||
for range db.capacity {
|
||||
bucket := probe.Bucket()
|
||||
meta := db.metaMap.Get(bucket)
|
||||
|
||||
if IsEmpty(meta) {
|
||||
// Definitely not in the table.
|
||||
return nil, 0, false, nil
|
||||
}
|
||||
|
||||
if IsTombstone(meta) {
|
||||
probe.Next()
|
||||
continue
|
||||
}
|
||||
|
||||
if !TagMatches(meta, hash) {
|
||||
probe.Next()
|
||||
continue
|
||||
}
|
||||
|
||||
// Tag matches — read the data page to confirm.
|
||||
page, err := db.readDataPage(bucket)
|
||||
if err != nil {
|
||||
return nil, 0, false, err
|
||||
}
|
||||
|
||||
storedID := page.PageIDBytes()
|
||||
if storedID == encodedID {
|
||||
return page, bucket, true, nil
|
||||
}
|
||||
|
||||
probe.Next()
|
||||
}
|
||||
|
||||
return nil, 0, false, nil
|
||||
}
|
||||
|
||||
// StorePage writes a page to the hash table. If the page already exists
|
||||
// (by probing), it is overwritten in-place. Otherwise, a new bucket is
|
||||
// allocated.
|
||||
func (db *DB) StorePage(pageID core.PageID, page *core.RawPage) (
|
||||
uint64, error,
|
||||
) {
|
||||
// Ensure the encoded PageID is in the page data.
|
||||
encodedID := pageID.Encode()
|
||||
page.SetPageIDBytes(encodedID)
|
||||
|
||||
hash := HashPageID(db.seed, pageID)
|
||||
probe := NewProbeSequence(hash, db.capacity)
|
||||
metaByte := MakeOccupied(hash)
|
||||
|
||||
var firstTombstone int64 = -1
|
||||
|
||||
for range db.capacity {
|
||||
bucket := probe.Bucket()
|
||||
meta := db.metaMap.Get(bucket)
|
||||
|
||||
if IsEmpty(meta) {
|
||||
// Use tombstone if we passed one, otherwise use this empty slot.
|
||||
target := bucket
|
||||
if firstTombstone >= 0 {
|
||||
target = uint64(firstTombstone)
|
||||
} else {
|
||||
db.occupied.Add(1)
|
||||
}
|
||||
db.metaMap.Set(target, metaByte)
|
||||
if err := db.writeDataPage(target, page); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return target, nil
|
||||
}
|
||||
|
||||
if IsTombstone(meta) {
|
||||
if firstTombstone < 0 {
|
||||
firstTombstone = int64(bucket)
|
||||
}
|
||||
probe.Next()
|
||||
continue
|
||||
}
|
||||
|
||||
if TagMatches(meta, hash) {
|
||||
// Check if this is the same page.
|
||||
existing, err := db.readDataPage(bucket)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if existing.PageIDBytes() == encodedID {
|
||||
// Overwrite in-place.
|
||||
if err := db.writeDataPage(bucket, page); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return bucket, nil
|
||||
}
|
||||
}
|
||||
|
||||
probe.Next()
|
||||
}
|
||||
|
||||
return 0, fmt.Errorf("bitbox: hash table full")
|
||||
}
|
||||
|
||||
// DeletePage removes a page from the hash table by setting its meta byte
|
||||
// to tombstone.
|
||||
func (db *DB) DeletePage(pageID core.PageID) (bool, error) {
|
||||
hash := HashPageID(db.seed, pageID)
|
||||
probe := NewProbeSequence(hash, db.capacity)
|
||||
encodedID := pageID.Encode()
|
||||
|
||||
for range db.capacity {
|
||||
bucket := probe.Bucket()
|
||||
meta := db.metaMap.Get(bucket)
|
||||
|
||||
if IsEmpty(meta) {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
if IsTombstone(meta) {
|
||||
probe.Next()
|
||||
continue
|
||||
}
|
||||
|
||||
if TagMatches(meta, hash) {
|
||||
existing, err := db.readDataPage(bucket)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
if existing.PageIDBytes() == encodedID {
|
||||
db.metaMap.Set(bucket, MetaTombstone)
|
||||
db.occupied.Add(-1)
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
|
||||
probe.Next()
|
||||
}
|
||||
|
||||
return false, nil
|
||||
}
|
||||
|
||||
// FlushMeta writes all dirty meta pages to disk and updates the header.
|
||||
func (db *DB) FlushMeta() error {
|
||||
for _, pageIdx := range db.metaMap.DirtyMetaPages() {
|
||||
if err := db.metaMap.WriteMetaPage(db.file, pageIdx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
db.metaMap.ClearDirty()
|
||||
|
||||
// Update occupied count in header.
|
||||
var buf [8]byte
|
||||
occ := max(db.occupied.Load(), 0)
|
||||
binary.LittleEndian.PutUint64(buf[:], uint64(occ))
|
||||
if _, err := db.file.WriteAt(buf[:], occupiedOffset); err != nil {
|
||||
return fmt.Errorf("bitbox: update occupied count: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Sync flushes all pending data to disk.
|
||||
func (db *DB) Sync() error {
|
||||
if err := db.FlushMeta(); err != nil {
|
||||
return err
|
||||
}
|
||||
return db.file.Sync()
|
||||
}
|
||||
|
||||
// --- internal I/O ---
|
||||
|
||||
func (db *DB) readDataPage(bucket uint64) (*core.RawPage, error) {
|
||||
page := new(core.RawPage)
|
||||
offset := db.offsets.DataPageOffset(bucket)
|
||||
if _, err := db.file.ReadAt(page[:], offset); err != nil {
|
||||
return nil, fmt.Errorf("bitbox: read data page at bucket %d: %w",
|
||||
bucket, err)
|
||||
}
|
||||
return page, nil
|
||||
}
|
||||
|
||||
func (db *DB) writeDataPage(bucket uint64, page *core.RawPage) error {
|
||||
offset := db.offsets.DataPageOffset(bucket)
|
||||
if _, err := db.file.WriteAt(page[:], offset); err != nil {
|
||||
return fmt.Errorf("bitbox: write data page at bucket %d: %w",
|
||||
bucket, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -1,127 +0,0 @@
|
|||
// Package bitbox implements an on-disk open-addressing hash table that maps
|
||||
// PageIDs to 4096-byte pages. It is the storage backend for the NOMT trie.
|
||||
package bitbox
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/ethereum/go-ethereum/nomt/core"
|
||||
)
|
||||
|
||||
const (
|
||||
// pageSize is the size of a disk page.
|
||||
pageSize = core.PageSize // 4096
|
||||
|
||||
// metaBytesPerPage is the number of meta bytes that fit in one page.
|
||||
metaBytesPerPage = pageSize
|
||||
|
||||
// headerSize is the size of the HT file header in bytes.
|
||||
// Layout: [seed 16] [capacity 8] [occupied 8] = 32 bytes, padded to
|
||||
// one full page.
|
||||
headerSize = pageSize
|
||||
|
||||
// seedOffset is the offset of the 16-byte seed in the header.
|
||||
seedOffset = 0
|
||||
// capacityOffset is the offset of the 8-byte capacity in the header.
|
||||
capacityOffset = 16
|
||||
// occupiedOffset is the offset of the 8-byte occupied count.
|
||||
occupiedOffset = 24
|
||||
)
|
||||
|
||||
// HTOffsets holds precomputed file offsets for the hash table file layout.
|
||||
//
|
||||
// File layout:
|
||||
//
|
||||
// [header: 1 page] [meta pages: ceil(capacity/4096)] [data pages: capacity * 4096]
|
||||
type HTOffsets struct {
|
||||
// Capacity is the number of buckets in the hash table.
|
||||
Capacity uint64
|
||||
// MetaPages is ceil(Capacity / 4096).
|
||||
MetaPages uint64
|
||||
}
|
||||
|
||||
// NewHTOffsets creates an HTOffsets for the given capacity.
|
||||
func NewHTOffsets(capacity uint64) HTOffsets {
|
||||
return HTOffsets{
|
||||
Capacity: capacity,
|
||||
MetaPages: (capacity + metaBytesPerPage - 1) / metaBytesPerPage,
|
||||
}
|
||||
}
|
||||
|
||||
// MetaByteOffset returns the file offset for the meta byte of a given bucket.
|
||||
func (o *HTOffsets) MetaByteOffset(bucket uint64) int64 {
|
||||
return int64(headerSize) + int64(bucket)
|
||||
}
|
||||
|
||||
// DataPageOffset returns the file offset for the data page of a given bucket.
|
||||
func (o *HTOffsets) DataPageOffset(bucket uint64) int64 {
|
||||
dataStart := int64(headerSize) + int64(o.MetaPages)*pageSize
|
||||
return dataStart + int64(bucket)*pageSize
|
||||
}
|
||||
|
||||
// TotalFileSize returns the total size of the HT file in bytes.
|
||||
func (o *HTOffsets) TotalFileSize() int64 {
|
||||
return int64(headerSize) + int64(o.MetaPages)*pageSize +
|
||||
int64(o.Capacity)*pageSize
|
||||
}
|
||||
|
||||
// CreateHTFile creates a new hash table file with the given capacity and seed.
|
||||
// The file is pre-allocated to its full size.
|
||||
func CreateHTFile(path string, capacity uint64, seed [16]byte) (
|
||||
*os.File, HTOffsets, error,
|
||||
) {
|
||||
offsets := NewHTOffsets(capacity)
|
||||
|
||||
f, err := os.Create(path)
|
||||
if err != nil {
|
||||
return nil, offsets, fmt.Errorf("bitbox: create HT file: %w", err)
|
||||
}
|
||||
|
||||
// Pre-allocate.
|
||||
totalSize := offsets.TotalFileSize()
|
||||
if err := f.Truncate(totalSize); err != nil {
|
||||
f.Close()
|
||||
return nil, offsets, fmt.Errorf("bitbox: truncate HT file: %w", err)
|
||||
}
|
||||
|
||||
// Write header.
|
||||
var header [headerSize]byte
|
||||
copy(header[seedOffset:], seed[:])
|
||||
binary.LittleEndian.PutUint64(header[capacityOffset:], capacity)
|
||||
binary.LittleEndian.PutUint64(header[occupiedOffset:], 0)
|
||||
|
||||
if _, err := f.WriteAt(header[:], 0); err != nil {
|
||||
f.Close()
|
||||
return nil, offsets, fmt.Errorf("bitbox: write header: %w", err)
|
||||
}
|
||||
|
||||
return f, offsets, nil
|
||||
}
|
||||
|
||||
// OpenHTFile opens an existing hash table file and reads its header.
|
||||
func OpenHTFile(path string) (
|
||||
*os.File, HTOffsets, [16]byte, uint64, error,
|
||||
) {
|
||||
f, err := os.OpenFile(path, os.O_RDWR, 0)
|
||||
if err != nil {
|
||||
return nil, HTOffsets{}, [16]byte{}, 0,
|
||||
fmt.Errorf("bitbox: open HT file: %w", err)
|
||||
}
|
||||
|
||||
var header [headerSize]byte
|
||||
if _, err := f.ReadAt(header[:], 0); err != nil {
|
||||
f.Close()
|
||||
return nil, HTOffsets{}, [16]byte{}, 0,
|
||||
fmt.Errorf("bitbox: read header: %w", err)
|
||||
}
|
||||
|
||||
var seed [16]byte
|
||||
copy(seed[:], header[seedOffset:seedOffset+16])
|
||||
capacity := binary.LittleEndian.Uint64(header[capacityOffset:])
|
||||
occupied := binary.LittleEndian.Uint64(header[occupiedOffset:])
|
||||
offsets := NewHTOffsets(capacity)
|
||||
|
||||
return f, offsets, seed, occupied, nil
|
||||
}
|
||||
|
|
@ -1,118 +0,0 @@
|
|||
package bitbox
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
)
|
||||
|
||||
// Meta byte constants.
|
||||
const (
|
||||
// MetaEmpty marks an empty bucket.
|
||||
MetaEmpty byte = 0x00
|
||||
// MetaTombstone marks a deleted bucket (still probed through).
|
||||
MetaTombstone byte = 0x7F
|
||||
)
|
||||
|
||||
// IsOccupied reports whether a meta byte indicates an occupied bucket.
|
||||
// Occupied bytes have bit 7 set (value >= 0x80).
|
||||
func IsOccupied(b byte) bool {
|
||||
return b&0x80 != 0
|
||||
}
|
||||
|
||||
// IsEmpty reports whether a meta byte indicates an empty bucket.
|
||||
func IsEmpty(b byte) bool {
|
||||
return b == MetaEmpty
|
||||
}
|
||||
|
||||
// IsTombstone reports whether a meta byte indicates a tombstone.
|
||||
func IsTombstone(b byte) bool {
|
||||
return b == MetaTombstone
|
||||
}
|
||||
|
||||
// MakeOccupied creates an occupied meta byte from a hash value.
|
||||
// It takes the top 7 bits of the hash and sets bit 7 to 1.
|
||||
func MakeOccupied(hash uint64) byte {
|
||||
return 0x80 | byte(hash>>57)
|
||||
}
|
||||
|
||||
// TagMatches reports whether an occupied meta byte could match a given hash.
|
||||
func TagMatches(metaByte byte, hash uint64) bool {
|
||||
return IsOccupied(metaByte) && metaByte == MakeOccupied(hash)
|
||||
}
|
||||
|
||||
// MetaMap holds an in-memory copy of all meta bytes for the hash table.
|
||||
type MetaMap struct {
|
||||
data []byte
|
||||
dirty []bool // per meta-page dirty tracking
|
||||
}
|
||||
|
||||
// NewMetaMap creates a MetaMap for the given capacity with all empty buckets.
|
||||
func NewMetaMap(capacity uint64) *MetaMap {
|
||||
metaPages := (capacity + metaBytesPerPage - 1) / metaBytesPerPage
|
||||
return &MetaMap{
|
||||
data: make([]byte, capacity),
|
||||
dirty: make([]bool, metaPages),
|
||||
}
|
||||
}
|
||||
|
||||
// LoadMetaMap reads all meta bytes from the HT file into memory.
|
||||
func LoadMetaMap(f *os.File, offsets HTOffsets) (*MetaMap, error) {
|
||||
mm := NewMetaMap(offsets.Capacity)
|
||||
|
||||
// Read all meta bytes at once.
|
||||
metaRegionSize := int64(offsets.MetaPages) * pageSize
|
||||
buf := make([]byte, metaRegionSize)
|
||||
if _, err := f.ReadAt(buf, int64(headerSize)); err != nil {
|
||||
return nil, fmt.Errorf("bitbox: load meta map: %w", err)
|
||||
}
|
||||
|
||||
// Copy only the capacity-many bytes (the rest is padding).
|
||||
copy(mm.data, buf[:offsets.Capacity])
|
||||
return mm, nil
|
||||
}
|
||||
|
||||
// Get returns the meta byte for a bucket.
|
||||
func (m *MetaMap) Get(bucket uint64) byte {
|
||||
return m.data[bucket]
|
||||
}
|
||||
|
||||
// Set writes a meta byte for a bucket and marks the containing page dirty.
|
||||
func (m *MetaMap) Set(bucket uint64, value byte) {
|
||||
m.data[bucket] = value
|
||||
m.dirty[bucket/metaBytesPerPage] = true
|
||||
}
|
||||
|
||||
// DirtyMetaPages returns the indices of meta pages that have been modified
|
||||
// since the last call to ClearDirty.
|
||||
func (m *MetaMap) DirtyMetaPages() []uint64 {
|
||||
pages := make([]uint64, 0, len(m.dirty))
|
||||
for i, d := range m.dirty {
|
||||
if d {
|
||||
pages = append(pages, uint64(i))
|
||||
}
|
||||
}
|
||||
return pages
|
||||
}
|
||||
|
||||
// ClearDirty resets all dirty flags.
|
||||
func (m *MetaMap) ClearDirty() {
|
||||
for i := range m.dirty {
|
||||
m.dirty[i] = false
|
||||
}
|
||||
}
|
||||
|
||||
// WriteMetaPage writes a single meta page (identified by index) to the file.
|
||||
func (m *MetaMap) WriteMetaPage(
|
||||
f *os.File, pageIdx uint64,
|
||||
) error {
|
||||
var buf [pageSize]byte
|
||||
start := pageIdx * metaBytesPerPage
|
||||
end := min(start+metaBytesPerPage, uint64(len(m.data)))
|
||||
copy(buf[:], m.data[start:end])
|
||||
|
||||
offset := int64(headerSize) + int64(pageIdx)*pageSize
|
||||
if _, err := f.WriteAt(buf[:], offset); err != nil {
|
||||
return fmt.Errorf("bitbox: write meta page %d: %w", pageIdx, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -1,81 +0,0 @@
|
|||
package bitbox
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
|
||||
"github.com/cespare/xxhash/v2"
|
||||
"github.com/ethereum/go-ethereum/nomt/core"
|
||||
)
|
||||
|
||||
// HashPageID computes the xxhash64 of seed||encodedPageID.
|
||||
func HashPageID(seed [16]byte, pageID core.PageID) uint64 {
|
||||
encoded := pageID.Encode()
|
||||
var buf [48]byte
|
||||
copy(buf[:16], seed[:])
|
||||
copy(buf[16:], encoded[:])
|
||||
return xxhash.Sum64(buf[:])
|
||||
}
|
||||
|
||||
// HashPageIDBytes computes the xxhash64 from seed and raw encoded page ID.
|
||||
func HashPageIDBytes(seed [16]byte, encodedPageID [32]byte) uint64 {
|
||||
var buf [48]byte
|
||||
copy(buf[:16], seed[:])
|
||||
copy(buf[16:], encodedPageID[:])
|
||||
return xxhash.Sum64(buf[:])
|
||||
}
|
||||
|
||||
// HashSeedFromBytes creates a [16]byte seed from a byte slice.
|
||||
func HashSeedFromBytes(b []byte) [16]byte {
|
||||
var seed [16]byte
|
||||
copy(seed[:], b)
|
||||
return seed
|
||||
}
|
||||
|
||||
// HashSeedFromUint64 creates a deterministic seed from two uint64 values.
|
||||
func HashSeedFromUint64(a, b uint64) [16]byte {
|
||||
var seed [16]byte
|
||||
binary.LittleEndian.PutUint64(seed[:8], a)
|
||||
binary.LittleEndian.PutUint64(seed[8:], b)
|
||||
return seed
|
||||
}
|
||||
|
||||
// ProbeSequence implements triangular probing over the hash table.
|
||||
//
|
||||
// Bucket(step) = (initial + step*(step+1)/2) mod capacity
|
||||
//
|
||||
// With a power-of-2 capacity, triangular probing visits every bucket before
|
||||
// repeating, guaranteeing termination.
|
||||
type ProbeSequence struct {
|
||||
hash uint64
|
||||
bucket uint64
|
||||
step uint64
|
||||
capacity uint64
|
||||
}
|
||||
|
||||
// NewProbeSequence creates a new probe sequence for the given hash and
|
||||
// capacity. The capacity MUST be a power of 2.
|
||||
func NewProbeSequence(hash, capacity uint64) ProbeSequence {
|
||||
initial := hash % capacity
|
||||
return ProbeSequence{
|
||||
hash: hash,
|
||||
bucket: initial,
|
||||
step: 0,
|
||||
capacity: capacity,
|
||||
}
|
||||
}
|
||||
|
||||
// Bucket returns the current bucket index.
|
||||
func (p *ProbeSequence) Bucket() uint64 {
|
||||
return p.bucket
|
||||
}
|
||||
|
||||
// Hash returns the hash used to seed this probe.
|
||||
func (p *ProbeSequence) Hash() uint64 {
|
||||
return p.hash
|
||||
}
|
||||
|
||||
// Next advances to the next bucket in the triangular probe sequence.
|
||||
func (p *ProbeSequence) Next() {
|
||||
p.step++
|
||||
p.bucket = (p.bucket + p.step) % p.capacity
|
||||
}
|
||||
|
|
@ -1,73 +0,0 @@
|
|||
package bitbox
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/ethereum/go-ethereum/nomt/core"
|
||||
)
|
||||
|
||||
// Recover replays the WAL file to restore the database to a consistent state.
|
||||
// Returns the sync sequence number from the WAL, or 0 if no recovery was
|
||||
// needed.
|
||||
func (db *DB) Recover(walPath string) (uint32, error) {
|
||||
data, err := ReadWALFile(walPath)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("bitbox/recover: %w", err)
|
||||
}
|
||||
|
||||
if len(data) == 0 {
|
||||
return 0, nil // No recovery needed.
|
||||
}
|
||||
|
||||
syncSeqn, entries, err := ReadWAL(data)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("bitbox/recover: parse: %w", err)
|
||||
}
|
||||
|
||||
for _, entry := range entries {
|
||||
switch entry.Kind {
|
||||
case WALEntryClear:
|
||||
db.metaMap.Set(entry.ClearBucket, MetaTombstone)
|
||||
|
||||
case WALEntryUpdate:
|
||||
// Read the existing data page at this bucket (or use a fresh one).
|
||||
page, readErr := db.readDataPage(entry.UpdateBucket)
|
||||
if readErr != nil {
|
||||
page = new(core.RawPage)
|
||||
}
|
||||
|
||||
// Apply the diff: unpack changed nodes into the page.
|
||||
entry.Diff.UnpackChangedNodes(entry.ChangedNodes, page)
|
||||
|
||||
// Set elided children and page ID.
|
||||
page.SetElidedChildren(entry.ElidedChildren)
|
||||
page.SetPageIDBytes(entry.PageID)
|
||||
|
||||
// Write data page.
|
||||
if err := db.writeDataPage(entry.UpdateBucket, page); err != nil {
|
||||
return 0, fmt.Errorf("bitbox/recover: write page: %w", err)
|
||||
}
|
||||
|
||||
// Update meta byte.
|
||||
hash := HashPageIDBytes(db.seed, entry.PageID)
|
||||
db.metaMap.Set(entry.UpdateBucket, MakeOccupied(hash))
|
||||
}
|
||||
}
|
||||
|
||||
// Write dirty meta pages.
|
||||
if err := db.FlushMeta(); err != nil {
|
||||
return 0, fmt.Errorf("bitbox/recover: flush meta: %w", err)
|
||||
}
|
||||
|
||||
// fsync HT file.
|
||||
if err := db.file.Sync(); err != nil {
|
||||
return 0, fmt.Errorf("bitbox/recover: fsync: %w", err)
|
||||
}
|
||||
|
||||
// Truncate WAL.
|
||||
if err := TruncateWALFile(walPath); err != nil {
|
||||
return 0, fmt.Errorf("bitbox/recover: truncate WAL: %w", err)
|
||||
}
|
||||
|
||||
return syncSeqn, nil
|
||||
}
|
||||
|
|
@ -1,131 +0,0 @@
|
|||
package bitbox
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/ethereum/go-ethereum/nomt/core"
|
||||
"github.com/ethereum/go-ethereum/nomt/merkle"
|
||||
)
|
||||
|
||||
// SyncPlan holds the pre-computed work for a sync operation.
|
||||
type SyncPlan struct {
|
||||
walData []byte
|
||||
dataWrites []dataWrite
|
||||
syncSeqn uint32
|
||||
}
|
||||
|
||||
type dataWrite struct {
|
||||
bucket uint64
|
||||
page *core.RawPage
|
||||
}
|
||||
|
||||
// BeginSync prepares a sync plan from a set of updated pages. It allocates
|
||||
// or reuses buckets, builds the WAL, and returns a SyncPlan.
|
||||
//
|
||||
// This is Phase 1 of the 3-phase sync protocol.
|
||||
func (db *DB) BeginSync(
|
||||
walPath string,
|
||||
syncSeqn uint32,
|
||||
updates []merkle.UpdatedPage,
|
||||
) (*SyncPlan, error) {
|
||||
wal := NewWALBuilder()
|
||||
writes := make([]dataWrite, 0, len(updates))
|
||||
|
||||
for _, up := range updates {
|
||||
if up.Diff.IsCleared() {
|
||||
// Page was cleared — tombstone its bucket.
|
||||
_, bucket, found, err := db.LoadPage(up.PageID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("bitbox/sync: load for clear: %w", err)
|
||||
}
|
||||
if found {
|
||||
db.metaMap.Set(bucket, MetaTombstone)
|
||||
db.occupied.Add(-1)
|
||||
wal.AddClear(bucket)
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
// Encode the PageID into the page data.
|
||||
encodedID := up.PageID.Encode()
|
||||
up.Page.SetPageIDBytes(encodedID)
|
||||
up.Page.SetElidedChildren(up.Page.ElidedChildren())
|
||||
|
||||
// Allocate or reuse a bucket.
|
||||
bucket, err := db.StorePage(up.PageID, up.Page)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("bitbox/sync: store page: %w", err)
|
||||
}
|
||||
|
||||
// Pack changed nodes from diff.
|
||||
changedNodes := up.Diff.PackChangedNodes(up.Page)
|
||||
|
||||
wal.AddUpdate(
|
||||
encodedID,
|
||||
up.Diff,
|
||||
changedNodes,
|
||||
up.Page.ElidedChildren(),
|
||||
bucket,
|
||||
)
|
||||
writes = append(writes, dataWrite{bucket: bucket, page: up.Page})
|
||||
}
|
||||
|
||||
walData := wal.Finish(syncSeqn)
|
||||
|
||||
return &SyncPlan{
|
||||
walData: walData,
|
||||
dataWrites: writes,
|
||||
syncSeqn: syncSeqn,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// WriteWAL writes the WAL to disk and fsyncs it.
|
||||
//
|
||||
// This is Phase 2 of the 3-phase sync protocol.
|
||||
func (db *DB) WriteWAL(walPath string, plan *SyncPlan) error {
|
||||
return WriteWALFile(walPath, plan.walData)
|
||||
}
|
||||
|
||||
// CommitSync writes dirty HT data + meta pages, fsyncs the HT file, and
|
||||
// truncates the WAL.
|
||||
//
|
||||
// This is Phase 3 of the 3-phase sync protocol.
|
||||
func (db *DB) CommitSync(walPath string, plan *SyncPlan) error {
|
||||
// Write data pages.
|
||||
for _, dw := range plan.dataWrites {
|
||||
if err := db.writeDataPage(dw.bucket, dw.page); err != nil {
|
||||
return fmt.Errorf("bitbox/sync: write data: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// Write dirty meta pages.
|
||||
if err := db.FlushMeta(); err != nil {
|
||||
return fmt.Errorf("bitbox/sync: flush meta: %w", err)
|
||||
}
|
||||
|
||||
// fsync the HT file.
|
||||
if err := db.file.Sync(); err != nil {
|
||||
return fmt.Errorf("bitbox/sync: fsync HT: %w", err)
|
||||
}
|
||||
|
||||
// Truncate WAL — no fsync needed.
|
||||
return TruncateWALFile(walPath)
|
||||
}
|
||||
|
||||
// FullSync runs all three phases of the sync protocol.
|
||||
func (db *DB) FullSync(
|
||||
walPath string,
|
||||
syncSeqn uint32,
|
||||
updates []merkle.UpdatedPage,
|
||||
) error {
|
||||
plan, err := db.BeginSync(walPath, syncSeqn, updates)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := db.WriteWAL(walPath, plan); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return db.CommitSync(walPath, plan)
|
||||
}
|
||||
|
|
@ -1,296 +0,0 @@
|
|||
package bitbox
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/ethereum/go-ethereum/nomt/core"
|
||||
)
|
||||
|
||||
// WAL entry type tags.
|
||||
const (
|
||||
walTagStart byte = 0x01
|
||||
walTagClear byte = 0x02
|
||||
walTagUpdate byte = 0x03
|
||||
walTagEnd byte = 0x04
|
||||
)
|
||||
|
||||
// WALEntryKind distinguishes the types of WAL entries.
|
||||
type WALEntryKind int
|
||||
|
||||
const (
|
||||
WALEntryClear WALEntryKind = iota
|
||||
WALEntryUpdate
|
||||
)
|
||||
|
||||
// WALEntry represents a single entry in the WAL.
|
||||
type WALEntry struct {
|
||||
Kind WALEntryKind
|
||||
|
||||
// For Clear entries:
|
||||
ClearBucket uint64
|
||||
|
||||
// For Update entries:
|
||||
PageID [32]byte
|
||||
Diff core.PageDiff
|
||||
ChangedNodes []core.Node
|
||||
ElidedChildren uint64
|
||||
UpdateBucket uint64
|
||||
}
|
||||
|
||||
// WALBuilder accumulates WAL entries in memory before serializing.
|
||||
type WALBuilder struct {
|
||||
entries []WALEntry
|
||||
}
|
||||
|
||||
// NewWALBuilder creates an empty WAL builder.
|
||||
func NewWALBuilder() *WALBuilder {
|
||||
return &WALBuilder{
|
||||
entries: make([]WALEntry, 0, 64),
|
||||
}
|
||||
}
|
||||
|
||||
// AddClear adds a CLEAR entry (tombstone a bucket).
|
||||
func (b *WALBuilder) AddClear(bucket uint64) {
|
||||
b.entries = append(b.entries, WALEntry{
|
||||
Kind: WALEntryClear,
|
||||
ClearBucket: bucket,
|
||||
})
|
||||
}
|
||||
|
||||
// AddUpdate adds an UPDATE entry.
|
||||
func (b *WALBuilder) AddUpdate(
|
||||
pageID [32]byte,
|
||||
diff core.PageDiff,
|
||||
changedNodes []core.Node,
|
||||
elidedChildren uint64,
|
||||
bucket uint64,
|
||||
) {
|
||||
b.entries = append(b.entries, WALEntry{
|
||||
Kind: WALEntryUpdate,
|
||||
PageID: pageID,
|
||||
Diff: diff,
|
||||
ChangedNodes: changedNodes,
|
||||
ElidedChildren: elidedChildren,
|
||||
UpdateBucket: bucket,
|
||||
})
|
||||
}
|
||||
|
||||
// Finish serializes the WAL with a START and END record, padded to a
|
||||
// multiple of pageSize.
|
||||
func (b *WALBuilder) Finish(syncSeqn uint32) []byte {
|
||||
// Estimate size: START(5) + entries + END(1).
|
||||
estimatedSize := 5 + 1
|
||||
for _, e := range b.entries {
|
||||
switch e.Kind {
|
||||
case WALEntryClear:
|
||||
estimatedSize += 1 + 8 // tag + bucket
|
||||
case WALEntryUpdate:
|
||||
estimatedSize += 1 + 32 + 16 + len(e.ChangedNodes)*32 + 8 + 8
|
||||
}
|
||||
}
|
||||
|
||||
buf := make([]byte, 0, estimatedSize+pageSize)
|
||||
|
||||
// START: tag(1) + syncSeqn(4)
|
||||
buf = append(buf, walTagStart)
|
||||
var seqBuf [4]byte
|
||||
binary.LittleEndian.PutUint32(seqBuf[:], syncSeqn)
|
||||
buf = append(buf, seqBuf[:]...)
|
||||
|
||||
// Entries.
|
||||
var u64Buf [8]byte
|
||||
for _, e := range b.entries {
|
||||
switch e.Kind {
|
||||
case WALEntryClear:
|
||||
buf = append(buf, walTagClear)
|
||||
binary.LittleEndian.PutUint64(u64Buf[:], e.ClearBucket)
|
||||
buf = append(buf, u64Buf[:]...)
|
||||
|
||||
case WALEntryUpdate:
|
||||
buf = append(buf, walTagUpdate)
|
||||
buf = append(buf, e.PageID[:]...)
|
||||
encoded := e.Diff.Encode()
|
||||
buf = append(buf, encoded[:]...)
|
||||
for _, n := range e.ChangedNodes {
|
||||
buf = append(buf, n[:]...)
|
||||
}
|
||||
binary.LittleEndian.PutUint64(u64Buf[:], e.ElidedChildren)
|
||||
buf = append(buf, u64Buf[:]...)
|
||||
binary.LittleEndian.PutUint64(u64Buf[:], e.UpdateBucket)
|
||||
buf = append(buf, u64Buf[:]...)
|
||||
}
|
||||
}
|
||||
|
||||
// END tag.
|
||||
buf = append(buf, walTagEnd)
|
||||
|
||||
// Pad to page boundary.
|
||||
if rem := len(buf) % pageSize; rem != 0 {
|
||||
padding := make([]byte, pageSize-rem)
|
||||
buf = append(buf, padding...)
|
||||
}
|
||||
|
||||
return buf
|
||||
}
|
||||
|
||||
// ReadWAL parses a WAL from raw bytes. Returns the sync sequence number
|
||||
// and the list of entries. Returns an error if the WAL is malformed.
|
||||
func ReadWAL(data []byte) (uint32, []WALEntry, error) {
|
||||
if len(data) == 0 {
|
||||
return 0, nil, nil // Empty WAL = no recovery needed.
|
||||
}
|
||||
|
||||
pos := 0
|
||||
read := func(n int) ([]byte, error) {
|
||||
if pos+n > len(data) {
|
||||
return nil, fmt.Errorf("bitbox/wal: unexpected EOF at offset %d", pos)
|
||||
}
|
||||
b := data[pos : pos+n]
|
||||
pos += n
|
||||
return b, nil
|
||||
}
|
||||
|
||||
readByte := func() (byte, error) {
|
||||
b, err := read(1)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return b[0], nil
|
||||
}
|
||||
|
||||
readU32 := func() (uint32, error) {
|
||||
b, err := read(4)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return binary.LittleEndian.Uint32(b), nil
|
||||
}
|
||||
|
||||
readU64 := func() (uint64, error) {
|
||||
b, err := read(8)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return binary.LittleEndian.Uint64(b), nil
|
||||
}
|
||||
|
||||
// Read START.
|
||||
tag, err := readByte()
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
if tag != walTagStart {
|
||||
return 0, nil, fmt.Errorf("bitbox/wal: expected START tag, got 0x%02x", tag)
|
||||
}
|
||||
|
||||
syncSeqn, err := readU32()
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
|
||||
var entries []WALEntry
|
||||
|
||||
for {
|
||||
tag, err := readByte()
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
|
||||
switch tag {
|
||||
case walTagEnd:
|
||||
return syncSeqn, entries, nil
|
||||
|
||||
case walTagClear:
|
||||
bucket, err := readU64()
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
entries = append(entries, WALEntry{
|
||||
Kind: WALEntryClear,
|
||||
ClearBucket: bucket,
|
||||
})
|
||||
|
||||
case walTagUpdate:
|
||||
pidBytes, err := read(32)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
var pageID [32]byte
|
||||
copy(pageID[:], pidBytes)
|
||||
|
||||
diffBytes, err := read(16)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
var diffBuf [16]byte
|
||||
copy(diffBuf[:], diffBytes)
|
||||
diff := core.DecodePageDiff(diffBuf)
|
||||
|
||||
nodeCount := diff.Count()
|
||||
nodes := make([]core.Node, nodeCount)
|
||||
for i := range nodeCount {
|
||||
nodeBytes, err := read(32)
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
copy(nodes[i][:], nodeBytes)
|
||||
}
|
||||
|
||||
elidedChildren, err := readU64()
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
bucket, err := readU64()
|
||||
if err != nil {
|
||||
return 0, nil, err
|
||||
}
|
||||
|
||||
entries = append(entries, WALEntry{
|
||||
Kind: WALEntryUpdate,
|
||||
PageID: pageID,
|
||||
Diff: diff,
|
||||
ChangedNodes: nodes,
|
||||
ElidedChildren: elidedChildren,
|
||||
UpdateBucket: bucket,
|
||||
})
|
||||
|
||||
default:
|
||||
return 0, nil, fmt.Errorf("bitbox/wal: unknown tag 0x%02x at offset %d",
|
||||
tag, pos-1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// WriteWALFile writes a WAL to a file, creating or truncating it.
|
||||
func WriteWALFile(path string, data []byte) error {
|
||||
if err := os.WriteFile(path, data, 0644); err != nil {
|
||||
return fmt.Errorf("bitbox/wal: write: %w", err)
|
||||
}
|
||||
// fsync via re-open.
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return fmt.Errorf("bitbox/wal: open for sync: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
return f.Sync()
|
||||
}
|
||||
|
||||
// ReadWALFile reads a WAL file. Returns nil data if the file doesn't exist
|
||||
// or is empty.
|
||||
func ReadWALFile(path string) ([]byte, error) {
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
if os.IsNotExist(err) {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, fmt.Errorf("bitbox/wal: read: %w", err)
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
// TruncateWALFile empties the WAL file.
|
||||
func TruncateWALFile(path string) error {
|
||||
return os.Truncate(path, 0)
|
||||
}
|
||||
|
|
@ -1,217 +0,0 @@
|
|||
package bitbox
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/ethereum/go-ethereum/nomt/core"
|
||||
"github.com/ethereum/go-ethereum/nomt/merkle"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// --- WAL Builder/Reader Tests ---
|
||||
|
||||
func TestWALEmptyRoundTrip(t *testing.T) {
|
||||
b := NewWALBuilder()
|
||||
data := b.Finish(42)
|
||||
|
||||
// Should be padded to page boundary.
|
||||
assert.Equal(t, 0, len(data)%pageSize)
|
||||
|
||||
seqn, entries, err := ReadWAL(data)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, uint32(42), seqn)
|
||||
assert.Empty(t, entries)
|
||||
}
|
||||
|
||||
func TestWALClearEntryRoundTrip(t *testing.T) {
|
||||
b := NewWALBuilder()
|
||||
b.AddClear(123)
|
||||
b.AddClear(456)
|
||||
data := b.Finish(1)
|
||||
|
||||
seqn, entries, err := ReadWAL(data)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, uint32(1), seqn)
|
||||
require.Len(t, entries, 2)
|
||||
|
||||
assert.Equal(t, WALEntryClear, entries[0].Kind)
|
||||
assert.Equal(t, uint64(123), entries[0].ClearBucket)
|
||||
assert.Equal(t, uint64(456), entries[1].ClearBucket)
|
||||
}
|
||||
|
||||
func TestWALUpdateEntryRoundTrip(t *testing.T) {
|
||||
var pageID [32]byte
|
||||
pageID[0] = 0xAB
|
||||
|
||||
var diff core.PageDiff
|
||||
diff.SetChanged(5)
|
||||
diff.SetChanged(70)
|
||||
|
||||
nodes := []core.Node{{0x01}, {0x02}}
|
||||
|
||||
b := NewWALBuilder()
|
||||
b.AddUpdate(pageID, diff, nodes, 0xFF, 99)
|
||||
data := b.Finish(7)
|
||||
|
||||
seqn, entries, err := ReadWAL(data)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, uint32(7), seqn)
|
||||
require.Len(t, entries, 1)
|
||||
|
||||
e := entries[0]
|
||||
assert.Equal(t, WALEntryUpdate, e.Kind)
|
||||
assert.Equal(t, pageID, e.PageID)
|
||||
assert.True(t, e.Diff.IsChanged(5))
|
||||
assert.True(t, e.Diff.IsChanged(70))
|
||||
require.Len(t, e.ChangedNodes, 2)
|
||||
assert.Equal(t, core.Node{0x01}, e.ChangedNodes[0])
|
||||
assert.Equal(t, core.Node{0x02}, e.ChangedNodes[1])
|
||||
assert.Equal(t, uint64(0xFF), e.ElidedChildren)
|
||||
assert.Equal(t, uint64(99), e.UpdateBucket)
|
||||
}
|
||||
|
||||
func TestWALMixedEntries(t *testing.T) {
|
||||
b := NewWALBuilder()
|
||||
b.AddClear(10)
|
||||
|
||||
var pid [32]byte
|
||||
var diff core.PageDiff
|
||||
diff.SetChanged(0)
|
||||
b.AddUpdate(pid, diff, []core.Node{{0xAA}}, 0, 20)
|
||||
|
||||
b.AddClear(30)
|
||||
data := b.Finish(100)
|
||||
|
||||
_, entries, err := ReadWAL(data)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, entries, 3)
|
||||
assert.Equal(t, WALEntryClear, entries[0].Kind)
|
||||
assert.Equal(t, WALEntryUpdate, entries[1].Kind)
|
||||
assert.Equal(t, WALEntryClear, entries[2].Kind)
|
||||
}
|
||||
|
||||
func TestReadWALEmpty(t *testing.T) {
|
||||
seqn, entries, err := ReadWAL(nil)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, uint32(0), seqn)
|
||||
assert.Nil(t, entries)
|
||||
}
|
||||
|
||||
func TestWALFilePersistence(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "test.wal")
|
||||
|
||||
b := NewWALBuilder()
|
||||
b.AddClear(42)
|
||||
data := b.Finish(5)
|
||||
|
||||
require.NoError(t, WriteWALFile(path, data))
|
||||
|
||||
loaded, err := ReadWALFile(path)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, data, loaded)
|
||||
|
||||
require.NoError(t, TruncateWALFile(path))
|
||||
loaded2, err := ReadWALFile(path)
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, loaded2)
|
||||
}
|
||||
|
||||
// --- Sync Controller Tests ---
|
||||
|
||||
func TestFullSyncCycle(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
htPath := filepath.Join(dir, "test.bitbox")
|
||||
walPath := filepath.Join(dir, "test.wal")
|
||||
|
||||
seed := HashSeedFromUint64(1, 2)
|
||||
db, err := Create(htPath, 1024, seed)
|
||||
require.NoError(t, err)
|
||||
defer db.Close()
|
||||
|
||||
rootID := core.RootPageID()
|
||||
page := new(core.RawPage)
|
||||
page.SetNodeAt(0, core.Node{0xAA})
|
||||
|
||||
var diff core.PageDiff
|
||||
diff.SetChanged(0)
|
||||
|
||||
updates := []merkle.UpdatedPage{{
|
||||
PageID: rootID,
|
||||
Page: page,
|
||||
Diff: diff,
|
||||
}}
|
||||
|
||||
require.NoError(t, db.FullSync(walPath, 1, updates))
|
||||
|
||||
// Verify page is persisted.
|
||||
loaded, _, found, err := db.LoadPage(rootID)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, found)
|
||||
assert.Equal(t, core.Node{0xAA}, loaded.NodeAt(0))
|
||||
}
|
||||
|
||||
// --- Recovery Tests ---
|
||||
|
||||
func TestRecoverFromWAL(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
htPath := filepath.Join(dir, "test.bitbox")
|
||||
walPath := filepath.Join(dir, "test.wal")
|
||||
|
||||
seed := HashSeedFromUint64(1, 2)
|
||||
|
||||
// Create DB and write a WAL but don't commit Phase 3.
|
||||
db, err := Create(htPath, 1024, seed)
|
||||
require.NoError(t, err)
|
||||
|
||||
rootID := core.RootPageID()
|
||||
page := new(core.RawPage)
|
||||
page.SetNodeAt(0, core.Node{0xBB})
|
||||
|
||||
var diff core.PageDiff
|
||||
diff.SetChanged(0)
|
||||
|
||||
updates := []merkle.UpdatedPage{{
|
||||
PageID: rootID,
|
||||
Page: page,
|
||||
Diff: diff,
|
||||
}}
|
||||
|
||||
// Phase 1 + 2 only (simulate crash before Phase 3).
|
||||
plan, err := db.BeginSync(walPath, 5, updates)
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, db.WriteWAL(walPath, plan))
|
||||
db.Close()
|
||||
|
||||
// Reopen and recover.
|
||||
db2, err := Open(htPath)
|
||||
require.NoError(t, err)
|
||||
defer db2.Close()
|
||||
|
||||
seqn, err := db2.Recover(walPath)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, uint32(5), seqn)
|
||||
|
||||
// Verify the page was recovered.
|
||||
loaded, _, found, err := db2.LoadPage(rootID)
|
||||
require.NoError(t, err)
|
||||
assert.True(t, found)
|
||||
assert.Equal(t, core.Node{0xBB}, loaded.NodeAt(0))
|
||||
}
|
||||
|
||||
func TestRecoverNoWAL(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
htPath := filepath.Join(dir, "test.bitbox")
|
||||
walPath := filepath.Join(dir, "test.wal")
|
||||
|
||||
seed := HashSeedFromUint64(1, 2)
|
||||
db, err := Create(htPath, 1024, seed)
|
||||
require.NoError(t, err)
|
||||
defer db.Close()
|
||||
|
||||
seqn, err := db.Recover(walPath)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, uint32(0), seqn, "no recovery needed")
|
||||
}
|
||||
195
nomt/db/db.go
195
nomt/db/db.go
|
|
@ -1,35 +1,37 @@
|
|||
// Package db provides the unified NOMT trie database combining Bitbox
|
||||
// Package db provides the NOMT trie database combining PebbleDB page
|
||||
// storage with the PageWalker merkle engine.
|
||||
//
|
||||
// This package handles only the trie structure (merkle pages). Flat
|
||||
// key-value storage (accounts, storage slots) stays on geth's PebbleDB.
|
||||
// Trie pages are stored as 4KB blobs in geth's ethdb under key prefix 0x04.
|
||||
// Flat key-value storage (accounts, storage slots) stays on geth's PebbleDB
|
||||
// under separate prefixes managed by triedb/nomtdb.
|
||||
package db
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"crypto/rand"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sort"
|
||||
"sync"
|
||||
|
||||
"github.com/ethereum/go-ethereum/nomt/bitbox"
|
||||
"github.com/ethereum/go-ethereum/ethdb"
|
||||
"github.com/ethereum/go-ethereum/nomt/core"
|
||||
"github.com/ethereum/go-ethereum/nomt/merkle"
|
||||
)
|
||||
|
||||
const (
|
||||
htFileName = "nomt.ht"
|
||||
walFileName = "nomt.wal"
|
||||
// nomtPagePrefix is the ethdb key prefix for NOMT trie pages.
|
||||
// Key format: 0x04 || PageID.Encode()[32] → RawPage[4032]
|
||||
nomtPagePrefix byte = 0x04
|
||||
|
||||
// nomtMetaPrefix is the ethdb key prefix for NOMT metadata.
|
||||
nomtMetaPrefix byte = 0x05
|
||||
)
|
||||
|
||||
// nomtMetaRootKey is the ethdb key for the persisted page tree root.
|
||||
var nomtMetaRootKey = []byte{nomtMetaPrefix, 'r', 'o', 'o', 't'}
|
||||
|
||||
// Config holds configuration for the NOMT database.
|
||||
type Config struct {
|
||||
// HTCapacity is the number of hash table buckets. Must be a power of 2.
|
||||
HTCapacity uint64
|
||||
|
||||
// NumWorkers is the number of parallel goroutines for trie updates.
|
||||
// Defaults to runtime.NumCPU() if zero.
|
||||
NumWorkers int
|
||||
|
|
@ -37,71 +39,34 @@ type Config struct {
|
|||
|
||||
// DefaultConfig returns a default configuration.
|
||||
func DefaultConfig() Config {
|
||||
return Config{
|
||||
HTCapacity: 1 << 20, // ~1M buckets = ~4GB
|
||||
}
|
||||
return Config{}
|
||||
}
|
||||
|
||||
// DB is the NOMT trie database.
|
||||
type DB struct {
|
||||
dataDir string
|
||||
bb *bitbox.DB
|
||||
diskdb ethdb.Database
|
||||
root core.Node
|
||||
syncSeqn uint32
|
||||
numWorkers int
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
// Open opens or creates a NOMT trie database at the given directory.
|
||||
func Open(dataDir string, config Config) (*DB, error) {
|
||||
if err := os.MkdirAll(dataDir, 0755); err != nil {
|
||||
return nil, fmt.Errorf("nomt/db: create datadir: %w", err)
|
||||
}
|
||||
|
||||
htPath := filepath.Join(dataDir, htFileName)
|
||||
walPath := filepath.Join(dataDir, walFileName)
|
||||
|
||||
var bb *bitbox.DB
|
||||
var err error
|
||||
|
||||
if _, statErr := os.Stat(htPath); os.IsNotExist(statErr) {
|
||||
// Create new database.
|
||||
var seed [16]byte
|
||||
if _, err := rand.Read(seed[:]); err != nil {
|
||||
return nil, fmt.Errorf("nomt/db: generate seed: %w", err)
|
||||
}
|
||||
bb, err = bitbox.Create(htPath, config.HTCapacity, seed)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("nomt/db: create bitbox: %w", err)
|
||||
}
|
||||
} else {
|
||||
// Open existing database.
|
||||
bb, err = bitbox.Open(htPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("nomt/db: open bitbox: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
// New creates or opens a NOMT trie database backed by the given ethdb.
|
||||
// The page tree root is loaded from persisted metadata if available.
|
||||
func New(diskdb ethdb.Database, config Config) (*DB, error) {
|
||||
numWorkers := config.NumWorkers
|
||||
if numWorkers <= 0 {
|
||||
numWorkers = runtime.NumCPU()
|
||||
}
|
||||
|
||||
db := &DB{
|
||||
dataDir: dataDir,
|
||||
bb: bb,
|
||||
diskdb: diskdb,
|
||||
root: core.Terminator,
|
||||
numWorkers: numWorkers,
|
||||
}
|
||||
|
||||
// Run WAL recovery.
|
||||
seqn, err := bb.Recover(walPath)
|
||||
if err != nil {
|
||||
bb.Close()
|
||||
return nil, fmt.Errorf("nomt/db: recover: %w", err)
|
||||
}
|
||||
if seqn > 0 {
|
||||
db.syncSeqn = seqn
|
||||
// Load persisted root.
|
||||
if data, err := diskdb.Get(nomtMetaRootKey); err == nil && len(data) == 32 {
|
||||
copy(db.root[:], data)
|
||||
}
|
||||
|
||||
return db, nil
|
||||
|
|
@ -114,20 +79,6 @@ func (db *DB) Root() core.Node {
|
|||
return db.root
|
||||
}
|
||||
|
||||
// SetRoot sets the current trie root (used when loading state from metadata).
|
||||
func (db *DB) SetRoot(root core.Node) {
|
||||
db.mu.Lock()
|
||||
defer db.mu.Unlock()
|
||||
db.root = root
|
||||
}
|
||||
|
||||
// SyncSeqn returns the current sync sequence number.
|
||||
func (db *DB) SyncSeqn() uint32 {
|
||||
db.mu.RLock()
|
||||
defer db.mu.RUnlock()
|
||||
return db.syncSeqn
|
||||
}
|
||||
|
||||
// Update applies a batch of stem key-value pairs to the trie.
|
||||
// The pairs are sorted internally before processing.
|
||||
func (db *DB) Update(ops []core.StemKeyValue) (core.Node, error) {
|
||||
|
|
@ -148,58 +99,76 @@ func (db *DB) UpdateSorted(ops []core.StemKeyValue) (core.Node, error) {
|
|||
defer db.mu.Unlock()
|
||||
|
||||
pageSetFactory := func() merkle.PageSet {
|
||||
return newBitboxPageSet(db.bb)
|
||||
return newPebblePageSet(db.diskdb)
|
||||
}
|
||||
out := merkle.ParallelUpdate(db.root, ops, db.numWorkers, pageSetFactory)
|
||||
|
||||
// Persist updated pages.
|
||||
walPath := filepath.Join(db.dataDir, walFileName)
|
||||
db.syncSeqn++
|
||||
if err := db.bb.FullSync(walPath, db.syncSeqn, out.Pages); err != nil {
|
||||
return core.Terminator, fmt.Errorf("nomt/db: sync: %w", err)
|
||||
// Persist updated pages via atomic batch write.
|
||||
batch := db.diskdb.NewBatch()
|
||||
for _, up := range out.Pages {
|
||||
key := nomtPageKey(up.PageID)
|
||||
if up.Diff.IsCleared() {
|
||||
if err := batch.Delete(key); err != nil {
|
||||
return core.Terminator, fmt.Errorf("nomt/db: delete page: %w", err)
|
||||
}
|
||||
} else {
|
||||
if err := batch.Put(key, up.Page[:]); err != nil {
|
||||
return core.Terminator, fmt.Errorf("nomt/db: put page: %w", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
// Persist root.
|
||||
if err := batch.Put(nomtMetaRootKey, out.Root[:]); err != nil {
|
||||
return core.Terminator, fmt.Errorf("nomt/db: put root: %w", err)
|
||||
}
|
||||
if err := batch.Write(); err != nil {
|
||||
return core.Terminator, fmt.Errorf("nomt/db: batch write: %w", err)
|
||||
}
|
||||
|
||||
db.root = out.Root
|
||||
return out.Root, nil
|
||||
}
|
||||
|
||||
// LoadPage loads a page from Bitbox storage by its PageID.
|
||||
// LoadPage loads a page from ethdb storage by its PageID.
|
||||
func (db *DB) LoadPage(pageID core.PageID) (*core.RawPage, error) {
|
||||
page, _, found, err := db.bb.LoadPage(pageID)
|
||||
data, err := db.diskdb.Get(nomtPageKey(pageID))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("nomt/db: load page: %w", err)
|
||||
return nil, nil // Not found.
|
||||
}
|
||||
if !found {
|
||||
return nil, nil
|
||||
if len(data) != core.PageSize {
|
||||
return nil, fmt.Errorf("nomt/db: page size mismatch: got %d, want %d", len(data), core.PageSize)
|
||||
}
|
||||
page := new(core.RawPage)
|
||||
copy(page[:], data)
|
||||
return page, nil
|
||||
}
|
||||
|
||||
// Close closes the database.
|
||||
// Close is a no-op — the ethdb lifecycle is managed by the caller.
|
||||
func (db *DB) Close() error {
|
||||
return db.bb.Close()
|
||||
return nil
|
||||
}
|
||||
|
||||
// --- BitboxPageSet ---
|
||||
// --- PebblePageSet ---
|
||||
|
||||
// bitboxPageSet implements merkle.PageSet backed by Bitbox disk storage.
|
||||
type bitboxPageSet struct {
|
||||
bb *bitbox.DB
|
||||
cache map[string]*core.RawPage
|
||||
// pebblePageSet implements merkle.PageSet backed by ethdb (PebbleDB).
|
||||
type pebblePageSet struct {
|
||||
diskdb ethdb.Database
|
||||
cache map[string]*core.RawPage
|
||||
}
|
||||
|
||||
func newBitboxPageSet(bb *bitbox.DB) *bitboxPageSet {
|
||||
return &bitboxPageSet{
|
||||
bb: bb,
|
||||
cache: make(map[string]*core.RawPage, 16),
|
||||
func newPebblePageSet(diskdb ethdb.Database) *pebblePageSet {
|
||||
return &pebblePageSet{
|
||||
diskdb: diskdb,
|
||||
cache: make(map[string]*core.RawPage, 16),
|
||||
}
|
||||
}
|
||||
|
||||
func (ps *bitboxPageSet) Get(pageID core.PageID) (
|
||||
func (ps *pebblePageSet) Get(pageID core.PageID) (
|
||||
*core.RawPage, merkle.PageOrigin, bool,
|
||||
) {
|
||||
key := pageIDKey(pageID)
|
||||
key := pageIDCacheKey(pageID)
|
||||
if cached, ok := ps.cache[key]; ok {
|
||||
// Return a copy so the walker can mutate freely.
|
||||
pageCopy := new(core.RawPage)
|
||||
*pageCopy = *cached
|
||||
return pageCopy, merkle.PageOrigin{
|
||||
|
|
@ -207,8 +176,8 @@ func (ps *bitboxPageSet) Get(pageID core.PageID) (
|
|||
}, true
|
||||
}
|
||||
|
||||
page, _, found, err := ps.bb.LoadPage(pageID)
|
||||
if err != nil || !found {
|
||||
data, err := ps.diskdb.Get(nomtPageKey(pageID))
|
||||
if err != nil || len(data) != core.PageSize {
|
||||
// Return a fresh page if not found — this handles the case
|
||||
// where the trie is being built from scratch or expanded
|
||||
// into new regions.
|
||||
|
|
@ -216,7 +185,11 @@ func (ps *bitboxPageSet) Get(pageID core.PageID) (
|
|||
return fresh, merkle.PageOrigin{Kind: merkle.PageOriginFresh}, true
|
||||
}
|
||||
|
||||
page := new(core.RawPage)
|
||||
copy(page[:], data)
|
||||
ps.cache[key] = page
|
||||
|
||||
// Return a copy so the walker can mutate freely.
|
||||
pageCopy := new(core.RawPage)
|
||||
*pageCopy = *page
|
||||
return pageCopy, merkle.PageOrigin{
|
||||
|
|
@ -224,26 +197,36 @@ func (ps *bitboxPageSet) Get(pageID core.PageID) (
|
|||
}, true
|
||||
}
|
||||
|
||||
func (ps *bitboxPageSet) Contains(pageID core.PageID) bool {
|
||||
key := pageIDKey(pageID)
|
||||
func (ps *pebblePageSet) Contains(pageID core.PageID) bool {
|
||||
key := pageIDCacheKey(pageID)
|
||||
if _, ok := ps.cache[key]; ok {
|
||||
return true
|
||||
}
|
||||
_, _, found, _ := ps.bb.LoadPage(pageID)
|
||||
return found
|
||||
has, _ := ps.diskdb.Has(nomtPageKey(pageID))
|
||||
return has
|
||||
}
|
||||
|
||||
func (ps *bitboxPageSet) Fresh(pageID core.PageID) *core.RawPage {
|
||||
func (ps *pebblePageSet) Fresh(pageID core.PageID) *core.RawPage {
|
||||
return new(core.RawPage)
|
||||
}
|
||||
|
||||
func (ps *bitboxPageSet) Insert(
|
||||
func (ps *pebblePageSet) Insert(
|
||||
pageID core.PageID, page *core.RawPage, origin merkle.PageOrigin,
|
||||
) {
|
||||
ps.cache[pageIDKey(pageID)] = page
|
||||
ps.cache[pageIDCacheKey(pageID)] = page
|
||||
}
|
||||
|
||||
func pageIDKey(id core.PageID) string {
|
||||
// nomtPageKey builds the ethdb key for a NOMT trie page.
|
||||
func nomtPageKey(id core.PageID) []byte {
|
||||
encoded := id.Encode()
|
||||
key := make([]byte, 1+len(encoded))
|
||||
key[0] = nomtPagePrefix
|
||||
copy(key[1:], encoded[:])
|
||||
return key
|
||||
}
|
||||
|
||||
// pageIDCacheKey returns a string key for the in-memory cache.
|
||||
func pageIDCacheKey(id core.PageID) string {
|
||||
encoded := id.Encode()
|
||||
return string(encoded[:])
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,32 +3,30 @@ package db
|
|||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/ethereum/go-ethereum/core/rawdb"
|
||||
"github.com/ethereum/go-ethereum/nomt/core"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestOpenClose(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
db, err := Open(dir, DefaultConfig())
|
||||
func newTestDB(t *testing.T) *DB {
|
||||
t.Helper()
|
||||
diskdb := rawdb.NewMemoryDatabase()
|
||||
db, err := New(diskdb, DefaultConfig())
|
||||
require.NoError(t, err)
|
||||
|
||||
assert.Equal(t, core.Terminator, db.Root())
|
||||
require.NoError(t, db.Close())
|
||||
t.Cleanup(func() { db.Close() })
|
||||
return db
|
||||
}
|
||||
|
||||
func TestOpenCreatesDirectory(t *testing.T) {
|
||||
dir := t.TempDir() + "/subdir"
|
||||
db, err := Open(dir, DefaultConfig())
|
||||
require.NoError(t, err)
|
||||
defer db.Close()
|
||||
func TestNewClose(t *testing.T) {
|
||||
db := newTestDB(t)
|
||||
assert.Equal(t, core.Terminator, db.Root())
|
||||
}
|
||||
|
||||
func TestReopenPreservesState(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
func TestReopenPreservesRoot(t *testing.T) {
|
||||
diskdb := rawdb.NewMemoryDatabase()
|
||||
|
||||
db, err := Open(dir, DefaultConfig())
|
||||
db, err := New(diskdb, DefaultConfig())
|
||||
require.NoError(t, err)
|
||||
|
||||
newRoot, err := db.Update([]core.StemKeyValue{
|
||||
|
|
@ -36,23 +34,17 @@ func TestReopenPreservesState(t *testing.T) {
|
|||
})
|
||||
require.NoError(t, err)
|
||||
require.False(t, core.IsTerminator(&newRoot))
|
||||
require.NoError(t, db.Close())
|
||||
|
||||
// Reopen and set the root.
|
||||
db2, err := Open(dir, DefaultConfig())
|
||||
// "Reopen" by creating a new DB on the same ethdb.
|
||||
db2, err := New(diskdb, DefaultConfig())
|
||||
require.NoError(t, err)
|
||||
defer db2.Close()
|
||||
|
||||
// Root is not automatically persisted (that's the geth integration's
|
||||
// job), but the pages should still be on disk.
|
||||
assert.Equal(t, core.Terminator, db2.Root())
|
||||
// Root is now persisted in PebbleDB, so it should be recovered.
|
||||
assert.Equal(t, newRoot, db2.Root())
|
||||
}
|
||||
|
||||
func TestUpdateSingleKey(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
db, err := Open(dir, DefaultConfig())
|
||||
require.NoError(t, err)
|
||||
defer db.Close()
|
||||
db := newTestDB(t)
|
||||
|
||||
newRoot, err := db.Update([]core.StemKeyValue{
|
||||
{Stem: makeStem(0x10), Hash: makeHash(0x42)},
|
||||
|
|
@ -64,10 +56,7 @@ func TestUpdateSingleKey(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestUpdateMultipleKeys(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
db, err := Open(dir, DefaultConfig())
|
||||
require.NoError(t, err)
|
||||
defer db.Close()
|
||||
db := newTestDB(t)
|
||||
|
||||
ops := []core.StemKeyValue{
|
||||
{Stem: makeStem(0x10), Hash: makeHash(0x01)},
|
||||
|
|
@ -86,11 +75,7 @@ func TestUpdateDeterministic(t *testing.T) {
|
|||
}
|
||||
|
||||
run := func() core.Node {
|
||||
dir := t.TempDir()
|
||||
db, err := Open(dir, DefaultConfig())
|
||||
require.NoError(t, err)
|
||||
defer db.Close()
|
||||
|
||||
db := newTestDB(t)
|
||||
root, err := db.Update(ops)
|
||||
require.NoError(t, err)
|
||||
return root
|
||||
|
|
@ -102,10 +87,7 @@ func TestUpdateDeterministic(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestUpdateEmptyOps(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
db, err := Open(dir, DefaultConfig())
|
||||
require.NoError(t, err)
|
||||
defer db.Close()
|
||||
db := newTestDB(t)
|
||||
|
||||
root, err := db.Update(nil)
|
||||
require.NoError(t, err)
|
||||
|
|
@ -113,10 +95,7 @@ func TestUpdateEmptyOps(t *testing.T) {
|
|||
}
|
||||
|
||||
func TestUpdateSortsByStem(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
db, err := Open(dir, DefaultConfig())
|
||||
require.NoError(t, err)
|
||||
defer db.Close()
|
||||
db := newTestDB(t)
|
||||
|
||||
// Provide stems in reverse order — should still work.
|
||||
ops := []core.StemKeyValue{
|
||||
|
|
@ -129,27 +108,6 @@ func TestUpdateSortsByStem(t *testing.T) {
|
|||
assert.False(t, core.IsTerminator(&root))
|
||||
}
|
||||
|
||||
func TestSyncSeqnIncrements(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
db, err := Open(dir, DefaultConfig())
|
||||
require.NoError(t, err)
|
||||
defer db.Close()
|
||||
|
||||
assert.Equal(t, uint32(0), db.SyncSeqn())
|
||||
|
||||
_, err = db.Update([]core.StemKeyValue{
|
||||
{Stem: makeStem(0x10), Hash: makeHash(0x01)},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, uint32(1), db.SyncSeqn())
|
||||
|
||||
_, err = db.Update([]core.StemKeyValue{
|
||||
{Stem: makeStem(0x80), Hash: makeHash(0x02)},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, uint32(2), db.SyncSeqn())
|
||||
}
|
||||
|
||||
func makeStem(b byte) core.StemPath {
|
||||
var sp core.StemPath
|
||||
for i := range sp {
|
||||
|
|
|
|||
|
|
@ -99,7 +99,7 @@ func TestAssignToWorkersMoreWorkersThanChildren(t *testing.T) {
|
|||
// --- Integration tests ---
|
||||
|
||||
// permissivePageSet wraps MemoryPageSet to return fresh pages for missing
|
||||
// entries (matching bitboxPageSet behavior). This is needed because the
|
||||
// entries (matching pebblePageSet behavior). This is needed because the
|
||||
// parallel workers descend into child pages that may not exist yet.
|
||||
type permissivePageSet struct {
|
||||
*MemoryPageSet
|
||||
|
|
|
|||
|
|
@ -29,14 +29,11 @@ func newBintrie(t *testing.T) *bintrie.BinaryTrie {
|
|||
return bt
|
||||
}
|
||||
|
||||
// newNomtTrieForCompat creates a NomtTrie with in-memory ethdb and temp Bitbox.
|
||||
// newNomtTrieForCompat creates a NomtTrie with in-memory ethdb.
|
||||
func newNomtTrieForCompat(t *testing.T) *NomtTrie {
|
||||
t.Helper()
|
||||
diskdb := rawdb.NewMemoryDatabase()
|
||||
backend := nomtdb.New(diskdb, &nomtdb.Config{
|
||||
DataDir: t.TempDir(),
|
||||
HTCapacity: 1 << 16,
|
||||
})
|
||||
backend := nomtdb.New(diskdb, nil)
|
||||
t.Cleanup(func() { backend.Close() })
|
||||
|
||||
tr, err := New(common.Hash{}, backend)
|
||||
|
|
|
|||
|
|
@ -14,15 +14,11 @@ import (
|
|||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// newTestTrie creates a NomtTrie backed by an in-memory ethdb and a temp
|
||||
// Bitbox directory. Returns the trie and a cleanup function.
|
||||
// newTestTrie creates a NomtTrie backed by an in-memory ethdb.
|
||||
func newTestTrie(t *testing.T) *NomtTrie {
|
||||
t.Helper()
|
||||
diskdb := rawdb.NewMemoryDatabase()
|
||||
backend := nomtdb.New(diskdb, &nomtdb.Config{
|
||||
DataDir: t.TempDir(),
|
||||
HTCapacity: 1 << 16,
|
||||
})
|
||||
backend := nomtdb.New(diskdb, nil)
|
||||
t.Cleanup(func() { backend.Close() })
|
||||
|
||||
tr, err := New(common.Hash{}, backend)
|
||||
|
|
|
|||
|
|
@ -2,9 +2,6 @@ package triecompare
|
|||
|
||||
import (
|
||||
"fmt"
|
||||
"math/bits"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
|
|
@ -33,19 +30,15 @@ func newBintrie(t testing.TB) *bintrie.BinaryTrie {
|
|||
return bt
|
||||
}
|
||||
|
||||
func newNomtTrieWithDir(t testing.TB, htCapacity uint64) (*nomttrie.NomtTrie, string) {
|
||||
func newNomtTrie(t testing.TB) *nomttrie.NomtTrie {
|
||||
t.Helper()
|
||||
diskdb := rawdb.NewMemoryDatabase()
|
||||
dir := t.TempDir()
|
||||
backend := nomtdb.New(diskdb, &nomtdb.Config{
|
||||
DataDir: dir,
|
||||
HTCapacity: htCapacity,
|
||||
})
|
||||
backend := nomtdb.New(diskdb, nil)
|
||||
t.Cleanup(func() { backend.Close() })
|
||||
|
||||
nt, err := nomttrie.New(common.Hash{}, backend)
|
||||
require.NoError(t, err)
|
||||
return nt, dir
|
||||
return nt
|
||||
}
|
||||
|
||||
// applyOp applies a single StateOp to both bintrie and nomttrie.
|
||||
|
|
@ -116,10 +109,9 @@ func TestRootEquality(t *testing.T) {
|
|||
for name, cfg := range configs {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
blocks := GenerateBlocks(cfg)
|
||||
htCap := estimateHTCapacity(cfg.NumAccounts, cfg.NumContracts, (cfg.MinSlots+cfg.MaxSlots)/2)
|
||||
|
||||
bt := newBintrie(t)
|
||||
nt, _ := newNomtTrieWithDir(t, htCap)
|
||||
nt := newNomtTrie(t)
|
||||
|
||||
for blockIdx, ops := range blocks {
|
||||
for _, op := range ops {
|
||||
|
|
@ -144,11 +136,7 @@ func TestRootEquality(t *testing.T) {
|
|||
func TestDeterminism(t *testing.T) {
|
||||
computeRoot := func() common.Hash {
|
||||
blocks := GenerateBlocks(smallConfig)
|
||||
htCap := estimateHTCapacity(
|
||||
smallConfig.NumAccounts, smallConfig.NumContracts,
|
||||
(smallConfig.MinSlots+smallConfig.MaxSlots)/2,
|
||||
)
|
||||
nt, _ := newNomtTrieWithDir(t, htCap)
|
||||
nt := newNomtTrie(t)
|
||||
bt := newBintrie(t)
|
||||
var root common.Hash
|
||||
for _, ops := range blocks {
|
||||
|
|
@ -185,10 +173,9 @@ func TestDistributionVariants(t *testing.T) {
|
|||
cfg.Seed = 123 // same seed for all
|
||||
|
||||
blocks := GenerateBlocks(cfg)
|
||||
htCap := estimateHTCapacity(cfg.NumAccounts, cfg.NumContracts, (cfg.MinSlots+cfg.MaxSlots)/2)
|
||||
|
||||
bt := newBintrie(t)
|
||||
nt, _ := newNomtTrieWithDir(t, htCap)
|
||||
nt := newNomtTrie(t)
|
||||
|
||||
var binRoot, nomtRoot common.Hash
|
||||
for _, ops := range blocks {
|
||||
|
|
@ -224,10 +211,9 @@ func TestIncrementalRootEquality(t *testing.T) {
|
|||
Seed: 99,
|
||||
}
|
||||
blocks := GenerateBlocks(cfg)
|
||||
htCap := estimateHTCapacity(cfg.NumAccounts, cfg.NumContracts, 3)
|
||||
|
||||
bt := newBintrie(t)
|
||||
nt, _ := newNomtTrieWithDir(t, htCap)
|
||||
nt := newNomtTrie(t)
|
||||
|
||||
for i, op := range blocks[0] {
|
||||
applyOp(t, bt, nt, op)
|
||||
|
|
@ -242,8 +228,8 @@ func TestIncrementalRootEquality(t *testing.T) {
|
|||
t.Logf("verified %d incremental hashes match", len(blocks[0]))
|
||||
}
|
||||
|
||||
// TestStorageFootprint populates state and measures storage used by each
|
||||
// implementation. Logs sizes and ratio.
|
||||
// TestStorageFootprint populates state and measures serialized node sizes
|
||||
// for bintrie. NOMT pages are now in ethdb, so only bintrie size is reported.
|
||||
func TestStorageFootprint(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("storage footprint test requires medium config")
|
||||
|
|
@ -251,10 +237,9 @@ func TestStorageFootprint(t *testing.T) {
|
|||
|
||||
cfg := mediumConfig
|
||||
blocks := GenerateBlocks(cfg)
|
||||
htCap := estimateHTCapacity(cfg.NumAccounts, cfg.NumContracts, (cfg.MinSlots+cfg.MaxSlots)/2)
|
||||
|
||||
bt := newBintrie(t)
|
||||
nt, nomtDir := newNomtTrieWithDir(t, htCap)
|
||||
nt := newNomtTrie(t)
|
||||
|
||||
for _, ops := range blocks {
|
||||
for _, op := range ops {
|
||||
|
|
@ -271,13 +256,7 @@ func TestStorageFootprint(t *testing.T) {
|
|||
_, ns := bt.Commit(false)
|
||||
binBytes := nodesetBytes(ns)
|
||||
|
||||
// NOMT: sum file sizes on disk.
|
||||
nomtBytes := dirSize(t, nomtDir)
|
||||
|
||||
ratio := float64(nomtBytes) / float64(max(binBytes, 1))
|
||||
t.Logf("bintrie serialized nodes: %s (%d bytes)", humanBytes(binBytes), binBytes)
|
||||
t.Logf("NOMT bitbox on disk: %s (%d bytes)", humanBytes(nomtBytes), nomtBytes)
|
||||
t.Logf("NOMT / bintrie ratio: %.2fx", ratio)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
|
@ -288,7 +267,6 @@ func BenchmarkUpdateAccount(b *testing.B) {
|
|||
cfg := smallConfig
|
||||
blocks := GenerateBlocks(cfg)
|
||||
ops := filterOps(blocks[0], OpUpdateAccount)
|
||||
htCap := estimateHTCapacity(cfg.NumAccounts, cfg.NumContracts, 10)
|
||||
|
||||
b.Run("bintrie", func(b *testing.B) {
|
||||
bt := newBintrie(b)
|
||||
|
|
@ -300,7 +278,7 @@ func BenchmarkUpdateAccount(b *testing.B) {
|
|||
})
|
||||
|
||||
b.Run("nomt", func(b *testing.B) {
|
||||
nt, _ := newNomtTrieWithDir(b, htCap)
|
||||
nt := newNomtTrie(b)
|
||||
b.ResetTimer()
|
||||
for i := range b.N {
|
||||
op := ops[i%len(ops)]
|
||||
|
|
@ -313,7 +291,6 @@ func BenchmarkUpdateStorage(b *testing.B) {
|
|||
cfg := smallConfig
|
||||
blocks := GenerateBlocks(cfg)
|
||||
ops := filterOps(blocks[0], OpUpdateStorage)
|
||||
htCap := estimateHTCapacity(cfg.NumAccounts, cfg.NumContracts, 10)
|
||||
|
||||
b.Run("bintrie", func(b *testing.B) {
|
||||
bt := newBintrie(b)
|
||||
|
|
@ -325,7 +302,7 @@ func BenchmarkUpdateStorage(b *testing.B) {
|
|||
})
|
||||
|
||||
b.Run("nomt", func(b *testing.B) {
|
||||
nt, _ := newNomtTrieWithDir(b, htCap)
|
||||
nt := newNomtTrie(b)
|
||||
b.ResetTimer()
|
||||
for i := range b.N {
|
||||
op := ops[i%len(ops)]
|
||||
|
|
@ -347,7 +324,6 @@ func BenchmarkHash(b *testing.B) {
|
|||
Seed: 77,
|
||||
}
|
||||
blocks := GenerateBlocks(cfg)
|
||||
htCap := estimateHTCapacity(size, 0, 0)
|
||||
|
||||
b.Run("bintrie", func(b *testing.B) {
|
||||
bt := newBintrie(b)
|
||||
|
|
@ -367,7 +343,7 @@ func BenchmarkHash(b *testing.B) {
|
|||
})
|
||||
|
||||
b.Run("nomt", func(b *testing.B) {
|
||||
nt, _ := newNomtTrieWithDir(b, htCap)
|
||||
nt := newNomtTrie(b)
|
||||
for _, op := range blocks[0] {
|
||||
_ = nt.UpdateAccount(op.Address, op.Account, op.CodeLen)
|
||||
}
|
||||
|
|
@ -388,7 +364,6 @@ func BenchmarkHash(b *testing.B) {
|
|||
func BenchmarkBlockWorkload(b *testing.B) {
|
||||
cfg := smallConfig
|
||||
blocks := GenerateBlocks(cfg)
|
||||
htCap := estimateHTCapacity(cfg.NumAccounts, cfg.NumContracts, 10)
|
||||
|
||||
// Use block 1 (mutations) as the repeated workload.
|
||||
workload := blocks[1]
|
||||
|
|
@ -411,7 +386,7 @@ func BenchmarkBlockWorkload(b *testing.B) {
|
|||
})
|
||||
|
||||
b.Run("nomt", func(b *testing.B) {
|
||||
nt, _ := newNomtTrieWithDir(b, htCap)
|
||||
nt := newNomtTrie(b)
|
||||
for _, op := range blocks[0] {
|
||||
applyOpSingleNomt(b, nt, op)
|
||||
}
|
||||
|
|
@ -480,35 +455,6 @@ func nodesetBytes(ns *trienode.NodeSet) int64 {
|
|||
return total
|
||||
}
|
||||
|
||||
// dirSize walks a directory and returns total file size in bytes.
|
||||
func dirSize(t testing.TB, dir string) int64 {
|
||||
t.Helper()
|
||||
var total int64
|
||||
err := filepath.Walk(dir, func(_ string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !info.IsDir() {
|
||||
total += info.Size()
|
||||
}
|
||||
return nil
|
||||
})
|
||||
require.NoError(t, err)
|
||||
return total
|
||||
}
|
||||
|
||||
// estimateHTCapacity returns a power-of-2 hash table capacity for ~50% load.
|
||||
// Each account uses ~1 stem; each contract uses 1 + ceil(avgSlots/256) stems.
|
||||
func estimateHTCapacity(numAccounts, numContracts, avgSlots int) uint64 {
|
||||
stems := numAccounts + numContracts
|
||||
if avgSlots > 0 {
|
||||
stems += numContracts * ((avgSlots + 255) / 256)
|
||||
}
|
||||
// 50% load factor → double the stem count, then round up to power of 2.
|
||||
target := max(uint64(stems*2), 64)
|
||||
return 1 << bits.Len64(target-1)
|
||||
}
|
||||
|
||||
// humanBytes formats byte counts for log output.
|
||||
func humanBytes(b int64) string {
|
||||
switch {
|
||||
|
|
|
|||
|
|
@ -8,15 +8,7 @@ package nomtdb
|
|||
|
||||
// Config holds configuration for the NOMT triedb backend.
|
||||
type Config struct {
|
||||
// DataDir is the directory for NOMT's Bitbox storage files.
|
||||
DataDir string
|
||||
|
||||
// HTCapacity is the number of hash table buckets. Must be a power of 2.
|
||||
// Defaults to 1<<20 (~1M buckets) if zero.
|
||||
HTCapacity uint64
|
||||
}
|
||||
|
||||
// Defaults is the default configuration for the NOMT backend.
|
||||
var Defaults = &Config{
|
||||
HTCapacity: 1 << 20,
|
||||
// NumWorkers is the number of parallel goroutines for trie updates.
|
||||
// Defaults to runtime.NumCPU() if zero.
|
||||
NumWorkers int
|
||||
}
|
||||
|
|
|
|||
|
|
@ -11,23 +11,22 @@ import (
|
|||
// Database is the NOMT triedb backend. It manages the NOMT trie engine for
|
||||
// page-based merkle storage and delegates flat state to geth's ethdb.
|
||||
type Database struct {
|
||||
diskdb ethdb.Database // geth's existing PebbleDB for flat state + metadata
|
||||
nomt *db.DB // NOMT trie engine (Bitbox page storage)
|
||||
diskdb ethdb.Database // geth's existing PebbleDB for flat state + pages
|
||||
nomt *db.DB // NOMT trie engine
|
||||
config *Config
|
||||
}
|
||||
|
||||
// New creates a new NOMT backend. The diskdb is used for flat state storage
|
||||
// (accounts, storage slots) and NOMT metadata. The NOMT engine opens its own
|
||||
// Bitbox files under config.DataDir.
|
||||
// New creates a new NOMT backend. The diskdb is used for flat state storage,
|
||||
// NOMT page storage, and metadata. Pass nil config for defaults.
|
||||
func New(diskdb ethdb.Database, config *Config) *Database {
|
||||
if config.HTCapacity == 0 {
|
||||
config.HTCapacity = Defaults.HTCapacity
|
||||
if config == nil {
|
||||
config = &Config{}
|
||||
}
|
||||
nomtDB, err := db.Open(config.DataDir, db.Config{
|
||||
HTCapacity: config.HTCapacity,
|
||||
nomtDB, err := db.New(diskdb, db.Config{
|
||||
NumWorkers: config.NumWorkers,
|
||||
})
|
||||
if err != nil {
|
||||
log.Crit("Failed to open NOMT database", "err", err)
|
||||
log.Crit("Failed to create NOMT database", "err", err)
|
||||
}
|
||||
return &Database{
|
||||
diskdb: diskdb,
|
||||
|
|
|
|||
Loading…
Reference in a new issue