From 278992863bd15636a352997a2a693d3f90594657 Mon Sep 17 00:00:00 2001 From: lightclient Date: Wed, 21 Jan 2026 10:08:41 -0700 Subject: [PATCH] all: update to final eraE spec --- cmd/geth/chaincmd.go | 2 +- cmd/utils/cmd.go | 53 ++-- cmd/utils/history_test.go | 2 +- internal/era/era.go | 40 ++- internal/era/execdb/builder.go | 232 ++++++++--------- internal/era/execdb/era_test.go | 387 +++++++++++++++++++++-------- internal/era/execdb/iterator.go | 3 +- internal/era/execdb/reader.go | 55 ++-- internal/era/onedb/builder.go | 39 +-- internal/era/onedb/builder_test.go | 2 +- internal/era/proof.go | 86 +------ 11 files changed, 531 insertions(+), 370 deletions(-) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 52d48d9b9b..d051dda573 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -580,7 +580,7 @@ func exportHistory(ctx *cli.Context) error { default: return fmt.Errorf("unknown archive format %q (use 'era1' or 'erae')", format) } - if err := utils.ExportHistory(chain, dir, uint64(first), uint64(last), uint64(era.MaxSize), newBuilder, filename); err != nil { + if err := utils.ExportHistory(chain, dir, uint64(first), uint64(last), newBuilder, filename); err != nil { utils.Fatalf("Export error: %v\n", err) } diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 3797d665ca..1457333c4b 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -416,7 +416,7 @@ func ExportAppendChain(blockchain *core.BlockChain, fn string, first uint64, las // ExportHistory exports blockchain history into the specified directory, // following the Era format. -func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64, newBuilder func(io.Writer) era.Builder, filename func(network string, epoch int, root common.Hash) string) error { +func ExportHistory(bc *core.BlockChain, dir string, first, last uint64, newBuilder func(io.Writer) era.Builder, filename func(network string, epoch int, lastBlockHash common.Hash) string) error { log.Info("Exporting blockchain history", "dir", dir) if head := bc.CurrentBlock().Number.Uint64(); head < last { log.Warn("Last block beyond head, setting last = head", "head", head, "last", last) @@ -435,12 +435,31 @@ func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64, ne reported = time.Now() h = sha256.New() buf = bytes.NewBuffer(nil) - td = new(big.Int) // TODO: should disallow non-zero "first" since we need to compute td from genesis now that we don't store it + td = new(big.Int) checksums []string ) - for batch := first; batch <= last; batch += step { - idx := int(batch / step) + // Compute initial TD by accumulating difficulty from genesis to first-1. + // This is necessary because TD is no longer stored in the database. Only + // compute if a segment of the export is pre-merge. + b := bc.GetBlockByNumber(first) + if b == nil { + return fmt.Errorf("block #%d not found", first) + } + if first > 0 && b.Difficulty().Sign() != 0 { + log.Info("Computing initial total difficulty", "from", 0, "to", first-1) + for i := uint64(0); i < first; i++ { + b := bc.GetBlockByNumber(i) + if b == nil { + return fmt.Errorf("block #%d not found while computing initial TD", i) + } + td.Add(td, b.Difficulty()) + } + log.Info("Initial total difficulty computed", "td", td) + } + + for batch := first; batch <= last; batch += uint64(era.MaxSize) { + idx := int(batch / uint64(era.MaxSize)) tmpPath := filepath.Join(dir, filename(network, idx, common.Hash{})) if err := func() error { @@ -452,28 +471,34 @@ func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64, ne builder := newBuilder(f) - for j := uint64(0); j < step && batch+j <= last; j++ { + for j := uint64(0); j < uint64(era.MaxSize) && batch+j <= last; j++ { n := batch + j - blk := bc.GetBlockByNumber(n) - if blk == nil { + block := bc.GetBlockByNumber(n) + if block == nil { return fmt.Errorf("block #%d not found", n) } - rcpt := bc.GetReceiptsByHash(blk.Hash()) - if rcpt == nil { + receipt := bc.GetReceiptsByHash(block.Hash()) + if receipt == nil { return fmt.Errorf("receipts for #%d missing", n) } - td.Add(td, blk.Difficulty()) - if err := builder.Add(blk, rcpt, new(big.Int).Set(td), nil); err != nil { + // For pre-merge blocks, pass accumulated TD. + // For post-merge blocks (difficulty == 0), pass nil TD. + var blockTD *big.Int + if block.Difficulty().Sign() != 0 { + td.Add(td, block.Difficulty()) + blockTD = new(big.Int).Set(td) + } + + if err := builder.Add(block, receipt, blockTD); err != nil { return err } } - root, err := builder.Finalize() + id, err := builder.Finalize() if err != nil { return err } - // TODO: this root shouldn't be used post merge! - final := filepath.Join(dir, filename(network, idx, root)) + final := filepath.Join(dir, filename(network, idx, id)) if err := os.Rename(tmpPath, final); err != nil { return err } diff --git a/cmd/utils/history_test.go b/cmd/utils/history_test.go index d1dbaab1f6..6631946129 100644 --- a/cmd/utils/history_test.go +++ b/cmd/utils/history_test.go @@ -101,7 +101,7 @@ func TestHistoryImportAndExport(t *testing.T) { dir := t.TempDir() // Export history to temp directory. - if err := ExportHistory(chain, dir, 0, count, step, tt.builder, tt.filename); err != nil { + if err := ExportHistory(chain, dir, 0, count, tt.builder, tt.filename); err != nil { t.Fatalf("error exporting history: %v", err) } diff --git a/internal/era/era.go b/internal/era/era.go index 4e391640b9..56dcf78ffe 100644 --- a/internal/era/era.go +++ b/internal/era/era.go @@ -37,21 +37,53 @@ type ReadAtSeekCloser interface { io.Closer } -// Iterator represents the iterator interface for various types of era stores. +// Iterator provides sequential access to blocks in an era file. type Iterator interface { + // Next advances to the next block. Returns true if a block is available, + // false when iteration is complete or an error occurred. Next() bool + + // Number returns the block number of the current block. Number() uint64 + + // Block returns the current block. Block() (*types.Block, error) + + // BlockAndReceipts returns the current block and its receipts. BlockAndReceipts() (*types.Block, types.Receipts, error) + + // Receipts returns the receipts for the current block. Receipts() (types.Receipts, error) + + // Error returns any error encountered during iteration. Error() error } -// Builder represents the interface for various types of era formats. +// Builder constructs era files from blocks and receipts. +// +// Builders handle three epoch types automatically: +// - Pre-merge: all blocks have difficulty > 0, TD is stored for each block +// - Transition: starts pre-merge, ends post-merge; TD stored for all blocks +// - Post-merge: all blocks have difficulty == 0, no TD stored type Builder interface { - Add(block *types.Block, receipts types.Receipts, td *big.Int, proof Proof) error - AddRLP(header, body, receipts, proof []byte, number uint64, hash common.Hash, td, difficulty *big.Int) error + // Add appends a block and its receipts to the era file. + // For pre-merge blocks, td must be provided. + // For post-merge blocks, td should be nil. + Add(block *types.Block, receipts types.Receipts, td *big.Int) error + + // AddRLP appends RLP-encoded block components to the era file. + // For pre-merge blocks, td and difficulty must be provided. + // For post-merge blocks, td and difficulty should be nil. + AddRLP(header, body, receipts []byte, number uint64, hash common.Hash, td, difficulty *big.Int) error + + // Finalize writes all collected entries and returns the epoch identifier. + // For Era1 (onedb): returns the accumulator root. + // For EraE (execdb): returns the last block hash. Finalize() (common.Hash, error) + + // Accumulator returns the accumulator root after Finalize has been called. + // Returns nil for post-merge epochs where no accumulator exists. + Accumulator() *common.Hash } // Era represents the interface for reading era data. diff --git a/internal/era/execdb/builder.go b/internal/era/execdb/builder.go index 21e6e72ebb..2f7634e330 100644 --- a/internal/era/execdb/builder.go +++ b/internal/era/execdb/builder.go @@ -16,31 +16,40 @@ package execdb +// EraE file format specification. +// // The format can be summarized with the following expression: -// eraE := Version | CompressedHeader* | CompressedBody* | CompressedReceipts* | TotalDifficulty* | Proofs* | other-entries* | Accumulator | BlockIndex +// +// eraE := Version | CompressedHeader* | CompressedBody* | CompressedSlimReceipts* | TotalDifficulty* | other-entries* | Accumulator? | ComponentIndex +// // Each basic element is its own e2store entry: - -// Version = { type: 0x3265, data: nil } -// CompressedHeader = { type: 0x03, data: snappyFramed(rlp(header)) } -// CompressedBody = { type: 0x04, data: snappyFramed(rlp(body)) } -// CompressedReceipts = { type: 0x05, data: snappyFramed(rlp([tx-type, post-state-or-status, cumulative-gas, logs])) } -// TotalDifficulty = { type: 0x06, data: uint256(header.total_difficulty) } -// Proofs = { type: 0x07 data: snappyFramed(rlp([BlockProofHistoricalHashesAccumulator, BlockProofHistoricalRoots, BlockProofHistoricalSummaries]))} -// AccumulatorRoot = { type: 0x08, data: hash_tree_root(List(HeaderRecord, 8192)) } -// BlockIndex = { type: 0x3266, data: block-index } -// TotalDifficulty is little-endian encoded. - -// AccumulatorRoot is only defined for epochs with pre-merge data. -// HeaderRecord is defined in the Portal Network specification[^5]. - -// BlockIndex stores relative offsets to each compressed block entry. The format is: - -// block-index := starting-number | index | index | index ... | count -// All values in the block index are little-endian uint64. - -// starting-number is the first block number in the archive. Every index is a defined relative to index's location in the file. The total number of block entries in the file is recorded in count. - -// Due to the accumulator size limit of 8192, the maximum number of blocks in an Era batch is also 8192. This is also the value of SLOTS_PER_HISTORICAL_ROOT[^6] on the Beacon chain, so it is nice to align on the value. +// +// Version = { type: 0x3265, data: nil } +// CompressedHeader = { type: 0x03, data: snappyFramed(rlp(header)) } +// CompressedBody = { type: 0x04, data: snappyFramed(rlp(body)) } +// CompressedSlimReceipts = { type: 0x08, data: snappyFramed(rlp([tx-type, post-state-or-status, cumulative-gas, logs])) } +// TotalDifficulty = { type: 0x06, data: uint256 (header.total_difficulty) } +// AccumulatorRoot = { type: 0x07, data: hash_tree_root(List(HeaderRecord, 8192)) } +// ComponentIndex = { type: 0x3267, data: component-index } +// +// Notes: +// - TotalDifficulty is present for pre-merge and merge transition epochs. +// For pure post-merge epochs, TotalDifficulty is omitted entirely. +// - In merge transition epochs, post-merge blocks store the final total +// difficulty (the TD at which the merge occurred). +// - AccumulatorRoot is only written for pre-merge epochs. +// - HeaderRecord is defined in the Portal Network specification. +// - Proofs (type 0x09) are defined in the spec but not yet supported in this implementation. +// +// ComponentIndex stores relative offsets to each block's components: +// +// component-index := starting-number | indexes | indexes | ... | component-count | count +// indexes := header-offset | body-offset | receipts-offset | td-offset? +// +// All values are little-endian uint64. +// +// Due to the accumulator size limit of 8192, the maximum number of blocks in an +// EraE file is also 8192. import ( "bytes" @@ -58,20 +67,23 @@ import ( "github.com/golang/snappy" ) -// Builder is used to build an Era2 e2store file. It collects block entries and writes them to the underlying e2store.Writer. +// Builder is used to build an EraE e2store file. It collects block entries and +// writes them to the underlying e2store.Writer. type Builder struct { w *e2store.Writer headers [][]byte - hashes []common.Hash + hashes []common.Hash // only pre-merge block hashes, for accumulator bodies [][]byte receipts [][]byte - proofs [][]byte tds []*big.Int - startNum *uint64 - merged bool - written uint64 + startNum *uint64 + ttd *big.Int // terminal total difficulty + last common.Hash // hash of last block added + accumulator *common.Hash // accumulator root, set by Finalize (nil for post-merge) + + written uint64 buf *bytes.Buffer snappy *snappy.Writer @@ -84,8 +96,8 @@ func NewBuilder(w io.Writer) era.Builder { } } -// Add writes a block entry, its reciepts, and optionally its proofs as well into the e2store file. -func (b *Builder) Add(block *types.Block, receipts types.Receipts, td *big.Int, proof era.Proof) error { +// Add writes a block entry and its receipts into the e2store file. +func (b *Builder) Add(block *types.Block, receipts types.Receipts, td *big.Int) error { eh, err := rlp.EncodeToBytes(block.Header()) if err != nil { return fmt.Errorf("encode header: %w", err) @@ -104,83 +116,84 @@ func (b *Builder) Add(block *types.Block, receipts types.Receipts, td *big.Int, return fmt.Errorf("encode receipts: %w", err) } - var ep []byte - if proof != nil { - ep, err = rlp.EncodeToBytes(ep) - if err != nil { - return fmt.Errorf("encode proof: %w", err) - } - } - var difficulty *big.Int - if !b.merged { - difficulty = block.Difficulty() - } - - return b.AddRLP(eh, eb, er, ep, block.Number().Uint64(), block.Hash(), td, difficulty) + return b.AddRLP(eh, eb, er, block.Number().Uint64(), block.Hash(), td, block.Difficulty()) } // AddRLP takes the RLP encoded block components and writes them to the underlying e2store file. -func (b *Builder) AddRLP(header []byte, body []byte, receipts []byte, proof []byte, number uint64, blockHash common.Hash, td, difficulty *big.Int) error { - if b.startNum == nil { - b.startNum = new(uint64) - *b.startNum = number - b.merged = difficulty.Sign() == 0 - } +// The builder automatically handles transition epochs where both pre and post-merge blocks exist. +func (b *Builder) AddRLP(header, body, receipts []byte, number uint64, blockHash common.Hash, td, difficulty *big.Int) error { if len(b.headers) >= era.MaxSize { return fmt.Errorf("exceeds max size %d", era.MaxSize) } - if !b.merged && (td == nil || difficulty == nil || difficulty.Sign() == 0) { - return fmt.Errorf("era pre-merge, but difficulty values not supplied") - } - if b.merged && (td != nil || difficulty != nil) { - return fmt.Errorf("era already merged, but given non-nil difficulty values") - } - if len(b.headers) != 0 && len(b.proofs) == 0 && proof != nil { - return fmt.Errorf("unexpected proof for block %d: first block had none", number) - } - if len(b.headers) != 0 && len(b.proofs) != 0 && proof == nil { - return fmt.Errorf("block %d missing proof: proofs required for every block", number) + // Set starting block number on first add. + if b.startNum == nil { + b.startNum = new(uint64) + *b.startNum = number } + // After the merge, difficulty must be nil. + post := (b.tds == nil && len(b.headers) > 0) || b.ttd != nil + if post && difficulty != nil { + return fmt.Errorf("post-merge epoch: cannot accept total difficulty for block %d", number) + } + + // If this marks the start of the transition, record final total + // difficulty value. + if len(b.tds) > 0 && difficulty == nil { + b.ttd = new(big.Int).Set(b.tds[len(b.tds)-1]) + } + + // Record block data. b.headers = append(b.headers, header) b.bodies = append(b.bodies, body) b.receipts = append(b.receipts, receipts) + b.last = blockHash - if !b.merged { - if difficulty.Sign() != 0 { - b.hashes = append(b.hashes, blockHash) - } + // Conditionally write the total difficulty and block hashes. + // - Pre-merge: store total difficulty and block hashes. + // - Transition: only store total difficulty. + // - Post-merge: store neither. + if difficulty != nil { + b.hashes = append(b.hashes, blockHash) b.tds = append(b.tds, new(big.Int).Set(td)) + } else if b.ttd != nil { + b.tds = append(b.tds, new(big.Int).Set(b.ttd)) + } else { + // Post-merge: no TD or block hashes stored. } - if proof != nil { - b.proofs = append(b.proofs, proof) - } return nil } +// Accumulator returns the accumulator root after Finalize has been called. +// Returns nil for post-merge epochs where no accumulator exists. +func (b *Builder) Accumulator() *common.Hash { + return b.accumulator +} + type offsets struct { headers []uint64 bodies []uint64 receipts []uint64 tds []uint64 - proofs []uint64 } -// Finalize writes all collected block entries to the e2store file and returns the accumulator root hash. -// It also writes the index table at the end of the file, which contains offsets to each block entry. +// Finalize writes all collected block entries to the e2store file. +// For pre-merge or transition epochs, the accumulator root is computed over +// pre-merge blocks and written. For pure post-merge epochs, no accumulator +// is written. Always returns the last block hash as the epoch identifier. func (b *Builder) Finalize() (common.Hash, error) { if b.startNum == nil { return common.Hash{}, errors.New("no blocks added, cannot finalize") } - // Write Era2 version before writing any blocks. + // Write version before writing any blocks. if n, err := b.w.Write(era.TypeVersion, nil); err != nil { return common.Hash{}, fmt.Errorf("write version entry: %w", err) } else { b.written += uint64(n) } - // Convert int values to byte-level LE representation. + // Convert TD values to byte-level LE representation. var tds [][]byte for _, td := range b.tds { tds = append(tds, uint256LE(td)) @@ -201,7 +214,6 @@ func (b *Builder) Finalize() (common.Hash, error) { {era.TypeCompressedBody, b.bodies, true, &o.bodies}, {era.TypeCompressedSlimReceipts, b.receipts, true, &o.receipts}, {era.TypeTotalDifficulty, tds, false, &o.tds}, - {era.TypeProof, b.proofs, true, &o.proofs}, } { for _, data := range section.data { *section.offsets = append(*section.offsets, b.written) @@ -221,12 +233,11 @@ func (b *Builder) Finalize() (common.Hash, error) { } } - // Compute and write accumlator root only when the first block the epoch is - // pre-merge, otherwise omit. - var accRoot common.Hash - if !b.merged { - var err error - accRoot, err = era.ComputeAccumulator(b.hashes, b.tds[:len(b.hashes)]) + // Compute and write accumulator root only for epochs that started pre-merge. + // The accumulator is computed over only the pre-merge blocks (b.hashes). + // Pure post-merge epochs have no accumulator. + if len(b.tds) > 0 { + accRoot, err := era.ComputeAccumulator(b.hashes, b.tds[:len(b.hashes)]) if err != nil { return common.Hash{}, fmt.Errorf("compute accumulator: %w", err) } @@ -235,11 +246,18 @@ func (b *Builder) Finalize() (common.Hash, error) { } else { b.written += uint64(n) } + b.accumulator = &accRoot + if err := b.writeIndex(&o); err != nil { + return common.Hash{}, err + } + return b.last, nil } - // TODO: accumulator root should only be returned when it's relevant - // (pre-merge)? - return accRoot, b.writeIndex(&o) + // Pure post-merge epoch: no accumulator. + if err := b.writeIndex(&o); err != nil { + return common.Hash{}, err + } + return b.last, nil } // uin256LE writes 32 byte big integers to little endian. @@ -269,45 +287,37 @@ func (b *Builder) snappyWrite(typ uint16, in []byte) error { return nil } -// writeIndex takes all the offset table and writes it to the file. The index table contains all offsets to specific block entries +// writeIndex writes the component index to the file. func (b *Builder) writeIndex(o *offsets) error { - count := uint64(len(o.headers)) - componentCount := uint64(3) + count := len(o.headers) + + // Post-merge, we only index headers, bodies, and receipts. Pre-merge, we also + // need to index the total difficulties. + componentCount := 3 if len(o.tds) > 0 { componentCount++ } - if len(o.proofs) > 0 { - componentCount++ - } - index := make([]byte, 8+count*8*componentCount+16) // 8 for start block, 8 per property per block, 16 for the number of properties and the number of blocks - binary.LittleEndian.PutUint64(index, *b.startNum) + // Offsets are stored relative to the index position (negative, stored as uint64). base := int64(b.written) rel := func(abs uint64) uint64 { return uint64(int64(abs) - base) } - for i := uint64(0); i < count; i++ { - basePosition := 8 + i*componentCount*8 - binary.LittleEndian.PutUint64(index[basePosition:], rel(o.headers[i])) - binary.LittleEndian.PutUint64(index[basePosition+8:], rel(o.bodies[i])) - binary.LittleEndian.PutUint64(index[basePosition+16:], rel(o.receipts[i])) + var buf bytes.Buffer + write := func(v uint64) { binary.Write(&buf, binary.LittleEndian, v) } - pos := uint64(24) + write(*b.startNum) + for i := range o.headers { + write(rel(o.headers[i])) + write(rel(o.bodies[i])) + write(rel(o.receipts[i])) if len(o.tds) > 0 { - binary.LittleEndian.PutUint64(index[basePosition+pos:], rel(o.tds[i])) - pos += 8 - } - if len(o.proofs) > 0 { - binary.LittleEndian.PutUint64(index[basePosition+pos:], rel(o.proofs[i])) + write(rel(o.tds[i])) } } - end := 8 + count*componentCount*8 + write(uint64(componentCount)) + write(uint64(count)) - binary.LittleEndian.PutUint64(index[end+0:], componentCount) - binary.LittleEndian.PutUint64(index[end+8:], count) - if n, err := b.w.Write(era.TypeComponentIndex, index); err != nil { - return err - } else { - b.written += uint64(n) - } - return nil + n, err := b.w.Write(era.TypeComponentIndex, buf.Bytes()) + b.written += uint64(n) + return err } diff --git a/internal/era/execdb/era_test.go b/internal/era/execdb/era_test.go index fad6818723..6d141aa22d 100644 --- a/internal/era/execdb/era_test.go +++ b/internal/era/execdb/era_test.go @@ -30,137 +30,318 @@ import ( "github.com/ethereum/go-ethereum/rlp" ) -type testchain struct { - headers [][]byte - bodies [][]byte - receipts [][]byte - tds []*big.Int -} - -func TestEra1Builder(t *testing.T) { +func TestEraE(t *testing.T) { t.Parallel() - // Get temp directory. - f, err := os.CreateTemp(t.TempDir(), "era1-test") + tests := []struct { + name string + start uint64 + preMerge int + postMerge int + accumulator bool // whether accumulator should exist + }{ + { + name: "pre-merge", + start: 0, + preMerge: 128, + postMerge: 0, + accumulator: true, + }, + { + name: "post-merge", + start: 0, + preMerge: 0, + postMerge: 64, + accumulator: false, + }, + { + name: "transition", + start: 0, + preMerge: 32, + postMerge: 32, + accumulator: true, + }, + { + name: "non-zero-start", + start: 8192, + preMerge: 64, + postMerge: 0, + accumulator: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + f, err := os.CreateTemp(t.TempDir(), "erae-test") + if err != nil { + t.Fatalf("error creating temp file: %v", err) + } + defer f.Close() + + // Build test data. + type blockData struct { + header, body, receipts []byte + hash common.Hash + td *big.Int + difficulty *big.Int + } + var ( + builder = NewBuilder(f) + blocks []blockData + totalBlocks = tt.preMerge + tt.postMerge + finalTD = big.NewInt(int64(tt.preMerge)) + ) + + // Add pre-merge blocks. + for i := 0; i < tt.preMerge; i++ { + num := tt.start + uint64(i) + blk := blockData{ + header: mustEncode(&types.Header{Number: big.NewInt(int64(num)), Difficulty: big.NewInt(1)}), + body: mustEncode(&types.Body{Transactions: []*types.Transaction{types.NewTransaction(0, common.Address{byte(i)}, nil, 0, nil, nil)}}), + receipts: mustEncode([]types.ReceiptForStorage{{CumulativeGasUsed: uint64(i)}}), + hash: common.Hash{byte(i)}, + td: big.NewInt(int64(i + 1)), + difficulty: big.NewInt(1), + } + blocks = append(blocks, blk) + if err := builder.AddRLP(blk.header, blk.body, blk.receipts, num, blk.hash, blk.td, blk.difficulty); err != nil { + t.Fatalf("error adding pre-merge block %d: %v", i, err) + } + } + + // Add post-merge blocks. + for i := 0; i < tt.postMerge; i++ { + idx := tt.preMerge + i + num := tt.start + uint64(idx) + blk := blockData{ + header: mustEncode(&types.Header{Number: big.NewInt(int64(num)), Difficulty: big.NewInt(0)}), + body: mustEncode(&types.Body{}), + receipts: mustEncode([]types.ReceiptForStorage{}), + hash: common.Hash{byte(idx)}, + } + blocks = append(blocks, blk) + if err := builder.AddRLP(blk.header, blk.body, blk.receipts, num, blk.hash, nil, nil); err != nil { + t.Fatalf("error adding post-merge block %d: %v", idx, err) + } + } + + // Finalize and check return values. + epochID, err := builder.Finalize() + if err != nil { + t.Fatalf("error finalizing: %v", err) + } + + // Verify epoch ID is always the last block hash. + expectedLastHash := blocks[len(blocks)-1].hash + if epochID != expectedLastHash { + t.Fatalf("wrong epoch ID: want %s, got %s", expectedLastHash.Hex(), epochID.Hex()) + } + + // Verify accumulator presence. + if tt.accumulator { + if builder.Accumulator() == nil { + t.Fatal("expected non-nil accumulator") + } + } else { + if builder.Accumulator() != nil { + t.Fatalf("expected nil accumulator, got %s", builder.Accumulator().Hex()) + } + } + + // Open and verify the era file. + e, err := Open(f.Name()) + if err != nil { + t.Fatalf("failed to open era: %v", err) + } + defer e.Close() + + // Verify metadata. + if e.Start() != tt.start { + t.Fatalf("wrong start block: want %d, got %d", tt.start, e.Start()) + } + if e.Count() != uint64(totalBlocks) { + t.Fatalf("wrong block count: want %d, got %d", totalBlocks, e.Count()) + } + + // Verify accumulator in file. + if tt.accumulator { + accRoot, err := e.Accumulator() + if err != nil { + t.Fatalf("error getting accumulator: %v", err) + } + if accRoot != *builder.Accumulator() { + t.Fatalf("accumulator mismatch: builder has %s, file contains %s", + builder.Accumulator().Hex(), accRoot.Hex()) + } + } else { + if _, err := e.Accumulator(); err == nil { + t.Fatal("expected error when reading accumulator from post-merge epoch") + } + } + + // Verify blocks via raw iterator. + it, err := NewRawIterator(e) + if err != nil { + t.Fatalf("failed to make iterator: %v", err) + } + for i := 0; i < totalBlocks; i++ { + if !it.Next() { + t.Fatalf("expected more entries at %d", i) + } + if it.Error() != nil { + t.Fatalf("unexpected error: %v", it.Error()) + } + + // Check header. + rawHeader, err := io.ReadAll(it.Header) + if err != nil { + t.Fatalf("error reading header: %v", err) + } + if !bytes.Equal(rawHeader, blocks[i].header) { + t.Fatalf("mismatched header at %d", i) + } + + // Check body. + rawBody, err := io.ReadAll(it.Body) + if err != nil { + t.Fatalf("error reading body: %v", err) + } + if !bytes.Equal(rawBody, blocks[i].body) { + t.Fatalf("mismatched body at %d", i) + } + + // Check receipts. + rawReceipts, err := io.ReadAll(it.Receipts) + if err != nil { + t.Fatalf("error reading receipts: %v", err) + } + if !bytes.Equal(rawReceipts, blocks[i].receipts) { + t.Fatalf("mismatched receipts at %d", i) + } + + // Check TD (only for epochs that have TD stored). + if tt.preMerge > 0 && it.TotalDifficulty != nil { + rawTd, err := io.ReadAll(it.TotalDifficulty) + if err != nil { + t.Fatalf("error reading TD: %v", err) + } + slices.Reverse(rawTd) + td := new(big.Int).SetBytes(rawTd) + var expectedTD *big.Int + if i < tt.preMerge { + expectedTD = blocks[i].td + } else { + // Post-merge blocks in transition epoch use final TD. + expectedTD = finalTD + } + if td.Cmp(expectedTD) != 0 { + t.Fatalf("mismatched TD at %d: want %s, got %s", i, expectedTD, td) + } + } + } + + // Verify random access. + for _, blockNum := range []uint64{tt.start, tt.start + uint64(totalBlocks) - 1} { + blk, err := e.GetBlockByNumber(blockNum) + if err != nil { + t.Fatalf("error getting block %d: %v", blockNum, err) + } + if blk.Number().Uint64() != blockNum { + t.Fatalf("wrong block number: want %d, got %d", blockNum, blk.Number().Uint64()) + } + } + + // Verify out-of-range access fails. + if _, err := e.GetBlockByNumber(tt.start + uint64(totalBlocks)); err == nil { + t.Fatal("expected error for out-of-range block") + } + if tt.start > 0 { + if _, err := e.GetBlockByNumber(tt.start - 1); err == nil { + t.Fatal("expected error for block before start") + } + } + + // Verify high-level iterator. + hlIt, err := e.Iterator() + if err != nil { + t.Fatalf("failed to create iterator: %v", err) + } + count := 0 + for hlIt.Next() { + blk, err := hlIt.Block() + if err != nil { + t.Fatalf("error getting block: %v", err) + } + if blk.Number().Uint64() != tt.start+uint64(count) { + t.Fatalf("wrong block number: want %d, got %d", tt.start+uint64(count), blk.Number().Uint64()) + } + count++ + } + if hlIt.Error() != nil { + t.Fatalf("iterator error: %v", hlIt.Error()) + } + if count != totalBlocks { + t.Fatalf("wrong iteration count: want %d, got %d", totalBlocks, count) + } + }) + } +} + +// TestInitialTD tests the InitialTD calculation separately since it requires +// specific TD/difficulty values. +func TestInitialTD(t *testing.T) { + t.Parallel() + + f, err := os.CreateTemp(t.TempDir(), "erae-initial-td-test") if err != nil { t.Fatalf("error creating temp file: %v", err) } defer f.Close() - var ( - builder = NewBuilder(f) - chain = testchain{} - ) - for i := 0; i < 128; i++ { - chain.headers = append(chain.headers, mustEncode(&types.Header{Number: big.NewInt(int64(i))})) - chain.bodies = append(chain.bodies, mustEncode(&types.Body{Transactions: []*types.Transaction{types.NewTransaction(0, common.Address{byte(i)}, nil, 0, nil, nil)}})) - chain.receipts = append(chain.receipts, mustEncode([]types.ReceiptForStorage{{CumulativeGasUsed: uint64(i)}})) - chain.tds = append(chain.tds, big.NewInt(int64(i))) + builder := NewBuilder(f) + + // First block: difficulty=5, TD=10, so initial TD = 10-5 = 5. + header := mustEncode(&types.Header{Number: big.NewInt(0), Difficulty: big.NewInt(5)}) + body := mustEncode(&types.Body{}) + receipts := mustEncode([]types.ReceiptForStorage{}) + + if err := builder.AddRLP(header, body, receipts, 0, common.Hash{0}, big.NewInt(10), big.NewInt(5)); err != nil { + t.Fatalf("error adding block: %v", err) } - // Write blocks to Era1. - for i := 0; i < len(chain.headers); i++ { - var ( - header = chain.headers[i] - body = chain.bodies[i] - receipts = chain.receipts[i] - hash = common.Hash{byte(i)} - td = chain.tds[i] - ) - if err = builder.AddRLP(header, body, receipts, nil, uint64(i), hash, td, big.NewInt(1)); err != nil { - t.Fatalf("error adding entry: %v", err) - } + // Second block: difficulty=3, TD=13. + header2 := mustEncode(&types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(3)}) + if err := builder.AddRLP(header2, body, receipts, 1, common.Hash{1}, big.NewInt(13), big.NewInt(3)); err != nil { + t.Fatalf("error adding block: %v", err) } - // Finalize Era1. if _, err := builder.Finalize(); err != nil { - t.Fatalf("error finalizing era1: %v", err) + t.Fatalf("error finalizing: %v", err) } - // Verify Era1 contents. e, err := Open(f.Name()) if err != nil { t.Fatalf("failed to open era: %v", err) } defer e.Close() - it, err := NewRawIterator(e) + + initialTD, err := e.InitialTD() if err != nil { - t.Fatalf("failed to make iterator: %s", err) + t.Fatalf("error getting initial TD: %v", err) } - for i := uint64(0); i < uint64(len(chain.headers)); i++ { - if !it.Next() { - t.Fatalf("expected more entries, have %d want %d", i, len(chain.headers)) - } - if it.Error() != nil { - t.Fatalf("unexpected error %v", it.Error()) - } - // Check headers. - rawHeader, err := io.ReadAll(it.Header) - if err != nil { - t.Fatalf("error reading header from iterator: %v", err) - } - if !bytes.Equal(rawHeader, chain.headers[i]) { - t.Fatalf("mismatched header: want %s, got %s", chain.headers[i], rawHeader) - } - // Check bodies. - body, err := io.ReadAll(it.Body) - if err != nil { - t.Fatalf("error reading body: %v", err) - } - if !bytes.Equal(body, chain.bodies[i]) { - t.Fatalf("mismatched body: want %s, got %s", chain.bodies[i], body) - } - - // Check receipts. - rawReceipts, err := io.ReadAll(it.Receipts) - if err != nil { - t.Fatalf("error reading receipts from iterator: %v", err) - } - if !bytes.Equal(rawReceipts, chain.receipts[i]) { - t.Fatalf("mismatched receipts: want %s, got %s", chain.receipts[i], rawReceipts) - } - receipts, err := getReceiptsByNumber(e, i) - if err != nil { - t.Fatalf("error reading receipts: %v", err) - } - encReceipts, err := rlp.EncodeToBytes(receipts) - if err != nil { - t.Fatalf("error encoding receipts: %v", err) - } - if !bytes.Equal(encReceipts, chain.receipts[i]) { - t.Fatalf("mismatched receipts: want %s, got %s", chain.receipts[i], encReceipts) - } - - // Check total difficulty. - rawTd, err := io.ReadAll(it.TotalDifficulty) - if err != nil { - t.Fatalf("error reading td: %v", err) - } - slices.Reverse(rawTd) - td := new(big.Int).SetBytes(rawTd) - if td.Cmp(chain.tds[i]) != 0 { - t.Fatalf("mismatched tds: want %s, got %s", chain.tds[i], td) - } + // Initial TD should be TD[0] - Difficulty[0] = 10 - 5 = 5. + if initialTD.Cmp(big.NewInt(5)) != 0 { + t.Fatalf("wrong initial TD: want 5, got %s", initialTD) } } func mustEncode(obj any) []byte { b, err := rlp.EncodeToBytes(obj) if err != nil { - panic(fmt.Sprintf("failed in encode obj: %v", err)) + panic(fmt.Sprintf("failed to encode obj: %v", err)) } return b } - -func getReceiptsByNumber(e *Era, number uint64) ([]*types.ReceiptForStorage, error) { - r, err := e.GetRawReceiptsByNumber(number) - if err != nil { - return nil, err - } - var receipts []*types.ReceiptForStorage - if err := rlp.DecodeBytes(r, &receipts); err != nil { - return nil, err - } - return receipts, nil -} diff --git a/internal/era/execdb/iterator.go b/internal/era/execdb/iterator.go index bbf9796fc0..257993312e 100644 --- a/internal/era/execdb/iterator.go +++ b/internal/era/execdb/iterator.go @@ -194,7 +194,8 @@ func (it *RawIterator) Next() bool { return false } - if int(header)+3 < int(it.e.m.components) { + // Check if TD component is present in this file (pre-merge or merge-transition epoch). + if int(td) < int(it.e.m.components) { tdOffset, err := it.e.tdOff(it.next) if err != nil { it.setErr(err) diff --git a/internal/era/execdb/reader.go b/internal/era/execdb/reader.go index 503b3e4c55..76c510521d 100644 --- a/internal/era/execdb/reader.go +++ b/internal/era/execdb/reader.go @@ -39,9 +39,10 @@ type Era struct { m metadata // metadata for the Era file } -// Filename returns a recognizable filename for era file. -func Filename(network string, epoch int, root common.Hash) string { - return fmt.Sprintf("%s-%05d-%s.erae", network, epoch, root.Hex()[2:10]) +// Filename returns a recognizable filename for an EraE file. +// The filename uses the last block hash to uniquely identify the epoch's content. +func Filename(network string, epoch int, lastBlockHash common.Hash) string { + return fmt.Sprintf("%s-%05d-%s.erae", network, epoch, lastBlockHash.Hex()[2:10]) } // Open accesses the era file. @@ -185,46 +186,28 @@ func (e *Era) GetRawReceiptsByNumber(blockNum uint64) ([]byte, error) { } // InitialTD returns initial total difficulty before the difficulty of the -// first block of the Era is applied. +// first block of the Era is applied. Returns an error if TD is not available +// (e.g., post-merge epoch). func (e *Era) InitialTD() (*big.Int, error) { - var ( - r io.Reader - header types.Header - rawTd []byte - n int64 - off int64 - err error - ) + // Check if TD component exists. + if int(td) >= int(e.m.components) { + return nil, fmt.Errorf("total difficulty not available in this epoch") + } - // Read first header. - if off, err = e.headerOff(e.m.start); err != nil { - return nil, err - } - if r, n, err = newSnappyReader(e.s, era.TypeCompressedHeader, off); err != nil { - return nil, err - } - if err := rlp.Decode(r, &header); err != nil { - return nil, err - } - off += n - - // Skip over header and body. - off, err = e.s.SkipN(off, 2) + // Get first header to read its difficulty. + header, err := e.GetHeader(e.m.start) if err != nil { - return nil, err + return nil, fmt.Errorf("read first header: %w", err) } - // Read total difficulty after first block. - if r, _, err = e.s.ReaderAt(era.TypeTotalDifficulty, off); err != nil { - return nil, err - } - rawTd, err = io.ReadAll(r) + // Get TD after first block using the index. + firstTD, err := e.GetTD(e.m.start) if err != nil { - return nil, err + return nil, fmt.Errorf("read first TD: %w", err) } - slices.Reverse(rawTd) - td := new(big.Int).SetBytes(rawTd) - return td.Sub(td, header.Difficulty), nil + + // Initial TD = TD[0] - Difficulty[0] + return new(big.Int).Sub(firstTD, header.Difficulty), nil } // Accumulator reads the accumulator entry in the EraE file if it exists. diff --git a/internal/era/onedb/builder.go b/internal/era/onedb/builder.go index 948d1f446e..6497bf4f60 100644 --- a/internal/era/onedb/builder.go +++ b/internal/era/onedb/builder.go @@ -73,13 +73,15 @@ import ( // Due to the accumulator size limit of 8192, the maximum number of blocks in // an Era1 batch is also 8192. type Builder struct { - w *e2store.Writer - startNum *uint64 - startTd *big.Int - indexes []uint64 - hashes []common.Hash - tds []*big.Int - written int + w *e2store.Writer + startNum *uint64 + startTd *big.Int + indexes []uint64 + hashes []common.Hash + tds []*big.Int + accumulator *common.Hash // accumulator root, set by Finalize + + written int buf *bytes.Buffer snappy *snappy.Writer @@ -97,10 +99,7 @@ func NewBuilder(w io.Writer) era.Builder { // Add writes a compressed block entry and compressed receipts entry to the // underlying e2store file. -func (b *Builder) Add(block *types.Block, receipts types.Receipts, td *big.Int, proof era.Proof) error { - if proof != nil { - return fmt.Errorf("proof not allowed in era1 format") - } +func (b *Builder) Add(block *types.Block, receipts types.Receipts, td *big.Int) error { eh, err := rlp.EncodeToBytes(block.Header()) if err != nil { return err @@ -113,15 +112,12 @@ func (b *Builder) Add(block *types.Block, receipts types.Receipts, td *big.Int, if err != nil { return err } - return b.AddRLP(eh, eb, er, nil, block.NumberU64(), block.Hash(), td, block.Difficulty()) + return b.AddRLP(eh, eb, er, block.NumberU64(), block.Hash(), td, block.Difficulty()) } // AddRLP writes a compressed block entry and compressed receipts entry to the // underlying e2store file. -func (b *Builder) AddRLP(header, body, receipts, proof []byte, number uint64, hash common.Hash, td, difficulty *big.Int) error { - if proof != nil { - return fmt.Errorf("proof not allowed in era1 format") - } +func (b *Builder) AddRLP(header, body, receipts []byte, number uint64, hash common.Hash, td, difficulty *big.Int) error { // Write Era1 version entry before first block. if b.startNum == nil { n, err := b.w.Write(era.TypeVersion, nil) @@ -164,7 +160,8 @@ func (b *Builder) AddRLP(header, body, receipts, proof []byte, number uint64, ha } // Finalize computes the accumulator and block index values, then writes the -// corresponding e2store entries. +// corresponding e2store entries. Era1 always has an accumulator, so this +// always returns a valid hash. func (b *Builder) Finalize() (common.Hash, error) { if b.startNum == nil { return common.Hash{}, errors.New("finalize called on empty builder") @@ -179,6 +176,8 @@ func (b *Builder) Finalize() (common.Hash, error) { if err != nil { return common.Hash{}, fmt.Errorf("error writing accumulator: %w", err) } + b.accumulator = &root + // Get beginning of index entry to calculate block relative offset. base := int64(b.written) @@ -210,6 +209,12 @@ func (b *Builder) Finalize() (common.Hash, error) { return root, nil } +// Accumulator returns the accumulator root after Finalize has been called. +// For Era1, this always returns a non-nil value since all blocks are pre-merge. +func (b *Builder) Accumulator() *common.Hash { + return b.accumulator +} + // snappyWrite is a small helper to take care snappy encoding and writing an e2store entry. func (b *Builder) snappyWrite(typ uint16, in []byte) error { var ( diff --git a/internal/era/onedb/builder_test.go b/internal/era/onedb/builder_test.go index 0e64f68481..bc7a1d9e63 100644 --- a/internal/era/onedb/builder_test.go +++ b/internal/era/onedb/builder_test.go @@ -67,7 +67,7 @@ func TestEra1Builder(t *testing.T) { hash = common.Hash{byte(i)} td = chain.tds[i] ) - if err = builder.AddRLP(header, body, receipts, nil, uint64(i), hash, td, big.NewInt(1)); err != nil { + if err = builder.AddRLP(header, body, receipts, uint64(i), hash, td, big.NewInt(1)); err != nil { t.Fatalf("error adding entry: %v", err) } } diff --git a/internal/era/proof.go b/internal/era/proof.go index 3c75aa5c6e..873dfef40a 100644 --- a/internal/era/proof.go +++ b/internal/era/proof.go @@ -18,95 +18,19 @@ package era import ( "io" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/rlp" ) -type variant uint16 +type ProofVariant uint16 const ( - proofNone variant = iota - proofHistoricalHashesAccumulator - proofHistoricalRoots - proofCapella - proofDeneb + ProofNone ProofVariant = iota ) -type BlockProofHistoricalHashesAccumulator [15]common.Hash // 15 * 32 = 480 bytes - -// BlockProofHistoricalRoots – Altair / Bellatrix historical_roots branch. -type BlockProofHistoricalRoots struct { - BeaconBlockProof [14]common.Hash // 448 - BeaconBlockRoot common.Hash // 32 - ExecutionBlockProof [11]common.Hash // 352 - Slot uint64 // 8 => 840 bytes -} - -// BlockProofHistoricalSummariesCapella – Capella historical_summaries branch. -type BlockProofHistoricalSummariesCapella struct { - BeaconBlockProof [13]common.Hash // 416 - BeaconBlockRoot common.Hash // 32 - ExecutionBlockProof [11]common.Hash // 352 - Slot uint64 // 8 => 808 bytes -} - -// BlockProofHistoricalSummariesDeneb – Deneb historical_summaries branch. -type BlockProofHistoricalSummariesDeneb struct { - BeaconBlockProof [13]common.Hash // 416 - BeaconBlockRoot common.Hash // 32 - ExecutionBlockProof [12]common.Hash // 384 - Slot uint64 // 8 => 840 bytes -} - -// Proof is the interface for all block proof types in the era2 package. +// Proof is the interface for all block proof types in the package. +// It's a stub for later integration into Era. type Proof interface { EncodeRLP(w io.Writer) error DecodeRlP(s *rlp.Stream) error - Variant() variant + Variant() ProofVariant } - -type hhaAlias BlockProofHistoricalHashesAccumulator - -// EncodeRLP encodes the BlockProofHistoricalHashesAccumulator into RLP format. -func (p *BlockProofHistoricalHashesAccumulator) EncodeRLP(w io.Writer) error { - payload := []interface{}{uint16(proofHistoricalHashesAccumulator), hhaAlias(*p)} - return rlp.Encode(w, payload) -} - -// Variant returns the variant type of the BlockProofHistoricalHashesAccumulator. -func (p *BlockProofHistoricalHashesAccumulator) Variant() variant { - return proofHistoricalHashesAccumulator -} - -type rootsAlias BlockProofHistoricalRoots - -// EncodeRLP encodes the BlockProofHistoricalRoots into RLP format. -func (p *BlockProofHistoricalRoots) EncodeRLP(w io.Writer) error { - payload := []interface{}{uint16(proofHistoricalRoots), rootsAlias(*p)} - return rlp.Encode(w, payload) -} - -// Variant returns the variant type of the BlockProofHistoricalRoots. -func (*BlockProofHistoricalRoots) Variant() variant { return proofHistoricalRoots } - -type capellaAlias BlockProofHistoricalSummariesCapella - -// EncodeRLP encodes the BlockProofHistoricalSummariesCapella into RLP format. -func (p *BlockProofHistoricalSummariesCapella) EncodeRLP(w io.Writer) error { - payload := []interface{}{uint16(proofCapella), capellaAlias(*p)} - return rlp.Encode(w, payload) -} - -// Variant returns the variant type of the BlockProofHistoricalSummariesCapella. -func (*BlockProofHistoricalSummariesCapella) Variant() variant { return proofCapella } - -type denebAlias BlockProofHistoricalSummariesDeneb - -// EncodeRLP encodes the BlockProofHistoricalSummariesDeneb into RLP format. -func (p *BlockProofHistoricalSummariesDeneb) EncodeRLP(w io.Writer) error { - payload := []interface{}{uint16(proofDeneb), denebAlias(*p)} - return rlp.Encode(w, payload) -} - -// Variant returns the variant type of the BlockProofHistoricalSummariesDeneb. -func (*BlockProofHistoricalSummariesDeneb) Variant() variant { return proofDeneb }