Working on implementation

internal/era2/builder2.go

@@ -0,0 +1,153 @@
+package era
+
+// The format can be summarized with the following expression:
+
+//    eraE := Version | CompressedHeader* | CompressedBody* | CompressedReceipts* | TotalDifficulty* | Proofs* | other-entries* | Accumulator | BlockIndex
+// Each basic element is its own e2store entry:
+
+//    Version            = { type: 0x3265, data: nil }
+//    CompressedHeader   = { type: 0x03,   data: snappyFramed(rlp(header)) }
+//    CompressedBody     = { type: 0x04,   data: snappyFramed(rlp(body)) }
+//    CompressedReceipts = { type: 0x05,   data: snappyFramed(rlp([tx-type, post-state-or-status, cumulative-gas, logs])) }
+//    TotalDifficulty    = { type: 0x06,   data: uint256(header.total_difficulty) }
+//    Proofs             = { type: 0x07    data: snappyFramed(rlp([BlockProofHistoricalHashesAccumulator, BlockProofHistoricalRoots, BlockProofHistoricalSummaries]))}
+//    AccumulatorRoot    = { type: 0x08,   data: hash_tree_root(List(HeaderRecord, 8192)) }
+//    BlockIndex         = { type: 0x3266, data: block-index }
+// TotalDifficulty is little-endian encoded.
+
+// AccumulatorRoot is only defined for epochs with pre-merge data.
+// HeaderRecord is defined in the Portal Network specification[^5].
+
+// BlockIndex stores relative offsets to each compressed block entry. The format is:
+
+//    block-index := starting-number | index | index | index ... | count
+// All values in the block index are little-endian uint64.
+
+// starting-number is the first block number in the archive. Every index is a defined relative to index's location in the file. The total number of block entries in the file is recorded in count.
+
+// Due to the accumulator size limit of 8192, the maximum number of blocks in an Era batch is also 8192. This is also the value of SLOTS_PER_HISTORICAL_ROOT[^6] on the Beacon chain, so it is nice to align on the value.
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/internal/era/e2store"
+	"github.com/golang/snappy"
+)
+
+const (
+	TypeVersion            uint16 = 0x3265
+	TypeCompressedHeader   uint16 = 0x08
+	TypeCompressedBody     uint16 = 0x09
+	TypeCompressedReceipts uint16 = 0x0a
+	TypeTotalDifficulty    uint16 = 0x0b
+	TypeProof              uint16 = 0x0c
+	TypeAccumulator        uint16 = 0x0d
+	TypeBlockIndex         uint16 = 0x3267
+
+	MaxEraESize = 8192
+)
+
+type Builder2 struct {
+	w   *e2store.Writer
+	buf *bytes.Buffer
+	sn  *snappy.Writer
+
+	// buffered entries per type:
+	headersRLP  [][]byte
+	bodiesRLP   [][]byte
+	receiptsRLP [][]byte
+	proofsRLP   [][]byte
+	tds         []*big.Int
+
+	headeroffsets  []uint64
+	bodyoffsets    []uint64
+	receiptoffsets []uint64
+	proofoffsets   []uint64
+	startTd        *big.Int
+
+	startNum     *uint64
+	hashes       []common.Hash
+	writtenBytes uint64
+}
+
+// NewBuilder returns a new EraE Builder writing into the given io.Writer.
+func NewBuilder2(w io.Writer) *Builder2 {
+	buf := bytes.NewBuffer(nil)
+	return &Builder2{
+		w:   e2store.NewWriter(w),
+		buf: buf,
+		sn:  snappy.NewBufferedWriter(buf),
+	}
+}
+
+func (b *Builder) Add(block *types.Block, receipts types.Receipts, td *big.Int, proofRLP []byte) error {
+	if len(b.headersRLP) >= MaxEraESize {
+		return fmt.Errorf("exceeds maximum batch size of %d", MaxEraESize)
+	}
+
+	headerb, err := b.encodeHeader(block.Header())
+	if err != nil {
+		return fmt.Errorf("failed to encode header: %w", err)
+	}
+	bodyb, err := b.encodeBody(block.Body())
+	if err != nil {
+		return fmt.Errorf("failed to encode body: %w", err)
+	}
+	receiptsb, err := b.encodeReceipts(receipts)
+	if err != nil {
+		return fmt.Errorf("failed to encode receipts: %w", err)
+	}
+
+	tdbytes := uint256LE(td)
+
+	if b.startNum == nil {
+		start := block.NumberU64()
+		b.startNum = &start
+		_, err := b.w.Write(TypeVersion, nil)
+		if err != nil {
+			return fmt.Errorf("failed to write version entry: %w", err)
+		}
+		b.writtenBytes += 8
+	}
+
+	b.headersRLP = append(b.headersRLP, headerb)
+	b.bodiesRLP = append(b.bodiesRLP, bodyb)
+	b.receiptsRLP = append(b.receiptsRLP, receiptsb)
+	b.tds = append(b.tds, tdbytes)
+	b.proofsRLP = append(b.proofsRLP, proofRLP)
+	b.hashes = append(b.hashes, block.Hash())
+	return nil
+}
+
+func (b *Builder2) Finalize(common.Hash, error) {
+	if b.startNum == nil {
+		return fmt.Errorf("no blocks added, cannot finalize")
+	}
+
+	offs := snappy.Encode(b.buf, b.headersRLP)
+}
+
+func uint256LE(v *big.Int) []byte {
+	b := v.FillBytes(make([]byte, 32))
+	for i := 0; i < 16; i++ {
+		b[i], b[31-i] = b[31-i], b[i]
+	}
+	return b
+}
+
+func decodeBigs(raw [][]byte) []*big.Int {
+	out := make([]*big.Int, len(raw))
+	for i, le := range raw {
+		be := make([]byte, 32)
+		for j := 0; j < 32; j++ {
+			be[j] = le[31-j]
+		}
+		out[i] = new(big.Int).SetBytes(be)
+	}
+	return out
+}

internal/era2/era2.go

@@ -0,0 +1,38 @@
+package era
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+)
+
+type BlockProofHistoricalHashesAccumulator [15]common.Hash // 15 * 32 = 480 bytes
+
+// BlockProofHistoricalRoots – Altair / Bellatrix historical_roots branch.
+type BlockProofHistoricalRoots struct {
+	BeaconBlockProof    [14]common.Hash // 448
+	BeaconBlockRoot     common.Hash     // 32
+	ExecutionBlockProof [11]common.Hash // 352
+	Slot                uint64          // 8  => 840 bytes
+}
+
+// BlockProofHistoricalSummariesCapella – Capella historical_summaries branch.
+type BlockProofHistoricalSummariesCapella struct {
+	BeaconBlockProof    [13]common.Hash // 416
+	BeaconBlockRoot     common.Hash     // 32
+	ExecutionBlockProof [11]common.Hash // 352
+	Slot                uint64          // 8  => 808 bytes
+}
+
+// BlockProofHistoricalSummariesDeneb – Deneb historical_summaries branch.
+type BlockProofHistoricalSummariesDeneb struct {
+	BeaconBlockProof    [13]common.Hash // 416
+	BeaconBlockRoot     common.Hash     // 32
+	ExecutionBlockProof [12]common.Hash // 384
+	Slot                uint64          // 8  => 840 bytes
+}
+
+type Proofs struct {
+	HistoricalHashesAccumulator BlockProofHistoricalHashesAccumulator
+	HistoricalRootsProof        *BlockProofHistoricalRoots
+	SummariesProofCapella       *BlockProofHistoricalSummariesCapella
+	SummariesProofDeneb         *BlockProofHistoricalSummariesDeneb
+}

internal/era2/iterator.go

@@ -0,0 +1,197 @@
+// Copyright 2024 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package era
+
+import (
+	"errors"
+	"io"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/rlp"
+)
+
+// Iterator wraps RawIterator and returns decoded Era1 entries.
+type Iterator struct {
+	inner *RawIterator
+}
+
+// NewIterator returns a new Iterator instance. Next must be immediately
+// called on new iterators to load the first item.
+func NewIterator(e *Era) (*Iterator, error) {
+	inner, err := NewRawIterator(e)
+	if err != nil {
+		return nil, err
+	}
+	return &Iterator{inner}, nil
+}
+
+// Next moves the iterator to the next block entry. It returns false when all
+// items have been read or an error has halted its progress. Block, Receipts,
+// and BlockAndReceipts should no longer be called after false is returned.
+func (it *Iterator) Next() bool {
+	return it.inner.Next()
+}
+
+// Number returns the current number block the iterator will return.
+func (it *Iterator) Number() uint64 {
+	return it.inner.next - 1
+}
+
+// Error returns the error status of the iterator. It should be called before
+// reading from any of the iterator's values.
+func (it *Iterator) Error() error {
+	return it.inner.Error()
+}
+
+// Block returns the block for the iterator's current position.
+func (it *Iterator) Block() (*types.Block, error) {
+	if it.inner.Header == nil || it.inner.Body == nil {
+		return nil, errors.New("header and body must be non-nil")
+	}
+	var (
+		header types.Header
+		body   types.Body
+	)
+	if err := rlp.Decode(it.inner.Header, &header); err != nil {
+		return nil, err
+	}
+	if err := rlp.Decode(it.inner.Body, &body); err != nil {
+		return nil, err
+	}
+	return types.NewBlockWithHeader(&header).WithBody(body), nil
+}
+
+// Receipts returns the receipts for the iterator's current position.
+func (it *Iterator) Receipts() (types.Receipts, error) {
+	if it.inner.Receipts == nil {
+		return nil, errors.New("receipts must be non-nil")
+	}
+	var receipts types.Receipts
+	err := rlp.Decode(it.inner.Receipts, &receipts)
+	return receipts, err
+}
+
+// BlockAndReceipts returns the block and receipts for the iterator's current
+// position.
+func (it *Iterator) BlockAndReceipts() (*types.Block, types.Receipts, error) {
+	b, err := it.Block()
+	if err != nil {
+		return nil, nil, err
+	}
+	r, err := it.Receipts()
+	if err != nil {
+		return nil, nil, err
+	}
+	return b, r, nil
+}
+
+// TotalDifficulty returns the total difficulty for the iterator's current
+// position.
+func (it *Iterator) TotalDifficulty() (*big.Int, error) {
+	td, err := io.ReadAll(it.inner.TotalDifficulty)
+	if err != nil {
+		return nil, err
+	}
+	return new(big.Int).SetBytes(reverseOrder(td)), nil
+}
+
+// RawIterator reads an RLP-encode Era1 entries.
+type RawIterator struct {
+	e    *Era   // backing Era1
+	next uint64 // next block to read
+	err  error  // last error
+
+	Header          io.Reader
+	Body            io.Reader
+	Receipts        io.Reader
+	TotalDifficulty io.Reader
+}
+
+// NewRawIterator returns a new RawIterator instance. Next must be immediately
+// called on new iterators to load the first item.
+func NewRawIterator(e *Era) (*RawIterator, error) {
+	return &RawIterator{
+		e:    e,
+		next: e.m.start,
+	}, nil
+}
+
+// Next moves the iterator to the next block entry. It returns false when all
+// items have been read or an error has halted its progress. Header, Body,
+// Receipts, TotalDifficulty will be set to nil in the case returning false or
+// finding an error and should therefore no longer be read from.
+func (it *RawIterator) Next() bool {
+	// Clear old errors.
+	it.err = nil
+	if it.e.m.start+it.e.m.count <= it.next {
+		it.clear()
+		return false
+	}
+	off, err := it.e.readOffset(it.next)
+	if err != nil {
+		// Error here means block index is corrupted, so don't
+		// continue.
+		it.clear()
+		it.err = err
+		return false
+	}
+	var n int64
+	if it.Header, n, it.err = newSnappyReader(it.e.s, TypeCompressedHeader, off); it.err != nil {
+		it.clear()
+		return true
+	}
+	off += n
+	if it.Body, n, it.err = newSnappyReader(it.e.s, TypeCompressedBody, off); it.err != nil {
+		it.clear()
+		return true
+	}
+	off += n
+	if it.Receipts, n, it.err = newSnappyReader(it.e.s, TypeCompressedReceipts, off); it.err != nil {
+		it.clear()
+		return true
+	}
+	off += n
+	if it.TotalDifficulty, _, it.err = it.e.s.ReaderAt(TypeTotalDifficulty, off); it.err != nil {
+		it.clear()
+		return true
+	}
+	it.next += 1
+	return true
+}
+
+// Number returns the current number block the iterator will return.
+func (it *RawIterator) Number() uint64 {
+	return it.next - 1
+}
+
+// Error returns the error status of the iterator. It should be called before
+// reading from any of the iterator's values.
+func (it *RawIterator) Error() error {
+	if it.err == io.EOF {
+		return nil
+	}
+	return it.err
+}
+
+// clear sets all the outputs to nil.
+func (it *RawIterator) clear() {
+	it.Header = nil
+	it.Body = nil
+	it.Receipts = nil
+	it.TotalDifficulty = nil
+}