diff --git a/internal/era2/builder2.go b/internal/era2/builder2.go new file mode 100644 index 0000000000..3d6cb0a7f8 --- /dev/null +++ b/internal/era2/builder2.go @@ -0,0 +1,153 @@ +package era + +// The format can be summarized with the following expression: + +// eraE := Version | CompressedHeader* | CompressedBody* | CompressedReceipts* | TotalDifficulty* | Proofs* | other-entries* | Accumulator | BlockIndex +// Each basic element is its own e2store entry: + +// Version = { type: 0x3265, data: nil } +// CompressedHeader = { type: 0x03, data: snappyFramed(rlp(header)) } +// CompressedBody = { type: 0x04, data: snappyFramed(rlp(body)) } +// CompressedReceipts = { type: 0x05, data: snappyFramed(rlp([tx-type, post-state-or-status, cumulative-gas, logs])) } +// TotalDifficulty = { type: 0x06, data: uint256(header.total_difficulty) } +// Proofs = { type: 0x07 data: snappyFramed(rlp([BlockProofHistoricalHashesAccumulator, BlockProofHistoricalRoots, BlockProofHistoricalSummaries]))} +// AccumulatorRoot = { type: 0x08, data: hash_tree_root(List(HeaderRecord, 8192)) } +// BlockIndex = { type: 0x3266, data: block-index } +// TotalDifficulty is little-endian encoded. + +// AccumulatorRoot is only defined for epochs with pre-merge data. +// HeaderRecord is defined in the Portal Network specification[^5]. + +// BlockIndex stores relative offsets to each compressed block entry. The format is: + +// block-index := starting-number | index | index | index ... | count +// All values in the block index are little-endian uint64. + +// starting-number is the first block number in the archive. Every index is a defined relative to index's location in the file. The total number of block entries in the file is recorded in count. + +// Due to the accumulator size limit of 8192, the maximum number of blocks in an Era batch is also 8192. This is also the value of SLOTS_PER_HISTORICAL_ROOT[^6] on the Beacon chain, so it is nice to align on the value. + +import ( + "bytes" + "fmt" + "io" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/internal/era/e2store" + "github.com/golang/snappy" +) + +const ( + TypeVersion uint16 = 0x3265 + TypeCompressedHeader uint16 = 0x08 + TypeCompressedBody uint16 = 0x09 + TypeCompressedReceipts uint16 = 0x0a + TypeTotalDifficulty uint16 = 0x0b + TypeProof uint16 = 0x0c + TypeAccumulator uint16 = 0x0d + TypeBlockIndex uint16 = 0x3267 + + MaxEraESize = 8192 +) + +type Builder2 struct { + w *e2store.Writer + buf *bytes.Buffer + sn *snappy.Writer + + // buffered entries per type: + headersRLP [][]byte + bodiesRLP [][]byte + receiptsRLP [][]byte + proofsRLP [][]byte + tds []*big.Int + + headeroffsets []uint64 + bodyoffsets []uint64 + receiptoffsets []uint64 + proofoffsets []uint64 + startTd *big.Int + + startNum *uint64 + hashes []common.Hash + writtenBytes uint64 +} + +// NewBuilder returns a new EraE Builder writing into the given io.Writer. +func NewBuilder2(w io.Writer) *Builder2 { + buf := bytes.NewBuffer(nil) + return &Builder2{ + w: e2store.NewWriter(w), + buf: buf, + sn: snappy.NewBufferedWriter(buf), + } +} + +func (b *Builder) Add(block *types.Block, receipts types.Receipts, td *big.Int, proofRLP []byte) error { + if len(b.headersRLP) >= MaxEraESize { + return fmt.Errorf("exceeds maximum batch size of %d", MaxEraESize) + } + + headerb, err := b.encodeHeader(block.Header()) + if err != nil { + return fmt.Errorf("failed to encode header: %w", err) + } + bodyb, err := b.encodeBody(block.Body()) + if err != nil { + return fmt.Errorf("failed to encode body: %w", err) + } + receiptsb, err := b.encodeReceipts(receipts) + if err != nil { + return fmt.Errorf("failed to encode receipts: %w", err) + } + + tdbytes := uint256LE(td) + + if b.startNum == nil { + start := block.NumberU64() + b.startNum = &start + _, err := b.w.Write(TypeVersion, nil) + if err != nil { + return fmt.Errorf("failed to write version entry: %w", err) + } + b.writtenBytes += 8 + } + + b.headersRLP = append(b.headersRLP, headerb) + b.bodiesRLP = append(b.bodiesRLP, bodyb) + b.receiptsRLP = append(b.receiptsRLP, receiptsb) + b.tds = append(b.tds, tdbytes) + b.proofsRLP = append(b.proofsRLP, proofRLP) + b.hashes = append(b.hashes, block.Hash()) + return nil +} + +func (b *Builder2) Finalize(common.Hash, error) { + if b.startNum == nil { + return fmt.Errorf("no blocks added, cannot finalize") + } + + offs := snappy.Encode(b.buf, b.headersRLP) +} + +func uint256LE(v *big.Int) []byte { + b := v.FillBytes(make([]byte, 32)) + for i := 0; i < 16; i++ { + b[i], b[31-i] = b[31-i], b[i] + } + return b +} + +func decodeBigs(raw [][]byte) []*big.Int { + out := make([]*big.Int, len(raw)) + for i, le := range raw { + be := make([]byte, 32) + for j := 0; j < 32; j++ { + be[j] = le[31-j] + } + out[i] = new(big.Int).SetBytes(be) + } + return out +} diff --git a/internal/era2/era2.go b/internal/era2/era2.go new file mode 100644 index 0000000000..e8a9d89e53 --- /dev/null +++ b/internal/era2/era2.go @@ -0,0 +1,38 @@ +package era + +import ( + "github.com/ethereum/go-ethereum/common" +) + +type BlockProofHistoricalHashesAccumulator [15]common.Hash // 15 * 32 = 480 bytes + +// BlockProofHistoricalRoots – Altair / Bellatrix historical_roots branch. +type BlockProofHistoricalRoots struct { + BeaconBlockProof [14]common.Hash // 448 + BeaconBlockRoot common.Hash // 32 + ExecutionBlockProof [11]common.Hash // 352 + Slot uint64 // 8 => 840 bytes +} + +// BlockProofHistoricalSummariesCapella – Capella historical_summaries branch. +type BlockProofHistoricalSummariesCapella struct { + BeaconBlockProof [13]common.Hash // 416 + BeaconBlockRoot common.Hash // 32 + ExecutionBlockProof [11]common.Hash // 352 + Slot uint64 // 8 => 808 bytes +} + +// BlockProofHistoricalSummariesDeneb – Deneb historical_summaries branch. +type BlockProofHistoricalSummariesDeneb struct { + BeaconBlockProof [13]common.Hash // 416 + BeaconBlockRoot common.Hash // 32 + ExecutionBlockProof [12]common.Hash // 384 + Slot uint64 // 8 => 840 bytes +} + +type Proofs struct { + HistoricalHashesAccumulator BlockProofHistoricalHashesAccumulator + HistoricalRootsProof *BlockProofHistoricalRoots + SummariesProofCapella *BlockProofHistoricalSummariesCapella + SummariesProofDeneb *BlockProofHistoricalSummariesDeneb +} diff --git a/internal/era2/iterator.go b/internal/era2/iterator.go new file mode 100644 index 0000000000..3c4f82d850 --- /dev/null +++ b/internal/era2/iterator.go @@ -0,0 +1,197 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package era + +import ( + "errors" + "io" + "math/big" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" +) + +// Iterator wraps RawIterator and returns decoded Era1 entries. +type Iterator struct { + inner *RawIterator +} + +// NewIterator returns a new Iterator instance. Next must be immediately +// called on new iterators to load the first item. +func NewIterator(e *Era) (*Iterator, error) { + inner, err := NewRawIterator(e) + if err != nil { + return nil, err + } + return &Iterator{inner}, nil +} + +// Next moves the iterator to the next block entry. It returns false when all +// items have been read or an error has halted its progress. Block, Receipts, +// and BlockAndReceipts should no longer be called after false is returned. +func (it *Iterator) Next() bool { + return it.inner.Next() +} + +// Number returns the current number block the iterator will return. +func (it *Iterator) Number() uint64 { + return it.inner.next - 1 +} + +// Error returns the error status of the iterator. It should be called before +// reading from any of the iterator's values. +func (it *Iterator) Error() error { + return it.inner.Error() +} + +// Block returns the block for the iterator's current position. +func (it *Iterator) Block() (*types.Block, error) { + if it.inner.Header == nil || it.inner.Body == nil { + return nil, errors.New("header and body must be non-nil") + } + var ( + header types.Header + body types.Body + ) + if err := rlp.Decode(it.inner.Header, &header); err != nil { + return nil, err + } + if err := rlp.Decode(it.inner.Body, &body); err != nil { + return nil, err + } + return types.NewBlockWithHeader(&header).WithBody(body), nil +} + +// Receipts returns the receipts for the iterator's current position. +func (it *Iterator) Receipts() (types.Receipts, error) { + if it.inner.Receipts == nil { + return nil, errors.New("receipts must be non-nil") + } + var receipts types.Receipts + err := rlp.Decode(it.inner.Receipts, &receipts) + return receipts, err +} + +// BlockAndReceipts returns the block and receipts for the iterator's current +// position. +func (it *Iterator) BlockAndReceipts() (*types.Block, types.Receipts, error) { + b, err := it.Block() + if err != nil { + return nil, nil, err + } + r, err := it.Receipts() + if err != nil { + return nil, nil, err + } + return b, r, nil +} + +// TotalDifficulty returns the total difficulty for the iterator's current +// position. +func (it *Iterator) TotalDifficulty() (*big.Int, error) { + td, err := io.ReadAll(it.inner.TotalDifficulty) + if err != nil { + return nil, err + } + return new(big.Int).SetBytes(reverseOrder(td)), nil +} + +// RawIterator reads an RLP-encode Era1 entries. +type RawIterator struct { + e *Era // backing Era1 + next uint64 // next block to read + err error // last error + + Header io.Reader + Body io.Reader + Receipts io.Reader + TotalDifficulty io.Reader +} + +// NewRawIterator returns a new RawIterator instance. Next must be immediately +// called on new iterators to load the first item. +func NewRawIterator(e *Era) (*RawIterator, error) { + return &RawIterator{ + e: e, + next: e.m.start, + }, nil +} + +// Next moves the iterator to the next block entry. It returns false when all +// items have been read or an error has halted its progress. Header, Body, +// Receipts, TotalDifficulty will be set to nil in the case returning false or +// finding an error and should therefore no longer be read from. +func (it *RawIterator) Next() bool { + // Clear old errors. + it.err = nil + if it.e.m.start+it.e.m.count <= it.next { + it.clear() + return false + } + off, err := it.e.readOffset(it.next) + if err != nil { + // Error here means block index is corrupted, so don't + // continue. + it.clear() + it.err = err + return false + } + var n int64 + if it.Header, n, it.err = newSnappyReader(it.e.s, TypeCompressedHeader, off); it.err != nil { + it.clear() + return true + } + off += n + if it.Body, n, it.err = newSnappyReader(it.e.s, TypeCompressedBody, off); it.err != nil { + it.clear() + return true + } + off += n + if it.Receipts, n, it.err = newSnappyReader(it.e.s, TypeCompressedReceipts, off); it.err != nil { + it.clear() + return true + } + off += n + if it.TotalDifficulty, _, it.err = it.e.s.ReaderAt(TypeTotalDifficulty, off); it.err != nil { + it.clear() + return true + } + it.next += 1 + return true +} + +// Number returns the current number block the iterator will return. +func (it *RawIterator) Number() uint64 { + return it.next - 1 +} + +// Error returns the error status of the iterator. It should be called before +// reading from any of the iterator's values. +func (it *RawIterator) Error() error { + if it.err == io.EOF { + return nil + } + return it.err +} + +// clear sets all the outputs to nil. +func (it *RawIterator) clear() { + it.Header = nil + it.Body = nil + it.Receipts = nil + it.TotalDifficulty = nil +}