mirror of
https://github.com/ethereum/go-ethereum.git
synced 2026-07-21 20:26:41 +00:00
305 lines
9 KiB
Go
305 lines
9 KiB
Go
package era2
|
|
|
|
// The format can be summarized with the following expression:
|
|
|
|
// eraE := Version | CompressedHeader* | CompressedBody* | CompressedReceipts* | TotalDifficulty* | Proofs* | other-entries* | Accumulator | BlockIndex
|
|
// Each basic element is its own e2store entry:
|
|
|
|
// Version = { type: 0x3265, data: nil }
|
|
// CompressedHeader = { type: 0x03, data: snappyFramed(rlp(header)) }
|
|
// CompressedBody = { type: 0x04, data: snappyFramed(rlp(body)) }
|
|
// CompressedReceipts = { type: 0x05, data: snappyFramed(rlp([tx-type, post-state-or-status, cumulative-gas, logs])) }
|
|
// TotalDifficulty = { type: 0x06, data: uint256(header.total_difficulty) }
|
|
// Proofs = { type: 0x07 data: snappyFramed(rlp([BlockProofHistoricalHashesAccumulator, BlockProofHistoricalRoots, BlockProofHistoricalSummaries]))}
|
|
// AccumulatorRoot = { type: 0x08, data: hash_tree_root(List(HeaderRecord, 8192)) }
|
|
// BlockIndex = { type: 0x3266, data: block-index }
|
|
// TotalDifficulty is little-endian encoded.
|
|
|
|
// AccumulatorRoot is only defined for epochs with pre-merge data.
|
|
// HeaderRecord is defined in the Portal Network specification[^5].
|
|
|
|
// BlockIndex stores relative offsets to each compressed block entry. The format is:
|
|
|
|
// block-index := starting-number | index | index | index ... | count
|
|
// All values in the block index are little-endian uint64.
|
|
|
|
// starting-number is the first block number in the archive. Every index is a defined relative to index's location in the file. The total number of block entries in the file is recorded in count.
|
|
|
|
// Due to the accumulator size limit of 8192, the maximum number of blocks in an Era batch is also 8192. This is also the value of SLOTS_PER_HISTORICAL_ROOT[^6] on the Beacon chain, so it is nice to align on the value.
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/binary"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"math/big"
|
|
|
|
"github.com/ethereum/go-ethereum/common"
|
|
"github.com/ethereum/go-ethereum/core/types"
|
|
"github.com/ethereum/go-ethereum/internal/era/e2store"
|
|
"github.com/ethereum/go-ethereum/rlp"
|
|
"github.com/golang/snappy"
|
|
)
|
|
|
|
const (
|
|
TypeVersion uint16 = 0x3265
|
|
TypeCompressedHeader uint16 = 0x08
|
|
TypeCompressedBody uint16 = 0x09
|
|
TypeCompressedReceipts uint16 = 0x0a
|
|
TypeTotalDifficulty uint16 = 0x0b
|
|
TypeProofHistoricalHashesAccumulator uint16 = 0x0c
|
|
TypeProofHistoricalRoots uint16 = 0x0d
|
|
TypeProofHistoricalSummariesCapella uint16 = 0x0e
|
|
TypeProofHistoricalSummariesDeneb uint16 = 0x0f
|
|
TypeAccumulatorRoot uint16 = 0x1a
|
|
TypeBlockIndex uint16 = 0x3267
|
|
MaxEraESize int = 8192
|
|
headerSize uint64 = 8
|
|
)
|
|
|
|
type proofvar uint16
|
|
|
|
type Builder struct {
|
|
w *e2store.Writer
|
|
buf *bytes.Buffer
|
|
sn *snappy.Writer
|
|
|
|
// buffered entries per type:
|
|
headersRLP [][]byte
|
|
bodiesRLP [][]byte
|
|
receiptsRLP [][]byte
|
|
proofsRLP [][]byte
|
|
tds [][]byte
|
|
tdsint []*big.Int
|
|
hashes []common.Hash
|
|
|
|
headeroffsets []uint64
|
|
bodyoffsets []uint64
|
|
receiptoffsets []uint64
|
|
proofoffsets []uint64
|
|
tdoff []uint64
|
|
startTd *big.Int
|
|
|
|
prooftype proofvar
|
|
|
|
startNum *uint64
|
|
writtenBytes uint64
|
|
}
|
|
|
|
// NewBuilder returns a new EraE Builder writing into the given io.Writer.
|
|
func NewBuilder(w io.Writer) *Builder {
|
|
buf := bytes.NewBuffer(nil)
|
|
return &Builder{
|
|
w: e2store.NewWriter(w),
|
|
buf: buf,
|
|
sn: snappy.NewBufferedWriter(buf),
|
|
}
|
|
}
|
|
|
|
func (b *Builder) Add(block *types.Block, receipts types.Receipts, td *big.Int, proofBytes []byte, proofty proofvar) error {
|
|
if len(b.headersRLP) >= MaxEraESize {
|
|
return fmt.Errorf("exceeds MaxEraESize %d", MaxEraESize)
|
|
}
|
|
|
|
if proofty != 0 && len(proofBytes) == 0 {
|
|
return fmt.Errorf("proof type %d requires proof bytes", proofty)
|
|
}
|
|
|
|
if len(b.headersRLP) != 0 && proofty != b.prooftype {
|
|
return fmt.Errorf("cannot mix proof types, expected %d, got %d", b.prooftype, proofty)
|
|
}
|
|
|
|
hdr, err := rlp.EncodeToBytes(block.Header())
|
|
if err != nil {
|
|
return fmt.Errorf("error encoding block header: %w", err)
|
|
}
|
|
bod, err := rlp.EncodeToBytes(block.Body())
|
|
if err != nil {
|
|
return fmt.Errorf("error encoding block header: %w", err)
|
|
}
|
|
rct, err := rlp.EncodeToBytes(receipts)
|
|
if err != nil {
|
|
return fmt.Errorf("error encoding block header: %w", err)
|
|
}
|
|
|
|
b.headersRLP = append(b.headersRLP, hdr)
|
|
b.bodiesRLP = append(b.bodiesRLP, bod)
|
|
b.receiptsRLP = append(b.receiptsRLP, rct)
|
|
b.tds = append(b.tds, uint256LE(td))
|
|
b.tdsint = append(b.tdsint, new(big.Int).Set(td))
|
|
b.hashes = append(b.hashes, block.Hash())
|
|
|
|
if b.prooftype != 0 {
|
|
b.proofsRLP = append(b.proofsRLP, proofBytes)
|
|
}
|
|
|
|
if b.startNum == nil {
|
|
sn := block.NumberU64()
|
|
b.startNum = &sn
|
|
if n, err := b.w.Write(TypeVersion, nil); err != nil {
|
|
return fmt.Errorf("error writing version entry: %w", err)
|
|
} else {
|
|
b.writtenBytes += uint64(n)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (b *Builder) Finalize() error {
|
|
if b.startNum == nil {
|
|
return errors.New("no blocks added, cannot finalize")
|
|
}
|
|
var err error
|
|
b.headeroffsets, err = b.writeSection(TypeCompressedHeader, b.headersRLP, true)
|
|
if err != nil {
|
|
return fmt.Errorf("error writing compressed headers: %w", err)
|
|
}
|
|
b.bodyoffsets, err = b.writeSection(TypeCompressedBody, b.bodiesRLP, true)
|
|
if err != nil {
|
|
return fmt.Errorf("error writing compressed bodies: %w", err)
|
|
}
|
|
b.receiptoffsets, err = b.writeSection(TypeCompressedReceipts, b.receiptsRLP, true)
|
|
if err != nil {
|
|
return fmt.Errorf("error writing compressed receipts: %w", err)
|
|
}
|
|
|
|
if len(b.tds) > 0 {
|
|
b.tdoff, err = b.writeSection(TypeTotalDifficulty, b.tds, false)
|
|
if err != nil {
|
|
return fmt.Errorf("error writing total difficulties: %w", err)
|
|
}
|
|
}
|
|
|
|
if b.prooftype != 0 {
|
|
b.proofoffsets, err = b.writeSection(uint16(b.prooftype), b.proofsRLP, true)
|
|
if err != nil {
|
|
return fmt.Errorf("error writing proofs: %w", err)
|
|
}
|
|
}
|
|
|
|
if len(b.hashes) > 0 {
|
|
accRoot, err := ComputeAccumulator(b.hashes, b.tdsint)
|
|
if err != nil {
|
|
return fmt.Errorf("error calculating accumulator root: %w", err)
|
|
}
|
|
if n, err := b.w.Write(TypeAccumulatorRoot, accRoot[:]); err != nil {
|
|
return fmt.Errorf("error writing accumulator root: %w", err)
|
|
} else {
|
|
b.writtenBytes += uint64(n)
|
|
}
|
|
}
|
|
|
|
return b.writeIndex()
|
|
|
|
}
|
|
|
|
func uint256LE(v *big.Int) []byte {
|
|
b := v.FillBytes(make([]byte, 32))
|
|
for i := 0; i < 16; i++ {
|
|
b[i], b[31-i] = b[31-i], b[i]
|
|
}
|
|
return b
|
|
}
|
|
|
|
func decodeBigs(raw [][]byte) []*big.Int {
|
|
out := make([]*big.Int, len(raw))
|
|
for i, le := range raw {
|
|
be := make([]byte, 32)
|
|
for j := 0; j < 32; j++ {
|
|
be[j] = le[31-j]
|
|
}
|
|
out[i] = new(big.Int).SetBytes(be)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// snappyWrite is a small helper to take care snappy encoding and writing an e2store entry.
|
|
func (b *Builder) snappyWrite(typ uint16, in []byte) error {
|
|
var (
|
|
buf = b.buf
|
|
s = b.sn
|
|
)
|
|
buf.Reset()
|
|
s.Reset(buf)
|
|
if _, err := b.sn.Write(in); err != nil {
|
|
return fmt.Errorf("error snappy encoding: %w", err)
|
|
}
|
|
if err := s.Flush(); err != nil {
|
|
return fmt.Errorf("error flushing snappy encoding: %w", err)
|
|
}
|
|
n, err := b.w.Write(typ, b.buf.Bytes())
|
|
b.writtenBytes += uint64(n)
|
|
if err != nil {
|
|
return fmt.Errorf("error writing e2store entry: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (b *Builder) writeSection(typ uint16, list [][]byte, useSnappy bool) ([]uint64, error) {
|
|
if len(list) == 0 {
|
|
return nil, errors.New("cannot write empty section")
|
|
}
|
|
|
|
buf := bytes.NewBuffer(nil)
|
|
offs := make([]uint64, len(list))
|
|
for i, data := range list {
|
|
offs[i] = b.writtenBytes + headerSize + uint64(buf.Len())
|
|
if useSnappy {
|
|
buf.Write(snappy.Encode(nil, data))
|
|
} else {
|
|
buf.Write(data)
|
|
}
|
|
}
|
|
|
|
if n, err := b.w.Write(typ, buf.Bytes()); err != nil {
|
|
return nil, fmt.Errorf("error writing section %d: %w", typ, err)
|
|
} else {
|
|
b.writtenBytes += uint64(n)
|
|
}
|
|
|
|
return offs, nil
|
|
}
|
|
|
|
func (b *Builder) writeIndex() error {
|
|
count := uint64(len(b.headeroffsets))
|
|
compcount := uint64(3)
|
|
if len(b.tds) > 0 {
|
|
compcount++
|
|
}
|
|
if len(b.proofsRLP) > 0 {
|
|
compcount++
|
|
}
|
|
|
|
idx := make([]byte, 8+count*8*compcount+16) //8 for start block, 8 per property per block, 16 for the number of properties and the number of blocks
|
|
binary.LittleEndian.PutUint64(idx, *b.startNum)
|
|
base := int64(b.writtenBytes)
|
|
pos := 8
|
|
rel := func(abs uint64) uint64 { return uint64(int64(abs) - base) }
|
|
for i := uint64(0); i < count; i++ {
|
|
binary.LittleEndian.PutUint64(idx[pos:], rel(b.headeroffsets[i]))
|
|
pos += 8
|
|
binary.LittleEndian.PutUint64(idx[pos:], rel(b.bodyoffsets[i]))
|
|
pos += 8
|
|
binary.LittleEndian.PutUint64(idx[pos:], rel(b.receiptoffsets[i]))
|
|
pos += 8
|
|
if len(b.tds) > 0 {
|
|
binary.LittleEndian.PutUint64(idx[pos+24:], rel(b.tdoff[i]))
|
|
pos += 8
|
|
}
|
|
if len(b.proofsRLP) > 0 {
|
|
binary.LittleEndian.PutUint64(idx[pos:], rel(b.proofoffsets[i]))
|
|
pos += 8
|
|
}
|
|
}
|
|
|
|
binary.LittleEndian.PutUint64(idx[pos:], compcount)
|
|
pos += 8
|
|
binary.LittleEndian.PutUint64(idx[pos:], count)
|
|
if n, err := b.w.Write(TypeBlockIndex, idx); err != nil {
|
|
return err
|
|
} else {
|
|
b.writtenBytes += uint64(n)
|
|
}
|
|
return nil
|
|
}
|