Updated with all comments.

Fixed all issues including extra logic regarding proof types, modularizing some functions and refactoring code for correctness and readability.

internal/era2/builder.go

@@ -48,11 +48,8 @@ const (
 	TypeCompressedBody     uint16 = 0x09
 	TypeCompressedReceipts uint16 = 0x0a
 	TypeTotalDifficulty    uint16 = 0x0b
-	TypeProofHistoricalHashesAccumulator uint16 = 0x0c
+	TypeProof              uint16 = 0x0c
-	TypeProofHistoricalRoots             uint16 = 0x0d
+	TypeAccumulatorRoot    uint16 = 0x0d
-	TypeProofHistoricalSummariesCapella  uint16 = 0x0e
-	TypeProofHistoricalSummariesDeneb    uint16 = 0x0f
-	TypeAccumulatorRoot                  uint16 = 0x1a
 	TypeBlockIndex         uint16 = 0x3267
 	MaxEraESize            int    = 8192
 	headerSize             uint64 = 8
@@ -61,11 +58,11 @@ const (
 type proofvar uint16
 
 const (
-	ProofNone    proofvar = 0
+	ProofNone proofvar = iota
-	ProofHHA     proofvar = proofvar(TypeProofHistoricalHashesAccumulator)
+	ProofHHA
-	ProofRoots   proofvar = proofvar(TypeProofHistoricalRoots)
+	ProofRoots
-	ProofCapella proofvar = proofvar(TypeProofHistoricalSummariesCapella)
+	ProofCapella
-	ProofDeneb   proofvar = proofvar(TypeProofHistoricalSummariesDeneb)
+	ProofDeneb
 )
 
 // Proof bundles variant + compressed bytes.
@@ -79,11 +76,10 @@ type Builder struct {
 	buf *bytes.Buffer
 	sn  *snappy.Writer
 
-	// buffered entries per type:
+	headers  [][]byte
-	headersRLP  [][]byte
+	bodies   [][]byte
-	bodiesRLP   [][]byte
+	receipts [][]byte
-	receiptsRLP [][]byte
+	proofs   [][]byte
-	proofsRLP   [][]byte
 	tds      [][]byte
 	tdsint   []*big.Int
 	hashes   []common.Hash
@@ -93,7 +89,6 @@ type Builder struct {
 	receiptoffsets []uint64
 	proofoffsets   []uint64
 	tdoff          []uint64
-	startTd        *big.Int
 
 	prooftype proofvar
 
@@ -101,6 +96,7 @@ type Builder struct {
 	writtenBytes uint64
 }
 
+// NewBuilder returns a new Builder instance.
 func NewBuilder(w io.Writer) *Builder {
 	buf := bytes.NewBuffer(nil)
 	return &Builder{
@@ -110,56 +106,69 @@ func NewBuilder(w io.Writer) *Builder {
 	}
 }
 
-func (b *Builder) Add(header types.Header, body types.Body, receipts types.Receipts, blockhash common.Hash, blocknum uint64, td *big.Int, proof *Proof) error {
+// Add writes a block entry, its reciepts, and optionally its proofs as well into the e2store file.
-	if len(b.headersRLP) >= MaxEraESize {
+func (b *Builder) Add(header types.Header, body types.Body, receipts types.Receipts, td *big.Int, proof *Proof) error {
-		return fmt.Errorf("exceeds MaxEraESize %d", MaxEraESize)
-	}
-
 	var pv proofvar = ProofNone
 	var pData []byte
 	if proof != nil {
-		pv = proof.Variant
+		pv, pData = proof.Variant, proof.Data
-		pData = proof.Data
 		if pv == ProofNone || len(pData) == 0 {
 			return fmt.Errorf("invalid proof: variant=%d len=%d", pv, len(pData))
 		}
 	}
 
-	if len(b.headersRLP) == 0 {
+	if len(b.headers) == 0 {
 		b.prooftype = pv
 	} else if pv != b.prooftype {
-		return fmt.Errorf("cannot mix proof variants: have %d want %d", b.prooftype, pv)
+		return fmt.Errorf("cannot mix proof variants: first=%d now=%d", b.prooftype, pv)
 	}
 
-	hdr, err := rlp.EncodeToBytes(&header)
+	eh, err := rlp.EncodeToBytes(&header)
 	if err != nil {
-		return fmt.Errorf("error encoding block header: %w", err)
+		return fmt.Errorf("encode header: %w", err)
 	}
-	bod, err := rlp.EncodeToBytes(&body)
+	eb, err := rlp.EncodeToBytes(&body)
 	if err != nil {
-		return fmt.Errorf("error encoding block header: %w", err)
+		return fmt.Errorf("encode body: %w", err)
 	}
-	rct, err := rlp.EncodeToBytes(receipts)
+	er, err := rlp.EncodeToBytes(receipts)
 	if err != nil {
-		return fmt.Errorf("error encoding block header: %w", err)
+		return fmt.Errorf("encode receipts: %w", err)
 	}
 
-	b.headersRLP = append(b.headersRLP, hdr)
+	var ep []byte
-	b.bodiesRLP = append(b.bodiesRLP, bod)
+	if pv != ProofNone {
-	b.receiptsRLP = append(b.receiptsRLP, rct)
+		ep, err = rlp.EncodeToBytes([]interface{}{uint16(pv), proof.Data})
+		if err != nil {
+			return fmt.Errorf("encode proof: %w", err)
+		}
+	}
+
+	return b.AddRLP(eh, eb, er, ep, header.Number.Uint64(), header.Hash(), td)
+}
+
+// AddRLP takes the RLP encoded block components and writes them to the underlying e2store file
+func (b *Builder) AddRLP(headerRLP []byte, bodyRLP []byte, receipts []byte, proof []byte, blockNum uint64, blockHash common.Hash, td *big.Int) error {
+	if len(b.headers) >= MaxEraESize {
+		return fmt.Errorf("exceeds MaxEraESize %d", MaxEraESize)
+	}
+
+	b.headers = append(b.headers, headerRLP)
+	b.bodies = append(b.bodies, bodyRLP)
+	b.receipts = append(b.receipts, receipts)
 	b.tds = append(b.tds, uint256LE(td))
 	b.tdsint = append(b.tdsint, new(big.Int).Set(td))
-	b.hashes = append(b.hashes, blockhash)
+	b.hashes = append(b.hashes, blockHash)
-
+	if proof != nil {
-	if b.prooftype != ProofNone {
+		b.proofs = append(b.proofs, proof)
-		b.proofsRLP = append(b.proofsRLP, proof.Data)
 	}
 
+	//Write Era2 version before writing any blocks.
 	if b.startNum == nil {
-		sn := blocknum
+		b.startNum = new(uint64)
-		b.startNum = &sn
+		*b.startNum = blockNum
 		if n, err := b.w.Write(TypeVersion, nil); err != nil {
-			return fmt.Errorf("error writing version entry: %w", err)
+			return fmt.Errorf("write version entry: %w", err)
 		} else {
 			b.writtenBytes += uint64(n)
 		}
@@ -167,18 +176,19 @@ func (b *Builder) Add(header types.Header, body types.Body, receipts types.Recei
 	return nil
 }
 
+// Finalize flushes all entries present in cache to the underlying e2store file.
 func (b *Builder) Finalize() (common.Hash, error) {
 	if b.startNum == nil {
 		return common.Hash{}, errors.New("no blocks added, cannot finalize")
 	}
 	var err error
-	if err := b.flushKind(TypeCompressedHeader, b.headersRLP, true, &b.headeroffsets); err != nil {
+	if err := b.flushKind(TypeCompressedHeader, b.headers, true, &b.headeroffsets); err != nil {
 		return common.Hash{}, err
 	}
-	if err := b.flushKind(TypeCompressedBody, b.bodiesRLP, true, &b.bodyoffsets); err != nil {
+	if err := b.flushKind(TypeCompressedBody, b.bodies, true, &b.bodyoffsets); err != nil {
 		return common.Hash{}, err
 	}
-	if err := b.flushKind(TypeCompressedReceipts, b.receiptsRLP, true, &b.receiptoffsets); err != nil {
+	if err := b.flushKind(TypeCompressedReceipts, b.receipts, true, &b.receiptoffsets); err != nil {
 		return common.Hash{}, err
 	}
 
@@ -188,7 +198,7 @@ func (b *Builder) Finalize() (common.Hash, error) {
 		}
 	}
 	if b.prooftype != ProofNone {
-		if err := b.flushKind(uint16(b.prooftype), b.proofsRLP, true, &b.proofoffsets); err != nil {
+		if err := b.flushKind(TypeProof, b.proofs, true, &b.proofoffsets); err != nil {
 			return common.Hash{}, err
 		}
 	}
@@ -208,6 +218,7 @@ func (b *Builder) Finalize() (common.Hash, error) {
 	return accRoot, b.writeIndex()
 }
 
+// Writes 32 byte big integers to little endian
 func uint256LE(v *big.Int) []byte {
 	b := v.FillBytes(make([]byte, 32))
 	for i := 0; i < 16; i++ {
@@ -216,18 +227,7 @@ func uint256LE(v *big.Int) []byte {
 	return b
 }
 
-func decodeBigs(raw [][]byte) []*big.Int {
+// Compresses into snappy encoding
-	out := make([]*big.Int, len(raw))
-	for i, le := range raw {
-		be := make([]byte, 32)
-		for j := 0; j < 32; j++ {
-			be[j] = le[31-j]
-		}
-		out[i] = new(big.Int).SetBytes(be)
-	}
-	return out
-}
-
 func (b *Builder) snappyWrite(typ uint16, in []byte) error {
 	var (
 		buf = b.buf
@@ -249,6 +249,7 @@ func (b *Builder) snappyWrite(typ uint16, in []byte) error {
 	return nil
 }
 
+// Add entry takes the e2store object and writes it into the file
 func (b *Builder) addEntry(typ uint16, payload []byte, snappyIt bool) (uint64, error) {
 	offset := b.writtenBytes
 	var err error
@@ -266,6 +267,7 @@ func (b *Builder) addEntry(typ uint16, payload []byte, snappyIt bool) (uint64, e
 	return offset, nil
 }
 
+// flush kind takes all entries of the cached component and flushes it to the file
 func (b *Builder) flushKind(typ uint16, list [][]byte, useSnappy bool, dst *[]uint64) error {
 	for _, data := range list {
 		off, err := b.addEntry(typ, data, useSnappy)
@@ -277,13 +279,15 @@ func (b *Builder) flushKind(typ uint16, list [][]byte, useSnappy bool, dst *[]ui
 	return nil
 }
 
+// write index takes all the offset table and writes it to the file
+// the index table contains all offsets to specific block entries
 func (b *Builder) writeIndex() error {
 	count := uint64(len(b.headeroffsets))
 	compcount := uint64(3)
 	if len(b.tds) > 0 {
 		compcount++
 	}
-	if len(b.proofsRLP) > 0 {
+	if len(b.proofs) > 0 {
 		compcount++
 	}
 
@@ -303,7 +307,7 @@ func (b *Builder) writeIndex() error {
 			binary.LittleEndian.PutUint64(idx[pos:], rel(b.tdoff[i]))
 			pos += 8
 		}
-		if len(b.proofsRLP) > 0 {
+		if len(b.proofs) > 0 {
 			binary.LittleEndian.PutUint64(idx[pos:], rel(b.proofoffsets[i]))
 			pos += 8
 		}

internal/era2/era.go

@@ -6,7 +6,6 @@ import (
 	"io"
 	"math/big"
 	"os"
-	"sync"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
@@ -15,11 +14,11 @@ import (
 	"github.com/klauspost/compress/snappy"
 )
 
-type meta struct {
+type metadata struct {
 	start      uint64 // start block number
 	count      uint64 // number of blocks in the era
-	compcount uint64 // number of properties
+	components uint64 // number of properties
-	filelen   int64  // length of the file in bytes
+	length     int64  // length of the file in bytes
 }
 
 type BlockProofHistoricalHashesAccumulator [15]common.Hash // 15 * 32 = 480 bytes
@@ -48,31 +47,28 @@ type BlockProofHistoricalSummariesDeneb struct {
 	Slot                uint64          // 8  => 840 bytes
 }
 
-// BlockAccumulatorRoot is the SSZ hash tree root of the Era2 block accumulator.
-
 type ReadAtSeekCloser interface {
 	io.ReaderAt
 	io.Seeker
 	io.Closer
 }
 
-type Era2 struct {
+type Era struct {
 	f                                                 ReadAtSeekCloser
 	s                                                 *e2store.Reader
-	m                                                 meta // metadata for the era2 file
+	m                                                 metadata // metadata for the Era file
-	mu                                                *sync.Mutex
 	headeroff, bodyoff, receiptsoff, tdoff, proofsoff []uint64 // offsets for each entry type
 	indstart                                          int64
 	rootheader                                        uint64 // offset of the root header in the file if present
-	prooftype                                         uint16
 }
 
-func Open(path string) (*Era2, error) {
+// Opens era file
+func Open(path string) (*Era, error) {
 	f, err := os.Open(path)
 	if err != nil {
 		return nil, err
 	}
-	e := &Era2{f: f, s: e2store.NewReader(f)}
+	e := &Era{f: f, s: e2store.NewReader(f)}
 	if err := e.loadIndex(); err != nil {
 		f.Close()
 		return nil, err
@@ -80,7 +76,8 @@ func Open(path string) (*Era2, error) {
 	return e, nil
 }
 
-func (e *Era2) Close() error {
+// Closes era file
+func (e *Era) Close() error {
 	if e.f == nil {
 		return nil
 	}
@@ -89,15 +86,18 @@ func (e *Era2) Close() error {
 	return err
 }
 
-func (e *Era2) Start() uint64 {
+// retrieves starting block number
+func (e *Era) Start() uint64 {
 	return e.m.start
 }
 
-func (e *Era2) Count() uint64 {
+// retrieves count of blocks present
+func (e *Era) Count() uint64 {
 	return e.m.count
 }
 
-func (e *Era2) GetBlockByNumber(blockNum uint64) (*types.Block, error) {
+// retrieves the block if present within the era file
+func (e *Era) GetBlockByNumber(blockNum uint64) (*types.Block, error) {
 	h, err := e.getHeader(blockNum)
 	if err != nil {
 		return nil, err
@@ -109,7 +109,8 @@ func (e *Era2) GetBlockByNumber(blockNum uint64) (*types.Block, error) {
 	return types.NewBlockWithHeader(h).WithBody(*b), nil
 }
 
-func (e *Era2) getHeader(blockNum uint64) (*types.Header, error) {
+// retrieves header from era file through the cached offset table
+func (e *Era) getHeader(blockNum uint64) (*types.Header, error) {
 	if blockNum < e.m.start || blockNum >= e.m.start+e.m.count {
 		return nil, fmt.Errorf("block number %d out of range [%d, %d)", blockNum, e.m.start, e.m.start+e.m.count)
 	}
@@ -123,7 +124,8 @@ func (e *Era2) getHeader(blockNum uint64) (*types.Header, error) {
 	return &h, rlp.Decode(r, &h)
 }
 
-func (e *Era2) getBody(blockNum uint64) (*types.Body, error) {
+// retrieves body from era file through cached offset table
+func (e *Era) getBody(blockNum uint64) (*types.Body, error) {
 	if blockNum < e.m.start || blockNum >= e.m.start+e.m.count {
 		return nil, fmt.Errorf("block number %d out of range [%d, %d)", blockNum, e.m.start, e.m.start+e.m.count)
 	}
@@ -137,7 +139,8 @@ func (e *Era2) getBody(blockNum uint64) (*types.Body, error) {
 	return &b, rlp.Decode(r, &b)
 }
 
-func (e *Era2) getTD(blockNum uint64) (*big.Int, error) {
+// retrieves td from era file through cached offset table
+func (e *Era) getTD(blockNum uint64) (*big.Int, error) {
 	if blockNum < e.m.start || blockNum >= e.m.start+e.m.count {
 		return nil, fmt.Errorf("block number %d out of range [%d, %d)",
 			blockNum, e.m.start, e.m.start+e.m.count)
@@ -155,7 +158,8 @@ func (e *Era2) getTD(blockNum uint64) (*big.Int, error) {
 	return td, nil
 }
 
-func (e *Era2) GetRawBodyFrameByNumber(blockNum uint64) ([]byte, error) {
+// retrieves the raw body frame in bytes of a specific block
+func (e *Era) GetRawBodyFrameByNumber(blockNum uint64) ([]byte, error) {
 	if blockNum < e.m.start || blockNum >= e.m.start+e.m.count {
 		return nil, fmt.Errorf("block number %d out of range [%d, %d)", blockNum, e.m.start, e.m.start+e.m.count)
 	}
@@ -166,7 +170,8 @@ func (e *Era2) GetRawBodyFrameByNumber(blockNum uint64) ([]byte, error) {
 	return io.ReadAll(r)
 }
 
-func (e *Era2) GetRawReceiptsFrameByNumber(blockNum uint64) ([]byte, error) {
+// retrieves the raw receipts frame in bytes of a specific block
+func (e *Era) GetRawReceiptsFrameByNumber(blockNum uint64) ([]byte, error) {
 	if blockNum < e.m.start || blockNum >= e.m.start+e.m.count {
 		return nil, fmt.Errorf("block number %d out of range [%d, %d)", blockNum, e.m.start, e.m.start+e.m.count)
 	}
@@ -177,46 +182,38 @@ func (e *Era2) GetRawReceiptsFrameByNumber(blockNum uint64) ([]byte, error) {
 	return io.ReadAll(r)
 }
 
-func (e *Era2) GetRawProofFrameByNumber(blockNum uint64) ([]byte, error) {
+// retrieves the raw proof frame in bytes of a specific block proof
+func (e *Era) GetRawProofFrameByNumber(blockNum uint64) ([]byte, error) {
 	if len(e.proofsoff) == 0 {
 		return nil, fmt.Errorf("proofs section not present")
 	}
 	if blockNum < e.m.start || blockNum >= e.m.start+e.m.count {
 		return nil, fmt.Errorf("block number %d out of range [%d, %d)", blockNum, e.m.start, e.m.start+e.m.count)
 	}
-	r, _, err := e.s.ReaderAt(e.prooftype, int64(e.proofsoff[blockNum-e.m.start]))
+	r, _, err := e.s.ReaderAt(TypeProof, int64(e.proofsoff[blockNum-e.m.start]))
 	if err != nil {
 		return nil, err
 	}
 	return io.ReadAll(r)
 }
 
-func (e *Era2) rawPayload(abs uint64) ([]byte, error) {
+// loads in the index table containing all offsets and caches it
-	sr := io.NewSectionReader(e.f, int64(abs), e.indstart-int64(abs))
+func (e *Era) loadIndex() error {
-	return io.ReadAll(sr)
-}
-
-func (e *Era2) snappyPayload(abs uint64) (io.Reader, error) {
-	sr := io.NewSectionReader(e.f, int64(abs), e.indstart-int64(abs))
-	return snappy.NewReader(sr), nil
-}
-
-func (e *Era2) loadIndex() error {
 	var err error
-	e.m.filelen, err = e.f.Seek(0, io.SeekEnd)
+	e.m.length, err = e.f.Seek(0, io.SeekEnd)
 	if err != nil {
 		return err
 	}
 
 	b := make([]byte, 16)
-	if _, err = e.f.ReadAt(b, e.m.filelen-16); err != nil {
+	if _, err = e.f.ReadAt(b, e.m.length-16); err != nil {
 		return err
 	}
-	e.m.compcount = binary.LittleEndian.Uint64(b[0:8])
+	e.m.components = binary.LittleEndian.Uint64(b[0:8])
 	e.m.count = binary.LittleEndian.Uint64(b[8:16])
 
-	payloadlen := 8 + 8*e.m.count*e.m.compcount + 16 // 8 for start block, 8 per property per block, 16 for the number of properties and the number of blocks
+	payloadlen := 8 + 8*e.m.count*e.m.components + 16 // 8 for start block, 8 per property per block, 16 for the number of properties and the number of blocks
-	tlvstart := e.m.filelen - int64(payloadlen) - 8
+	tlvstart := e.m.length - int64(payloadlen) - 8
 	e.indstart = tlvstart
 
 	_, err = e.f.ReadAt(b[:8], tlvstart+8)
@@ -225,7 +222,7 @@ func (e *Era2) loadIndex() error {
 	}
 
 	e.m.start = binary.LittleEndian.Uint64(b[:8])
-	num := int(e.m.compcount)
+	num := int(e.m.components)
 
 	totaloffsets := num * int(e.m.count)
 	offBytes := make([]byte, totaloffsets*8)
@@ -262,14 +259,6 @@ func (e *Era2) loadIndex() error {
 		}
 	}
 
-	if len(e.proofsoff) > 0 {
-		typ, _, perr := e.s.ReadMetadataAt(int64(e.proofsoff[0]))
-		if perr != nil {
-			return fmt.Errorf("read proof header: %w", perr)
-		}
-		e.prooftype = typ
-	}
-
 	var off int64 = 0 // start at byte-0 of file
 
 	for off < e.indstart { // never enter the Block-Index TLV
@@ -286,7 +275,9 @@ func (e *Era2) loadIndex() error {
 	return nil
 }
 
-func (e *Era2) BatchRange(first, count uint64, wantHdr, wantBody, wantRec, wantPrf bool) (hdrs []*types.Header, bods []*types.Body, recs []types.Receipts, prfs [][]byte, err error) {
+// retrieves a range of block from any start block to n number of blocks
+// the components retrieved can be customized depending on the properties wanted
+func (e *Era) BatchRange(first, count uint64, wantHdr, wantBody, wantRec, wantPrf bool) (hdrs []*types.Header, bods []*types.Body, recs []types.Receipts, prfs [][]byte, err error) {
 	if count == 0 {
 		err = fmt.Errorf("count must be > 0")
 		return
@@ -351,7 +342,7 @@ func (e *Era2) BatchRange(first, count uint64, wantHdr, wantBody, wantRec, wantP
 				err = fmt.Errorf("proofs section not present")
 				return
 			}
-			r, _, er := e.s.ReaderAt(e.prooftype, int64(e.proofsoff[id])) // type already checked when writing
+			r, _, er := e.s.ReaderAt(TypeProof, int64(e.proofsoff[id]))
 			if er != nil {
 				err = er
 				return
@@ -361,10 +352,3 @@ func (e *Era2) BatchRange(first, count uint64, wantHdr, wantBody, wantRec, wantP
 	}
 	return
 }
-
-func reverseOrder32(b []byte) []byte {
-	for i := 0; i < 16; i++ {
-		b[i], b[32-i-1] = b[32-i-1], b[i]
-	}
-	return b
-}

internal/era2/era_test.go

@@ -63,10 +63,9 @@ func TestEra2Builder(t *testing.T) {
 			header   = chain.headers[i]
 			body     = chain.bodies[i]
 			receipts = chain.receipts[i]
-			hash     = common.Hash{byte(i)}
 			td       = chain.tds[i]
 		)
-		if err = builder.Add(header, body, receipts, hash, uint64(i), td, nil); err != nil {
+		if err = builder.Add(header, body, receipts, td, nil); err != nil {
 			t.Fatalf("error adding entry: %v", err)
 		}
 	}
@@ -124,6 +123,14 @@ func TestEra2Builder(t *testing.T) {
 		if !bytes.Equal(decRcpt, mustEncode(chain.receipts[i])) {
 			t.Fatalf("receipts frame %d mismatch", i)
 		}
+
+		td, err := era.getTD(bn)
+		if err != nil {
+			t.Fatalf("getTD %d: %v", i, err)
+		}
+		if td.Cmp(chain.tds[i]) != 0 {
+			t.Fatalf("td %d mismatch: want %v got %v", i, chain.tds[i], td)
+		}
 	}
 }