From 02b439cdead1cb60982574b48a6309a5d5c0abd6 Mon Sep 17 00:00:00 2001 From: shantichanal <158101918+shantichanal@users.noreply.github.com> Date: Thu, 10 Jul 2025 00:49:03 +0200 Subject: [PATCH] Updated with all comments. Fixed all issues including extra logic regarding proof types, modularizing some functions and refactoring code for correctness and readability. --- internal/era2/{builder2.go => builder.go} | 144 +++++++++++----------- internal/era2/{era2.go => era.go} | 102 +++++++-------- internal/era2/era_test.go | 11 +- 3 files changed, 126 insertions(+), 131 deletions(-) rename internal/era2/{builder2.go => builder.go} (69%) rename internal/era2/{era2.go => era.go} (74%) diff --git a/internal/era2/builder2.go b/internal/era2/builder.go similarity index 69% rename from internal/era2/builder2.go rename to internal/era2/builder.go index 14f3ce1b2e..38ecd371b1 100644 --- a/internal/era2/builder2.go +++ b/internal/era2/builder.go @@ -43,29 +43,26 @@ import ( ) const ( - TypeVersion uint16 = 0x3265 - TypeCompressedHeader uint16 = 0x08 - TypeCompressedBody uint16 = 0x09 - TypeCompressedReceipts uint16 = 0x0a - TypeTotalDifficulty uint16 = 0x0b - TypeProofHistoricalHashesAccumulator uint16 = 0x0c - TypeProofHistoricalRoots uint16 = 0x0d - TypeProofHistoricalSummariesCapella uint16 = 0x0e - TypeProofHistoricalSummariesDeneb uint16 = 0x0f - TypeAccumulatorRoot uint16 = 0x1a - TypeBlockIndex uint16 = 0x3267 - MaxEraESize int = 8192 - headerSize uint64 = 8 + TypeVersion uint16 = 0x3265 + TypeCompressedHeader uint16 = 0x08 + TypeCompressedBody uint16 = 0x09 + TypeCompressedReceipts uint16 = 0x0a + TypeTotalDifficulty uint16 = 0x0b + TypeProof uint16 = 0x0c + TypeAccumulatorRoot uint16 = 0x0d + TypeBlockIndex uint16 = 0x3267 + MaxEraESize int = 8192 + headerSize uint64 = 8 ) type proofvar uint16 const ( - ProofNone proofvar = 0 - ProofHHA proofvar = proofvar(TypeProofHistoricalHashesAccumulator) - ProofRoots proofvar = proofvar(TypeProofHistoricalRoots) - ProofCapella proofvar = proofvar(TypeProofHistoricalSummariesCapella) - ProofDeneb proofvar = proofvar(TypeProofHistoricalSummariesDeneb) + ProofNone proofvar = iota + ProofHHA + ProofRoots + ProofCapella + ProofDeneb ) // Proof bundles variant + compressed bytes. @@ -79,21 +76,19 @@ type Builder struct { buf *bytes.Buffer sn *snappy.Writer - // buffered entries per type: - headersRLP [][]byte - bodiesRLP [][]byte - receiptsRLP [][]byte - proofsRLP [][]byte - tds [][]byte - tdsint []*big.Int - hashes []common.Hash + headers [][]byte + bodies [][]byte + receipts [][]byte + proofs [][]byte + tds [][]byte + tdsint []*big.Int + hashes []common.Hash headeroffsets []uint64 bodyoffsets []uint64 receiptoffsets []uint64 proofoffsets []uint64 tdoff []uint64 - startTd *big.Int prooftype proofvar @@ -101,6 +96,7 @@ type Builder struct { writtenBytes uint64 } +// NewBuilder returns a new Builder instance. func NewBuilder(w io.Writer) *Builder { buf := bytes.NewBuffer(nil) return &Builder{ @@ -110,56 +106,69 @@ func NewBuilder(w io.Writer) *Builder { } } -func (b *Builder) Add(header types.Header, body types.Body, receipts types.Receipts, blockhash common.Hash, blocknum uint64, td *big.Int, proof *Proof) error { - if len(b.headersRLP) >= MaxEraESize { - return fmt.Errorf("exceeds MaxEraESize %d", MaxEraESize) - } - +// Add writes a block entry, its reciepts, and optionally its proofs as well into the e2store file. +func (b *Builder) Add(header types.Header, body types.Body, receipts types.Receipts, td *big.Int, proof *Proof) error { var pv proofvar = ProofNone var pData []byte if proof != nil { - pv = proof.Variant - pData = proof.Data + pv, pData = proof.Variant, proof.Data if pv == ProofNone || len(pData) == 0 { return fmt.Errorf("invalid proof: variant=%d len=%d", pv, len(pData)) } } - if len(b.headersRLP) == 0 { + if len(b.headers) == 0 { b.prooftype = pv } else if pv != b.prooftype { - return fmt.Errorf("cannot mix proof variants: have %d want %d", b.prooftype, pv) + return fmt.Errorf("cannot mix proof variants: first=%d now=%d", b.prooftype, pv) } - hdr, err := rlp.EncodeToBytes(&header) + eh, err := rlp.EncodeToBytes(&header) if err != nil { - return fmt.Errorf("error encoding block header: %w", err) + return fmt.Errorf("encode header: %w", err) } - bod, err := rlp.EncodeToBytes(&body) + eb, err := rlp.EncodeToBytes(&body) if err != nil { - return fmt.Errorf("error encoding block header: %w", err) + return fmt.Errorf("encode body: %w", err) } - rct, err := rlp.EncodeToBytes(receipts) + er, err := rlp.EncodeToBytes(receipts) if err != nil { - return fmt.Errorf("error encoding block header: %w", err) + return fmt.Errorf("encode receipts: %w", err) } - b.headersRLP = append(b.headersRLP, hdr) - b.bodiesRLP = append(b.bodiesRLP, bod) - b.receiptsRLP = append(b.receiptsRLP, rct) + var ep []byte + if pv != ProofNone { + ep, err = rlp.EncodeToBytes([]interface{}{uint16(pv), proof.Data}) + if err != nil { + return fmt.Errorf("encode proof: %w", err) + } + } + + return b.AddRLP(eh, eb, er, ep, header.Number.Uint64(), header.Hash(), td) +} + +// AddRLP takes the RLP encoded block components and writes them to the underlying e2store file +func (b *Builder) AddRLP(headerRLP []byte, bodyRLP []byte, receipts []byte, proof []byte, blockNum uint64, blockHash common.Hash, td *big.Int) error { + if len(b.headers) >= MaxEraESize { + return fmt.Errorf("exceeds MaxEraESize %d", MaxEraESize) + } + + b.headers = append(b.headers, headerRLP) + b.bodies = append(b.bodies, bodyRLP) + b.receipts = append(b.receipts, receipts) b.tds = append(b.tds, uint256LE(td)) b.tdsint = append(b.tdsint, new(big.Int).Set(td)) - b.hashes = append(b.hashes, blockhash) - - if b.prooftype != ProofNone { - b.proofsRLP = append(b.proofsRLP, proof.Data) + b.hashes = append(b.hashes, blockHash) + if proof != nil { + b.proofs = append(b.proofs, proof) } + //Write Era2 version before writing any blocks. if b.startNum == nil { - sn := blocknum - b.startNum = &sn + b.startNum = new(uint64) + *b.startNum = blockNum if n, err := b.w.Write(TypeVersion, nil); err != nil { - return fmt.Errorf("error writing version entry: %w", err) + return fmt.Errorf("write version entry: %w", err) } else { b.writtenBytes += uint64(n) } @@ -167,18 +176,19 @@ func (b *Builder) Add(header types.Header, body types.Body, receipts types.Recei return nil } +// Finalize flushes all entries present in cache to the underlying e2store file. func (b *Builder) Finalize() (common.Hash, error) { if b.startNum == nil { return common.Hash{}, errors.New("no blocks added, cannot finalize") } var err error - if err := b.flushKind(TypeCompressedHeader, b.headersRLP, true, &b.headeroffsets); err != nil { + if err := b.flushKind(TypeCompressedHeader, b.headers, true, &b.headeroffsets); err != nil { return common.Hash{}, err } - if err := b.flushKind(TypeCompressedBody, b.bodiesRLP, true, &b.bodyoffsets); err != nil { + if err := b.flushKind(TypeCompressedBody, b.bodies, true, &b.bodyoffsets); err != nil { return common.Hash{}, err } - if err := b.flushKind(TypeCompressedReceipts, b.receiptsRLP, true, &b.receiptoffsets); err != nil { + if err := b.flushKind(TypeCompressedReceipts, b.receipts, true, &b.receiptoffsets); err != nil { return common.Hash{}, err } @@ -188,7 +198,7 @@ func (b *Builder) Finalize() (common.Hash, error) { } } if b.prooftype != ProofNone { - if err := b.flushKind(uint16(b.prooftype), b.proofsRLP, true, &b.proofoffsets); err != nil { + if err := b.flushKind(TypeProof, b.proofs, true, &b.proofoffsets); err != nil { return common.Hash{}, err } } @@ -208,6 +218,7 @@ func (b *Builder) Finalize() (common.Hash, error) { return accRoot, b.writeIndex() } +// Writes 32 byte big integers to little endian func uint256LE(v *big.Int) []byte { b := v.FillBytes(make([]byte, 32)) for i := 0; i < 16; i++ { @@ -216,18 +227,7 @@ func uint256LE(v *big.Int) []byte { return b } -func decodeBigs(raw [][]byte) []*big.Int { - out := make([]*big.Int, len(raw)) - for i, le := range raw { - be := make([]byte, 32) - for j := 0; j < 32; j++ { - be[j] = le[31-j] - } - out[i] = new(big.Int).SetBytes(be) - } - return out -} - +// Compresses into snappy encoding func (b *Builder) snappyWrite(typ uint16, in []byte) error { var ( buf = b.buf @@ -249,6 +249,7 @@ func (b *Builder) snappyWrite(typ uint16, in []byte) error { return nil } +// Add entry takes the e2store object and writes it into the file func (b *Builder) addEntry(typ uint16, payload []byte, snappyIt bool) (uint64, error) { offset := b.writtenBytes var err error @@ -266,6 +267,7 @@ func (b *Builder) addEntry(typ uint16, payload []byte, snappyIt bool) (uint64, e return offset, nil } +// flush kind takes all entries of the cached component and flushes it to the file func (b *Builder) flushKind(typ uint16, list [][]byte, useSnappy bool, dst *[]uint64) error { for _, data := range list { off, err := b.addEntry(typ, data, useSnappy) @@ -277,13 +279,15 @@ func (b *Builder) flushKind(typ uint16, list [][]byte, useSnappy bool, dst *[]ui return nil } +// write index takes all the offset table and writes it to the file +// the index table contains all offsets to specific block entries func (b *Builder) writeIndex() error { count := uint64(len(b.headeroffsets)) compcount := uint64(3) if len(b.tds) > 0 { compcount++ } - if len(b.proofsRLP) > 0 { + if len(b.proofs) > 0 { compcount++ } @@ -303,7 +307,7 @@ func (b *Builder) writeIndex() error { binary.LittleEndian.PutUint64(idx[pos:], rel(b.tdoff[i])) pos += 8 } - if len(b.proofsRLP) > 0 { + if len(b.proofs) > 0 { binary.LittleEndian.PutUint64(idx[pos:], rel(b.proofoffsets[i])) pos += 8 } diff --git a/internal/era2/era2.go b/internal/era2/era.go similarity index 74% rename from internal/era2/era2.go rename to internal/era2/era.go index 030c2400e1..aa0440f93a 100644 --- a/internal/era2/era2.go +++ b/internal/era2/era.go @@ -6,7 +6,6 @@ import ( "io" "math/big" "os" - "sync" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" @@ -15,11 +14,11 @@ import ( "github.com/klauspost/compress/snappy" ) -type meta struct { - start uint64 // start block number - count uint64 // number of blocks in the era - compcount uint64 // number of properties - filelen int64 // length of the file in bytes +type metadata struct { + start uint64 // start block number + count uint64 // number of blocks in the era + components uint64 // number of properties + length int64 // length of the file in bytes } type BlockProofHistoricalHashesAccumulator [15]common.Hash // 15 * 32 = 480 bytes @@ -48,31 +47,28 @@ type BlockProofHistoricalSummariesDeneb struct { Slot uint64 // 8 => 840 bytes } -// BlockAccumulatorRoot is the SSZ hash tree root of the Era2 block accumulator. - type ReadAtSeekCloser interface { io.ReaderAt io.Seeker io.Closer } -type Era2 struct { +type Era struct { f ReadAtSeekCloser s *e2store.Reader - m meta // metadata for the era2 file - mu *sync.Mutex + m metadata // metadata for the Era file headeroff, bodyoff, receiptsoff, tdoff, proofsoff []uint64 // offsets for each entry type indstart int64 rootheader uint64 // offset of the root header in the file if present - prooftype uint16 } -func Open(path string) (*Era2, error) { +// Opens era file +func Open(path string) (*Era, error) { f, err := os.Open(path) if err != nil { return nil, err } - e := &Era2{f: f, s: e2store.NewReader(f)} + e := &Era{f: f, s: e2store.NewReader(f)} if err := e.loadIndex(); err != nil { f.Close() return nil, err @@ -80,7 +76,8 @@ func Open(path string) (*Era2, error) { return e, nil } -func (e *Era2) Close() error { +// Closes era file +func (e *Era) Close() error { if e.f == nil { return nil } @@ -89,15 +86,18 @@ func (e *Era2) Close() error { return err } -func (e *Era2) Start() uint64 { +// retrieves starting block number +func (e *Era) Start() uint64 { return e.m.start } -func (e *Era2) Count() uint64 { +// retrieves count of blocks present +func (e *Era) Count() uint64 { return e.m.count } -func (e *Era2) GetBlockByNumber(blockNum uint64) (*types.Block, error) { +// retrieves the block if present within the era file +func (e *Era) GetBlockByNumber(blockNum uint64) (*types.Block, error) { h, err := e.getHeader(blockNum) if err != nil { return nil, err @@ -109,7 +109,8 @@ func (e *Era2) GetBlockByNumber(blockNum uint64) (*types.Block, error) { return types.NewBlockWithHeader(h).WithBody(*b), nil } -func (e *Era2) getHeader(blockNum uint64) (*types.Header, error) { +// retrieves header from era file through the cached offset table +func (e *Era) getHeader(blockNum uint64) (*types.Header, error) { if blockNum < e.m.start || blockNum >= e.m.start+e.m.count { return nil, fmt.Errorf("block number %d out of range [%d, %d)", blockNum, e.m.start, e.m.start+e.m.count) } @@ -123,7 +124,8 @@ func (e *Era2) getHeader(blockNum uint64) (*types.Header, error) { return &h, rlp.Decode(r, &h) } -func (e *Era2) getBody(blockNum uint64) (*types.Body, error) { +// retrieves body from era file through cached offset table +func (e *Era) getBody(blockNum uint64) (*types.Body, error) { if blockNum < e.m.start || blockNum >= e.m.start+e.m.count { return nil, fmt.Errorf("block number %d out of range [%d, %d)", blockNum, e.m.start, e.m.start+e.m.count) } @@ -137,7 +139,8 @@ func (e *Era2) getBody(blockNum uint64) (*types.Body, error) { return &b, rlp.Decode(r, &b) } -func (e *Era2) getTD(blockNum uint64) (*big.Int, error) { +// retrieves td from era file through cached offset table +func (e *Era) getTD(blockNum uint64) (*big.Int, error) { if blockNum < e.m.start || blockNum >= e.m.start+e.m.count { return nil, fmt.Errorf("block number %d out of range [%d, %d)", blockNum, e.m.start, e.m.start+e.m.count) @@ -155,7 +158,8 @@ func (e *Era2) getTD(blockNum uint64) (*big.Int, error) { return td, nil } -func (e *Era2) GetRawBodyFrameByNumber(blockNum uint64) ([]byte, error) { +// retrieves the raw body frame in bytes of a specific block +func (e *Era) GetRawBodyFrameByNumber(blockNum uint64) ([]byte, error) { if blockNum < e.m.start || blockNum >= e.m.start+e.m.count { return nil, fmt.Errorf("block number %d out of range [%d, %d)", blockNum, e.m.start, e.m.start+e.m.count) } @@ -166,7 +170,8 @@ func (e *Era2) GetRawBodyFrameByNumber(blockNum uint64) ([]byte, error) { return io.ReadAll(r) } -func (e *Era2) GetRawReceiptsFrameByNumber(blockNum uint64) ([]byte, error) { +// retrieves the raw receipts frame in bytes of a specific block +func (e *Era) GetRawReceiptsFrameByNumber(blockNum uint64) ([]byte, error) { if blockNum < e.m.start || blockNum >= e.m.start+e.m.count { return nil, fmt.Errorf("block number %d out of range [%d, %d)", blockNum, e.m.start, e.m.start+e.m.count) } @@ -177,46 +182,38 @@ func (e *Era2) GetRawReceiptsFrameByNumber(blockNum uint64) ([]byte, error) { return io.ReadAll(r) } -func (e *Era2) GetRawProofFrameByNumber(blockNum uint64) ([]byte, error) { +// retrieves the raw proof frame in bytes of a specific block proof +func (e *Era) GetRawProofFrameByNumber(blockNum uint64) ([]byte, error) { if len(e.proofsoff) == 0 { return nil, fmt.Errorf("proofs section not present") } if blockNum < e.m.start || blockNum >= e.m.start+e.m.count { return nil, fmt.Errorf("block number %d out of range [%d, %d)", blockNum, e.m.start, e.m.start+e.m.count) } - r, _, err := e.s.ReaderAt(e.prooftype, int64(e.proofsoff[blockNum-e.m.start])) + r, _, err := e.s.ReaderAt(TypeProof, int64(e.proofsoff[blockNum-e.m.start])) if err != nil { return nil, err } return io.ReadAll(r) } -func (e *Era2) rawPayload(abs uint64) ([]byte, error) { - sr := io.NewSectionReader(e.f, int64(abs), e.indstart-int64(abs)) - return io.ReadAll(sr) -} - -func (e *Era2) snappyPayload(abs uint64) (io.Reader, error) { - sr := io.NewSectionReader(e.f, int64(abs), e.indstart-int64(abs)) - return snappy.NewReader(sr), nil -} - -func (e *Era2) loadIndex() error { +// loads in the index table containing all offsets and caches it +func (e *Era) loadIndex() error { var err error - e.m.filelen, err = e.f.Seek(0, io.SeekEnd) + e.m.length, err = e.f.Seek(0, io.SeekEnd) if err != nil { return err } b := make([]byte, 16) - if _, err = e.f.ReadAt(b, e.m.filelen-16); err != nil { + if _, err = e.f.ReadAt(b, e.m.length-16); err != nil { return err } - e.m.compcount = binary.LittleEndian.Uint64(b[0:8]) + e.m.components = binary.LittleEndian.Uint64(b[0:8]) e.m.count = binary.LittleEndian.Uint64(b[8:16]) - payloadlen := 8 + 8*e.m.count*e.m.compcount + 16 // 8 for start block, 8 per property per block, 16 for the number of properties and the number of blocks - tlvstart := e.m.filelen - int64(payloadlen) - 8 + payloadlen := 8 + 8*e.m.count*e.m.components + 16 // 8 for start block, 8 per property per block, 16 for the number of properties and the number of blocks + tlvstart := e.m.length - int64(payloadlen) - 8 e.indstart = tlvstart _, err = e.f.ReadAt(b[:8], tlvstart+8) @@ -225,7 +222,7 @@ func (e *Era2) loadIndex() error { } e.m.start = binary.LittleEndian.Uint64(b[:8]) - num := int(e.m.compcount) + num := int(e.m.components) totaloffsets := num * int(e.m.count) offBytes := make([]byte, totaloffsets*8) @@ -262,14 +259,6 @@ func (e *Era2) loadIndex() error { } } - if len(e.proofsoff) > 0 { - typ, _, perr := e.s.ReadMetadataAt(int64(e.proofsoff[0])) - if perr != nil { - return fmt.Errorf("read proof header: %w", perr) - } - e.prooftype = typ - } - var off int64 = 0 // start at byte-0 of file for off < e.indstart { // never enter the Block-Index TLV @@ -286,7 +275,9 @@ func (e *Era2) loadIndex() error { return nil } -func (e *Era2) BatchRange(first, count uint64, wantHdr, wantBody, wantRec, wantPrf bool) (hdrs []*types.Header, bods []*types.Body, recs []types.Receipts, prfs [][]byte, err error) { +// retrieves a range of block from any start block to n number of blocks +// the components retrieved can be customized depending on the properties wanted +func (e *Era) BatchRange(first, count uint64, wantHdr, wantBody, wantRec, wantPrf bool) (hdrs []*types.Header, bods []*types.Body, recs []types.Receipts, prfs [][]byte, err error) { if count == 0 { err = fmt.Errorf("count must be > 0") return @@ -351,7 +342,7 @@ func (e *Era2) BatchRange(first, count uint64, wantHdr, wantBody, wantRec, wantP err = fmt.Errorf("proofs section not present") return } - r, _, er := e.s.ReaderAt(e.prooftype, int64(e.proofsoff[id])) // type already checked when writing + r, _, er := e.s.ReaderAt(TypeProof, int64(e.proofsoff[id])) if er != nil { err = er return @@ -361,10 +352,3 @@ func (e *Era2) BatchRange(first, count uint64, wantHdr, wantBody, wantRec, wantP } return } - -func reverseOrder32(b []byte) []byte { - for i := 0; i < 16; i++ { - b[i], b[32-i-1] = b[32-i-1], b[i] - } - return b -} diff --git a/internal/era2/era_test.go b/internal/era2/era_test.go index 1b49e033cb..218b477c0b 100644 --- a/internal/era2/era_test.go +++ b/internal/era2/era_test.go @@ -63,10 +63,9 @@ func TestEra2Builder(t *testing.T) { header = chain.headers[i] body = chain.bodies[i] receipts = chain.receipts[i] - hash = common.Hash{byte(i)} td = chain.tds[i] ) - if err = builder.Add(header, body, receipts, hash, uint64(i), td, nil); err != nil { + if err = builder.Add(header, body, receipts, td, nil); err != nil { t.Fatalf("error adding entry: %v", err) } } @@ -124,6 +123,14 @@ func TestEra2Builder(t *testing.T) { if !bytes.Equal(decRcpt, mustEncode(chain.receipts[i])) { t.Fatalf("receipts frame %d mismatch", i) } + + td, err := era.getTD(bn) + if err != nil { + t.Fatalf("getTD %d: %v", i, err) + } + if td.Cmp(chain.tds[i]) != 0 { + t.Fatalf("td %d mismatch: want %v got %v", i, chain.tds[i], td) + } } }