forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-07-21 20:26:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Updated with all comments.

Browse source 
Fixed all issues including extra logic regarding proof types, modularizing some functions and refactoring code for correctness and readability.
This commit is contained in:
shantichanal 2025-07-10 00:49:03 +02:00 committed by lightclient
parent 65a08322cb
commit 02b439cdea
No known key found for this signature in database
GPG key ID: 657913021EF45A6A
3 changed files with 126 additions and 131 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

144
internal/era2/builder2.go → internal/era2/builder.go
View file

@ -43,29 +43,26 @@ import (
)
const (
TypeVersion uint16 = 0x3265
TypeCompressedHeader uint16 = 0x08
TypeCompressedBody uint16 = 0x09
TypeCompressedReceipts uint16 = 0x0a
TypeTotalDifficulty uint16 = 0x0b
TypeProofHistoricalHashesAccumulator uint16 = 0x0c
TypeProofHistoricalRoots uint16 = 0x0d
TypeProofHistoricalSummariesCapella uint16 = 0x0e
TypeProofHistoricalSummariesDeneb uint16 = 0x0f
TypeAccumulatorRoot uint16 = 0x1a
TypeBlockIndex uint16 = 0x3267
MaxEraESize int = 8192
headerSize uint64 = 8
TypeVersion uint16 = 0x3265
TypeCompressedHeader uint16 = 0x08
TypeCompressedBody uint16 = 0x09
TypeCompressedReceipts uint16 = 0x0a
TypeTotalDifficulty uint16 = 0x0b
TypeProof uint16 = 0x0c
TypeAccumulatorRoot uint16 = 0x0d
TypeBlockIndex uint16 = 0x3267
MaxEraESize int = 8192
headerSize uint64 = 8
)
type proofvar uint16
const (
ProofNone proofvar = 0
ProofHHA proofvar = proofvar(TypeProofHistoricalHashesAccumulator)
ProofRoots proofvar = proofvar(TypeProofHistoricalRoots)
ProofCapella proofvar = proofvar(TypeProofHistoricalSummariesCapella)
ProofDeneb proofvar = proofvar(TypeProofHistoricalSummariesDeneb)
ProofNone proofvar = iota
ProofHHA
ProofRoots
ProofCapella
ProofDeneb
)
// Proof bundles variant + compressed bytes.
@ -79,21 +76,19 @@ type Builder struct {
buf *bytes.Buffer
sn *snappy.Writer
// buffered entries per type:
headersRLP [][]byte
bodiesRLP [][]byte
receiptsRLP [][]byte
proofsRLP [][]byte
tds [][]byte
tdsint []*big.Int
hashes []common.Hash
headers [][]byte
bodies [][]byte
receipts [][]byte
proofs [][]byte
tds [][]byte
tdsint []*big.Int
hashes []common.Hash
headeroffsets []uint64
bodyoffsets []uint64
receiptoffsets []uint64
proofoffsets []uint64
tdoff []uint64
startTd *big.Int
prooftype proofvar
@ -101,6 +96,7 @@ type Builder struct {
writtenBytes uint64
}
// NewBuilder returns a new Builder instance.
func NewBuilder(w io.Writer) *Builder {
buf := bytes.NewBuffer(nil)
return &Builder{
@ -110,56 +106,69 @@ func NewBuilder(w io.Writer) *Builder {
}
}
func (b *Builder) Add(header types.Header, body types.Body, receipts types.Receipts, blockhash common.Hash, blocknum uint64, td *big.Int, proof *Proof) error {
if len(b.headersRLP) >= MaxEraESize {
return fmt.Errorf("exceeds MaxEraESize %d", MaxEraESize)
}
// Add writes a block entry, its reciepts, and optionally its proofs as well into the e2store file.
func (b *Builder) Add(header types.Header, body types.Body, receipts types.Receipts, td *big.Int, proof *Proof) error {
var pv proofvar = ProofNone
var pData []byte
if proof != nil {
pv = proof.Variant
pData = proof.Data
pv, pData = proof.Variant, proof.Data
if pv == ProofNone || len(pData) == 0 {
return fmt.Errorf("invalid proof: variant=%d len=%d", pv, len(pData))
}
}
if len(b.headersRLP) == 0 {
if len(b.headers) == 0 {
b.prooftype = pv
} else if pv != b.prooftype {
return fmt.Errorf("cannot mix proof variants: have %d want %d", b.prooftype, pv)
return fmt.Errorf("cannot mix proof variants: first=%d now=%d", b.prooftype, pv)
}
hdr, err := rlp.EncodeToBytes(&header)
eh, err := rlp.EncodeToBytes(&header)
if err != nil {
return fmt.Errorf("error encoding block header: %w", err)
return fmt.Errorf("encode header: %w", err)
}
bod, err := rlp.EncodeToBytes(&body)
eb, err := rlp.EncodeToBytes(&body)
if err != nil {
return fmt.Errorf("error encoding block header: %w", err)
return fmt.Errorf("encode body: %w", err)
}
rct, err := rlp.EncodeToBytes(receipts)
er, err := rlp.EncodeToBytes(receipts)
if err != nil {
return fmt.Errorf("error encoding block header: %w", err)
return fmt.Errorf("encode receipts: %w", err)
}
b.headersRLP = append(b.headersRLP, hdr)
b.bodiesRLP = append(b.bodiesRLP, bod)
b.receiptsRLP = append(b.receiptsRLP, rct)
var ep []byte
if pv != ProofNone {
ep, err = rlp.EncodeToBytes([]interface{}{uint16(pv), proof.Data})
if err != nil {
return fmt.Errorf("encode proof: %w", err)
}
}
return b.AddRLP(eh, eb, er, ep, header.Number.Uint64(), header.Hash(), td)
}
// AddRLP takes the RLP encoded block components and writes them to the underlying e2store file
func (b *Builder) AddRLP(headerRLP []byte, bodyRLP []byte, receipts []byte, proof []byte, blockNum uint64, blockHash common.Hash, td *big.Int) error {
if len(b.headers) >= MaxEraESize {
return fmt.Errorf("exceeds MaxEraESize %d", MaxEraESize)
}
b.headers = append(b.headers, headerRLP)
b.bodies = append(b.bodies, bodyRLP)
b.receipts = append(b.receipts, receipts)
b.tds = append(b.tds, uint256LE(td))
b.tdsint = append(b.tdsint, new(big.Int).Set(td))
b.hashes = append(b.hashes, blockhash)
if b.prooftype != ProofNone {
b.proofsRLP = append(b.proofsRLP, proof.Data)
b.hashes = append(b.hashes, blockHash)
if proof != nil {
b.proofs = append(b.proofs, proof)
}
//Write Era2 version before writing any blocks.
if b.startNum == nil {
sn := blocknum
b.startNum = &sn
b.startNum = new(uint64)
*b.startNum = blockNum
if n, err := b.w.Write(TypeVersion, nil); err != nil {
return fmt.Errorf("error writing version entry: %w", err)
return fmt.Errorf("write version entry: %w", err)
} else {
b.writtenBytes += uint64(n)
}
@ -167,18 +176,19 @@ func (b *Builder) Add(header types.Header, body types.Body, receipts types.Recei
return nil
}
// Finalize flushes all entries present in cache to the underlying e2store file.
func (b *Builder) Finalize() (common.Hash, error) {
if b.startNum == nil {
return common.Hash{}, errors.New("no blocks added, cannot finalize")
}
var err error
if err := b.flushKind(TypeCompressedHeader, b.headersRLP, true, &b.headeroffsets); err != nil {
if err := b.flushKind(TypeCompressedHeader, b.headers, true, &b.headeroffsets); err != nil {
return common.Hash{}, err
}
if err := b.flushKind(TypeCompressedBody, b.bodiesRLP, true, &b.bodyoffsets); err != nil {
if err := b.flushKind(TypeCompressedBody, b.bodies, true, &b.bodyoffsets); err != nil {
return common.Hash{}, err
}
if err := b.flushKind(TypeCompressedReceipts, b.receiptsRLP, true, &b.receiptoffsets); err != nil {
if err := b.flushKind(TypeCompressedReceipts, b.receipts, true, &b.receiptoffsets); err != nil {
return common.Hash{}, err
}
@ -188,7 +198,7 @@ func (b *Builder) Finalize() (common.Hash, error) {
}
}
if b.prooftype != ProofNone {
if err := b.flushKind(uint16(b.prooftype), b.proofsRLP, true, &b.proofoffsets); err != nil {
if err := b.flushKind(TypeProof, b.proofs, true, &b.proofoffsets); err != nil {
return common.Hash{}, err
}
}
@ -208,6 +218,7 @@ func (b *Builder) Finalize() (common.Hash, error) {
return accRoot, b.writeIndex()
}
// Writes 32 byte big integers to little endian
func uint256LE(v *big.Int) []byte {
b := v.FillBytes(make([]byte, 32))
for i := 0; i < 16; i++ {
@ -216,18 +227,7 @@ func uint256LE(v *big.Int) []byte {
return b
}
func decodeBigs(raw [][]byte) []*big.Int {
out := make([]*big.Int, len(raw))
for i, le := range raw {
be := make([]byte, 32)
for j := 0; j < 32; j++ {
be[j] = le[31-j]
}
out[i] = new(big.Int).SetBytes(be)
}
return out
}
// Compresses into snappy encoding
func (b *Builder) snappyWrite(typ uint16, in []byte) error {
var (
buf = b.buf
@ -249,6 +249,7 @@ func (b *Builder) snappyWrite(typ uint16, in []byte) error {
return nil
}
// Add entry takes the e2store object and writes it into the file
func (b *Builder) addEntry(typ uint16, payload []byte, snappyIt bool) (uint64, error) {
offset := b.writtenBytes
var err error
@ -266,6 +267,7 @@ func (b *Builder) addEntry(typ uint16, payload []byte, snappyIt bool) (uint64, e
return offset, nil
}
// flush kind takes all entries of the cached component and flushes it to the file
func (b *Builder) flushKind(typ uint16, list [][]byte, useSnappy bool, dst *[]uint64) error {
for _, data := range list {
off, err := b.addEntry(typ, data, useSnappy)
@ -277,13 +279,15 @@ func (b *Builder) flushKind(typ uint16, list [][]byte, useSnappy bool, dst *[]ui
return nil
}
// write index takes all the offset table and writes it to the file
// the index table contains all offsets to specific block entries
func (b *Builder) writeIndex() error {
count := uint64(len(b.headeroffsets))
compcount := uint64(3)
if len(b.tds) > 0 {
compcount++
}
if len(b.proofsRLP) > 0 {
if len(b.proofs) > 0 {
compcount++
}
@ -303,7 +307,7 @@ func (b *Builder) writeIndex() error {
binary.LittleEndian.PutUint64(idx[pos:], rel(b.tdoff[i]))
pos += 8
}
if len(b.proofsRLP) > 0 {
if len(b.proofs) > 0 {
binary.LittleEndian.PutUint64(idx[pos:], rel(b.proofoffsets[i]))
pos += 8
}

102
internal/era2/era2.go → internal/era2/era.go
View file

@ -6,7 +6,6 @@ import (
"io"
"math/big"
"os"
"sync"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
@ -15,11 +14,11 @@ import (
"github.com/klauspost/compress/snappy"
)
type meta struct {
start uint64 // start block number
count uint64 // number of blocks in the era
compcount uint64 // number of properties
filelen int64 // length of the file in bytes
type metadata struct {
start uint64 // start block number
count uint64 // number of blocks in the era
components uint64 // number of properties
length int64 // length of the file in bytes
}
type BlockProofHistoricalHashesAccumulator [15]common.Hash // 15 * 32 = 480 bytes
@ -48,31 +47,28 @@ type BlockProofHistoricalSummariesDeneb struct {
Slot uint64 // 8 => 840 bytes
}
// BlockAccumulatorRoot is the SSZ hash tree root of the Era2 block accumulator.
type ReadAtSeekCloser interface {
io.ReaderAt
io.Seeker
io.Closer
}
type Era2 struct {
type Era struct {
f ReadAtSeekCloser
s *e2store.Reader
m meta // metadata for the era2 file
mu *sync.Mutex
m metadata // metadata for the Era file
headeroff, bodyoff, receiptsoff, tdoff, proofsoff []uint64 // offsets for each entry type
indstart int64
rootheader uint64 // offset of the root header in the file if present
prooftype uint16
}
func Open(path string) (*Era2, error) {
// Opens era file
func Open(path string) (*Era, error) {
f, err := os.Open(path)
if err != nil {
return nil, err
}
e := &Era2{f: f, s: e2store.NewReader(f)}
e := &Era{f: f, s: e2store.NewReader(f)}
if err := e.loadIndex(); err != nil {
f.Close()
return nil, err
@ -80,7 +76,8 @@ func Open(path string) (*Era2, error) {
return e, nil
}
func (e *Era2) Close() error {
// Closes era file
func (e *Era) Close() error {
if e.f == nil {
return nil
}
@ -89,15 +86,18 @@ func (e *Era2) Close() error {
return err
}
func (e *Era2) Start() uint64 {
// retrieves starting block number
func (e *Era) Start() uint64 {
return e.m.start
}
func (e *Era2) Count() uint64 {
// retrieves count of blocks present
func (e *Era) Count() uint64 {
return e.m.count
}
func (e *Era2) GetBlockByNumber(blockNum uint64) (*types.Block, error) {
// retrieves the block if present within the era file
func (e *Era) GetBlockByNumber(blockNum uint64) (*types.Block, error) {
h, err := e.getHeader(blockNum)
if err != nil {
return nil, err
@ -109,7 +109,8 @@ func (e *Era2) GetBlockByNumber(blockNum uint64) (*types.Block, error) {
return types.NewBlockWithHeader(h).WithBody(*b), nil
}
func (e *Era2) getHeader(blockNum uint64) (*types.Header, error) {
// retrieves header from era file through the cached offset table
func (e *Era) getHeader(blockNum uint64) (*types.Header, error) {
if blockNum < e.m.start || blockNum >= e.m.start+e.m.count {
return nil, fmt.Errorf("block number %d out of range [%d, %d)", blockNum, e.m.start, e.m.start+e.m.count)
}
@ -123,7 +124,8 @@ func (e *Era2) getHeader(blockNum uint64) (*types.Header, error) {
return &h, rlp.Decode(r, &h)
}
func (e *Era2) getBody(blockNum uint64) (*types.Body, error) {
// retrieves body from era file through cached offset table
func (e *Era) getBody(blockNum uint64) (*types.Body, error) {
if blockNum < e.m.start || blockNum >= e.m.start+e.m.count {
return nil, fmt.Errorf("block number %d out of range [%d, %d)", blockNum, e.m.start, e.m.start+e.m.count)
}
@ -137,7 +139,8 @@ func (e *Era2) getBody(blockNum uint64) (*types.Body, error) {
return &b, rlp.Decode(r, &b)
}
func (e *Era2) getTD(blockNum uint64) (*big.Int, error) {
// retrieves td from era file through cached offset table
func (e *Era) getTD(blockNum uint64) (*big.Int, error) {
if blockNum < e.m.start || blockNum >= e.m.start+e.m.count {
return nil, fmt.Errorf("block number %d out of range [%d, %d)",
blockNum, e.m.start, e.m.start+e.m.count)
@ -155,7 +158,8 @@ func (e *Era2) getTD(blockNum uint64) (*big.Int, error) {
return td, nil
}
func (e *Era2) GetRawBodyFrameByNumber(blockNum uint64) ([]byte, error) {
// retrieves the raw body frame in bytes of a specific block
func (e *Era) GetRawBodyFrameByNumber(blockNum uint64) ([]byte, error) {
if blockNum < e.m.start || blockNum >= e.m.start+e.m.count {
return nil, fmt.Errorf("block number %d out of range [%d, %d)", blockNum, e.m.start, e.m.start+e.m.count)
}
@ -166,7 +170,8 @@ func (e *Era2) GetRawBodyFrameByNumber(blockNum uint64) ([]byte, error) {
return io.ReadAll(r)
}
func (e *Era2) GetRawReceiptsFrameByNumber(blockNum uint64) ([]byte, error) {
// retrieves the raw receipts frame in bytes of a specific block
func (e *Era) GetRawReceiptsFrameByNumber(blockNum uint64) ([]byte, error) {
if blockNum < e.m.start || blockNum >= e.m.start+e.m.count {
return nil, fmt.Errorf("block number %d out of range [%d, %d)", blockNum, e.m.start, e.m.start+e.m.count)
}
@ -177,46 +182,38 @@ func (e *Era2) GetRawReceiptsFrameByNumber(blockNum uint64) ([]byte, error) {
return io.ReadAll(r)
}
func (e *Era2) GetRawProofFrameByNumber(blockNum uint64) ([]byte, error) {
// retrieves the raw proof frame in bytes of a specific block proof
func (e *Era) GetRawProofFrameByNumber(blockNum uint64) ([]byte, error) {
if len(e.proofsoff) == 0 {
return nil, fmt.Errorf("proofs section not present")
}
if blockNum < e.m.start || blockNum >= e.m.start+e.m.count {
return nil, fmt.Errorf("block number %d out of range [%d, %d)", blockNum, e.m.start, e.m.start+e.m.count)
}
r, _, err := e.s.ReaderAt(e.prooftype, int64(e.proofsoff[blockNum-e.m.start]))
r, _, err := e.s.ReaderAt(TypeProof, int64(e.proofsoff[blockNum-e.m.start]))
if err != nil {
return nil, err
}
return io.ReadAll(r)
}
func (e *Era2) rawPayload(abs uint64) ([]byte, error) {
sr := io.NewSectionReader(e.f, int64(abs), e.indstart-int64(abs))
return io.ReadAll(sr)
}
func (e *Era2) snappyPayload(abs uint64) (io.Reader, error) {
sr := io.NewSectionReader(e.f, int64(abs), e.indstart-int64(abs))
return snappy.NewReader(sr), nil
}
func (e *Era2) loadIndex() error {
// loads in the index table containing all offsets and caches it
func (e *Era) loadIndex() error {
var err error
e.m.filelen, err = e.f.Seek(0, io.SeekEnd)
e.m.length, err = e.f.Seek(0, io.SeekEnd)
if err != nil {
return err
}
b := make([]byte, 16)
if _, err = e.f.ReadAt(b, e.m.filelen-16); err != nil {
if _, err = e.f.ReadAt(b, e.m.length-16); err != nil {
return err
}
e.m.compcount = binary.LittleEndian.Uint64(b[0:8])
e.m.components = binary.LittleEndian.Uint64(b[0:8])
e.m.count = binary.LittleEndian.Uint64(b[8:16])
payloadlen := 8 + 8*e.m.count*e.m.compcount + 16 // 8 for start block, 8 per property per block, 16 for the number of properties and the number of blocks
tlvstart := e.m.filelen - int64(payloadlen) - 8
payloadlen := 8 + 8*e.m.count*e.m.components + 16 // 8 for start block, 8 per property per block, 16 for the number of properties and the number of blocks
tlvstart := e.m.length - int64(payloadlen) - 8
e.indstart = tlvstart
_, err = e.f.ReadAt(b[:8], tlvstart+8)
@ -225,7 +222,7 @@ func (e *Era2) loadIndex() error {
}
e.m.start = binary.LittleEndian.Uint64(b[:8])
num := int(e.m.compcount)
num := int(e.m.components)
totaloffsets := num * int(e.m.count)
offBytes := make([]byte, totaloffsets*8)
@ -262,14 +259,6 @@ func (e *Era2) loadIndex() error {
}
}
if len(e.proofsoff) > 0 {
typ, _, perr := e.s.ReadMetadataAt(int64(e.proofsoff[0]))
if perr != nil {
return fmt.Errorf("read proof header: %w", perr)
}
e.prooftype = typ
}
var off int64 = 0 // start at byte-0 of file
for off < e.indstart { // never enter the Block-Index TLV
@ -286,7 +275,9 @@ func (e *Era2) loadIndex() error {
return nil
}
func (e *Era2) BatchRange(first, count uint64, wantHdr, wantBody, wantRec, wantPrf bool) (hdrs []*types.Header, bods []*types.Body, recs []types.Receipts, prfs [][]byte, err error) {
// retrieves a range of block from any start block to n number of blocks
// the components retrieved can be customized depending on the properties wanted
func (e *Era) BatchRange(first, count uint64, wantHdr, wantBody, wantRec, wantPrf bool) (hdrs []*types.Header, bods []*types.Body, recs []types.Receipts, prfs [][]byte, err error) {
if count == 0 {
err = fmt.Errorf("count must be > 0")
return
@ -351,7 +342,7 @@ func (e *Era2) BatchRange(first, count uint64, wantHdr, wantBody, wantRec, wantP
err = fmt.Errorf("proofs section not present")
return
}
r, _, er := e.s.ReaderAt(e.prooftype, int64(e.proofsoff[id])) // type already checked when writing
r, _, er := e.s.ReaderAt(TypeProof, int64(e.proofsoff[id]))
if er != nil {
err = er
return
@ -361,10 +352,3 @@ func (e *Era2) BatchRange(first, count uint64, wantHdr, wantBody, wantRec, wantP
}
return
}
func reverseOrder32(b []byte) []byte {
for i := 0; i < 16; i++ {
b[i], b[32-i-1] = b[32-i-1], b[i]
}
return b
}

11
internal/era2/era_test.go
View file

@ -63,10 +63,9 @@ func TestEra2Builder(t *testing.T) {
header = chain.headers[i]
body = chain.bodies[i]
receipts = chain.receipts[i]
hash = common.Hash{byte(i)}
td = chain.tds[i]
)
if err = builder.Add(header, body, receipts, hash, uint64(i), td, nil); err != nil {
if err = builder.Add(header, body, receipts, td, nil); err != nil {
t.Fatalf("error adding entry: %v", err)
}
}
@ -124,6 +123,14 @@ func TestEra2Builder(t *testing.T) {
if !bytes.Equal(decRcpt, mustEncode(chain.receipts[i])) {
t.Fatalf("receipts frame %d mismatch", i)
}
td, err := era.getTD(bn)
if err != nil {
t.Fatalf("getTD %d: %v", i, err)
}
if td.Cmp(chain.tds[i]) != 0 {
t.Fatalf("td %d mismatch: want %v got %v", i, chain.tds[i], td)
}
}
}