finished builder

internal/era2/accumulator.go

@@ -0,0 +1,91 @@
+// Copyright 2024 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package era
+
+import (
+	"errors"
+	"fmt"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/common"
+	ssz "github.com/ferranbt/fastssz"
+)
+
+// ComputeAccumulator calculates the SSZ hash tree root of the Era1
+// accumulator of header records.
+func ComputeAccumulator(hashes []common.Hash, tds []*big.Int) (common.Hash, error) {
+	if len(hashes) != len(tds) {
+		return common.Hash{}, errors.New("must have equal number hashes as td values")
+	}
+	if len(hashes) > MaxEraESize {
+		return common.Hash{}, fmt.Errorf("too many records: have %d, max %d", len(hashes), MaxEraESize)
+	}
+	hh := ssz.NewHasher()
+	for i := range hashes {
+		rec := headerRecord{hashes[i], tds[i]}
+		root, err := rec.HashTreeRoot()
+		if err != nil {
+			return common.Hash{}, err
+		}
+		hh.Append(root[:])
+	}
+	hh.MerkleizeWithMixin(0, uint64(len(hashes)), uint64(MaxEraESize))
+	return hh.HashRoot()
+}
+
+// headerRecord is an individual record for a historical header.
+//
+// See https://github.com/ethereum/portal-network-specs/blob/master/history/history-network.md#the-historical-hashes-accumulator
+// for more information.
+type headerRecord struct {
+	Hash            common.Hash
+	TotalDifficulty *big.Int
+}
+
+// GetTree completes the ssz.HashRoot interface, but is unused.
+func (h *headerRecord) GetTree() (*ssz.Node, error) {
+	return nil, nil
+}
+
+// HashTreeRoot ssz hashes the headerRecord object.
+func (h *headerRecord) HashTreeRoot() ([32]byte, error) {
+	return ssz.HashWithDefaultHasher(h)
+}
+
+// HashTreeRootWith ssz hashes the headerRecord object with a hasher.
+func (h *headerRecord) HashTreeRootWith(hh ssz.HashWalker) (err error) {
+	hh.PutBytes(h.Hash[:])
+	td := bigToBytes32(h.TotalDifficulty)
+	hh.PutBytes(td[:])
+	hh.Merkleize(0)
+	return
+}
+
+// bigToBytes32 converts a big.Int into a little-endian 32-byte array.
+func bigToBytes32(n *big.Int) (b [32]byte) {
+	n.FillBytes(b[:])
+	reverseOrder(b[:])
+	return
+}
+
+// reverseOrder reverses the byte order of a slice.
+func reverseOrder(b []byte) []byte {
+	for i := 0; i < 16; i++ {
+		b[i], b[32-i-1] = b[32-i-1], b[i]
+	}
+	return b
+}

internal/era2/builder2.go

@@ -29,13 +29,16 @@ package era
 
 import (
 	"bytes"
+	"encoding/binary"
 	"errors"
 	"fmt"
 	"io"
 	"math/big"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/internal/era/e2store"
+	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/golang/snappy"
 )
 
@@ -55,6 +58,8 @@ const (
 	headerSize                           uint64 = 8
 )
 
+type proofvar uint16
+
 type Builder2 struct {
 	w   *e2store.Writer
 	buf *bytes.Buffer
@@ -65,18 +70,20 @@ type Builder2 struct {
 	bodiesRLP   [][]byte
 	receiptsRLP [][]byte
 	proofsRLP   [][]byte
-	tds         []*big.Int
+	tds         [][]byte
+	tdsint      []*big.Int
+	hashes      []common.Hash
 
 	headeroffsets  []uint64
 	bodyoffsets    []uint64
 	receiptoffsets []uint64
 	proofoffsets   []uint64
+	tdoff          []uint64
 	startTd        *big.Int
 
-	prooftype uint16
+	prooftype proofvar
 
 	startNum     *uint64
-	hashes       []common.Hash
 	writtenBytes uint64
 }
 
@@ -90,38 +97,102 @@ func NewBuilder2(w io.Writer) *Builder2 {
 	}
 }
 
-// func (b *Builder2) Add(block *types.Block, receipts types.Receipts, td *big.Int, proof[]byte) error {
+func (b *Builder2) Add(block *types.Block, receipts types.Receipts, td *big.Int, proofBytes []byte, proofty proofvar) error {
-// 	if len(b.headersRLP) >= MaxEraESize {
+	if len(b.headersRLP) >= MaxEraESize {
-// 		return fmt.Errorf("exceeds maximum batch size of %d", MaxEraESize)
+		return fmt.Errorf("exceeds MaxEraESize %d", MaxEraESize)
-// 	}
+	}
 
-// 	tdbytes := uint256LE(td)
+	if proofty != 0 && len(proofBytes) == 0 {
+		return fmt.Errorf("proof type %d requires proof bytes", proofty)
+	}
 
-// 	if b.startNum == nil {
+	if len(b.headersRLP) != 0 && proofty != b.prooftype {
-// 		start := block.NumberU64()
+		return fmt.Errorf("cannot mix proof types, expected %d, got %d", b.prooftype, proofty)
-// 		b.startNum = &start
+	}
-// 		_, err := b.w.Write(TypeVersion, nil)
-// 		if err != nil {
-// 			return fmt.Errorf("failed to write version entry: %w", err)
-// 		}
-// 		b.writtenBytes += 8
-// 	}
 
-// 	b.headersRLP = append(b.headersRLP, headerb)
+	hdr, err := rlp.EncodeToBytes(block.Header())
-// 	b.bodiesRLP = append(b.bodiesRLP, bodyb)
+	if err != nil {
-// 	b.receiptsRLP = append(b.receiptsRLP, receiptsb)
+		return fmt.Errorf("error encoding block header: %w", err)
-// 	b.tds = append(b.tds, tdbytes)
+	}
-// 	b.proofsRLP = append(b.proofsRLP, proofRLP)
+	bod, err := rlp.EncodeToBytes(block.Body())
-// 	b.hashes = append(b.hashes, block.Hash())
+	if err != nil {
-// 	return nil
+		return fmt.Errorf("error encoding block header: %w", err)
-// }
+	}
+	rct, err := rlp.EncodeToBytes(receipts)
+	if err != nil {
+		return fmt.Errorf("error encoding block header: %w", err)
+	}
 
-// func (b *Builder2) Finalize(common.Hash, error) {
+	b.headersRLP = append(b.headersRLP, hdr)
-// 	if b.startNum == nil {
+	b.bodiesRLP = append(b.bodiesRLP, bod)
-// 		return fmt.Errorf("no blocks added, cannot finalize")
+	b.receiptsRLP = append(b.receiptsRLP, rct)
-// 	}
+	b.tds = append(b.tds, uint256LE(td))
+	b.tdsint = append(b.tdsint, new(big.Int).Set(td))
+	b.hashes = append(b.hashes, block.Hash())
 
-// }
+	if b.prooftype != 0 {
+		b.proofsRLP = append(b.proofsRLP, proofBytes)
+	}
+
+	if b.startNum == nil {
+		sn := block.NumberU64()
+		b.startNum = &sn
+		if n, err := b.w.Write(TypeVersion, nil); err != nil {
+			return fmt.Errorf("error writing version entry: %w", err)
+		} else {
+			b.writtenBytes += uint64(n)
+		}
+	}
+	return nil
+}
+
+func (b *Builder2) Finalize() error {
+	if b.startNum == nil {
+		return errors.New("no blocks added, cannot finalize")
+	}
+	var err error
+	b.headeroffsets, err = b.writeSection(TypeCompressedHeader, b.headersRLP, true)
+	if err != nil {
+		return fmt.Errorf("error writing compressed headers: %w", err)
+	}
+	b.bodyoffsets, err = b.writeSection(TypeCompressedBody, b.bodiesRLP, true)
+	if err != nil {
+		return fmt.Errorf("error writing compressed bodies: %w", err)
+	}
+	b.receiptoffsets, err = b.writeSection(TypeCompressedReceipts, b.receiptsRLP, true)
+	if err != nil {
+		return fmt.Errorf("error writing compressed receipts: %w", err)
+	}
+
+	if len(b.tds) > 0 {
+		b.tdoff, err = b.writeSection(TypeTotalDifficulty, b.tds, false)
+		if err != nil {
+			return fmt.Errorf("error writing total difficulties: %w", err)
+		}
+	}
+
+	if b.prooftype != 0 {
+		b.proofoffsets, err = b.writeSection(uint16(b.prooftype), b.proofsRLP, true)
+		if err != nil {
+			return fmt.Errorf("error writing proofs: %w", err)
+		}
+	}
+
+	if len(b.hashes) > 0 {
+		accRoot, err := ComputeAccumulator(b.hashes, b.tdsint)
+		if err != nil {
+			return fmt.Errorf("error calculating accumulator root: %w", err)
+		}
+		if n, err := b.w.Write(TypeAccumulatorRoot, accRoot[:]); err != nil {
+			return fmt.Errorf("error writing accumulator root: %w", err)
+		} else {
+			b.writtenBytes += uint64(n)
+		}
+	}
+
+	return b.writeIndex()
+
+}
 
 func uint256LE(v *big.Int) []byte {
 	b := v.FillBytes(make([]byte, 32))
@@ -200,4 +271,35 @@ func (b *Builder2) writeIndex() error {
 		compcount++
 	}
 
+	idx := make([]byte, 8+count*8*compcount+16) //8 for start block, 8 per property per block, 16 for the number of properties and the number of blocks
+	binary.LittleEndian.PutUint64(idx, *b.startNum)
+	base := int64(b.writtenBytes)
+	pos := 8
+	rel := func(abs uint64) uint64 { return uint64(int64(abs) - base) }
+	for i := uint64(0); i < count; i++ {
+		binary.LittleEndian.PutUint64(idx[pos:], rel(b.headeroffsets[i]))
+		pos += 8
+		binary.LittleEndian.PutUint64(idx[pos:], rel(b.bodyoffsets[i]))
+		pos += 8
+		binary.LittleEndian.PutUint64(idx[pos:], rel(b.receiptoffsets[i]))
+		pos += 8
+		if len(b.tds) > 0 {
+			binary.LittleEndian.PutUint64(idx[pos+24:], rel(b.tdoff[i]))
+			pos += 8
+		}
+		if len(b.proofsRLP) > 0 {
+			binary.LittleEndian.PutUint64(idx[pos:], rel(b.proofoffsets[i]))
+			pos += 8
+		}
+	}
+
+	binary.LittleEndian.PutUint64(idx[pos:], compcount)
+	pos += 8
+	binary.LittleEndian.PutUint64(idx[pos:], count)
+	if n, err := b.w.Write(TypeBlockIndex, idx); err != nil {
+		return err
+	} else {
+		b.writtenBytes += uint64(n)
+	}
+	return nil
 }