cmd/utils,internal/era: some refactoring, use ReceiptForStorage in eraE

cmd/utils/cmd.go

@@ -45,7 +45,6 @@ import (
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/internal/debug"
 	"github.com/ethereum/go-ethereum/internal/era"
-	"github.com/ethereum/go-ethereum/internal/era/onedb"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/params"
@@ -431,16 +430,12 @@ func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64, ne
 		return fmt.Errorf("error creating output directory: %w", err)
 	}
 
-	td := new(big.Int)
-	for n := uint64(0); n < first; n++ {
-		td.Add(td, bc.GetHeaderByNumber(n).Difficulty)
-	}
-
 	var (
 		start     = time.Now()
 		reported  = time.Now()
 		h         = sha256.New()
 		buf       = bytes.NewBuffer(nil)
+		td        = new(big.Int) // TODO: should disallow non-zero "first" since we need to compute td from genesis now that we don't store it
 		checksums []string
 	)
 
@@ -449,13 +444,13 @@ func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64, ne
 		tmpPath := filepath.Join(dir, filename(network, idx, common.Hash{}))
 
 		if err := func() error {
-			fh, err := os.Create(tmpPath)
+			f, err := os.Create(tmpPath)
 			if err != nil {
 				return err
 			}
-			defer fh.Close()
+			defer f.Close()
 
-			bldr := newBuilder(fh)
+			builder := newBuilder(f)
 
 			for j := uint64(0); j < step && batch+j <= last; j++ {
 				n := batch + j
@@ -469,26 +464,26 @@ func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64, ne
 				}
 				td.Add(td, blk.Difficulty())
 
-				if err := bldr.Add(blk, rcpt, new(big.Int).Set(td), nil); err != nil {
+				if err := builder.Add(blk, rcpt, new(big.Int).Set(td), nil); err != nil {
 					return err
 				}
 			}
-
+			root, err := builder.Finalize()
-			root, err := bldr.Finalize()
 			if err != nil {
 				return err
 			}
-			final := filepath.Join(dir, onedb.Filename(network, idx, root))
+			// TODO: this root shouldn't be used post merge!
+			final := filepath.Join(dir, filename(network, idx, root))
 			if err := os.Rename(tmpPath, final); err != nil {
 				return err
 			}
 
-			if _, err := fh.Seek(0, io.SeekStart); err != nil {
+			if _, err := f.Seek(0, io.SeekStart); err != nil {
 				return err
 			}
 			h.Reset()
 			buf.Reset()
-			if _, err := io.Copy(h, fh); err != nil {
+			if _, err := io.Copy(h, f); err != nil {
 				return err
 			}
 			checksums = append(checksums, common.BytesToHash(h.Sum(buf.Bytes()[:])).Hex())

cmd/utils/history_test.go

@@ -33,6 +33,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/internal/era"
+	"github.com/ethereum/go-ethereum/internal/era/execdb"
 	"github.com/ethereum/go-ethereum/internal/era/onedb"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/trie"
@@ -45,136 +46,148 @@ var (
 )
 
 func TestHistoryImportAndExport(t *testing.T) {
-	var (
+	for _, tt := range []struct {
-		key, _  = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
+		name     string
-		address = crypto.PubkeyToAddress(key.PublicKey)
+		builder  func(io.Writer) era.Builder
-		genesis = &core.Genesis{
+		filename func(network string, epoch int, root common.Hash) string
-			Config: params.TestChainConfig,
+		from     func(f era.ReadAtSeekCloser) (era.Era, error)
-			Alloc:  types.GenesisAlloc{address: {Balance: big.NewInt(1000000000000000000)}},
+	}{
-		}
+		{"era1", onedb.NewBuilder, onedb.Filename, onedb.From},
-		signer = types.LatestSigner(genesis.Config)
+		{"erae", execdb.NewBuilder, execdb.Filename, execdb.From},
-	)
+	} {
-
+		t.Run(tt.name, func(t *testing.T) {
-	// Generate chain.
-	db, blocks, _ := core.GenerateChainWithGenesis(genesis, ethash.NewFaker(), int(count), func(i int, g *core.BlockGen) {
-		if i == 0 {
-			return
-		}
-		tx, err := types.SignNewTx(key, signer, &types.DynamicFeeTx{
-			ChainID:    genesis.Config.ChainID,
-			Nonce:      uint64(i - 1),
-			GasTipCap:  common.Big0,
-			GasFeeCap:  g.PrevBlock(0).BaseFee(),
-			Gas:        50000,
-			To:         &common.Address{0xaa},
-			Value:      big.NewInt(int64(i)),
-			Data:       nil,
-			AccessList: nil,
-		})
-		if err != nil {
-			t.Fatalf("error creating tx: %v", err)
-		}
-		g.AddTx(tx)
-	})
-
-	// Initialize BlockChain.
-	chain, err := core.NewBlockChain(db, genesis, ethash.NewFaker(), nil)
-	if err != nil {
-		t.Fatalf("unable to initialize chain: %v", err)
-	}
-	if _, err := chain.InsertChain(blocks); err != nil {
-		t.Fatalf("error inserting chain: %v", err)
-	}
-
-	// Make temp directory for era files.
-	dir := t.TempDir()
-
-	// Export history to temp directory.
-	if err := ExportHistory(chain, dir, 0, count, step, onedb.NewBuilder, onedb.Filename); err != nil {
-		t.Fatalf("error exporting history: %v", err)
-	}
-
-	// Read checksums.
-	b, err := os.ReadFile(filepath.Join(dir, "checksums.txt"))
-	if err != nil {
-		t.Fatalf("failed to read checksums: %v", err)
-	}
-	checksums := strings.Split(string(b), "\n")
-
-	// Verify each Era.
-	entries, _ := era.ReadDir(dir, "mainnet")
-	for i, filename := range entries {
-		func() {
-			f, err := os.Open(filepath.Join(dir, filename))
-			if err != nil {
-				t.Fatalf("error opening era file: %v", err)
-			}
 			var (
-				h   = sha256.New()
+				key, _  = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
-				buf = bytes.NewBuffer(nil)
+				address = crypto.PubkeyToAddress(key.PublicKey)
+				genesis = &core.Genesis{
+					Config: params.TestChainConfig,
+					Alloc:  types.GenesisAlloc{address: {Balance: big.NewInt(1000000000000000000)}},
+				}
+				signer = types.LatestSigner(genesis.Config)
 			)
-			if _, err := io.Copy(h, f); err != nil {
+
-				t.Fatalf("unable to recalculate checksum: %v", err)
+			// Generate chain.
-			}
+			db, blocks, _ := core.GenerateChainWithGenesis(genesis, ethash.NewFaker(), int(count), func(i int, g *core.BlockGen) {
-			if got, want := common.BytesToHash(h.Sum(buf.Bytes()[:])).Hex(), checksums[i]; got != want {
+				if i == 0 {
-				t.Fatalf("checksum %d does not match: got %s, want %s", i, got, want)
+					return
-			}
-			e, err := onedb.From(f)
-			if err != nil {
-				t.Fatalf("error opening era: %v", err)
-			}
-			defer e.Close()
-			it, err := onedb.NewIterator(e)
-			if err != nil {
-				t.Fatalf("error making era reader: %v", err)
-			}
-			for j := 0; it.Next(); j++ {
-				n := i*int(step) + j
-				if it.Error() != nil {
-					t.Fatalf("error reading block entry %d: %v", n, it.Error())
 				}
-				block, receipts, err := it.BlockAndReceipts()
+				tx, err := types.SignNewTx(key, signer, &types.DynamicFeeTx{
+					ChainID:    genesis.Config.ChainID,
+					Nonce:      uint64(i - 1),
+					GasTipCap:  common.Big0,
+					GasFeeCap:  g.PrevBlock(0).BaseFee(),
+					Gas:        50000,
+					To:         &common.Address{0xaa},
+					Value:      big.NewInt(int64(i)),
+					Data:       nil,
+					AccessList: nil,
+				})
 				if err != nil {
-					t.Fatalf("error reading block entry %d: %v", n, err)
+					t.Fatalf("error creating tx: %v", err)
-				}
-				want := chain.GetBlockByNumber(uint64(n))
-				if want, got := uint64(n), block.NumberU64(); want != got {
-					t.Fatalf("blocks out of order: want %d, got %d", want, got)
-				}
-				if want.Hash() != block.Hash() {
-					t.Fatalf("block hash mismatch %d: want %s, got %s", n, want.Hash().Hex(), block.Hash().Hex())
-				}
-				if got := types.DeriveSha(block.Transactions(), trie.NewStackTrie(nil)); got != want.TxHash() {
-					t.Fatalf("tx hash %d mismatch: want %s, got %s", n, want.TxHash(), got)
-				}
-				if got := types.CalcUncleHash(block.Uncles()); got != want.UncleHash() {
-					t.Fatalf("uncle hash %d mismatch: want %s, got %s", n, want.UncleHash(), got)
-				}
-				if got := types.DeriveSha(receipts, trie.NewStackTrie(nil)); got != want.ReceiptHash() {
-					t.Fatalf("receipt root %d mismatch: want %s, got %s", n, want.ReceiptHash(), got)
 				}
+				g.AddTx(tx)
+			})
+
+			// Initialize BlockChain.
+			chain, err := core.NewBlockChain(db, genesis, ethash.NewFaker(), nil)
+			if err != nil {
+				t.Fatalf("unable to initialize chain: %v", err)
+			}
+			if _, err := chain.InsertChain(blocks); err != nil {
+				t.Fatalf("error inserting chain: %v", err)
 			}
-		}()
-	}
 
-	// Now import Era.
+			// Make temp directory for era files.
-	db2, err := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{})
+			dir := t.TempDir()
-	if err != nil {
-		panic(err)
-	}
-	t.Cleanup(func() {
-		db2.Close()
-	})
 
-	genesis.MustCommit(db2, triedb.NewDatabase(db2, triedb.HashDefaults))
+			// Export history to temp directory.
-	imported, err := core.NewBlockChain(db2, genesis, ethash.NewFaker(), nil)
+			if err := ExportHistory(chain, dir, 0, count, step, tt.builder, tt.filename); err != nil {
-	if err != nil {
+				t.Fatalf("error exporting history: %v", err)
-		t.Fatalf("unable to initialize chain: %v", err)
+			}
-	}
+
-	if err := ImportHistory(imported, dir, "mainnet", onedb.From); err != nil {
+			// Read checksums.
-		t.Fatalf("failed to import chain: %v", err)
+			b, err := os.ReadFile(filepath.Join(dir, "checksums.txt"))
-	}
+			if err != nil {
-	if have, want := imported.CurrentHeader(), chain.CurrentHeader(); have.Hash() != want.Hash() {
+				t.Fatalf("failed to read checksums: %v", err)
-		t.Fatalf("imported chain does not match expected, have (%d, %s) want (%d, %s)", have.Number, have.Hash(), want.Number, want.Hash())
+			}
+			checksums := strings.Split(string(b), "\n")
+
+			// Verify each Era.
+			entries, _ := era.ReadDir(dir, "mainnet")
+			for i, filename := range entries {
+				func() {
+					f, err := os.Open(filepath.Join(dir, filename))
+					if err != nil {
+						t.Fatalf("error opening era file: %v", err)
+					}
+					var (
+						h   = sha256.New()
+						buf = bytes.NewBuffer(nil)
+					)
+					if _, err := io.Copy(h, f); err != nil {
+						t.Fatalf("unable to recalculate checksum: %v", err)
+					}
+					if got, want := common.BytesToHash(h.Sum(buf.Bytes()[:])).Hex(), checksums[i]; got != want {
+						t.Fatalf("checksum %d does not match: got %s, want %s", i, got, want)
+					}
+					e, err := tt.from(f)
+					if err != nil {
+						t.Fatalf("error opening era: %v", err)
+					}
+					defer e.Close()
+					it, err := e.Iterator()
+					if err != nil {
+						t.Fatalf("error making era reader: %v", err)
+					}
+					for j := 0; it.Next(); j++ {
+						n := i*int(step) + j
+						if it.Error() != nil {
+							t.Fatalf("error reading block entry %d: %v", n, it.Error())
+						}
+						block, receipts, err := it.BlockAndReceipts()
+						if err != nil {
+							t.Fatalf("error reading block entry %d: %v", n, err)
+						}
+						want := chain.GetBlockByNumber(uint64(n))
+						if want, got := uint64(n), block.NumberU64(); want != got {
+							t.Fatalf("blocks out of order: want %d, got %d", want, got)
+						}
+						if want.Hash() != block.Hash() {
+							t.Fatalf("block hash mismatch %d: want %s, got %s", n, want.Hash().Hex(), block.Hash().Hex())
+						}
+						if got := types.DeriveSha(block.Transactions(), trie.NewStackTrie(nil)); got != want.TxHash() {
+							t.Fatalf("tx hash %d mismatch: want %s, got %s", n, want.TxHash(), got)
+						}
+						if got := types.CalcUncleHash(block.Uncles()); got != want.UncleHash() {
+							t.Fatalf("uncle hash %d mismatch: want %s, got %s", n, want.UncleHash(), got)
+						}
+						if got := types.DeriveSha(receipts, trie.NewStackTrie(nil)); got != want.ReceiptHash() {
+							t.Fatalf("receipt root %d mismatch: want %s, got %s", n, want.ReceiptHash(), got)
+						}
+					}
+				}()
+			}
+
+			// Now import Era.
+			db2, err := rawdb.Open(rawdb.NewMemoryDatabase(), rawdb.OpenOptions{})
+			if err != nil {
+				panic(err)
+			}
+			t.Cleanup(func() {
+				db2.Close()
+			})
+
+			genesis.MustCommit(db2, triedb.NewDatabase(db2, triedb.HashDefaults))
+			imported, err := core.NewBlockChain(db2, genesis, ethash.NewFaker(), nil)
+			if err != nil {
+				t.Fatalf("unable to initialize chain: %v", err)
+			}
+			if err := ImportHistory(imported, dir, "mainnet", tt.from); err != nil {
+				t.Fatalf("failed to import chain: %v", err)
+			}
+			if have, want := imported.CurrentHeader(), chain.CurrentHeader(); have.Hash() != want.Hash() {
+				t.Fatalf("imported chain does not match expected, have (%d, %s) want (%d, %s)", have.Number, have.Hash(), want.Number, want.Hash())
+			}
+		})
 	}
 }

internal/era/era.go

@@ -78,7 +78,6 @@ func ReadDir(dir, network string) ([]string, error) {
 		eras    []string
 		dirType string
 	)
-	fmt.Println("entries", entries)
 	for _, entry := range entries {
 		ext := path.Ext(entry.Name())
 		if ext != ".erae" && ext != ".era1" {

internal/era/execdb/builder.go

@@ -58,190 +58,175 @@ import (
 	"github.com/golang/snappy"
 )
 
-// Temporary buffer for writing blocks until the Finalize method is called.
+// Builder is used to build an Era2 e2store file. It collects block entries and writes them to the underlying e2store.Writer.
-type buffer struct {
+type Builder struct {
+	w *e2store.Writer
+
 	headers  [][]byte
+	hashes   []common.Hash
 	bodies   [][]byte
 	receipts [][]byte
 	proofs   [][]byte
 	tds      []*big.Int
-}
 
-// The offsets holds the offsets of the different block components in the e2store file. Eventually these offsets will be used to write the index table at the end of the file.
+	startNum *uint64
-type offsets struct {
+	merged   bool
-	headers      []uint64
+	written  uint64
-	bodys        []uint64
-	receipts     []uint64
-	proofoffsets []uint64
-	tdoff        []uint64
-}
 
-// Builder is used to build an Era2 e2store file. It collects block entries and writes them to the underlying e2store.Writer.
+	buf    *bytes.Buffer
-type Builder struct {
+	snappy *snappy.Writer
-	w   *e2store.Writer
-	tmp *bytes.Buffer
-
-	buff buffer
-	off  offsets
-
-	hashes        []common.Hash
-	startNum      *uint64
-	written       uint64
-	expectsProofs bool
-	isPreMerge    bool
-	finalTD       *big.Int // final total difficulty, used for pre-merge and merge straddling files
 }
 
 // NewBuilder returns a new Builder instance.
 func NewBuilder(w io.Writer) era.Builder {
-	tmp := bytes.NewBuffer(nil)
 	return &Builder{
-		w:   e2store.NewWriter(w),
+		w: e2store.NewWriter(w),
-		tmp: tmp,
 	}
 }
 
 // Add writes a block entry, its reciepts, and optionally its proofs as well into the e2store file.
 func (b *Builder) Add(block *types.Block, receipts types.Receipts, td *big.Int, proof era.Proof) error {
-	header := block.Header()
+	eh, err := rlp.EncodeToBytes(block.Header())
-	body := block.Body()
-	if len(b.buff.headers) == 0 { // first block determines wether proofs are expected
-		b.expectsProofs = proof != nil
-	} else if b.expectsProofs && proof == nil { // every later block must follow this policy
-		return fmt.Errorf("block %d missing proof: proofs required for every block", header.Number.Uint64())
-	} else if !b.expectsProofs && proof != nil {
-		return fmt.Errorf("unexpected proof for block %d: first block had none", header.Number.Uint64())
-	}
-
-	eh, err := rlp.EncodeToBytes(&header)
 	if err != nil {
 		return fmt.Errorf("encode header: %w", err)
 	}
-	eb, err := rlp.EncodeToBytes(&body)
+	eb, err := rlp.EncodeToBytes(block.Body())
 	if err != nil {
 		return fmt.Errorf("encode body: %w", err)
 	}
-	er, err := rlp.EncodeToBytes(receipts)
+
+	rs := make([]*types.ReceiptForStorage, len(receipts))
+	for i, receipt := range receipts {
+		rs[i] = (*types.ReceiptForStorage)(receipt)
+	}
+	er, err := rlp.EncodeToBytes(rs)
 	if err != nil {
 		return fmt.Errorf("encode receipts: %w", err)
 	}
 
 	var ep []byte
-	var buffer bytes.Buffer
-
 	if proof != nil {
-		if err := proof.EncodeRLP(&buffer); err != nil {
+		ep, err = rlp.EncodeToBytes(ep)
+		if err != nil {
 			return fmt.Errorf("encode proof: %w", err)
 		}
-		ep = buffer.Bytes()
+	}
+	var difficulty *big.Int
+	if !b.merged {
+		difficulty = block.Difficulty()
 	}
 
-	return b.AddRLP(
+	return b.AddRLP(eh, eb, er, ep, block.Number().Uint64(), block.Hash(), td, difficulty)
-		eh, eb, er, ep,
-		header.Number.Uint64(),
-		header.Hash(), td, nil,
-	)
 }
 
 // AddRLP takes the RLP encoded block components and writes them to the underlying e2store file.
-func (b *Builder) AddRLP(headerRLP []byte, bodyRLP []byte, receipts []byte, proof []byte, blockNum uint64, blockHash common.Hash, td, difficulty *big.Int) error {
+func (b *Builder) AddRLP(header []byte, body []byte, receipts []byte, proof []byte, number uint64, blockHash common.Hash, td, difficulty *big.Int) error {
-	if difficulty != nil {
+	if b.startNum == nil {
-		return fmt.Errorf("block difficulty not allowed in erae format")
+		b.startNum = new(uint64)
+		*b.startNum = number
+		b.merged = difficulty.Sign() == 0
 	}
-	if len(b.buff.headers) >= era.MaxSize {
+	if len(b.headers) >= era.MaxSize {
 		return fmt.Errorf("exceeds max size %d", era.MaxSize)
 	}
-	if len(b.buff.headers) == 0 && td != nil {
+	if !b.merged && (td == nil || difficulty == nil || difficulty.Sign() == 0) {
-		if td.Sign() > 0 {
+		return fmt.Errorf("era pre-merge, but difficulty values not supplied")
-			b.isPreMerge = true
+	}
-		}
+	if b.merged && (td != nil || difficulty != nil) {
+		return fmt.Errorf("era already merged, but given non-nil difficulty values")
+	}
+	if len(b.headers) != 0 && len(b.proofs) == 0 && proof != nil {
+		return fmt.Errorf("unexpected proof for block %d: first block had none", number)
+	}
+	if len(b.headers) != 0 && len(b.proofs) != 0 && proof == nil {
+		return fmt.Errorf("block %d missing proof: proofs required for every block", number)
 	}
 
-	b.buff.headers = append(b.buff.headers, headerRLP)
+	b.headers = append(b.headers, header)
-	b.buff.bodies = append(b.buff.bodies, bodyRLP)
+	b.bodies = append(b.bodies, body)
-	b.buff.receipts = append(b.buff.receipts, receipts)
+	b.receipts = append(b.receipts, receipts)
-	if td != nil {
+
-		if b.isPreMerge && td.Sign() > 0 {
+	if !b.merged {
-			b.buff.tds = append(b.buff.tds, new(big.Int).Set(td))
+		if difficulty.Sign() != 0 {
-			b.finalTD = new(big.Int).Set(td)
 			b.hashes = append(b.hashes, blockHash)
-		} else if b.isPreMerge {
-			b.buff.tds = append(b.buff.tds, new(big.Int).Set(b.finalTD))
 		}
+		b.tds = append(b.tds, new(big.Int).Set(td))
 	}
 
 	if proof != nil {
-		b.buff.proofs = append(b.buff.proofs, proof)
+		b.proofs = append(b.proofs, proof)
-	}
-
-	// Write Era2 version before writing any blocks.
-	if b.startNum == nil {
-		b.startNum = new(uint64)
-		*b.startNum = blockNum
-		if n, err := b.w.Write(era.TypeVersion, nil); err != nil {
-			return fmt.Errorf("write version entry: %w", err)
-		} else {
-			b.written += uint64(n)
-		}
 	}
 	return nil
 }
 
+type offsets struct {
+	headers  []uint64
+	bodies   []uint64
+	receipts []uint64
+	tds      []uint64
+	proofs   []uint64
+}
+
 // Finalize writes all collected block entries to the e2store file and returns the accumulator root hash.
 // It also writes the index table at the end of the file, which contains offsets to each block entry.
 func (b *Builder) Finalize() (common.Hash, error) {
 	if b.startNum == nil {
 		return common.Hash{}, errors.New("no blocks added, cannot finalize")
 	}
-	for _, data := range b.buff.headers {
+	// Write Era2 version before writing any blocks.
-		off, err := b.addEntry(era.TypeCompressedHeader, data, true)
+	if n, err := b.w.Write(era.TypeVersion, nil); err != nil {
-		if err != nil {
+		return common.Hash{}, fmt.Errorf("write version entry: %w", err)
-			return common.Hash{}, fmt.Errorf("headers: %w", err)
+	} else {
-		}
+		b.written += uint64(n)
-		b.off.headers = append(b.off.headers, off)
 	}
 
-	for _, data := range b.buff.bodies {
+	// Convert int values to byte-level LE representation.
-		off, err := b.addEntry(era.TypeCompressedBody, data, true)
+	var tds [][]byte
-		if err != nil {
+	for _, td := range b.tds {
-			return common.Hash{}, fmt.Errorf("bodies: %w", err)
+		tds = append(tds, uint256LE(td))
-		}
-		b.off.bodys = append(b.off.bodys, off)
 	}
 
-	for _, data := range b.buff.receipts {
+	// Create snappy writer.
-		off, err := b.addEntry(era.TypeCompressedSlimReceipts, data, true)
+	b.buf = bytes.NewBuffer(nil)
-		if err != nil {
+	b.snappy = snappy.NewBufferedWriter(b.buf)
-			return common.Hash{}, fmt.Errorf("receipts: %w", err)
-		}
-		b.off.receipts = append(b.off.receipts, off)
-	}
 
-	if len(b.buff.tds) > 0 {
+	var o offsets
-		for _, data := range b.buff.tds {
+	for _, section := range []struct {
-			littleEndian := uint256LE(data)
+		typ        uint16
-			off, err := b.addEntry(era.TypeTotalDifficulty, littleEndian, false)
+		data       [][]byte
-			if err != nil {
+		compressed bool
-				return common.Hash{}, fmt.Errorf("total-difficulty: %w", err)
+		offsets    *[]uint64
+	}{
+		{era.TypeCompressedHeader, b.headers, true, &o.headers},
+		{era.TypeCompressedBody, b.bodies, true, &o.bodies},
+		{era.TypeCompressedSlimReceipts, b.receipts, true, &o.receipts},
+		{era.TypeTotalDifficulty, tds, false, &o.tds},
+		{era.TypeProof, b.proofs, true, &o.proofs},
+	} {
+		for _, data := range section.data {
+			*section.offsets = append(*section.offsets, b.written)
+			if section.compressed {
+				// Write snappy compressed data.
+				if err := b.snappyWrite(section.typ, data); err != nil {
+					return common.Hash{}, err
+				}
+			} else {
+				// Directly write uncompressed data.
+				n, err := b.w.Write(section.typ, data)
+				if err != nil {
+					return common.Hash{}, err
+				}
+				b.written += uint64(n)
 			}
-			b.off.tdoff = append(b.off.tdoff, off)
-		}
-	}
-
-	if len(b.buff.proofs) > 0 {
-		for _, data := range b.buff.proofs {
-			off, err := b.addEntry(era.TypeProof, data, true)
-			if err != nil {
-				return common.Hash{}, fmt.Errorf("proofs: %w", err)
-			}
-			b.off.proofoffsets = append(b.off.proofoffsets, off)
 		}
 	}
 
+	// Compute and write accumlator root only when the first block the epoch is
+	// pre-merge, otherwise omit.
 	var accRoot common.Hash
-	if len(b.hashes) > 0 {
+	if !b.merged {
 		var err error
-		accRoot, err = era.ComputeAccumulator(b.hashes, b.buff.tds[:len(b.hashes)])
+		accRoot, err = era.ComputeAccumulator(b.hashes, b.tds[:len(b.hashes)])
 		if err != nil {
 			return common.Hash{}, fmt.Errorf("compute accumulator: %w", err)
 		}
@@ -252,7 +237,9 @@ func (b *Builder) Finalize() (common.Hash, error) {
 		}
 	}
 
-	return accRoot, b.writeIndex()
+	// TODO: accumulator root should only be returned when it's relevant
+	// (pre-merge)?
+	return accRoot, b.writeIndex(&o)
 }
 
 // uin256LE writes 32 byte big integers to little endian.
@@ -266,17 +253,15 @@ func uint256LE(v *big.Int) []byte {
 
 // SnappyWrite compresses the input data using snappy and writes it to the e2store file.
 func (b *Builder) snappyWrite(typ uint16, in []byte) error {
-	var tmp = b.tmp
+	b.buf.Reset()
-	snappy := snappy.NewBufferedWriter(b.tmp)
+	b.snappy.Reset(b.buf)
-	tmp.Reset()
+	if _, err := b.snappy.Write(in); err != nil {
-	snappy.Reset(tmp)
-	if _, err := snappy.Write(in); err != nil {
 		return fmt.Errorf("error snappy encoding: %w", err)
 	}
-	if err := snappy.Flush(); err != nil {
+	if err := b.snappy.Flush(); err != nil {
 		return fmt.Errorf("error flushing snappy encoding: %w", err)
 	}
-	n, err := b.w.Write(typ, b.tmp.Bytes())
+	n, err := b.w.Write(typ, b.buf.Bytes())
 	b.written += uint64(n)
 	if err != nil {
 		return fmt.Errorf("error writing e2store entry: %w", err)
@@ -284,32 +269,14 @@ func (b *Builder) snappyWrite(typ uint16, in []byte) error {
 	return nil
 }
 
-// addEntry takes the e2store object and writes it into the file.
-func (b *Builder) addEntry(typ uint16, payload []byte, compressed bool) (uint64, error) {
-	offset := b.written
-	var err error
-	if compressed {
-		if err = b.snappyWrite(typ, payload); err != nil {
-			return 0, err
-		}
-	} else {
-		var n int
-		if n, err = b.w.Write(typ, payload); err != nil {
-			return 0, err
-		}
-		b.written += uint64(n)
-	}
-	return offset, nil
-}
-
 // writeIndex takes all the offset table and writes it to the file. The index table contains all offsets to specific block entries
-func (b *Builder) writeIndex() error {
+func (b *Builder) writeIndex(o *offsets) error {
-	count := uint64(len(b.off.headers))
+	count := uint64(len(o.headers))
 	componentCount := uint64(3)
-	if len(b.buff.tds) > 0 {
+	if len(o.tds) > 0 {
 		componentCount++
 	}
-	if len(b.buff.proofs) > 0 {
+	if len(o.proofs) > 0 {
 		componentCount++
 	}
 
@@ -320,17 +287,17 @@ func (b *Builder) writeIndex() error {
 	for i := uint64(0); i < count; i++ {
 		basePosition := 8 + i*componentCount*8
 
-		binary.LittleEndian.PutUint64(index[basePosition:], rel(b.off.headers[i]))
+		binary.LittleEndian.PutUint64(index[basePosition:], rel(o.headers[i]))
-		binary.LittleEndian.PutUint64(index[basePosition+8:], rel(b.off.bodys[i]))
+		binary.LittleEndian.PutUint64(index[basePosition+8:], rel(o.bodies[i]))
-		binary.LittleEndian.PutUint64(index[basePosition+16:], rel(b.off.receipts[i]))
+		binary.LittleEndian.PutUint64(index[basePosition+16:], rel(o.receipts[i]))
 
 		pos := uint64(24)
-		if len(b.buff.tds) > 0 {
+		if len(o.tds) > 0 {
-			binary.LittleEndian.PutUint64(index[basePosition+pos:], rel(b.off.tdoff[i]))
+			binary.LittleEndian.PutUint64(index[basePosition+pos:], rel(o.tds[i]))
 			pos += 8
 		}
-		if len(b.buff.proofs) > 0 {
+		if len(o.proofs) > 0 {
-			binary.LittleEndian.PutUint64(index[basePosition+pos:], rel(b.off.proofoffsets[i]))
+			binary.LittleEndian.PutUint64(index[basePosition+pos:], rel(o.proofs[i]))
 		}
 	}
 	end := 8 + count*componentCount*8

internal/era/execdb/era_test.go

@@ -22,25 +22,26 @@ import (
 	"io"
 	"math/big"
 	"os"
+	"slices"
 	"testing"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/klauspost/compress/snappy"
 )
 
 type testchain struct {
-	headers  []types.Header
+	headers  [][]byte
-	bodies   []types.Body
+	bodies   [][]byte
-	receipts []types.Receipts
+	receipts [][]byte
 	tds      []*big.Int
 }
 
-func TestEra2Builder(t *testing.T) {
+func TestEra1Builder(t *testing.T) {
 	t.Parallel()
+
 	// Get temp directory.
-	f, err := os.CreateTemp(t.TempDir(), "era2-test")
+	f, err := os.CreateTemp(t.TempDir(), "era1-test")
 	if err != nil {
 		t.Fatalf("error creating temp file: %v", err)
 	}
@@ -51,10 +52,10 @@ func TestEra2Builder(t *testing.T) {
 		chain   = testchain{}
 	)
 	for i := 0; i < 128; i++ {
-		chain.headers = append(chain.headers, types.Header{Number: big.NewInt(int64(i))})
+		chain.headers = append(chain.headers, mustEncode(&types.Header{Number: big.NewInt(int64(i))}))
-		chain.bodies = append(chain.bodies, types.Body{Transactions: []*types.Transaction{types.NewTransaction(0, common.Address{byte(i)}, nil, 0, nil, nil)}})
+		chain.bodies = append(chain.bodies, mustEncode(&types.Body{Transactions: []*types.Transaction{types.NewTransaction(0, common.Address{byte(i)}, nil, 0, nil, nil)}}))
-		chain.receipts = append(chain.receipts, types.Receipts{{CumulativeGasUsed: uint64(i)}})
+		chain.receipts = append(chain.receipts, mustEncode([]types.ReceiptForStorage{{CumulativeGasUsed: uint64(i)}}))
-		chain.tds = append(chain.tds, big.NewInt(int64(i+1)))
+		chain.tds = append(chain.tds, big.NewInt(int64(i)))
 	}
 
 	// Write blocks to Era1.
@@ -63,9 +64,10 @@ func TestEra2Builder(t *testing.T) {
 			header   = chain.headers[i]
 			body     = chain.bodies[i]
 			receipts = chain.receipts[i]
+			hash     = common.Hash{byte(i)}
 			td       = chain.tds[i]
 		)
-		if err = builder.Add(types.NewBlockWithHeader(&header).WithBody(body), receipts, td, nil); err != nil {
+		if err = builder.AddRLP(header, body, receipts, nil, uint64(i), hash, td, big.NewInt(1)); err != nil {
 			t.Fatalf("error adding entry: %v", err)
 		}
 	}
@@ -75,104 +77,70 @@ func TestEra2Builder(t *testing.T) {
 		t.Fatalf("error finalizing era1: %v", err)
 	}
 
-	// 3. open reader
+	// Verify Era1 contents.
-	t.Logf("filename: %s", f.Name())
+	e, err := Open(f.Name())
-	era, err := Open(f.Name())
 	if err != nil {
-		t.Fatalf("open era: %v", err)
+		t.Fatalf("failed to open era: %v", err)
 	}
-	defer era.Close()
+	defer e.Close()
-
+	it, err := NewRawIterator(e)
-	hdrs, err := era.GetHeaders(0, 128)
 	if err != nil {
-		t.Fatalf("BatchHeaders full: %v", err)
+		t.Fatalf("failed to make iterator: %s", err)
 	}
-	bods, err := era.GetBodies(0, 128)
+	for i := uint64(0); i < uint64(len(chain.headers)); i++ {
-	if err != nil {
+		if !it.Next() {
-		t.Fatalf("BatchBodies full: %v", err)
+			t.Fatalf("expected more entries, have %d want %d", i, len(chain.headers))
-	}
-	recs, err := era.GetReceipts(0, 128)
-	if err != nil {
-		t.Fatalf("BatchReceipts full: %v", err)
-	}
-
-	for i := 0; i < 128; i++ {
-		if hdrs[i].Hash() != chain.headers[i].Hash() {
-			t.Fatalf("batch header %d mismatch", i)
 		}
-		if !bytes.Equal(
+		if it.Error() != nil {
-			mustEncode(chain.bodies[i]),
+			t.Fatalf("unexpected error %v", it.Error())
-			mustEncode(bods[i]),
-		) {
-			t.Fatalf("batch body %d mismatch", i)
 		}
-		if !bytes.Equal(
+		// Check headers.
-			mustEncode(chain.receipts[i]),
+		rawHeader, err := io.ReadAll(it.Header)
-			mustEncode(recs[i]),
-		) {
-			t.Fatalf("batch receipts %d mismatch", i)
-		}
-	}
-
-	const start, cnt = 50, 10
-	winHdrs, err := era.GetHeaders(start, cnt)
-	if err != nil {
-		t.Fatalf("BatchHeaders window: %v", err)
-	}
-	for j := 0; j < cnt; j++ {
-		idx := start + j
-		if winHdrs[j].Hash() != chain.headers[idx].Hash() {
-			t.Fatalf("window header %d mismatch", idx)
-		}
-	}
-
-	for i := 0; i < 128; i++ {
-		bn := uint64(i)
-		gotBlock, err := era.GetBlockByNumber(bn)
 		if err != nil {
-			t.Fatalf("get block %d: %v", i, err)
+			t.Fatalf("error reading header from iterator: %v", err)
 		}
-		if chain.headers[i].Hash() != gotBlock.Header().Hash() {
+		if !bytes.Equal(rawHeader, chain.headers[i]) {
-			t.Fatalf("header %d mismatch", i)
+			t.Fatalf("mismatched header: want %s, got %s", chain.headers[i], rawHeader)
-		}
-		if !bytes.Equal(mustEncode(chain.bodies[i]), mustEncode(gotBlock.Body())) {
-			t.Fatalf("body %d mismatch", i)
 		}
 
-		rawBody, err := era.GetRawBodyByNumber(bn)
+		// Check bodies.
+		body, err := io.ReadAll(it.Body)
 		if err != nil {
-			t.Fatalf("raw body %d: %v", i, err)
+			t.Fatalf("error reading body: %v", err)
 		}
-		decBody, err := io.ReadAll(
+		if !bytes.Equal(body, chain.bodies[i]) {
-			snappy.NewReader(bytes.NewReader(rawBody)),
+			t.Fatalf("mismatched body: want %s, got %s", chain.bodies[i], body)
-		)
-		if err != nil {
-			t.Fatalf("snappy decode body %d: %v", i, err)
-		}
-		if !bytes.Equal(decBody, mustEncode(chain.bodies[i])) {
-			t.Fatalf("body frame %d mismatch", i)
 		}
 
-		rawRcpt, err := era.GetRawReceiptsByNumber(bn)
+		// Check receipts.
+		rawReceipts, err := io.ReadAll(it.Receipts)
 		if err != nil {
-			t.Fatalf("raw receipts %d: %v", i, err)
+			t.Fatalf("error reading receipts from iterator: %v", err)
 		}
-		decRcpt, err := io.ReadAll(
+		if !bytes.Equal(rawReceipts, chain.receipts[i]) {
-			snappy.NewReader(bytes.NewReader(rawRcpt)),
+			t.Fatalf("mismatched receipts: want %s, got %s", chain.receipts[i], rawReceipts)
-		)
+		}
+		receipts, err := getReceiptsByNumber(e, i)
 		if err != nil {
-			t.Fatalf("snappy decode receipts %d: %v", i, err)
+			t.Fatalf("error reading receipts: %v", err)
 		}
-		if !bytes.Equal(decRcpt, mustEncode(chain.receipts[i])) {
+		encReceipts, err := rlp.EncodeToBytes(receipts)
-			t.Fatalf("receipts frame %d mismatch", i)
+		if err != nil {
+			t.Fatalf("error encoding receipts: %v", err)
+		}
+		if !bytes.Equal(encReceipts, chain.receipts[i]) {
+			t.Fatalf("mismatched receipts: want %s, got %s", chain.receipts[i], encReceipts)
 		}
 
-		td, err := era.getTD(bn)
+		// Check total difficulty.
+		rawTd, err := io.ReadAll(it.TotalDifficulty)
 		if err != nil {
-			t.Fatalf("getTD %d: %v", i, err)
+			t.Fatalf("error reading td: %v", err)
 		}
+		slices.Reverse(rawTd)
+		td := new(big.Int).SetBytes(rawTd)
 		if td.Cmp(chain.tds[i]) != 0 {
-			t.Fatalf("td %d mismatch: want %v got %v", i, chain.tds[i], td)
+			t.Fatalf("mismatched tds: want %s, got %s", chain.tds[i], td)
 		}
 	}
 }
@@ -184,3 +152,15 @@ func mustEncode(obj any) []byte {
 	}
 	return b
 }
+
+func getReceiptsByNumber(e *Era, number uint64) ([]*types.ReceiptForStorage, error) {
+	r, err := e.GetRawReceiptsByNumber(number)
+	if err != nil {
+		return nil, err
+	}
+	var receipts []*types.ReceiptForStorage
+	if err := rlp.DecodeBytes(r, &receipts); err != nil {
+		return nil, err
+	}
+	return receipts, nil
+}

internal/era/execdb/iterator.go

@@ -31,6 +31,7 @@ import (
 
 type Iterator struct {
 	inner *RawIterator
+	block *types.Block // cache for decoded block
 }
 
 // NewIterator returns a header/body/receipt iterator over the archive.
@@ -44,7 +45,10 @@ func NewIterator(e era.Era) (era.Iterator, error) {
 }
 
 // Next advances to the next block entry.
-func (it *Iterator) Next() bool { return it.inner.Next() }
+func (it *Iterator) Next() bool {
+	it.block = nil
+	return it.inner.Next()
+}
 
 // Number is the number of the block currently loaded.
 func (it *Iterator) Number() uint64 { return it.inner.next - 1 }
@@ -55,6 +59,9 @@ func (it *Iterator) Error() error { return it.inner.Error() }
 
 // Block decodes the current header+body into a *types.Block.
 func (it *Iterator) Block() (*types.Block, error) {
+	if it.block != nil {
+		return it.block, nil
+	}
 	if it.inner.Header == nil || it.inner.Body == nil {
 		return nil, errors.New("header and body must be non‑nil")
 	}
@@ -68,16 +75,35 @@ func (it *Iterator) Block() (*types.Block, error) {
 	if err := rlp.Decode(it.inner.Body, &b); err != nil {
 		return nil, err
 	}
-	return types.NewBlockWithHeader(&h).WithBody(b), nil
+	it.block = types.NewBlockWithHeader(&h).WithBody(b)
+	return it.block, nil
 }
 
 // Receipts decodes receipts for the current block.
 func (it *Iterator) Receipts() (types.Receipts, error) {
+	block, err := it.Block()
+	if err != nil {
+		return nil, err
+	}
 	if it.inner.Receipts == nil {
 		return nil, errors.New("receipts must be non‑nil")
 	}
-	var r types.Receipts
+	var rs []*types.ReceiptForStorage
-	return r, rlp.Decode(it.inner.Receipts, &r)
+	if err := rlp.Decode(it.inner.Receipts, &rs); err != nil {
+		return nil, err
+	}
+
+	if len(rs) != len(block.Transactions()) {
+		return nil, errors.New("number of txs does not match receipts")
+	}
+
+	receipts := make([]*types.Receipt, len(rs))
+	for i, receipt := range rs {
+		receipts[i] = (*types.Receipt)(receipt)
+		receipts[i].Type = block.Transactions()[i].Type()
+		receipts[i].Bloom = types.CreateBloom(receipts[i])
+	}
+	return receipts, nil
 }
 
 // BlockAndReceipts is a convenience wrapper.
@@ -162,7 +188,7 @@ func (it *RawIterator) Next() bool {
 		it.setErr(err)
 		return false
 	}
-	it.Receipts, _, err = newSnappyReader(it.e.s, era.TypeCompressedReceipts, int64(receiptsOffset))
+	it.Receipts, _, err = newSnappyReader(it.e.s, era.TypeCompressedSlimReceipts, int64(receiptsOffset))
 	if err != nil {
 		it.setErr(err)
 		return false

internal/era/execdb/reader.go

@@ -166,6 +166,7 @@ func (e *Era) GetRawBodyByNumber(blockNum uint64) ([]byte, error) {
 	if err != nil {
 		return nil, err
 	}
+	r = snappy.NewReader(r)
 	return io.ReadAll(r)
 }
 
@@ -179,19 +180,7 @@ func (e *Era) GetRawReceiptsByNumber(blockNum uint64) ([]byte, error) {
 	if err != nil {
 		return nil, err
 	}
-	return io.ReadAll(r)
+	r = snappy.NewReader(r)
-}
-
-// GetRawProofFrameByNumber returns the RLP-encoded receipts for the given block number.
-func (e *Era) GetRawProofFrameByNumber(blockNum uint64) ([]byte, error) {
-	off, err := e.proofOff(blockNum)
-	if err != nil {
-		return nil, err
-	}
-	r, _, err := e.s.ReaderAt(era.TypeProof, int64(off))
-	if err != nil {
-		return nil, err
-	}
 	return io.ReadAll(r)
 }
 
@@ -251,93 +240,6 @@ func (e *Era) indexOffset(n uint64, component componentType) (uint64, error) {
 	return uint64(int64(rel) + indstart), nil
 }
 
-// GetHeaders returns RLP-decoded headers for a range of blocks.
-func (e *Era) GetHeaders(first, count uint64) ([]*types.Header, error) {
-	if count == 0 {
-		return nil, fmt.Errorf("count must be > 0")
-	}
-	if first < e.m.start || first+count > e.m.start+e.m.count {
-		return nil, fmt.Errorf("range [%d,%d) out of bounds", first, first+count)
-	}
-
-	out := make([]*types.Header, count)
-	for i := uint64(0); i < count; i++ {
-		n := first + i
-		off, err := e.headerOff(n)
-		if err != nil {
-			return nil, err
-		}
-		r, _, err := e.s.ReaderAt(era.TypeCompressedHeader, int64(off))
-		if err != nil {
-			return nil, err
-		}
-		var h types.Header
-		if err := rlp.Decode(snappy.NewReader(r), &h); err != nil {
-			return nil, err
-		}
-		out[i] = &h
-	}
-	return out, nil
-}
-
-// GetHeaders returns RLP-decoded headers for a range of blocks.
-func (e *Era) GetBodies(first, count uint64) ([]*types.Body, error) {
-	if count == 0 {
-		return nil, fmt.Errorf("count must be > 0")
-	}
-	if first < e.m.start || first+count > e.m.start+e.m.count {
-		return nil, fmt.Errorf("range [%d,%d) out of bounds", first, first+count)
-	}
-
-	out := make([]*types.Body, count)
-	for i := uint64(0); i < count; i++ {
-		n := first + i
-		off, err := e.bodyOff(n)
-		if err != nil {
-			return nil, err
-		}
-		r, _, err := e.s.ReaderAt(era.TypeCompressedBody, int64(off))
-		if err != nil {
-			return nil, err
-		}
-		var b types.Body
-		if err := rlp.Decode(snappy.NewReader(r), &b); err != nil {
-			return nil, err
-		}
-		out[i] = &b
-	}
-	return out, nil
-}
-
-// GetReceipts returns RLP-decoded receipts for a range of blocks.
-func (e *Era) GetReceipts(first, count uint64) ([]types.Receipts, error) {
-	if count == 0 {
-		return nil, fmt.Errorf("count must be > 0")
-	}
-	if first < e.m.start || first+count > e.m.start+e.m.count {
-		return nil, fmt.Errorf("range [%d,%d) out of bounds", first, first+count)
-	}
-
-	out := make([]types.Receipts, count)
-	for i := uint64(0); i < count; i++ {
-		n := first + i
-		off, err := e.receiptOff(n)
-		if err != nil {
-			return nil, err
-		}
-		r, _, err := e.s.ReaderAt(era.TypeCompressedSlimReceipts, int64(off))
-		if err != nil {
-			return nil, err
-		}
-		var rc types.Receipts
-		if err := rlp.Decode(snappy.NewReader(r), &rc); err != nil {
-			return nil, err
-		}
-		out[i] = rc
-	}
-	return out, nil
-}
-
 // metadata contains the information about the era file that is written into the file.
 type metadata struct {
 	start      uint64 // start block number