mirror of
https://github.com/ethereum/go-ethereum.git
synced 2026-07-19 11:20:45 +00:00
all: refactor era1 and era2 into era subpackages of onedb and execdb
This commit is contained in:
lightclient
parent
c965d78614
commit
5cc66c2f47
20 changed files with 656 additions and 799 deletions
|
|
@ -30,6 +30,7 @@ import (
|
|||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/internal/era"
|
||||
"github.com/ethereum/go-ethereum/internal/era/onedb"
|
||||
"github.com/ethereum/go-ethereum/internal/ethapi"
|
||||
"github.com/ethereum/go-ethereum/internal/flags"
|
||||
"github.com/ethereum/go-ethereum/params"
|
||||
|
|
@ -53,7 +54,7 @@ var (
|
|||
eraSizeFlag = &cli.IntFlag{
|
||||
Name: "size",
|
||||
Usage: "number of blocks per era",
|
||||
Value: era.MaxEra1Size,
|
||||
Value: era.MaxSize,
|
||||
}
|
||||
txsFlag = &cli.BoolFlag{
|
||||
Name: "txs",
|
||||
|
|
@ -164,19 +165,19 @@ func info(ctx *cli.Context) error {
|
|||
}
|
||||
|
||||
// open opens an era1 file at a certain epoch.
|
||||
func open(ctx *cli.Context, epoch uint64) (*era.Era, error) {
|
||||
func open(ctx *cli.Context, epoch uint64) (*onedb.Era, error) {
|
||||
var (
|
||||
dir = ctx.String(dirFlag.Name)
|
||||
network = ctx.String(networkFlag.Name)
|
||||
)
|
||||
entries, err := era.ReadDir(dir, network)
|
||||
entries, err := onedb.ReadDir(dir, network)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error reading era dir: %w", err)
|
||||
}
|
||||
if epoch >= uint64(len(entries)) {
|
||||
return nil, fmt.Errorf("epoch out-of-bounds: last %d, want %d", len(entries)-1, epoch)
|
||||
}
|
||||
return era.Open(filepath.Join(dir, entries[epoch]))
|
||||
return onedb.Open(filepath.Join(dir, entries[epoch]))
|
||||
}
|
||||
|
||||
// verify checks each era1 file in a directory to ensure it is well-formed and
|
||||
|
|
@ -198,7 +199,7 @@ func verify(ctx *cli.Context) error {
|
|||
reported = time.Now()
|
||||
)
|
||||
|
||||
entries, err := era.ReadDir(dir, network)
|
||||
entries, err := onedb.ReadDir(dir, network)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error reading %s: %w", dir, err)
|
||||
}
|
||||
|
|
@ -212,7 +213,7 @@ func verify(ctx *cli.Context) error {
|
|||
// Wrap in function so defers don't stack.
|
||||
err := func() error {
|
||||
name := entries[i]
|
||||
e, err := era.Open(filepath.Join(dir, name))
|
||||
e, err := onedb.Open(filepath.Join(dir, name))
|
||||
if err != nil {
|
||||
return fmt.Errorf("error opening era1 file %s: %w", name, err)
|
||||
}
|
||||
|
|
@ -243,7 +244,7 @@ func verify(ctx *cli.Context) error {
|
|||
}
|
||||
|
||||
// checkAccumulator verifies the accumulator matches the data in the Era.
|
||||
func checkAccumulator(e *era.Era) error {
|
||||
func checkAccumulator(e *onedb.Era) error {
|
||||
var (
|
||||
err error
|
||||
want common.Hash
|
||||
|
|
@ -257,7 +258,7 @@ func checkAccumulator(e *era.Era) error {
|
|||
if td, err = e.InitialTD(); err != nil {
|
||||
return fmt.Errorf("error reading total difficulty: %w", err)
|
||||
}
|
||||
it, err := era.NewIterator(e)
|
||||
it, err := onedb.NewIterator(e)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error making era iterator: %w", err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -43,7 +43,6 @@ import (
|
|||
"github.com/ethereum/go-ethereum/internal/debug"
|
||||
"github.com/ethereum/go-ethereum/internal/era"
|
||||
"github.com/ethereum/go-ethereum/internal/era/eradl"
|
||||
"github.com/ethereum/go-ethereum/internal/era2"
|
||||
"github.com/ethereum/go-ethereum/internal/flags"
|
||||
"github.com/ethereum/go-ethereum/log"
|
||||
"github.com/ethereum/go-ethereum/node"
|
||||
|
|
|
|||
|
|
@ -44,7 +44,8 @@ import (
|
|||
"github.com/ethereum/go-ethereum/eth/ethconfig"
|
||||
"github.com/ethereum/go-ethereum/ethdb"
|
||||
"github.com/ethereum/go-ethereum/internal/debug"
|
||||
"github.com/ethereum/go-ethereum/internal/era2"
|
||||
era2 "github.com/ethereum/go-ethereum/internal/era/execdb"
|
||||
"github.com/ethereum/go-ethereum/internal/era/onedb"
|
||||
"github.com/ethereum/go-ethereum/log"
|
||||
"github.com/ethereum/go-ethereum/node"
|
||||
"github.com/ethereum/go-ethereum/params"
|
||||
|
|
@ -252,11 +253,11 @@ func readList(filename string) ([]string, error) {
|
|||
// ImportHistory imports Era1 files containing historical block information,
|
||||
// starting from genesis. The assumption is held that the provided chain
|
||||
// segment in Era1 file should all be canonical and verified.
|
||||
func ImportHistory(chain *core.BlockChain, dir string, network string, format Format) error {
|
||||
func ImportHistory(chain *core.BlockChain, dir string, network string) error {
|
||||
if chain.CurrentSnapBlock().Number.BitLen() != 0 {
|
||||
return errors.New("history import only supported when starting from genesis")
|
||||
}
|
||||
entries, err := format.ReadDir(dir, network)
|
||||
entries, err := onedb.ReadDir(dir, network)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error reading %s: %w", dir, err)
|
||||
}
|
||||
|
|
@ -299,12 +300,12 @@ func ImportHistory(chain *core.BlockChain, dir string, network string, format Fo
|
|||
f.Close()
|
||||
return fmt.Errorf("%s checksum mismatch: have %s want %s", file, got, want)
|
||||
}
|
||||
// rewind for reading
|
||||
if _, err := f.Seek(0, io.SeekStart); err != nil {
|
||||
f.Close()
|
||||
return fmt.Errorf("rewind %s: %w", file, err)
|
||||
// Import all block data from Era1.
|
||||
e, err := onedb.From(f)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error opening era: %w", err)
|
||||
}
|
||||
it, err := format.NewIterator(f)
|
||||
it, err := onedb.NewIterator(e)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error creating iterator: %w", err)
|
||||
}
|
||||
|
|
@ -515,7 +516,7 @@ func ExportAppendChain(blockchain *core.BlockChain, fn string, first uint64, las
|
|||
|
||||
// ExportHistory exports blockchain history into the specified directory,
|
||||
// following the Era format.
|
||||
func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64, f Format) error {
|
||||
func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64) error {
|
||||
log.Info("Exporting blockchain history", "dir", dir)
|
||||
if head := bc.CurrentBlock().Number.Uint64(); head < last {
|
||||
log.Warn("Last block beyond head, setting last = head", "head", head, "last", last)
|
||||
|
|
@ -544,7 +545,7 @@ func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64, f
|
|||
|
||||
for batch := first; batch <= last; batch += step {
|
||||
idx := int(batch / step)
|
||||
tmpPath := filepath.Join(dir, f.Filename(network, idx, common.Hash{}))
|
||||
tmpPath := filepath.Join(dir, onedb.Filename(network, idx, common.Hash{}))
|
||||
|
||||
if err := func() error {
|
||||
fh, err := os.Create(tmpPath)
|
||||
|
|
@ -553,7 +554,7 @@ func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64, f
|
|||
}
|
||||
defer fh.Close()
|
||||
|
||||
bldr := f.NewBuilder(fh)
|
||||
bldr := onedb.NewBuilder(fh)
|
||||
|
||||
for j := uint64(0); j < step && batch+j <= last; j++ {
|
||||
n := batch + j
|
||||
|
|
@ -576,7 +577,7 @@ func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64, f
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
final := filepath.Join(dir, f.Filename(network, idx, root))
|
||||
final := filepath.Join(dir, onedb.Filename(network, idx, root))
|
||||
if err := os.Rename(tmpPath, final); err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,82 +0,0 @@
|
|||
package utils
|
||||
|
||||
import (
|
||||
"io"
|
||||
"math/big"
|
||||
"os"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/internal/era"
|
||||
"github.com/ethereum/go-ethereum/internal/era2"
|
||||
)
|
||||
|
||||
// Iterator is the common, read‑only view returned by both Era‑1 and Era‑E.
|
||||
type Iterator interface {
|
||||
Next() bool
|
||||
Number() uint64
|
||||
Block() (*types.Block, error)
|
||||
Receipts() (types.Receipts, error)
|
||||
Error() error
|
||||
}
|
||||
|
||||
// Builder (unchanged) – writes one archive file.
|
||||
type Builder interface {
|
||||
Add(blk *types.Block, rcpts types.Receipts, td *big.Int, proof era2.Proof) error
|
||||
Finalize() (common.Hash, error)
|
||||
}
|
||||
|
||||
// Format now encapsulates *all* format‑specific behaviour, both read & write.
|
||||
type Format interface {
|
||||
// ----- writer side -----
|
||||
Filename(network string, epoch int, root common.Hash) string
|
||||
NewBuilder(w io.Writer) Builder
|
||||
|
||||
// ----- reader side -----
|
||||
ReadDir(dir, network string) ([]string, error)
|
||||
NewIterator(f *os.File) (Iterator, error)
|
||||
}
|
||||
|
||||
type era1Builder struct{ *era.Builder }
|
||||
|
||||
func (b *era1Builder) Add(blk *types.Block, rc types.Receipts, td *big.Int, proof era2.Proof) error {
|
||||
return b.Builder.Add(blk, rc, td)
|
||||
}
|
||||
|
||||
type era1Format struct{}
|
||||
|
||||
func (era1Format) Filename(n string, e int, h common.Hash) string { return era.Filename(n, e, h) }
|
||||
func (era1Format) NewBuilder(w io.Writer) Builder { return &era1Builder{era.NewBuilder(w)} }
|
||||
func (era1Format) ReadDir(dir, net string) ([]string, error) { return era.ReadDir(dir, net) }
|
||||
func (era1Format) NewIterator(f *os.File) (Iterator, error) {
|
||||
e, err := era.From(f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return era.NewIterator(e)
|
||||
}
|
||||
|
||||
var Era1 Format = era1Format{}
|
||||
|
||||
type eraeBuilder struct{ *era2.Builder }
|
||||
|
||||
func (b *eraeBuilder) Add(blk *types.Block, rc types.Receipts, td *big.Int, proof era2.Proof) error {
|
||||
return b.Builder.Add(blk, rc, td, nil) // no proofs yet
|
||||
}
|
||||
|
||||
type eraeFormat struct{}
|
||||
|
||||
func (eraeFormat) Filename(n string, e int, h common.Hash) string { return era2.Filename(n, e, h) }
|
||||
func (eraeFormat) NewBuilder(w io.Writer) Builder { return &eraeBuilder{era2.NewBuilder(w)} }
|
||||
func (eraeFormat) ReadDir(dir, net string) ([]string, error) { return era2.ReadDir(dir, net) }
|
||||
func (eraeFormat) NewIterator(f *os.File) (Iterator, error) {
|
||||
e, err := era2.From(f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return era2.NewIterator(e)
|
||||
}
|
||||
|
||||
var EraE Format = eraeFormat{}
|
||||
|
||||
// Exported singleton
|
||||
|
|
@ -32,7 +32,7 @@ import (
|
|||
"github.com/ethereum/go-ethereum/core/rawdb"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/crypto"
|
||||
"github.com/ethereum/go-ethereum/internal/era2"
|
||||
"github.com/ethereum/go-ethereum/internal/era/onedb"
|
||||
"github.com/ethereum/go-ethereum/params"
|
||||
"github.com/ethereum/go-ethereum/trie"
|
||||
"github.com/ethereum/go-ethereum/triedb"
|
||||
|
|
@ -89,7 +89,7 @@ func TestHistoryImportAndExport(t *testing.T) {
|
|||
dir := t.TempDir()
|
||||
|
||||
// Export history to temp directory.
|
||||
if err := ExportHistory(chain, dir, 0, count, step, Era1); err != nil {
|
||||
if err := ExportHistory(chain, dir, 0, count, step); err != nil {
|
||||
t.Fatalf("error exporting history: %v", err)
|
||||
}
|
||||
|
||||
|
|
@ -101,7 +101,7 @@ func TestHistoryImportAndExport(t *testing.T) {
|
|||
checksums := strings.Split(string(b), "\n")
|
||||
|
||||
// Verify each Era.
|
||||
entries, _ := era2.ReadDir(dir, "mainnet")
|
||||
entries, _ := onedb.ReadDir(dir, "mainnet")
|
||||
for i, filename := range entries {
|
||||
func() {
|
||||
f, err := os.Open(filepath.Join(dir, filename))
|
||||
|
|
@ -118,12 +118,12 @@ func TestHistoryImportAndExport(t *testing.T) {
|
|||
if got, want := common.BytesToHash(h.Sum(buf.Bytes()[:])).Hex(), checksums[i]; got != want {
|
||||
t.Fatalf("checksum %d does not match: got %s, want %s", i, got, want)
|
||||
}
|
||||
e, err := era2.From(f)
|
||||
e, err := onedb.From(f)
|
||||
if err != nil {
|
||||
t.Fatalf("error opening era: %v", err)
|
||||
}
|
||||
defer e.Close()
|
||||
it, err := era2.NewIterator(e)
|
||||
it, err := onedb.NewIterator(e)
|
||||
if err != nil {
|
||||
t.Fatalf("error making era reader: %v", err)
|
||||
}
|
||||
|
|
@ -170,7 +170,7 @@ func TestHistoryImportAndExport(t *testing.T) {
|
|||
if err != nil {
|
||||
t.Fatalf("unable to initialize chain: %v", err)
|
||||
}
|
||||
if err := ImportHistory(imported, dir, "mainnet", EraE); err != nil {
|
||||
if err := ImportHistory(imported, dir, "mainnet"); err != nil {
|
||||
t.Fatalf("failed to import chain: %v", err)
|
||||
}
|
||||
if have, want := imported.CurrentHeader(), chain.CurrentHeader(); have.Hash() != want.Hash() {
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ import (
|
|||
|
||||
"github.com/ethereum/go-ethereum/common/lru"
|
||||
"github.com/ethereum/go-ethereum/internal/era"
|
||||
"github.com/ethereum/go-ethereum/internal/era/onedb"
|
||||
"github.com/ethereum/go-ethereum/log"
|
||||
"github.com/ethereum/go-ethereum/rlp"
|
||||
)
|
||||
|
|
@ -51,7 +52,7 @@ type Store struct {
|
|||
type fileCacheEntry struct {
|
||||
refcount int // reference count. This is protected by Store.mu!
|
||||
opened chan struct{} // signals opening of file has completed
|
||||
file *era.Era // the file
|
||||
file *onedb.Era // the file
|
||||
err error // error from opening the file
|
||||
}
|
||||
|
||||
|
|
@ -102,7 +103,7 @@ func (db *Store) Close() {
|
|||
|
||||
// GetRawBody returns the raw body for a given block number.
|
||||
func (db *Store) GetRawBody(number uint64) ([]byte, error) {
|
||||
epoch := number / uint64(era.MaxEra1Size)
|
||||
epoch := number / uint64(era.MaxSize)
|
||||
entry := db.getEraByEpoch(epoch)
|
||||
if entry.err != nil {
|
||||
if errors.Is(entry.err, fs.ErrNotExist) {
|
||||
|
|
@ -117,7 +118,7 @@ func (db *Store) GetRawBody(number uint64) ([]byte, error) {
|
|||
|
||||
// GetRawReceipts returns the raw receipts for a given block number.
|
||||
func (db *Store) GetRawReceipts(number uint64) ([]byte, error) {
|
||||
epoch := number / uint64(era.MaxEra1Size)
|
||||
epoch := number / uint64(era.MaxSize)
|
||||
entry := db.getEraByEpoch(epoch)
|
||||
if entry.err != nil {
|
||||
if errors.Is(entry.err, fs.ErrNotExist) {
|
||||
|
|
@ -249,7 +250,7 @@ func (db *Store) getCacheEntry(epoch uint64) (stat fileCacheStatus, entry *fileC
|
|||
}
|
||||
|
||||
// fileOpened is called after an era file has been successfully opened.
|
||||
func (db *Store) fileOpened(epoch uint64, entry *fileCacheEntry, file *era.Era) {
|
||||
func (db *Store) fileOpened(epoch uint64, entry *fileCacheEntry, file *onedb.Era) {
|
||||
db.mu.Lock()
|
||||
defer db.mu.Unlock()
|
||||
|
||||
|
|
@ -282,7 +283,7 @@ func (db *Store) fileFailedToOpen(epoch uint64, entry *fileCacheEntry, err error
|
|||
entry.err = err
|
||||
}
|
||||
|
||||
func (db *Store) openEraFile(epoch uint64) (*era.Era, error) {
|
||||
func (db *Store) openEraFile(epoch uint64) (*onedb.Era, error) {
|
||||
// File name scheme is <network>-<epoch>-<root>.
|
||||
glob := fmt.Sprintf("*-%05d-*.era1", epoch)
|
||||
matches, err := filepath.Glob(filepath.Join(db.datadir, glob))
|
||||
|
|
@ -297,14 +298,14 @@ func (db *Store) openEraFile(epoch uint64) (*era.Era, error) {
|
|||
}
|
||||
filename := matches[0]
|
||||
|
||||
e, err := era.Open(filename)
|
||||
e, err := onedb.Open(filename)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Sanity-check start block.
|
||||
if e.Start()%uint64(era.MaxEra1Size) != 0 {
|
||||
if e.Start()%uint64(era.MaxSize) != 0 {
|
||||
e.Close()
|
||||
return nil, fmt.Errorf("pre-merge era1 file has invalid boundary. %d %% %d != 0", e.Start(), era.MaxEra1Size)
|
||||
return nil, fmt.Errorf("pre-merge era1 file has invalid boundary. %d %% %d != 0", e.Start(), era.MaxSize)
|
||||
}
|
||||
log.Debug("Opened era1 file", "epoch", epoch)
|
||||
return e, nil
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ import (
|
|||
"errors"
|
||||
"fmt"
|
||||
"math/big"
|
||||
"slices"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
ssz "github.com/ferranbt/fastssz"
|
||||
|
|
@ -31,8 +32,8 @@ func ComputeAccumulator(hashes []common.Hash, tds []*big.Int) (common.Hash, erro
|
|||
if len(hashes) != len(tds) {
|
||||
return common.Hash{}, errors.New("must have equal number hashes as td values")
|
||||
}
|
||||
if len(hashes) > MaxEra1Size {
|
||||
return common.Hash{}, fmt.Errorf("too many records: have %d, max %d", len(hashes), MaxEra1Size)
|
||||
if len(hashes) > MaxSize {
|
||||
return common.Hash{}, fmt.Errorf("too many records: have %d, max %d", len(hashes), MaxSize)
|
||||
}
|
||||
hh := ssz.NewHasher()
|
||||
for i := range hashes {
|
||||
|
|
@ -43,7 +44,7 @@ func ComputeAccumulator(hashes []common.Hash, tds []*big.Int) (common.Hash, erro
|
|||
}
|
||||
hh.Append(root[:])
|
||||
}
|
||||
hh.MerkleizeWithMixin(0, uint64(len(hashes)), uint64(MaxEra1Size))
|
||||
hh.MerkleizeWithMixin(0, uint64(len(hashes)), uint64(MaxSize))
|
||||
return hh.HashRoot()
|
||||
}
|
||||
|
||||
|
|
@ -69,23 +70,15 @@ func (h *headerRecord) HashTreeRoot() ([32]byte, error) {
|
|||
// HashTreeRootWith ssz hashes the headerRecord object with a hasher.
|
||||
func (h *headerRecord) HashTreeRootWith(hh ssz.HashWalker) (err error) {
|
||||
hh.PutBytes(h.Hash[:])
|
||||
td := bigToBytes32(h.TotalDifficulty)
|
||||
td := BigToBytes32(h.TotalDifficulty)
|
||||
hh.PutBytes(td[:])
|
||||
hh.Merkleize(0)
|
||||
return
|
||||
}
|
||||
|
||||
// bigToBytes32 converts a big.Int into a little-endian 32-byte array.
|
||||
func bigToBytes32(n *big.Int) (b [32]byte) {
|
||||
func BigToBytes32(n *big.Int) (b [32]byte) {
|
||||
n.FillBytes(b[:])
|
||||
reverseOrder(b[:])
|
||||
slices.Reverse(b[:])
|
||||
return
|
||||
}
|
||||
|
||||
// reverseOrder reverses the byte order of a slice.
|
||||
func reverseOrder(b []byte) []byte {
|
||||
for i := 0; i < 16; i++ {
|
||||
b[i], b[32-i-1] = b[32-i-1], b[i]
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,317 +1,47 @@
|
|||
// Copyright 2024 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package era
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/big"
|
||||
"os"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/internal/era/e2store"
|
||||
"github.com/ethereum/go-ethereum/rlp"
|
||||
"github.com/golang/snappy"
|
||||
)
|
||||
|
||||
// Type constants for the e2store entries in the Era1 and EraE formats.
|
||||
var (
|
||||
TypeVersion uint16 = 0x3265
|
||||
TypeCompressedHeader uint16 = 0x03
|
||||
TypeCompressedBody uint16 = 0x04
|
||||
TypeCompressedReceipts uint16 = 0x05
|
||||
TypeTotalDifficulty uint16 = 0x06
|
||||
TypeAccumulator uint16 = 0x07
|
||||
TypeBlockIndex uint16 = 0x3266
|
||||
TypeVersion uint16 = 0x3265
|
||||
TypeCompressedHeader uint16 = 0x03
|
||||
TypeCompressedBody uint16 = 0x04
|
||||
TypeCompressedReceipts uint16 = 0x05
|
||||
TypeTotalDifficulty uint16 = 0x06
|
||||
TypeAccumulator uint16 = 0x07
|
||||
TypeCompressedSlimReceipts uint16 = 0x08 // uses eth/69 encoding
|
||||
TypeProof uint16 = 0x09
|
||||
TypeBlockIndex uint16 = 0x3266
|
||||
TypeComponentIndex uint16 = 0x3267
|
||||
|
||||
MaxEra1Size = 8192
|
||||
MaxSize = 8192
|
||||
// headerSize uint64 = 8
|
||||
)
|
||||
|
||||
// Filename returns a recognizable Era1-formatted file name for the specified
|
||||
// epoch and network.
|
||||
func Filename(network string, epoch int, root common.Hash) string {
|
||||
return fmt.Sprintf("%s-%05d-%s.era1", network, epoch, root.Hex()[2:10])
|
||||
}
|
||||
|
||||
// ReadDir reads all the era1 files in a directory for a given network.
|
||||
// Format: <network>-<epoch>-<hexroot>.era1
|
||||
func ReadDir(dir, network string) ([]string, error) {
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error reading directory %s: %w", dir, err)
|
||||
}
|
||||
var (
|
||||
next = uint64(0)
|
||||
eras []string
|
||||
)
|
||||
for _, entry := range entries {
|
||||
if path.Ext(entry.Name()) != ".era1" {
|
||||
continue
|
||||
}
|
||||
parts := strings.Split(entry.Name(), "-")
|
||||
if len(parts) != 3 || parts[0] != network {
|
||||
// Invalid era1 filename, skip.
|
||||
continue
|
||||
}
|
||||
if epoch, err := strconv.ParseUint(parts[1], 10, 64); err != nil {
|
||||
return nil, fmt.Errorf("malformed era1 filename: %s", entry.Name())
|
||||
} else if epoch != next {
|
||||
return nil, fmt.Errorf("missing epoch %d", next)
|
||||
}
|
||||
next += 1
|
||||
eras = append(eras, entry.Name())
|
||||
}
|
||||
return eras, nil
|
||||
}
|
||||
|
||||
type ReadAtSeekCloser interface {
|
||||
io.ReaderAt
|
||||
io.Seeker
|
||||
io.Closer
|
||||
}
|
||||
|
||||
// Era reads and Era1 file.
|
||||
type Era struct {
|
||||
f ReadAtSeekCloser // backing era1 file
|
||||
s *e2store.Reader // e2store reader over f
|
||||
m metadata // start, count, length info
|
||||
mu *sync.Mutex // lock for buf
|
||||
buf [8]byte // buffer reading entry offsets
|
||||
// Iterator represents the iterator interface for various types of era stores.
|
||||
type Iterator interface {
|
||||
Next() bool
|
||||
Number() uint64
|
||||
Block() (*types.Block, error)
|
||||
Receipts() (types.Receipts, error)
|
||||
Error() error
|
||||
}
|
||||
|
||||
// From returns an Era backed by f.
|
||||
func From(f ReadAtSeekCloser) (*Era, error) {
|
||||
m, err := readMetadata(f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Era{
|
||||
f: f,
|
||||
s: e2store.NewReader(f),
|
||||
m: m,
|
||||
mu: new(sync.Mutex),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Open returns an Era backed by the given filename.
|
||||
func Open(filename string) (*Era, error) {
|
||||
f, err := os.Open(filename)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return From(f)
|
||||
}
|
||||
|
||||
func (e *Era) Close() error {
|
||||
return e.f.Close()
|
||||
}
|
||||
|
||||
// GetBlockByNumber returns the block for the given block number.
|
||||
func (e *Era) GetBlockByNumber(num uint64) (*types.Block, error) {
|
||||
if e.m.start > num || e.m.start+e.m.count <= num {
|
||||
return nil, fmt.Errorf("out-of-bounds: %d not in [%d, %d)", num, e.m.start, e.m.start+e.m.count)
|
||||
}
|
||||
off, err := e.readOffset(num)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r, n, err := newSnappyReader(e.s, TypeCompressedHeader, off)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var header types.Header
|
||||
if err := rlp.Decode(r, &header); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
off += n
|
||||
r, _, err = newSnappyReader(e.s, TypeCompressedBody, off)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var body types.Body
|
||||
if err := rlp.Decode(r, &body); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return types.NewBlockWithHeader(&header).WithBody(body), nil
|
||||
}
|
||||
|
||||
// GetRawBodyByNumber returns the RLP-encoded body for the given block number.
|
||||
func (e *Era) GetRawBodyByNumber(num uint64) ([]byte, error) {
|
||||
if e.m.start > num || e.m.start+e.m.count <= num {
|
||||
return nil, fmt.Errorf("out-of-bounds: %d not in [%d, %d)", num, e.m.start, e.m.start+e.m.count)
|
||||
}
|
||||
off, err := e.readOffset(num)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
off, err = e.s.SkipN(off, 1)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r, _, err := newSnappyReader(e.s, TypeCompressedBody, off)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return io.ReadAll(r)
|
||||
}
|
||||
|
||||
// GetRawReceiptsByNumber returns the RLP-encoded receipts for the given block number.
|
||||
func (e *Era) GetRawReceiptsByNumber(num uint64) ([]byte, error) {
|
||||
if e.m.start > num || e.m.start+e.m.count <= num {
|
||||
return nil, fmt.Errorf("out-of-bounds: %d not in [%d, %d)", num, e.m.start, e.m.start+e.m.count)
|
||||
}
|
||||
off, err := e.readOffset(num)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Skip over header and body.
|
||||
off, err = e.s.SkipN(off, 2)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
r, _, err := newSnappyReader(e.s, TypeCompressedReceipts, off)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return io.ReadAll(r)
|
||||
}
|
||||
|
||||
// Accumulator reads the accumulator entry in the Era1 file.
|
||||
func (e *Era) Accumulator() (common.Hash, error) {
|
||||
entry, err := e.s.Find(TypeAccumulator)
|
||||
if err != nil {
|
||||
return common.Hash{}, err
|
||||
}
|
||||
return common.BytesToHash(entry.Value), nil
|
||||
}
|
||||
|
||||
// InitialTD returns initial total difficulty before the difficulty of the
|
||||
// first block of the Era1 is applied.
|
||||
func (e *Era) InitialTD() (*big.Int, error) {
|
||||
var (
|
||||
r io.Reader
|
||||
header types.Header
|
||||
rawTd []byte
|
||||
n int64
|
||||
off int64
|
||||
err error
|
||||
)
|
||||
|
||||
// Read first header.
|
||||
if off, err = e.readOffset(e.m.start); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if r, n, err = newSnappyReader(e.s, TypeCompressedHeader, off); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := rlp.Decode(r, &header); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
off += n
|
||||
|
||||
// Skip over header and body.
|
||||
off, err = e.s.SkipN(off, 2)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Read total difficulty after first block.
|
||||
if r, _, err = e.s.ReaderAt(TypeTotalDifficulty, off); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rawTd, err = io.ReadAll(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
td := new(big.Int).SetBytes(reverseOrder(rawTd))
|
||||
return td.Sub(td, header.Difficulty), nil
|
||||
}
|
||||
|
||||
// Start returns the listed start block.
|
||||
func (e *Era) Start() uint64 {
|
||||
return e.m.start
|
||||
}
|
||||
|
||||
// Count returns the total number of blocks in the Era1.
|
||||
func (e *Era) Count() uint64 {
|
||||
return e.m.count
|
||||
}
|
||||
|
||||
// readOffset reads a specific block's offset from the block index. The value n
|
||||
// is the absolute block number desired.
|
||||
func (e *Era) readOffset(n uint64) (int64, error) {
|
||||
var (
|
||||
blockIndexRecordOffset = e.m.length - 24 - int64(e.m.count)*8 // skips start, count, and header
|
||||
firstIndex = blockIndexRecordOffset + 16 // first index after header / start-num
|
||||
indexOffset = int64(n-e.m.start) * 8 // desired index * size of indexes
|
||||
offOffset = firstIndex + indexOffset // offset of block offset
|
||||
)
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
clear(e.buf[:])
|
||||
if _, err := e.f.ReadAt(e.buf[:], offOffset); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
// Since the block offset is relative from the start of the block index record
|
||||
// we need to add the record offset to it's offset to get the block's absolute
|
||||
// offset.
|
||||
return blockIndexRecordOffset + int64(binary.LittleEndian.Uint64(e.buf[:])), nil
|
||||
}
|
||||
|
||||
// newSnappyReader returns a snappy.Reader for the e2store entry value at off.
|
||||
func newSnappyReader(e *e2store.Reader, expectedType uint16, off int64) (io.Reader, int64, error) {
|
||||
r, n, err := e.ReaderAt(expectedType, off)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return snappy.NewReader(r), int64(n), err
|
||||
}
|
||||
|
||||
// metadata wraps the metadata in the block index.
|
||||
type metadata struct {
|
||||
start uint64
|
||||
count uint64
|
||||
length int64
|
||||
}
|
||||
|
||||
// readMetadata reads the metadata stored in an Era1 file's block index.
|
||||
func readMetadata(f ReadAtSeekCloser) (m metadata, err error) {
|
||||
// Determine length of reader.
|
||||
if m.length, err = f.Seek(0, io.SeekEnd); err != nil {
|
||||
return
|
||||
}
|
||||
b := make([]byte, 16)
|
||||
// Read count. It's the last 8 bytes of the file.
|
||||
if _, err = f.ReadAt(b[:8], m.length-8); err != nil {
|
||||
return
|
||||
}
|
||||
m.count = binary.LittleEndian.Uint64(b)
|
||||
// Read start. It's at the offset -sizeof(m.count) -
|
||||
// count*sizeof(indexEntry) - sizeof(m.start)
|
||||
if _, err = f.ReadAt(b[8:], m.length-16-int64(m.count*8)); err != nil {
|
||||
return
|
||||
}
|
||||
m.start = binary.LittleEndian.Uint64(b[8:])
|
||||
return
|
||||
// Builder represents the interface for various types of era formats.
|
||||
type Builder interface {
|
||||
Add(block *types.Block, receipts types.Receipts, td *big.Int, proof []byte) error
|
||||
Finalize() (common.Hash, error)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -86,8 +86,8 @@ func (l *Loader) DownloadAll(destDir string) error {
|
|||
|
||||
// DownloadBlockRange fetches the era1 files for the given block range.
|
||||
func (l *Loader) DownloadBlockRange(start, end uint64, destDir string) error {
|
||||
startEpoch := start / uint64(era.MaxEra1Size)
|
||||
endEpoch := end / uint64(era.MaxEra1Size)
|
||||
startEpoch := start / uint64(era.MaxSize)
|
||||
endEpoch := end / uint64(era.MaxSize)
|
||||
return l.DownloadEpochRange(startEpoch, endEpoch, destDir)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -52,25 +52,12 @@ import (
|
|||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/internal/era"
|
||||
"github.com/ethereum/go-ethereum/internal/era/e2store"
|
||||
"github.com/ethereum/go-ethereum/rlp"
|
||||
"github.com/golang/snappy"
|
||||
)
|
||||
|
||||
// Type constants for the e2store entries in the Era2 format.
|
||||
const (
|
||||
TypeVersion uint16 = 0x3265
|
||||
TypeCompressedHeader uint16 = 0x08
|
||||
TypeCompressedBody uint16 = 0x09
|
||||
TypeCompressedReceipts uint16 = 0x0a
|
||||
TypeTotalDifficulty uint16 = 0x0b
|
||||
TypeProof uint16 = 0x0c
|
||||
TypeAccumulatorRoot uint16 = 0x0d
|
||||
TypeBlockIndex uint16 = 0x3267
|
||||
MaxEraESize int = 8192
|
||||
headerSize uint64 = 8
|
||||
)
|
||||
|
||||
// Temporary buffer for writing blocks until the Finalize method is called.
|
||||
type buffer struct {
|
||||
headers [][]byte
|
||||
|
|
@ -113,7 +100,7 @@ func NewBuilder(w io.Writer) *Builder {
|
|||
}
|
||||
|
||||
// Add writes a block entry, its reciepts, and optionally its proofs as well into the e2store file.
|
||||
func (b *Builder) Add(block *types.Block, receipts types.Receipts, td *big.Int, proof Proof) error {
|
||||
func (b *Builder) Add(block *types.Block, receipts types.Receipts, td *big.Int, proof era.Proof) error {
|
||||
header := block.Header()
|
||||
body := block.Body()
|
||||
if len(b.buff.headers) == 0 { // first block determines wether proofs are expected
|
||||
|
|
@ -156,8 +143,8 @@ func (b *Builder) Add(block *types.Block, receipts types.Receipts, td *big.Int,
|
|||
|
||||
// AddRLP takes the RLP encoded block components and writes them to the underlying e2store file.
|
||||
func (b *Builder) AddRLP(headerRLP []byte, bodyRLP []byte, receipts []byte, proof []byte, blockNum uint64, blockHash common.Hash, td *big.Int) error {
|
||||
if len(b.buff.headers) >= MaxEraESize {
|
||||
return fmt.Errorf("exceeds MaxEraESize %d", MaxEraESize)
|
||||
if len(b.buff.headers) >= era.MaxSize {
|
||||
return fmt.Errorf("exceeds max size %d", era.MaxSize)
|
||||
}
|
||||
|
||||
b.buff.headers = append(b.buff.headers, headerRLP)
|
||||
|
|
@ -173,7 +160,7 @@ func (b *Builder) AddRLP(headerRLP []byte, bodyRLP []byte, receipts []byte, proo
|
|||
if b.startNum == nil {
|
||||
b.startNum = new(uint64)
|
||||
*b.startNum = blockNum
|
||||
if n, err := b.w.Write(TypeVersion, nil); err != nil {
|
||||
if n, err := b.w.Write(era.TypeVersion, nil); err != nil {
|
||||
return fmt.Errorf("write version entry: %w", err)
|
||||
} else {
|
||||
b.written += uint64(n)
|
||||
|
|
@ -189,7 +176,7 @@ func (b *Builder) Finalize() (common.Hash, error) {
|
|||
return common.Hash{}, errors.New("no blocks added, cannot finalize")
|
||||
}
|
||||
for _, data := range b.buff.headers {
|
||||
off, err := b.addEntry(TypeCompressedHeader, data, true)
|
||||
off, err := b.addEntry(era.TypeCompressedHeader, data, true)
|
||||
if err != nil {
|
||||
return common.Hash{}, fmt.Errorf("headers: %w", err)
|
||||
}
|
||||
|
|
@ -197,7 +184,7 @@ func (b *Builder) Finalize() (common.Hash, error) {
|
|||
}
|
||||
|
||||
for _, data := range b.buff.bodies {
|
||||
off, err := b.addEntry(TypeCompressedBody, data, true)
|
||||
off, err := b.addEntry(era.TypeCompressedBody, data, true)
|
||||
if err != nil {
|
||||
return common.Hash{}, fmt.Errorf("bodies: %w", err)
|
||||
}
|
||||
|
|
@ -205,7 +192,7 @@ func (b *Builder) Finalize() (common.Hash, error) {
|
|||
}
|
||||
|
||||
for _, data := range b.buff.receipts {
|
||||
off, err := b.addEntry(TypeCompressedReceipts, data, true)
|
||||
off, err := b.addEntry(era.TypeCompressedSlimReceipts, data, true)
|
||||
if err != nil {
|
||||
return common.Hash{}, fmt.Errorf("receipts: %w", err)
|
||||
}
|
||||
|
|
@ -215,7 +202,7 @@ func (b *Builder) Finalize() (common.Hash, error) {
|
|||
if len(b.buff.tds) > 0 {
|
||||
for _, data := range b.buff.tds {
|
||||
littleEndian := uint256LE(data)
|
||||
off, err := b.addEntry(TypeTotalDifficulty, littleEndian, false)
|
||||
off, err := b.addEntry(era.TypeTotalDifficulty, littleEndian, false)
|
||||
if err != nil {
|
||||
return common.Hash{}, fmt.Errorf("total-difficulty: %w", err)
|
||||
}
|
||||
|
|
@ -225,7 +212,7 @@ func (b *Builder) Finalize() (common.Hash, error) {
|
|||
|
||||
if len(b.buff.proofs) > 0 {
|
||||
for _, data := range b.buff.proofs {
|
||||
off, err := b.addEntry(TypeProof, data, true)
|
||||
off, err := b.addEntry(era.TypeProof, data, true)
|
||||
if err != nil {
|
||||
return common.Hash{}, fmt.Errorf("proofs: %w", err)
|
||||
}
|
||||
|
|
@ -236,11 +223,11 @@ func (b *Builder) Finalize() (common.Hash, error) {
|
|||
var accRoot common.Hash
|
||||
if len(b.hashes) > 0 {
|
||||
var err error
|
||||
accRoot, err = ComputeAccumulator(b.hashes, b.buff.tds)
|
||||
accRoot, err = era.ComputeAccumulator(b.hashes, b.buff.tds)
|
||||
if err != nil {
|
||||
return common.Hash{}, fmt.Errorf("compute accumulator: %w", err)
|
||||
}
|
||||
if n, err := b.w.Write(TypeAccumulatorRoot, accRoot[:]); err != nil {
|
||||
if n, err := b.w.Write(era.TypeAccumulator, accRoot[:]); err != nil {
|
||||
return common.Hash{}, fmt.Errorf("write accumulator: %w", err)
|
||||
} else {
|
||||
b.written += uint64(n)
|
||||
|
|
@ -332,7 +319,7 @@ func (b *Builder) writeIndex() error {
|
|||
|
||||
binary.LittleEndian.PutUint64(index[end+0:], componentCount)
|
||||
binary.LittleEndian.PutUint64(index[end+8:], count)
|
||||
if n, err := b.w.Write(TypeBlockIndex, index); err != nil {
|
||||
if n, err := b.w.Write(era.TypeComponentIndex, index); err != nil {
|
||||
return err
|
||||
} else {
|
||||
b.written += uint64(n)
|
||||
|
|
@ -20,8 +20,10 @@ import (
|
|||
"errors"
|
||||
"io"
|
||||
"math/big"
|
||||
"slices"
|
||||
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/internal/era"
|
||||
"github.com/ethereum/go-ethereum/internal/era/e2store"
|
||||
"github.com/ethereum/go-ethereum/rlp"
|
||||
"github.com/klauspost/compress/snappy"
|
||||
|
|
@ -100,7 +102,8 @@ func (it *Iterator) TotalDifficulty() (*big.Int, error) {
|
|||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return new(big.Int).SetBytes(reverseOrder(tdBytes)), nil
|
||||
slices.Reverse(tdBytes)
|
||||
return new(big.Int).SetBytes(tdBytes), nil
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------------
|
||||
|
|
@ -137,7 +140,7 @@ func (it *RawIterator) Next() bool {
|
|||
it.setErr(err)
|
||||
return false
|
||||
}
|
||||
it.Header, _, err = newSnappyReader(it.e.s, TypeCompressedHeader, int64(headerOffset))
|
||||
it.Header, _, err = newSnappyReader(it.e.s, era.TypeCompressedHeader, int64(headerOffset))
|
||||
if err != nil {
|
||||
it.setErr(err)
|
||||
return false
|
||||
|
|
@ -148,7 +151,7 @@ func (it *RawIterator) Next() bool {
|
|||
it.setErr(err)
|
||||
return false
|
||||
}
|
||||
it.Body, _, err = newSnappyReader(it.e.s, TypeCompressedBody, int64(bodyOffset))
|
||||
it.Body, _, err = newSnappyReader(it.e.s, era.TypeCompressedBody, int64(bodyOffset))
|
||||
if err != nil {
|
||||
it.setErr(err)
|
||||
return false
|
||||
|
|
@ -159,7 +162,7 @@ func (it *RawIterator) Next() bool {
|
|||
it.setErr(err)
|
||||
return false
|
||||
}
|
||||
it.Receipts, _, err = newSnappyReader(it.e.s, TypeCompressedReceipts, int64(receiptsOffset))
|
||||
it.Receipts, _, err = newSnappyReader(it.e.s, era.TypeCompressedReceipts, int64(receiptsOffset))
|
||||
if err != nil {
|
||||
it.setErr(err)
|
||||
return false
|
||||
|
|
@ -171,7 +174,7 @@ func (it *RawIterator) Next() bool {
|
|||
it.setErr(err)
|
||||
return false
|
||||
}
|
||||
it.TotalDifficulty, _, err = it.e.s.ReaderAt(TypeTotalDifficulty, int64(tdOffset))
|
||||
it.TotalDifficulty, _, err = it.e.s.ReaderAt(era.TypeTotalDifficulty, int64(tdOffset))
|
||||
if err != nil {
|
||||
it.setErr(err)
|
||||
return false
|
||||
|
|
@ -23,45 +23,21 @@ import (
|
|||
"math/big"
|
||||
"os"
|
||||
"path"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/internal/era"
|
||||
"github.com/ethereum/go-ethereum/internal/era/e2store"
|
||||
"github.com/ethereum/go-ethereum/rlp"
|
||||
"github.com/klauspost/compress/snappy"
|
||||
)
|
||||
|
||||
// metadata contains the information about the era file that is written into the file.
|
||||
type metadata struct {
|
||||
start uint64 // start block number
|
||||
count uint64 // number of blocks in the era
|
||||
components uint64 // number of properties
|
||||
length int64 // length of the file in bytes
|
||||
}
|
||||
|
||||
// componentType represents the integer form of a specific type that can be present in the era file.
|
||||
type componentType int
|
||||
|
||||
// TypeCompressedHeader, TypeCompressedBody, TypeCompressedReceipts, TypeTotalDifficulty, and TypeProof are the different types of components that can be present in the era file.
|
||||
const (
|
||||
header componentType = iota
|
||||
body
|
||||
receipts
|
||||
td
|
||||
proof
|
||||
)
|
||||
|
||||
type ReadAtSeekCloser interface {
|
||||
io.ReaderAt
|
||||
io.Seeker
|
||||
io.Closer
|
||||
}
|
||||
|
||||
// Era object represents an era file that contains blocks and their components.
|
||||
type Era struct {
|
||||
f ReadAtSeekCloser
|
||||
f era.ReadAtSeekCloser
|
||||
s *e2store.Reader
|
||||
m metadata // metadata for the Era file
|
||||
}
|
||||
|
|
@ -96,7 +72,7 @@ func (e *Era) Close() error {
|
|||
}
|
||||
|
||||
// From returns an Era backed by f.
|
||||
func From(f ReadAtSeekCloser) (*Era, error) {
|
||||
func From(f era.ReadAtSeekCloser) (*Era, error) {
|
||||
e := &Era{f: f, s: e2store.NewReader(f)}
|
||||
if err := e.loadIndex(); err != nil {
|
||||
f.Close()
|
||||
|
|
@ -166,7 +142,7 @@ func (e *Era) GetHeader(num uint64) (*types.Header, error) {
|
|||
return nil, err
|
||||
}
|
||||
|
||||
r, _, err := e.s.ReaderAt(TypeCompressedHeader, int64(off))
|
||||
r, _, err := e.s.ReaderAt(era.TypeCompressedHeader, int64(off))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -183,7 +159,7 @@ func (e *Era) GetBody(num uint64) (*types.Body, error) {
|
|||
return nil, err
|
||||
}
|
||||
|
||||
r, _, err := e.s.ReaderAt(TypeCompressedBody, int64(off))
|
||||
r, _, err := e.s.ReaderAt(era.TypeCompressedBody, int64(off))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -199,12 +175,13 @@ func (e *Era) getTD(blockNum uint64) (*big.Int, error) {
|
|||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r, _, err := e.s.ReaderAt(TypeTotalDifficulty, int64(off))
|
||||
r, _, err := e.s.ReaderAt(era.TypeTotalDifficulty, int64(off))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
buf, _ := io.ReadAll(r)
|
||||
td := new(big.Int).SetBytes(reverseOrder(buf))
|
||||
slices.Reverse(buf)
|
||||
td := new(big.Int).SetBytes(buf)
|
||||
return td, nil
|
||||
}
|
||||
|
||||
|
|
@ -214,7 +191,7 @@ func (e *Era) GetRawBodyFrameByNumber(blockNum uint64) ([]byte, error) {
|
|||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r, _, err := e.s.ReaderAt(TypeCompressedBody, int64(off))
|
||||
r, _, err := e.s.ReaderAt(era.TypeCompressedBody, int64(off))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -227,7 +204,7 @@ func (e *Era) GetRawReceiptsFrameByNumber(blockNum uint64) ([]byte, error) {
|
|||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r, _, err := e.s.ReaderAt(TypeCompressedReceipts, int64(off))
|
||||
r, _, err := e.s.ReaderAt(era.TypeCompressedSlimReceipts, int64(off))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -240,7 +217,7 @@ func (e *Era) GetRawProofFrameByNumber(blockNum uint64) ([]byte, error) {
|
|||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r, _, err := e.s.ReaderAt(TypeProof, int64(off))
|
||||
r, _, err := e.s.ReaderAt(era.TypeProof, int64(off))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -319,7 +296,7 @@ func (e *Era) GetHeaders(first, count uint64) ([]*types.Header, error) {
|
|||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r, _, err := e.s.ReaderAt(TypeCompressedHeader, int64(off))
|
||||
r, _, err := e.s.ReaderAt(era.TypeCompressedHeader, int64(off))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -348,7 +325,7 @@ func (e *Era) GetBodies(first, count uint64) ([]*types.Body, error) {
|
|||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r, _, err := e.s.ReaderAt(TypeCompressedBody, int64(off))
|
||||
r, _, err := e.s.ReaderAt(era.TypeCompressedBody, int64(off))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -377,7 +354,7 @@ func (e *Era) GetReceipts(first, count uint64) ([]types.Receipts, error) {
|
|||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r, _, err := e.s.ReaderAt(TypeCompressedReceipts, int64(off))
|
||||
r, _, err := e.s.ReaderAt(era.TypeCompressedSlimReceipts, int64(off))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
|
@ -389,3 +366,23 @@ func (e *Era) GetReceipts(first, count uint64) ([]types.Receipts, error) {
|
|||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// metadata contains the information about the era file that is written into the file.
|
||||
type metadata struct {
|
||||
start uint64 // start block number
|
||||
count uint64 // number of blocks in the era
|
||||
components uint64 // number of properties
|
||||
length int64 // length of the file in bytes
|
||||
}
|
||||
|
||||
// componentType represents the integer form of a specific type that can be present in the era file.
|
||||
type componentType int
|
||||
|
||||
// TypeCompressedHeader, TypeCompressedBody, TypeCompressedReceipts, TypeTotalDifficulty, and TypeProof are the different types of components that can be present in the era file.
|
||||
const (
|
||||
header componentType = iota
|
||||
body
|
||||
receipts
|
||||
td
|
||||
proof
|
||||
)
|
||||
|
|
@ -1,197 +1 @@
|
|||
// Copyright 2024 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package era
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"math/big"
|
||||
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/rlp"
|
||||
)
|
||||
|
||||
// Iterator wraps RawIterator and returns decoded Era1 entries.
|
||||
type Iterator struct {
|
||||
inner *RawIterator
|
||||
}
|
||||
|
||||
// NewIterator returns a new Iterator instance. Next must be immediately
|
||||
// called on new iterators to load the first item.
|
||||
func NewIterator(e *Era) (*Iterator, error) {
|
||||
inner, err := NewRawIterator(e)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Iterator{inner}, nil
|
||||
}
|
||||
|
||||
// Next moves the iterator to the next block entry. It returns false when all
|
||||
// items have been read or an error has halted its progress. Block, Receipts,
|
||||
// and BlockAndReceipts should no longer be called after false is returned.
|
||||
func (it *Iterator) Next() bool {
|
||||
return it.inner.Next()
|
||||
}
|
||||
|
||||
// Number returns the current number block the iterator will return.
|
||||
func (it *Iterator) Number() uint64 {
|
||||
return it.inner.next - 1
|
||||
}
|
||||
|
||||
// Error returns the error status of the iterator. It should be called before
|
||||
// reading from any of the iterator's values.
|
||||
func (it *Iterator) Error() error {
|
||||
return it.inner.Error()
|
||||
}
|
||||
|
||||
// Block returns the block for the iterator's current position.
|
||||
func (it *Iterator) Block() (*types.Block, error) {
|
||||
if it.inner.Header == nil || it.inner.Body == nil {
|
||||
return nil, errors.New("header and body must be non-nil")
|
||||
}
|
||||
var (
|
||||
header types.Header
|
||||
body types.Body
|
||||
)
|
||||
if err := rlp.Decode(it.inner.Header, &header); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := rlp.Decode(it.inner.Body, &body); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return types.NewBlockWithHeader(&header).WithBody(body), nil
|
||||
}
|
||||
|
||||
// Receipts returns the receipts for the iterator's current position.
|
||||
func (it *Iterator) Receipts() (types.Receipts, error) {
|
||||
if it.inner.Receipts == nil {
|
||||
return nil, errors.New("receipts must be non-nil")
|
||||
}
|
||||
var receipts types.Receipts
|
||||
err := rlp.Decode(it.inner.Receipts, &receipts)
|
||||
return receipts, err
|
||||
}
|
||||
|
||||
// BlockAndReceipts returns the block and receipts for the iterator's current
|
||||
// position.
|
||||
func (it *Iterator) BlockAndReceipts() (*types.Block, types.Receipts, error) {
|
||||
b, err := it.Block()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
r, err := it.Receipts()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return b, r, nil
|
||||
}
|
||||
|
||||
// TotalDifficulty returns the total difficulty for the iterator's current
|
||||
// position.
|
||||
func (it *Iterator) TotalDifficulty() (*big.Int, error) {
|
||||
td, err := io.ReadAll(it.inner.TotalDifficulty)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return new(big.Int).SetBytes(reverseOrder(td)), nil
|
||||
}
|
||||
|
||||
// RawIterator reads an RLP-encode Era1 entries.
|
||||
type RawIterator struct {
|
||||
e *Era // backing Era1
|
||||
next uint64 // next block to read
|
||||
err error // last error
|
||||
|
||||
Header io.Reader
|
||||
Body io.Reader
|
||||
Receipts io.Reader
|
||||
TotalDifficulty io.Reader
|
||||
}
|
||||
|
||||
// NewRawIterator returns a new RawIterator instance. Next must be immediately
|
||||
// called on new iterators to load the first item.
|
||||
func NewRawIterator(e *Era) (*RawIterator, error) {
|
||||
return &RawIterator{
|
||||
e: e,
|
||||
next: e.m.start,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Next moves the iterator to the next block entry. It returns false when all
|
||||
// items have been read or an error has halted its progress. Header, Body,
|
||||
// Receipts, TotalDifficulty will be set to nil in the case returning false or
|
||||
// finding an error and should therefore no longer be read from.
|
||||
func (it *RawIterator) Next() bool {
|
||||
// Clear old errors.
|
||||
it.err = nil
|
||||
if it.e.m.start+it.e.m.count <= it.next {
|
||||
it.clear()
|
||||
return false
|
||||
}
|
||||
off, err := it.e.readOffset(it.next)
|
||||
if err != nil {
|
||||
// Error here means block index is corrupted, so don't
|
||||
// continue.
|
||||
it.clear()
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
var n int64
|
||||
if it.Header, n, it.err = newSnappyReader(it.e.s, TypeCompressedHeader, off); it.err != nil {
|
||||
it.clear()
|
||||
return true
|
||||
}
|
||||
off += n
|
||||
if it.Body, n, it.err = newSnappyReader(it.e.s, TypeCompressedBody, off); it.err != nil {
|
||||
it.clear()
|
||||
return true
|
||||
}
|
||||
off += n
|
||||
if it.Receipts, n, it.err = newSnappyReader(it.e.s, TypeCompressedReceipts, off); it.err != nil {
|
||||
it.clear()
|
||||
return true
|
||||
}
|
||||
off += n
|
||||
if it.TotalDifficulty, _, it.err = it.e.s.ReaderAt(TypeTotalDifficulty, off); it.err != nil {
|
||||
it.clear()
|
||||
return true
|
||||
}
|
||||
it.next += 1
|
||||
return true
|
||||
}
|
||||
|
||||
// Number returns the current number block the iterator will return.
|
||||
func (it *RawIterator) Number() uint64 {
|
||||
return it.next - 1
|
||||
}
|
||||
|
||||
// Error returns the error status of the iterator. It should be called before
|
||||
// reading from any of the iterator's values.
|
||||
func (it *RawIterator) Error() error {
|
||||
if it.err == io.EOF {
|
||||
return nil
|
||||
}
|
||||
return it.err
|
||||
}
|
||||
|
||||
// clear sets all the outputs to nil.
|
||||
func (it *RawIterator) clear() {
|
||||
it.Header = nil
|
||||
it.Body = nil
|
||||
it.Receipts = nil
|
||||
it.TotalDifficulty = nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@
|
|||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package era
|
||||
package onedb
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
|
|
@ -26,6 +26,7 @@ import (
|
|||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/internal/era"
|
||||
"github.com/ethereum/go-ethereum/internal/era/e2store"
|
||||
"github.com/ethereum/go-ethereum/rlp"
|
||||
"github.com/golang/snappy"
|
||||
|
|
@ -96,7 +97,10 @@ func NewBuilder(w io.Writer) *Builder {
|
|||
|
||||
// Add writes a compressed block entry and compressed receipts entry to the
|
||||
// underlying e2store file.
|
||||
func (b *Builder) Add(block *types.Block, receipts types.Receipts, td *big.Int) error {
|
||||
func (b *Builder) Add(block *types.Block, receipts types.Receipts, td *big.Int, proof era.Proof) error {
|
||||
if proof != nil {
|
||||
return fmt.Errorf("proof not allowed in era1 format")
|
||||
}
|
||||
eh, err := rlp.EncodeToBytes(block.Header())
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -117,7 +121,7 @@ func (b *Builder) Add(block *types.Block, receipts types.Receipts, td *big.Int)
|
|||
func (b *Builder) AddRLP(header, body, receipts []byte, number uint64, hash common.Hash, td, difficulty *big.Int) error {
|
||||
// Write Era1 version entry before first block.
|
||||
if b.startNum == nil {
|
||||
n, err := b.w.Write(TypeVersion, nil)
|
||||
n, err := b.w.Write(era.TypeVersion, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -126,8 +130,8 @@ func (b *Builder) AddRLP(header, body, receipts []byte, number uint64, hash comm
|
|||
b.startTd = new(big.Int).Sub(td, difficulty)
|
||||
b.written += n
|
||||
}
|
||||
if len(b.indexes) >= MaxEra1Size {
|
||||
return fmt.Errorf("exceeds maximum batch size of %d", MaxEra1Size)
|
||||
if len(b.indexes) >= era.MaxSize {
|
||||
return fmt.Errorf("exceeds maximum batch size of %d", era.MaxSize)
|
||||
}
|
||||
|
||||
b.indexes = append(b.indexes, uint64(b.written))
|
||||
|
|
@ -135,19 +139,19 @@ func (b *Builder) AddRLP(header, body, receipts []byte, number uint64, hash comm
|
|||
b.tds = append(b.tds, td)
|
||||
|
||||
// Write block data.
|
||||
if err := b.snappyWrite(TypeCompressedHeader, header); err != nil {
|
||||
if err := b.snappyWrite(era.TypeCompressedHeader, header); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := b.snappyWrite(TypeCompressedBody, body); err != nil {
|
||||
if err := b.snappyWrite(era.TypeCompressedBody, body); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := b.snappyWrite(TypeCompressedReceipts, receipts); err != nil {
|
||||
if err := b.snappyWrite(era.TypeCompressedReceipts, receipts); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Also write total difficulty, but don't snappy encode.
|
||||
btd := bigToBytes32(td)
|
||||
n, err := b.w.Write(TypeTotalDifficulty, btd[:])
|
||||
btd := era.BigToBytes32(td)
|
||||
n, err := b.w.Write(era.TypeTotalDifficulty, btd[:])
|
||||
b.written += n
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -163,11 +167,11 @@ func (b *Builder) Finalize() (common.Hash, error) {
|
|||
return common.Hash{}, errors.New("finalize called on empty builder")
|
||||
}
|
||||
// Compute accumulator root and write entry.
|
||||
root, err := ComputeAccumulator(b.hashes, b.tds)
|
||||
root, err := era.ComputeAccumulator(b.hashes, b.tds)
|
||||
if err != nil {
|
||||
return common.Hash{}, fmt.Errorf("error calculating accumulator root: %w", err)
|
||||
}
|
||||
n, err := b.w.Write(TypeAccumulator, root[:])
|
||||
n, err := b.w.Write(era.TypeAccumulator, root[:])
|
||||
b.written += n
|
||||
if err != nil {
|
||||
return common.Hash{}, fmt.Errorf("error writing accumulator: %w", err)
|
||||
|
|
@ -196,7 +200,7 @@ func (b *Builder) Finalize() (common.Hash, error) {
|
|||
binary.LittleEndian.PutUint64(index[8+count*8:], uint64(count))
|
||||
|
||||
// Finally, write the block index entry.
|
||||
if _, err := b.w.Write(TypeBlockIndex, index); err != nil {
|
||||
if _, err := b.w.Write(era.TypeBlockIndex, index); err != nil {
|
||||
return common.Hash{}, fmt.Errorf("unable to write block index: %w", err)
|
||||
}
|
||||
|
||||
|
|
@ -14,7 +14,7 @@
|
|||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package era
|
||||
package onedb
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
|
|
@ -22,6 +22,7 @@ import (
|
|||
"io"
|
||||
"math/big"
|
||||
"os"
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
|
|
@ -136,7 +137,8 @@ func TestEra1Builder(t *testing.T) {
|
|||
if err != nil {
|
||||
t.Fatalf("error reading td: %v", err)
|
||||
}
|
||||
td := new(big.Int).SetBytes(reverseOrder(rawTd))
|
||||
slices.Reverse(rawTd)
|
||||
td := new(big.Int).SetBytes(rawTd)
|
||||
if td.Cmp(chain.tds[i]) != 0 {
|
||||
t.Fatalf("mismatched tds: want %s, got %s", chain.tds[i], td)
|
||||
}
|
||||
200
internal/era/onedb/iterator.go
Normal file
200
internal/era/onedb/iterator.go
Normal file
|
|
@ -0,0 +1,200 @@
|
|||
// Copyright 2024 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package onedb
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"io"
|
||||
"math/big"
|
||||
"slices"
|
||||
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/internal/era"
|
||||
"github.com/ethereum/go-ethereum/rlp"
|
||||
)
|
||||
|
||||
// Iterator wraps RawIterator and returns decoded Era1 entries.
|
||||
type Iterator struct {
|
||||
inner *RawIterator
|
||||
}
|
||||
|
||||
// NewIterator returns a new Iterator instance. Next must be immediately
|
||||
// called on new iterators to load the first item.
|
||||
func NewIterator(e *Era) (*Iterator, error) {
|
||||
inner, err := NewRawIterator(e)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Iterator{inner}, nil
|
||||
}
|
||||
|
||||
// Next moves the iterator to the next block entry. It returns false when all
|
||||
// items have been read or an error has halted its progress. Block, Receipts,
|
||||
// and BlockAndReceipts should no longer be called after false is returned.
|
||||
func (it *Iterator) Next() bool {
|
||||
return it.inner.Next()
|
||||
}
|
||||
|
||||
// Number returns the current number block the iterator will return.
|
||||
func (it *Iterator) Number() uint64 {
|
||||
return it.inner.next - 1
|
||||
}
|
||||
|
||||
// Error returns the error status of the iterator. It should be called before
|
||||
// reading from any of the iterator's values.
|
||||
func (it *Iterator) Error() error {
|
||||
return it.inner.Error()
|
||||
}
|
||||
|
||||
// Block returns the block for the iterator's current position.
|
||||
func (it *Iterator) Block() (*types.Block, error) {
|
||||
if it.inner.Header == nil || it.inner.Body == nil {
|
||||
return nil, errors.New("header and body must be non-nil")
|
||||
}
|
||||
var (
|
||||
header types.Header
|
||||
body types.Body
|
||||
)
|
||||
if err := rlp.Decode(it.inner.Header, &header); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := rlp.Decode(it.inner.Body, &body); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return types.NewBlockWithHeader(&header).WithBody(body), nil
|
||||
}
|
||||
|
||||
// Receipts returns the receipts for the iterator's current position.
|
||||
func (it *Iterator) Receipts() (types.Receipts, error) {
|
||||
if it.inner.Receipts == nil {
|
||||
return nil, errors.New("receipts must be non-nil")
|
||||
}
|
||||
var receipts types.Receipts
|
||||
err := rlp.Decode(it.inner.Receipts, &receipts)
|
||||
return receipts, err
|
||||
}
|
||||
|
||||
// BlockAndReceipts returns the block and receipts for the iterator's current
|
||||
// position.
|
||||
func (it *Iterator) BlockAndReceipts() (*types.Block, types.Receipts, error) {
|
||||
b, err := it.Block()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
r, err := it.Receipts()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return b, r, nil
|
||||
}
|
||||
|
||||
// TotalDifficulty returns the total difficulty for the iterator's current
|
||||
// position.
|
||||
func (it *Iterator) TotalDifficulty() (*big.Int, error) {
|
||||
td, err := io.ReadAll(it.inner.TotalDifficulty)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
slices.Reverse(td)
|
||||
return new(big.Int).SetBytes(td), nil
|
||||
}
|
||||
|
||||
// RawIterator reads an RLP-encode Era1 entries.
|
||||
type RawIterator struct {
|
||||
e *Era // backing Era1
|
||||
next uint64 // next block to read
|
||||
err error // last error
|
||||
|
||||
Header io.Reader
|
||||
Body io.Reader
|
||||
Receipts io.Reader
|
||||
TotalDifficulty io.Reader
|
||||
}
|
||||
|
||||
// NewRawIterator returns a new RawIterator instance. Next must be immediately
|
||||
// called on new iterators to load the first item.
|
||||
func NewRawIterator(e *Era) (*RawIterator, error) {
|
||||
return &RawIterator{
|
||||
e: e,
|
||||
next: e.m.start,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Next moves the iterator to the next block entry. It returns false when all
|
||||
// items have been read or an error has halted its progress. Header, Body,
|
||||
// Receipts, TotalDifficulty will be set to nil in the case returning false or
|
||||
// finding an error and should therefore no longer be read from.
|
||||
func (it *RawIterator) Next() bool {
|
||||
// Clear old errors.
|
||||
it.err = nil
|
||||
if it.e.m.start+it.e.m.count <= it.next {
|
||||
it.clear()
|
||||
return false
|
||||
}
|
||||
off, err := it.e.readOffset(it.next)
|
||||
if err != nil {
|
||||
// Error here means block index is corrupted, so don't
|
||||
// continue.
|
||||
it.clear()
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
var n int64
|
||||
if it.Header, n, it.err = newSnappyReader(it.e.s, era.TypeCompressedHeader, off); it.err != nil {
|
||||
it.clear()
|
||||
return true
|
||||
}
|
||||
off += n
|
||||
if it.Body, n, it.err = newSnappyReader(it.e.s, era.TypeCompressedBody, off); it.err != nil {
|
||||
it.clear()
|
||||
return true
|
||||
}
|
||||
off += n
|
||||
if it.Receipts, n, it.err = newSnappyReader(it.e.s, era.TypeCompressedReceipts, off); it.err != nil {
|
||||
it.clear()
|
||||
return true
|
||||
}
|
||||
off += n
|
||||
if it.TotalDifficulty, _, it.err = it.e.s.ReaderAt(era.TypeTotalDifficulty, off); it.err != nil {
|
||||
it.clear()
|
||||
return true
|
||||
}
|
||||
it.next += 1
|
||||
return true
|
||||
}
|
||||
|
||||
// Number returns the current number block the iterator will return.
|
||||
func (it *RawIterator) Number() uint64 {
|
||||
return it.next - 1
|
||||
}
|
||||
|
||||
// Error returns the error status of the iterator. It should be called before
|
||||
// reading from any of the iterator's values.
|
||||
func (it *RawIterator) Error() error {
|
||||
if it.err == io.EOF {
|
||||
return nil
|
||||
}
|
||||
return it.err
|
||||
}
|
||||
|
||||
// clear sets all the outputs to nil.
|
||||
func (it *RawIterator) clear() {
|
||||
it.Header = nil
|
||||
it.Body = nil
|
||||
it.Receipts = nil
|
||||
it.TotalDifficulty = nil
|
||||
}
|
||||
308
internal/era/onedb/reader.go
Normal file
308
internal/era/onedb/reader.go
Normal file
|
|
@ -0,0 +1,308 @@
|
|||
// Copyright 2025 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package onedb
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/big"
|
||||
"os"
|
||||
"path"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/internal/era"
|
||||
"github.com/ethereum/go-ethereum/internal/era/e2store"
|
||||
"github.com/ethereum/go-ethereum/rlp"
|
||||
"github.com/golang/snappy"
|
||||
)
|
||||
|
||||
// Filename returns a recognizable Era1-formatted file name for the specified
|
||||
// epoch and network.
|
||||
func Filename(network string, epoch int, root common.Hash) string {
|
||||
return fmt.Sprintf("%s-%05d-%s.era1", network, epoch, root.Hex()[2:10])
|
||||
}
|
||||
|
||||
// ReadDir reads all the era1 files in a directory for a given network.
|
||||
// Format: <network>-<epoch>-<hexroot>.era1
|
||||
func ReadDir(dir, network string) ([]string, error) {
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error reading directory %s: %w", dir, err)
|
||||
}
|
||||
var (
|
||||
next = uint64(0)
|
||||
eras []string
|
||||
)
|
||||
for _, entry := range entries {
|
||||
if path.Ext(entry.Name()) != ".era1" {
|
||||
continue
|
||||
}
|
||||
parts := strings.Split(entry.Name(), "-")
|
||||
if len(parts) != 3 || parts[0] != network {
|
||||
// Invalid era1 filename, skip.
|
||||
continue
|
||||
}
|
||||
if epoch, err := strconv.ParseUint(parts[1], 10, 64); err != nil {
|
||||
return nil, fmt.Errorf("malformed era1 filename: %s", entry.Name())
|
||||
} else if epoch != next {
|
||||
return nil, fmt.Errorf("missing epoch %d", next)
|
||||
}
|
||||
next += 1
|
||||
eras = append(eras, entry.Name())
|
||||
}
|
||||
return eras, nil
|
||||
}
|
||||
|
||||
type ReadAtSeekCloser interface {
|
||||
io.ReaderAt
|
||||
io.Seeker
|
||||
io.Closer
|
||||
}
|
||||
|
||||
// Era reads and Era1 file.
|
||||
type Era struct {
|
||||
f ReadAtSeekCloser // backing era1 file
|
||||
s *e2store.Reader // e2store reader over f
|
||||
m metadata // start, count, length info
|
||||
mu *sync.Mutex // lock for buf
|
||||
buf [8]byte // buffer reading entry offsets
|
||||
}
|
||||
|
||||
// From returns an Era backed by f.
|
||||
func From(f ReadAtSeekCloser) (*Era, error) {
|
||||
m, err := readMetadata(f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Era{
|
||||
f: f,
|
||||
s: e2store.NewReader(f),
|
||||
m: m,
|
||||
mu: new(sync.Mutex),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Open returns an Era backed by the given filename.
|
||||
func Open(filename string) (*Era, error) {
|
||||
f, err := os.Open(filename)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return From(f)
|
||||
}
|
||||
|
||||
func (e *Era) Close() error {
|
||||
return e.f.Close()
|
||||
}
|
||||
|
||||
// GetBlockByNumber returns the block for the given block number.
|
||||
func (e *Era) GetBlockByNumber(num uint64) (*types.Block, error) {
|
||||
if e.m.start > num || e.m.start+e.m.count <= num {
|
||||
return nil, fmt.Errorf("out-of-bounds: %d not in [%d, %d)", num, e.m.start, e.m.start+e.m.count)
|
||||
}
|
||||
off, err := e.readOffset(num)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r, n, err := newSnappyReader(e.s, era.TypeCompressedHeader, off)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var header types.Header
|
||||
if err := rlp.Decode(r, &header); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
off += n
|
||||
r, _, err = newSnappyReader(e.s, era.TypeCompressedBody, off)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var body types.Body
|
||||
if err := rlp.Decode(r, &body); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return types.NewBlockWithHeader(&header).WithBody(body), nil
|
||||
}
|
||||
|
||||
// GetRawBodyByNumber returns the RLP-encoded body for the given block number.
|
||||
func (e *Era) GetRawBodyByNumber(num uint64) ([]byte, error) {
|
||||
if e.m.start > num || e.m.start+e.m.count <= num {
|
||||
return nil, fmt.Errorf("out-of-bounds: %d not in [%d, %d)", num, e.m.start, e.m.start+e.m.count)
|
||||
}
|
||||
off, err := e.readOffset(num)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
off, err = e.s.SkipN(off, 1)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
r, _, err := newSnappyReader(e.s, era.TypeCompressedBody, off)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return io.ReadAll(r)
|
||||
}
|
||||
|
||||
// GetRawReceiptsByNumber returns the RLP-encoded receipts for the given block number.
|
||||
func (e *Era) GetRawReceiptsByNumber(num uint64) ([]byte, error) {
|
||||
if e.m.start > num || e.m.start+e.m.count <= num {
|
||||
return nil, fmt.Errorf("out-of-bounds: %d not in [%d, %d)", num, e.m.start, e.m.start+e.m.count)
|
||||
}
|
||||
off, err := e.readOffset(num)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Skip over header and body.
|
||||
off, err = e.s.SkipN(off, 2)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
r, _, err := newSnappyReader(e.s, era.TypeCompressedReceipts, off)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return io.ReadAll(r)
|
||||
}
|
||||
|
||||
// Accumulator reads the accumulator entry in the Era1 file.
|
||||
func (e *Era) Accumulator() (common.Hash, error) {
|
||||
entry, err := e.s.Find(era.TypeAccumulator)
|
||||
if err != nil {
|
||||
return common.Hash{}, err
|
||||
}
|
||||
return common.BytesToHash(entry.Value), nil
|
||||
}
|
||||
|
||||
// InitialTD returns initial total difficulty before the difficulty of the
|
||||
// first block of the Era1 is applied.
|
||||
func (e *Era) InitialTD() (*big.Int, error) {
|
||||
var (
|
||||
r io.Reader
|
||||
header types.Header
|
||||
rawTd []byte
|
||||
n int64
|
||||
off int64
|
||||
err error
|
||||
)
|
||||
|
||||
// Read first header.
|
||||
if off, err = e.readOffset(e.m.start); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if r, n, err = newSnappyReader(e.s, era.TypeCompressedHeader, off); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := rlp.Decode(r, &header); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
off += n
|
||||
|
||||
// Skip over header and body.
|
||||
off, err = e.s.SkipN(off, 2)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Read total difficulty after first block.
|
||||
if r, _, err = e.s.ReaderAt(era.TypeTotalDifficulty, off); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rawTd, err = io.ReadAll(r)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
slices.Reverse(rawTd)
|
||||
td := new(big.Int).SetBytes(rawTd)
|
||||
return td.Sub(td, header.Difficulty), nil
|
||||
}
|
||||
|
||||
// Start returns the listed start block.
|
||||
func (e *Era) Start() uint64 {
|
||||
return e.m.start
|
||||
}
|
||||
|
||||
// Count returns the total number of blocks in the Era1.
|
||||
func (e *Era) Count() uint64 {
|
||||
return e.m.count
|
||||
}
|
||||
|
||||
// readOffset reads a specific block's offset from the block index. The value n
|
||||
// is the absolute block number desired.
|
||||
func (e *Era) readOffset(n uint64) (int64, error) {
|
||||
var (
|
||||
blockIndexRecordOffset = e.m.length - 24 - int64(e.m.count)*8 // skips start, count, and header
|
||||
firstIndex = blockIndexRecordOffset + 16 // first index after header / start-num
|
||||
indexOffset = int64(n-e.m.start) * 8 // desired index * size of indexes
|
||||
offOffset = firstIndex + indexOffset // offset of block offset
|
||||
)
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
clear(e.buf[:])
|
||||
if _, err := e.f.ReadAt(e.buf[:], offOffset); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
// Since the block offset is relative from the start of the block index record
|
||||
// we need to add the record offset to it's offset to get the block's absolute
|
||||
// offset.
|
||||
return blockIndexRecordOffset + int64(binary.LittleEndian.Uint64(e.buf[:])), nil
|
||||
}
|
||||
|
||||
// newSnappyReader returns a snappy.Reader for the e2store entry value at off.
|
||||
func newSnappyReader(e *e2store.Reader, expectedType uint16, off int64) (io.Reader, int64, error) {
|
||||
r, n, err := e.ReaderAt(expectedType, off)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
return snappy.NewReader(r), int64(n), err
|
||||
}
|
||||
|
||||
// metadata wraps the metadata in the block index.
|
||||
type metadata struct {
|
||||
start uint64
|
||||
count uint64
|
||||
length int64
|
||||
}
|
||||
|
||||
// readMetadata reads the metadata stored in an Era1 file's block index.
|
||||
func readMetadata(f ReadAtSeekCloser) (m metadata, err error) {
|
||||
// Determine length of reader.
|
||||
if m.length, err = f.Seek(0, io.SeekEnd); err != nil {
|
||||
return
|
||||
}
|
||||
b := make([]byte, 16)
|
||||
// Read count. It's the last 8 bytes of the file.
|
||||
if _, err = f.ReadAt(b[:8], m.length-8); err != nil {
|
||||
return
|
||||
}
|
||||
m.count = binary.LittleEndian.Uint64(b)
|
||||
// Read start. It's at the offset -sizeof(m.count) -
|
||||
// count*sizeof(indexEntry) - sizeof(m.start)
|
||||
if _, err = f.ReadAt(b[8:], m.length-16-int64(m.count*8)); err != nil {
|
||||
return
|
||||
}
|
||||
m.start = binary.LittleEndian.Uint64(b[8:])
|
||||
return
|
||||
}
|
||||
|
|
@ -13,7 +13,7 @@
|
|||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
package era2
|
||||
package era
|
||||
|
||||
import (
|
||||
"io"
|
||||
|
|
@ -1,91 +0,0 @@
|
|||
// Copyright 2025 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package era2
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/big"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
ssz "github.com/ferranbt/fastssz"
|
||||
)
|
||||
|
||||
// ComputeAccumulator calculates the SSZ hash tree root of the Era1
|
||||
// accumulator of header records.
|
||||
func ComputeAccumulator(hashes []common.Hash, tds []*big.Int) (common.Hash, error) {
|
||||
if len(hashes) != len(tds) {
|
||||
return common.Hash{}, errors.New("must have equal number hashes as td values")
|
||||
}
|
||||
if len(hashes) > MaxEraESize {
|
||||
return common.Hash{}, fmt.Errorf("too many records: have %d, max %d", len(hashes), MaxEraESize)
|
||||
}
|
||||
hh := ssz.NewHasher()
|
||||
for i := range hashes {
|
||||
rec := headerRecord{hashes[i], tds[i]}
|
||||
root, err := rec.HashTreeRoot()
|
||||
if err != nil {
|
||||
return common.Hash{}, err
|
||||
}
|
||||
hh.Append(root[:])
|
||||
}
|
||||
hh.MerkleizeWithMixin(0, uint64(len(hashes)), uint64(MaxEraESize))
|
||||
return hh.HashRoot()
|
||||
}
|
||||
|
||||
// headerRecord is an individual record for a historical header.
|
||||
//
|
||||
// See https://github.com/ethereum/portal-network-specs/blob/master/history/history-network.md#the-historical-hashes-accumulator
|
||||
// for more information.
|
||||
type headerRecord struct {
|
||||
Hash common.Hash
|
||||
TotalDifficulty *big.Int
|
||||
}
|
||||
|
||||
// GetTree completes the ssz.HashRoot interface, but is unused.
|
||||
func (h *headerRecord) GetTree() (*ssz.Node, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// HashTreeRoot ssz hashes the headerRecord object.
|
||||
func (h *headerRecord) HashTreeRoot() ([32]byte, error) {
|
||||
return ssz.HashWithDefaultHasher(h)
|
||||
}
|
||||
|
||||
// HashTreeRootWith ssz hashes the headerRecord object with a hasher.
|
||||
func (h *headerRecord) HashTreeRootWith(hh ssz.HashWalker) (err error) {
|
||||
hh.PutBytes(h.Hash[:])
|
||||
td := bigToBytes32(h.TotalDifficulty)
|
||||
hh.PutBytes(td[:])
|
||||
hh.Merkleize(0)
|
||||
return
|
||||
}
|
||||
|
||||
// bigToBytes32 converts a big.Int into a little-endian 32-byte array.
|
||||
func bigToBytes32(n *big.Int) (b [32]byte) {
|
||||
n.FillBytes(b[:])
|
||||
reverseOrder(b[:])
|
||||
return
|
||||
}
|
||||
|
||||
// reverseOrder reverses the byte order of a slice.
|
||||
func reverseOrder(b []byte) []byte {
|
||||
for i := 0; i < 16; i++ {
|
||||
b[i], b[32-i-1] = b[32-i-1], b[i]
|
||||
}
|
||||
return b
|
||||
}
|
||||
Loading…
Reference in a new issue