mirror of
https://github.com/ethereum/go-ethereum.git
synced 2026-08-20 10:52:25 +00:00
internal/era,cmd/utils,cmd/era: change to iterator interface for reading era entries
This commit is contained in:
lightclient@protonmail.com
lightclient
parent
008ed82224
commit
3fa17e775f
6 changed files with 382 additions and 259 deletions
|
|
@ -19,7 +19,6 @@ package main
|
|||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"math/big"
|
||||
"os"
|
||||
"path"
|
||||
|
|
@ -111,19 +110,15 @@ func block(ctx *cli.Context) error {
|
|||
if err != nil {
|
||||
return fmt.Errorf("invalid block number: %w", err)
|
||||
}
|
||||
f, err := open(ctx, num/uint64(ctx.Int(eraSizeFlag.Name)))
|
||||
e, err := open(ctx, num/uint64(ctx.Int(eraSizeFlag.Name)))
|
||||
if err != nil {
|
||||
return fmt.Errorf("error opening era: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
r, err := era.NewReader(f)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error making era reader: %w", err)
|
||||
}
|
||||
defer e.Close()
|
||||
// Read block with number.
|
||||
block, err := r.ReadBlock(num)
|
||||
block, err := e.GetBlockByNumber(num)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error reading era: %w", err)
|
||||
return fmt.Errorf("error reading block %d: %w", num, err)
|
||||
}
|
||||
// Convert block to JSON and print.
|
||||
val, err := ethapi.RPCMarshalBlock(block, ctx.Bool(txsFlag.Name), ctx.Bool(txsFlag.Name), params.MainnetChainConfig)
|
||||
|
|
@ -144,20 +139,16 @@ func info(ctx *cli.Context) error {
|
|||
if err != nil {
|
||||
return fmt.Errorf("invalid epoch number: %w", err)
|
||||
}
|
||||
f, err := open(ctx, epoch)
|
||||
e, err := open(ctx, epoch)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
r, err := era.NewReader(f)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error creating era reader: %w", err)
|
||||
}
|
||||
acc, err := r.Accumulator()
|
||||
defer e.Close()
|
||||
acc, err := e.Accumulator()
|
||||
if err != nil {
|
||||
return fmt.Errorf("error reading accumulator: %w", err)
|
||||
}
|
||||
td, err := r.InitialTD()
|
||||
td, err := e.InitialTD()
|
||||
if err != nil {
|
||||
return fmt.Errorf("error reading total difficulty: %w", err)
|
||||
}
|
||||
|
|
@ -167,7 +158,7 @@ func info(ctx *cli.Context) error {
|
|||
StartBlock uint64 `json:"startBlock"`
|
||||
Count uint64 `json:"count"`
|
||||
}{
|
||||
acc, td, r.Start(), r.Count(),
|
||||
acc, td, e.Start(), e.Count(),
|
||||
}
|
||||
b, _ := json.MarshalIndent(info, "", " ")
|
||||
fmt.Println(string(b))
|
||||
|
|
@ -175,7 +166,7 @@ func info(ctx *cli.Context) error {
|
|||
}
|
||||
|
||||
// open opens an era1 file at a certain epoch.
|
||||
func open(ctx *cli.Context, epoch uint64) (*os.File, error) {
|
||||
func open(ctx *cli.Context, epoch uint64) (*era.Era, error) {
|
||||
var (
|
||||
dir = ctx.String(dirFlag.Name)
|
||||
network = ctx.String(networkFlag.Name)
|
||||
|
|
@ -187,7 +178,7 @@ func open(ctx *cli.Context, epoch uint64) (*os.File, error) {
|
|||
if epoch >= uint64(len(entries)) {
|
||||
return nil, fmt.Errorf("epoch out-of-bounds: last %d, want %d", len(entries)-1, epoch)
|
||||
}
|
||||
return os.Open(path.Join(dir, entries[epoch]))
|
||||
return era.Open(path.Join(dir, entries[epoch]))
|
||||
}
|
||||
|
||||
// verify checks each era1 file in a directory to ensure it is well-formed and
|
||||
|
|
@ -223,29 +214,21 @@ func verify(ctx *cli.Context) error {
|
|||
// Wrap in function so defers don't stack.
|
||||
err := func() error {
|
||||
name := entries[i]
|
||||
f, err := os.Open(path.Join(dir, name))
|
||||
e, err := era.Open(path.Join(dir, name))
|
||||
if err != nil {
|
||||
return fmt.Errorf("error opening era1 file %s: %w", name, err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
r, err := era.NewReader(f)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to make era reader: %w", err)
|
||||
}
|
||||
|
||||
defer e.Close()
|
||||
// Read accumulator and check against expected.
|
||||
if got, err := r.Accumulator(); err != nil {
|
||||
if got, err := e.Accumulator(); err != nil {
|
||||
return fmt.Errorf("error retrieving accumulator for %s: %w", name, err)
|
||||
} else if got != want {
|
||||
return fmt.Errorf("invalid root %s: got %s, want %s", name, got, want)
|
||||
}
|
||||
|
||||
// Recompute accumulator.
|
||||
if err := checkAccumulator(r); err != nil {
|
||||
if err := checkAccumulator(e); err != nil {
|
||||
return fmt.Errorf("error verify era1 file %s: %w", name, err)
|
||||
}
|
||||
|
||||
// Give the user some feedback that something is happening.
|
||||
if time.Since(reported) >= 8*time.Second {
|
||||
fmt.Printf("Verifying Era1 files \t\t verified=%d,\t elapsed=%s\n", i, common.PrettyDuration(time.Since(start)))
|
||||
|
|
@ -262,21 +245,25 @@ func verify(ctx *cli.Context) error {
|
|||
}
|
||||
|
||||
// checkAccumulator verifies the accumulator matches the data in the Era.
|
||||
func checkAccumulator(r *era.Reader) error {
|
||||
func checkAccumulator(e *era.Era) error {
|
||||
var (
|
||||
err error
|
||||
start = r.Start()
|
||||
start = e.Start()
|
||||
want common.Hash
|
||||
td *big.Int
|
||||
tds = make([]*big.Int, 0)
|
||||
hashes = make([]common.Hash, 0)
|
||||
)
|
||||
if want, err = r.Accumulator(); err != nil {
|
||||
if want, err = e.Accumulator(); err != nil {
|
||||
return fmt.Errorf("error reading accumulator: %w", err)
|
||||
}
|
||||
if td, err = r.InitialTD(); err != nil {
|
||||
if td, err = e.InitialTD(); err != nil {
|
||||
return fmt.Errorf("error reading total difficulty: %w", err)
|
||||
}
|
||||
it, err := era.NewIterator(e)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error making era iterator: %w", err)
|
||||
}
|
||||
// Starting at epoch 0, iterate through all available era1 files and
|
||||
// check the following:
|
||||
// * the block index is constructed correctly
|
||||
|
|
@ -284,13 +271,14 @@ func checkAccumulator(r *era.Reader) error {
|
|||
// * the accumulator is correct by recomputing it locally,
|
||||
// which verifies the blocks are all correct (via hash)
|
||||
// * the receipts root matches the value in the block
|
||||
for j := 0; ; j++ {
|
||||
// read() walks the block index, so we're able to
|
||||
for j := 0; it.Next(); j++ {
|
||||
// next() walks the block index, so we're able to
|
||||
// implicitly verify it.
|
||||
block, receipts, err := r.Read()
|
||||
if err == io.EOF {
|
||||
break
|
||||
} else if err != nil {
|
||||
if it.Error() != nil {
|
||||
return fmt.Errorf("error reading block %d: %w", start+uint64(j), err)
|
||||
}
|
||||
block, receipts, err := it.BlockAndReceipts()
|
||||
if it.Error() != nil {
|
||||
return fmt.Errorf("error reading block %d: %w", start+uint64(j), err)
|
||||
}
|
||||
tr := types.DeriveSha(block.Transactions(), trie.NewStackTrie(nil))
|
||||
|
|
|
|||
|
|
@ -285,20 +285,27 @@ func ImportHistory(chain *core.BlockChain, db ethdb.Database, dir string, networ
|
|||
buf.Reset()
|
||||
|
||||
// Import all block data from Era1.
|
||||
r, err := era.NewReader(f)
|
||||
e, err := era.From(f)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error opening era: %w", err)
|
||||
}
|
||||
it, err := era.NewIterator(e)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error making era reader: %w", err)
|
||||
}
|
||||
for j := 0; ; j += 1 {
|
||||
for j := 0; it.Next(); j++ {
|
||||
n := i*era.MaxEra1Size + j
|
||||
block, receipts, err := r.Read()
|
||||
if err == io.EOF {
|
||||
break
|
||||
} else if err != nil {
|
||||
block, err := it.Block()
|
||||
if err != nil {
|
||||
return fmt.Errorf("error reading block %d: %w", n, err)
|
||||
} else if block.Number().BitLen() == 0 {
|
||||
}
|
||||
if block.Number().BitLen() == 0 {
|
||||
continue // skip genesis
|
||||
}
|
||||
receipts, err := it.Receipts()
|
||||
if err != nil {
|
||||
return fmt.Errorf("error reading receipts %d: %w", n, err)
|
||||
}
|
||||
if status, err := chain.HeaderChain().InsertHeaderChain([]*types.Header{block.Header()}, start, forker); err != nil {
|
||||
return fmt.Errorf("error inserting header %d: %w", n, err)
|
||||
} else if status != core.CanonStatTy {
|
||||
|
|
|
|||
|
|
@ -107,42 +107,54 @@ func TestHistoryImportAndExport(t *testing.T) {
|
|||
// Verify each Era.
|
||||
entries, _ := era.ReadDir(dir, "mainnet")
|
||||
for i, filename := range entries {
|
||||
f, err := os.ReadFile(path.Join(dir, filename))
|
||||
f, err := os.Open(path.Join(dir, filename))
|
||||
if err != nil {
|
||||
t.Fatalf("error opening era file: %v", err)
|
||||
}
|
||||
if want, got := common.HexToHash(checksums[i]), common.Hash(sha256.Sum256(f)); want != got {
|
||||
|
||||
var (
|
||||
h = sha256.New()
|
||||
buf = bytes.NewBuffer(nil)
|
||||
)
|
||||
if _, err := io.Copy(h, f); err != nil {
|
||||
t.Fatalf("unable to recalculate checksum: %v", err)
|
||||
}
|
||||
if got, want := common.BytesToHash(h.Sum(buf.Bytes()[:])).Hex(), checksums[i]; got != want {
|
||||
t.Fatalf("checksum %d does not match: got %s, want %s", i, got, want)
|
||||
}
|
||||
r, err := era.NewReader(bytes.NewReader(f))
|
||||
|
||||
e, err := era.From(f)
|
||||
if err != nil {
|
||||
t.Fatalf("error opening era: %v", err)
|
||||
}
|
||||
it, err := era.NewIterator(e)
|
||||
if err != nil {
|
||||
t.Fatalf("error making era reader: %v", err)
|
||||
}
|
||||
for j := 0; ; j += 1 {
|
||||
block, receipts, err := r.Read()
|
||||
if err == io.EOF {
|
||||
break
|
||||
} else if err != nil {
|
||||
t.Fatalf("error reading era file %d: %v", i, err)
|
||||
for j := 0; it.Next(); j++ {
|
||||
n := i*int(step) + j
|
||||
if it.Error() != nil {
|
||||
t.Fatalf("error reading block entry %d: %v", n, err)
|
||||
}
|
||||
var (
|
||||
n = i*int(step) + j
|
||||
want = chain.GetBlockByNumber(uint64(n))
|
||||
)
|
||||
block, receipts, err := it.BlockAndReceipts()
|
||||
if err != nil {
|
||||
t.Fatalf("error reading block entry %d: %v", n, err)
|
||||
}
|
||||
want := chain.GetBlockByNumber(uint64(n))
|
||||
if want, got := uint64(n), block.NumberU64(); want != got {
|
||||
t.Fatalf("blocks out of order: want %d, got %d", want, got)
|
||||
}
|
||||
if want.Hash() != block.Hash() {
|
||||
t.Fatalf("block hash mismatch %d: want %s, got %s", i+j, want.Hash().Hex(), block.Hash().Hex())
|
||||
t.Fatalf("block hash mismatch %d: want %s, got %s", n, want.Hash().Hex(), block.Hash().Hex())
|
||||
}
|
||||
if got := types.DeriveSha(block.Transactions(), trie.NewStackTrie(nil)); got != want.TxHash() {
|
||||
t.Fatalf("tx hash %d mismatch: want %s, got %s", i+j, want.TxHash(), got)
|
||||
t.Fatalf("tx hash %d mismatch: want %s, got %s", n, want.TxHash(), got)
|
||||
}
|
||||
if got := types.CalcUncleHash(block.Uncles()); got != want.UncleHash() {
|
||||
t.Fatalf("uncle hash %d mismatch: want %s, got %s", i+j, want.UncleHash(), got)
|
||||
t.Fatalf("uncle hash %d mismatch: want %s, got %s", n, want.UncleHash(), got)
|
||||
}
|
||||
if got := types.DeriveSha(receipts, trie.NewStackTrie(nil)); got != want.ReceiptHash() {
|
||||
t.Fatalf("receipt root %d mismatch: want %s, got %s", i+j, want.ReceiptHash(), got)
|
||||
t.Fatalf("receipt root %d mismatch: want %s, got %s", n, want.ReceiptHash(), got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ import (
|
|||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
|
|
@ -83,197 +84,75 @@ func ReadDir(dir, network string) ([]string, error) {
|
|||
return eras, nil
|
||||
}
|
||||
|
||||
// Reader reads an Era1 archive.
|
||||
// See Builder documentation for a detailed explanation of the Era1 format.
|
||||
type Reader struct {
|
||||
r io.ReaderAt
|
||||
e *e2store.Reader
|
||||
|
||||
buf [8]byte // buffer reading entry offsets
|
||||
next uint64 // next block to read
|
||||
length int64 // total length of r
|
||||
metadata metadata // start, count info
|
||||
}
|
||||
|
||||
type ReadAtSeeker interface {
|
||||
type ReadAtSeekCloser interface {
|
||||
io.ReaderAt
|
||||
io.Seeker
|
||||
io.Closer
|
||||
}
|
||||
|
||||
// NewReader returns a new Reader instance.
|
||||
func NewReader(r ReadAtSeeker) (*Reader, error) {
|
||||
length, err := r.Seek(0, io.SeekEnd)
|
||||
// Era reads and Era1 file.
|
||||
type Era struct {
|
||||
f ReadAtSeekCloser // backing era1 file
|
||||
s *e2store.Reader // e2store reader over f
|
||||
m metadata // start, count, length info
|
||||
mu *sync.Mutex // lock for buf
|
||||
buf [8]byte // buffer reading entry offsets
|
||||
}
|
||||
|
||||
// From returns an Era backed by f.
|
||||
func From(f ReadAtSeekCloser) (*Era, error) {
|
||||
m, err := readMetadata(f)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m, err := readMetadata(r, length)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Reader{
|
||||
r: r,
|
||||
e: e2store.NewReader(r),
|
||||
next: m.start,
|
||||
length: length,
|
||||
metadata: m,
|
||||
return &Era{
|
||||
f: f,
|
||||
s: e2store.NewReader(f),
|
||||
m: m,
|
||||
mu: new(sync.Mutex),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// readOffset reads a specific block's offset from the block index. The value n
|
||||
// is the absolute block number desired.
|
||||
func (r *Reader) readOffset(n uint64) (int64, error) {
|
||||
var (
|
||||
firstIndex = -8 - int64(r.metadata.count)*8 // size of count - index entries
|
||||
indexOffset = int64(n-r.metadata.start) * 8 // desired index * size of indexes
|
||||
offOffset = r.length + firstIndex + indexOffset // offset of block offset
|
||||
)
|
||||
r.clearBuffer()
|
||||
if _, err := r.r.ReadAt(r.buf[:], offOffset); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
// Since the block offset is relative from its location + size of index
|
||||
// value (8), we need to add it to it's offset to get the block's
|
||||
// absolute offset.
|
||||
return offOffset + 8 + int64(binary.LittleEndian.Uint64(r.buf[:])), nil
|
||||
}
|
||||
|
||||
// Read reads one (block, receipts) tuple from an Era1 archive.
|
||||
func (r *Reader) Read() (*types.Block, types.Receipts, error) {
|
||||
block, receipts, err := r.ReadBlockAndReceipts(r.next)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
r.next += 1
|
||||
return block, receipts, nil
|
||||
}
|
||||
|
||||
// readBlob reads an entry of data.
|
||||
func (r *Reader) readEntry(n uint64, skip int) (*e2store.Entry, error) {
|
||||
if n < r.metadata.start || r.metadata.start+r.metadata.count < n {
|
||||
return nil, fmt.Errorf("request out-of-bounds: want %d, start: %d, count: %d", n, r.metadata.start, r.metadata.count)
|
||||
}
|
||||
// Read the specified block's offset from the block index.
|
||||
off, err := r.readOffset(n)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error reading block offset: %w", err)
|
||||
}
|
||||
// Skip to the requested entry.
|
||||
for i := 0; i < skip; i++ {
|
||||
if length, err := r.e.LengthAt(off); err != nil {
|
||||
return nil, err
|
||||
} else {
|
||||
off += length
|
||||
}
|
||||
}
|
||||
// Read entry.
|
||||
var entry e2store.Entry
|
||||
if _, err := r.e.ReadAt(&entry, off); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &entry, nil
|
||||
}
|
||||
|
||||
// readHeaderRLP reads the header number n RLP.
|
||||
func (r *Reader) readHeaderRLP(n uint64) ([]byte, error) {
|
||||
e, err := r.readEntry(n, 0)
|
||||
// Open returns an Era backed by the given filename.
|
||||
func Open(filename string) (*Era, error) {
|
||||
f, err := os.Open(filename)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if e.Type != TypeCompressedHeader {
|
||||
return nil, fmt.Errorf("expected header entry, got %x", e.Type)
|
||||
}
|
||||
return io.ReadAll(snappy.NewReader(bytes.NewReader(e.Value)))
|
||||
return From(f)
|
||||
}
|
||||
|
||||
// readBodyRLP reads the block body number n RLP.
|
||||
func (r *Reader) readBodyRLP(n uint64) ([]byte, error) {
|
||||
e, err := r.readEntry(n, 1)
|
||||
func (e *Era) Close() error {
|
||||
return e.f.Close()
|
||||
}
|
||||
|
||||
func (e *Era) GetBlockByNumber(num uint64) (*types.Block, error) {
|
||||
if e.m.start > num || e.m.start+e.m.count <= num {
|
||||
return nil, fmt.Errorf("out-of-bounds")
|
||||
}
|
||||
off, err := e.readOffset(num)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if e.Type != TypeCompressedBody {
|
||||
return nil, fmt.Errorf("expected body entry, got %x", e.Type)
|
||||
}
|
||||
return io.ReadAll(snappy.NewReader(bytes.NewReader(e.Value)))
|
||||
}
|
||||
|
||||
// readReceiptsRLP reads the receipts RLP associated with number n.
|
||||
func (r *Reader) readReceiptsRLP(n uint64) ([]byte, error) {
|
||||
e, err := r.readEntry(n, 2)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if e.Type != TypeCompressedReceipts {
|
||||
return nil, fmt.Errorf("expected receipts entry, got %x", e.Type)
|
||||
}
|
||||
return io.ReadAll(snappy.NewReader(bytes.NewReader(e.Value)))
|
||||
}
|
||||
|
||||
// readTotalDifficulty reads the total difficulty of block number n.
|
||||
func (r *Reader) readTotalDifficulty(n uint64) (*big.Int, error) {
|
||||
e, err := r.readEntry(n, 3)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if e.Type != TypeTotalDifficulty {
|
||||
return nil, fmt.Errorf("expected TD entry, got %x", e.Type)
|
||||
}
|
||||
return new(big.Int).SetBytes(reverseOrder(e.Value)), nil
|
||||
}
|
||||
|
||||
// ReadHeader reads the header number n.
|
||||
func (r *Reader) ReadHeader(n uint64) (*types.Header, error) {
|
||||
h, err := r.readHeaderRLP(n)
|
||||
r, n, err := newSnappyReader(e.s, off)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var header types.Header
|
||||
if err := rlp.DecodeBytes(h, &header); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &header, nil
|
||||
}
|
||||
|
||||
// ReadBlock reads the block number n.
|
||||
func (r *Reader) ReadBlock(n uint64) (*types.Block, error) {
|
||||
header, err := r.ReadHeader(n)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
b, err := r.readBodyRLP(n)
|
||||
if err != nil {
|
||||
if err := rlp.Decode(r, &header); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
off += int64(n)
|
||||
var body types.Body
|
||||
if err := rlp.DecodeBytes(b, &body); err != nil {
|
||||
if err := rlp.Decode(r, &body); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return types.NewBlockWithHeader(header).WithBody(body.Transactions, body.Uncles), nil
|
||||
}
|
||||
|
||||
// ReadBlockAndReceipts reads the block number n and associated receipts.
|
||||
func (r *Reader) ReadBlockAndReceipts(n uint64) (*types.Block, types.Receipts, error) {
|
||||
// Read block.
|
||||
block, err := r.ReadBlock(n)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
// Read receipts.
|
||||
rr, err := r.readReceiptsRLP(n)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
// Decode receipts.
|
||||
var receipts types.Receipts
|
||||
if err := rlp.DecodeBytes(rr, &receipts); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return block, receipts, err
|
||||
return types.NewBlockWithHeader(&header).WithBody(body.Transactions, body.Uncles), nil
|
||||
}
|
||||
|
||||
// Accumulator reads the accumulator entry in the Era1 file.
|
||||
func (r *Reader) Accumulator() (common.Hash, error) {
|
||||
entry, err := r.e.Find(TypeAccumulator)
|
||||
func (e *Era) Accumulator() (common.Hash, error) {
|
||||
entry, err := e.s.Find(TypeAccumulator)
|
||||
if err != nil {
|
||||
return common.Hash{}, err
|
||||
}
|
||||
|
|
@ -282,53 +161,126 @@ func (r *Reader) Accumulator() (common.Hash, error) {
|
|||
|
||||
// InitialTD returns initial total difficulty before the difficulty of the
|
||||
// first block of the Era1 is applied.
|
||||
func (r *Reader) InitialTD() (*big.Int, error) {
|
||||
h, err := r.ReadHeader(r.Start())
|
||||
func (e *Era) InitialTD() (*big.Int, error) {
|
||||
var (
|
||||
r io.Reader
|
||||
header types.Header
|
||||
rawTd []byte
|
||||
n int
|
||||
off int64
|
||||
err error
|
||||
)
|
||||
|
||||
// Read first header.
|
||||
if off, err = e.readOffset(e.m.start); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if r, n, err = newSnappyReader(e.s, off); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := rlp.Decode(r, header); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
off += int64(n)
|
||||
|
||||
// Skip over next two records.
|
||||
for i := 0; i < 2; i++ {
|
||||
length, err := e.s.LengthAt(off)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// Above seek also sets reader so next TD entry will be for this block.
|
||||
entry, err := r.e.Find(TypeTotalDifficulty)
|
||||
if err != nil {
|
||||
off += length
|
||||
}
|
||||
|
||||
// Read total difficulty after first block.
|
||||
if r, n, err = newReader(e.s, off); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
td := new(big.Int).SetBytes(reverseOrder(entry.Value))
|
||||
return td.Sub(td, h.Difficulty), nil
|
||||
if err := rlp.Decode(r, rawTd); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
td := new(big.Int).SetBytes(reverseOrder(rawTd))
|
||||
return td.Sub(td, header.Difficulty), nil
|
||||
}
|
||||
|
||||
// Start returns the listed start block.
|
||||
func (r *Reader) Start() uint64 {
|
||||
return r.metadata.start
|
||||
func (e *Era) Start() uint64 {
|
||||
return e.m.start
|
||||
}
|
||||
|
||||
// Count returns the total number of blocks in the Era1.
|
||||
func (r *Reader) Count() uint64 {
|
||||
return r.metadata.count
|
||||
func (e *Era) Count() uint64 {
|
||||
return e.m.count
|
||||
}
|
||||
|
||||
// readOffset reads a specific block's offset from the block index. The value n
|
||||
// is the absolute block number desired.
|
||||
func (e *Era) readOffset(n uint64) (int64, error) {
|
||||
var (
|
||||
firstIndex = -8 - int64(e.m.count)*8 // size of count - index entries
|
||||
indexOffset = int64(n-e.m.start) * 8 // desired index * size of indexes
|
||||
offOffset = e.m.length + firstIndex + indexOffset // offset of block offset
|
||||
)
|
||||
e.mu.Lock()
|
||||
defer e.mu.Unlock()
|
||||
clearBuffer(e.buf[:])
|
||||
if _, err := e.f.ReadAt(e.buf[:], offOffset); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
// Since the block offset is relative from its location + size of index
|
||||
// value (8), we need to add it to it's offset to get the block's
|
||||
// absolute offset.
|
||||
return offOffset + 8 + int64(binary.LittleEndian.Uint64(e.buf[:])), nil
|
||||
}
|
||||
|
||||
// newReader returns an io.Reader for the e2store entry value at off.
|
||||
func newReader(e *e2store.Reader, off int64) (io.Reader, int, error) {
|
||||
var (
|
||||
entry e2store.Entry
|
||||
n int
|
||||
err error
|
||||
)
|
||||
if n, err = e.ReadAt(&entry, off); err != nil {
|
||||
return nil, n, err
|
||||
}
|
||||
return bytes.NewReader(entry.Value), n, nil
|
||||
}
|
||||
|
||||
// newReader returns a snappy.Reader for the e2store entry value at off.
|
||||
func newSnappyReader(e *e2store.Reader, off int64) (io.Reader, int, error) {
|
||||
r, n, err := newReader(e, off)
|
||||
return snappy.NewReader(r), n, err
|
||||
}
|
||||
|
||||
// clearBuffer zeroes out the buffer.
|
||||
func (r *Reader) clearBuffer() {
|
||||
for i := 0; i < len(r.buf); i++ {
|
||||
r.buf[i] = 0
|
||||
func clearBuffer(buf []byte) {
|
||||
for i := 0; i < len(buf); i++ {
|
||||
buf[i] = 0
|
||||
}
|
||||
}
|
||||
|
||||
// metadata wraps the metadata in the block index.
|
||||
type metadata struct {
|
||||
start, count uint64
|
||||
start uint64
|
||||
count uint64
|
||||
length int64
|
||||
}
|
||||
|
||||
// readMetadata reads the metadata stored in an Era1 file's block index.
|
||||
func readMetadata(r io.ReaderAt, length int64) (m metadata, err error) {
|
||||
func readMetadata(f ReadAtSeekCloser) (m metadata, err error) {
|
||||
// Determine length of reader.
|
||||
if m.length, err = f.Seek(0, io.SeekEnd); err != nil {
|
||||
return
|
||||
}
|
||||
b := make([]byte, 16)
|
||||
// Read count. It's the last 8 bytes of the file.
|
||||
if _, err = r.ReadAt(b[:8], length-8); err != nil {
|
||||
if _, err = f.ReadAt(b[:8], m.length-8); err != nil {
|
||||
return
|
||||
}
|
||||
m.count = binary.LittleEndian.Uint64(b)
|
||||
// Read start. It's at the offset -sizeof(m.count) -
|
||||
// count*sizeof(indexEntry) - sizeof(m.start)
|
||||
if _, err = r.ReadAt(b[8:], length-16-int64(m.count*8)); err != nil {
|
||||
if _, err = f.ReadAt(b[8:], m.length-16-int64(m.count*8)); err != nil {
|
||||
return
|
||||
}
|
||||
m.start = binary.LittleEndian.Uint64(b[8:])
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ package era
|
|||
|
||||
import (
|
||||
"bytes"
|
||||
"io"
|
||||
"math/big"
|
||||
"os"
|
||||
"testing"
|
||||
|
|
@ -71,43 +72,52 @@ func TestEra1Builder(t *testing.T) {
|
|||
}
|
||||
|
||||
// Verify Era1 contents.
|
||||
r, err := NewReader(f)
|
||||
e, err := Open(f.Name())
|
||||
if err != nil {
|
||||
t.Fatalf("failed to make reader: %s", err)
|
||||
t.Fatalf("failed to open era: %v", err)
|
||||
}
|
||||
it, err := NewRawIterator(e)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to make iterator: %s", err)
|
||||
}
|
||||
for i := uint64(0); i < uint64(len(chain.headers)); i++ {
|
||||
if !it.Next() {
|
||||
t.Fatalf("expected more entries")
|
||||
}
|
||||
if it.Error() != nil {
|
||||
t.Fatalf("unexpected error %v", it.Error())
|
||||
}
|
||||
// Check headers.
|
||||
header, err := r.readHeaderRLP(i)
|
||||
header, err := io.ReadAll(it.Header)
|
||||
if err != nil {
|
||||
t.Fatalf("error reading from era1: %v", err)
|
||||
t.Fatalf("error reading header: %v", err)
|
||||
}
|
||||
if !bytes.Equal(header, chain.headers[i]) {
|
||||
t.Fatalf("mismatched header: want %s, got %s", chain.headers[i], header)
|
||||
}
|
||||
|
||||
// Check bodies.
|
||||
body, err := r.readBodyRLP(i)
|
||||
body, err := io.ReadAll(it.Body)
|
||||
if err != nil {
|
||||
t.Fatalf("error reading from era1: %v", err)
|
||||
t.Fatalf("error reading body: %v", err)
|
||||
}
|
||||
if !bytes.Equal(body, chain.bodies[i]) {
|
||||
t.Fatalf("mismatched body: want %s, got %s", chain.bodies[i], body)
|
||||
}
|
||||
|
||||
// Check receipts.
|
||||
receipts, err := r.readReceiptsRLP(i)
|
||||
receipts, err := io.ReadAll(it.Receipts)
|
||||
if err != nil {
|
||||
t.Fatalf("error reading from era1: %v", err)
|
||||
t.Fatalf("error reading receipts: %v", err)
|
||||
}
|
||||
if !bytes.Equal(receipts, chain.receipts[i]) {
|
||||
t.Fatalf("mismatched body: want %s, got %s", chain.receipts[i], receipts)
|
||||
t.Fatalf("mismatched receipts: want %s, got %s", chain.receipts[i], receipts)
|
||||
}
|
||||
|
||||
// Check total difficulty.
|
||||
td, err := r.readTotalDifficulty(i)
|
||||
rawTd, err := io.ReadAll(it.TotalDifficulty)
|
||||
if err != nil {
|
||||
t.Fatalf("error reading from era1: %v", err)
|
||||
t.Fatalf("error reading td: %v", err)
|
||||
}
|
||||
td := new(big.Int).SetBytes(reverseOrder(rawTd))
|
||||
if td.Cmp(chain.tds[i]) != 0 {
|
||||
t.Fatalf("mismatched tds: want %s, got %s", chain.tds[i], td)
|
||||
}
|
||||
|
|
|
|||
154
internal/era/iterator.go
Normal file
154
internal/era/iterator.go
Normal file
|
|
@ -0,0 +1,154 @@
|
|||
// Copyright 2023 The go-ethereum Authors
|
||||
// This file is part of go-ethereum.
|
||||
//
|
||||
// go-ethereum is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// go-ethereum is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU General Public License
|
||||
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package era
|
||||
|
||||
import (
|
||||
"io"
|
||||
"math/big"
|
||||
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/rlp"
|
||||
)
|
||||
|
||||
// Iterator wraps RawIterator and returns decoded Era1 entries.
|
||||
type Iterator struct {
|
||||
inner *RawIterator
|
||||
}
|
||||
|
||||
// NewIterator returns a new Iterator instance.
|
||||
func NewIterator(e *Era) (*Iterator, error) {
|
||||
inner, err := NewRawIterator(e)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Iterator{inner}, nil
|
||||
}
|
||||
|
||||
// Next moves the iterator to the next block entry.
|
||||
func (it *Iterator) Next() bool {
|
||||
return it.inner.Next()
|
||||
}
|
||||
|
||||
// Error returns the error status of the iterator.
|
||||
func (it *Iterator) Error() error {
|
||||
return it.inner.Error()
|
||||
}
|
||||
|
||||
// Block returns the block for the iterator's current position.
|
||||
func (it *Iterator) Block() (*types.Block, error) {
|
||||
var (
|
||||
header types.Header
|
||||
body types.Body
|
||||
)
|
||||
if err := rlp.Decode(it.inner.Header, &header); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := rlp.Decode(it.inner.Body, &body); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return types.NewBlockWithHeader(&header).WithBody(body.Transactions, body.Uncles), nil
|
||||
}
|
||||
|
||||
// Receipts returns the receipts for the iterator's current position.
|
||||
func (it *Iterator) Receipts() (types.Receipts, error) {
|
||||
var receipts types.Receipts
|
||||
err := rlp.Decode(it.inner.Receipts, &receipts)
|
||||
return receipts, err
|
||||
}
|
||||
|
||||
// BlockAndReceipts returns the block and receipts for the iterator's current
|
||||
// position.
|
||||
func (it *Iterator) BlockAndReceipts() (*types.Block, types.Receipts, error) {
|
||||
b, err := it.Block()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
r, err := it.Receipts()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
return b, r, nil
|
||||
}
|
||||
|
||||
// TotalDifficulty returns the total difficulty for the iterator's current
|
||||
// position.
|
||||
func (it *Iterator) TotalDifficulty() (*big.Int, error) {
|
||||
var td []byte
|
||||
if err := rlp.Decode(it.inner.TotalDifficulty, td); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return new(big.Int).SetBytes(reverseOrder(td)), nil
|
||||
}
|
||||
|
||||
// RawIterator reads an RLP-encode Era1 entries.
|
||||
type RawIterator struct {
|
||||
e *Era // backing Era1
|
||||
curOffset int64 // offset of current block
|
||||
next uint64 // next block to read
|
||||
err error // last error
|
||||
|
||||
Header io.Reader
|
||||
Body io.Reader
|
||||
Receipts io.Reader
|
||||
TotalDifficulty io.Reader
|
||||
}
|
||||
|
||||
// NewRawIterator returns a new Iterator instance.
|
||||
func NewRawIterator(e *Era) (*RawIterator, error) {
|
||||
return &RawIterator{
|
||||
e: e,
|
||||
next: e.m.start,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Next moves the iterator to the next block entry.
|
||||
func (it *RawIterator) Next() bool {
|
||||
if it.e.m.start+it.e.m.count <= it.next {
|
||||
return false
|
||||
}
|
||||
off, err := it.e.readOffset(it.next)
|
||||
if err == io.EOF {
|
||||
it.err = err
|
||||
return false
|
||||
}
|
||||
var n int
|
||||
if it.Header, n, it.err = newSnappyReader(it.e.s, off); it.err != nil {
|
||||
return true
|
||||
}
|
||||
off += int64(n)
|
||||
if it.Body, n, it.err = newSnappyReader(it.e.s, off); it.err != nil {
|
||||
return true
|
||||
}
|
||||
off += int64(n)
|
||||
if it.Receipts, n, it.err = newSnappyReader(it.e.s, off); it.err != nil {
|
||||
return true
|
||||
}
|
||||
off += int64(n)
|
||||
if it.TotalDifficulty, n, it.err = newReader(it.e.s, off); it.err != nil {
|
||||
return true
|
||||
}
|
||||
it.next += 1
|
||||
return true
|
||||
}
|
||||
|
||||
// Error returns the error status of the iterator.
|
||||
func (it *RawIterator) Error() error {
|
||||
if it.err == io.EOF {
|
||||
return nil
|
||||
}
|
||||
return it.err
|
||||
}
|
||||
Loading…
Reference in a new issue