From 3fa17e775f80515863429ca66520f289d373a6cc Mon Sep 17 00:00:00 2001 From: "lightclient@protonmail.com" Date: Fri, 2 Jun 2023 17:03:02 +0200 Subject: [PATCH] internal/era,cmd/utils,cmd/era: change to iterator interface for reading era entries --- cmd/era/main.go | 72 ++++----- cmd/utils/cmd.go | 21 ++- cmd/utils/history_test.go | 46 +++--- internal/era/era.go | 312 ++++++++++++++++---------------------- internal/era/era_test.go | 36 +++-- internal/era/iterator.go | 154 +++++++++++++++++++ 6 files changed, 382 insertions(+), 259 deletions(-) create mode 100644 internal/era/iterator.go diff --git a/cmd/era/main.go b/cmd/era/main.go index 4b624a4745..972377af71 100644 --- a/cmd/era/main.go +++ b/cmd/era/main.go @@ -19,7 +19,6 @@ package main import ( "encoding/json" "fmt" - "io" "math/big" "os" "path" @@ -111,19 +110,15 @@ func block(ctx *cli.Context) error { if err != nil { return fmt.Errorf("invalid block number: %w", err) } - f, err := open(ctx, num/uint64(ctx.Int(eraSizeFlag.Name))) + e, err := open(ctx, num/uint64(ctx.Int(eraSizeFlag.Name))) if err != nil { return fmt.Errorf("error opening era: %w", err) } - defer f.Close() - r, err := era.NewReader(f) - if err != nil { - return fmt.Errorf("error making era reader: %w", err) - } + defer e.Close() // Read block with number. - block, err := r.ReadBlock(num) + block, err := e.GetBlockByNumber(num) if err != nil { - return fmt.Errorf("error reading era: %w", err) + return fmt.Errorf("error reading block %d: %w", num, err) } // Convert block to JSON and print. val, err := ethapi.RPCMarshalBlock(block, ctx.Bool(txsFlag.Name), ctx.Bool(txsFlag.Name), params.MainnetChainConfig) @@ -144,20 +139,16 @@ func info(ctx *cli.Context) error { if err != nil { return fmt.Errorf("invalid epoch number: %w", err) } - f, err := open(ctx, epoch) + e, err := open(ctx, epoch) if err != nil { return err } - defer f.Close() - r, err := era.NewReader(f) - if err != nil { - return fmt.Errorf("error creating era reader: %w", err) - } - acc, err := r.Accumulator() + defer e.Close() + acc, err := e.Accumulator() if err != nil { return fmt.Errorf("error reading accumulator: %w", err) } - td, err := r.InitialTD() + td, err := e.InitialTD() if err != nil { return fmt.Errorf("error reading total difficulty: %w", err) } @@ -167,7 +158,7 @@ func info(ctx *cli.Context) error { StartBlock uint64 `json:"startBlock"` Count uint64 `json:"count"` }{ - acc, td, r.Start(), r.Count(), + acc, td, e.Start(), e.Count(), } b, _ := json.MarshalIndent(info, "", " ") fmt.Println(string(b)) @@ -175,7 +166,7 @@ func info(ctx *cli.Context) error { } // open opens an era1 file at a certain epoch. -func open(ctx *cli.Context, epoch uint64) (*os.File, error) { +func open(ctx *cli.Context, epoch uint64) (*era.Era, error) { var ( dir = ctx.String(dirFlag.Name) network = ctx.String(networkFlag.Name) @@ -187,7 +178,7 @@ func open(ctx *cli.Context, epoch uint64) (*os.File, error) { if epoch >= uint64(len(entries)) { return nil, fmt.Errorf("epoch out-of-bounds: last %d, want %d", len(entries)-1, epoch) } - return os.Open(path.Join(dir, entries[epoch])) + return era.Open(path.Join(dir, entries[epoch])) } // verify checks each era1 file in a directory to ensure it is well-formed and @@ -223,29 +214,21 @@ func verify(ctx *cli.Context) error { // Wrap in function so defers don't stack. err := func() error { name := entries[i] - f, err := os.Open(path.Join(dir, name)) + e, err := era.Open(path.Join(dir, name)) if err != nil { return fmt.Errorf("error opening era1 file %s: %w", name, err) } - defer f.Close() - - r, err := era.NewReader(f) - if err != nil { - return fmt.Errorf("unable to make era reader: %w", err) - } - + defer e.Close() // Read accumulator and check against expected. - if got, err := r.Accumulator(); err != nil { + if got, err := e.Accumulator(); err != nil { return fmt.Errorf("error retrieving accumulator for %s: %w", name, err) } else if got != want { return fmt.Errorf("invalid root %s: got %s, want %s", name, got, want) } - // Recompute accumulator. - if err := checkAccumulator(r); err != nil { + if err := checkAccumulator(e); err != nil { return fmt.Errorf("error verify era1 file %s: %w", name, err) } - // Give the user some feedback that something is happening. if time.Since(reported) >= 8*time.Second { fmt.Printf("Verifying Era1 files \t\t verified=%d,\t elapsed=%s\n", i, common.PrettyDuration(time.Since(start))) @@ -262,21 +245,25 @@ func verify(ctx *cli.Context) error { } // checkAccumulator verifies the accumulator matches the data in the Era. -func checkAccumulator(r *era.Reader) error { +func checkAccumulator(e *era.Era) error { var ( err error - start = r.Start() + start = e.Start() want common.Hash td *big.Int tds = make([]*big.Int, 0) hashes = make([]common.Hash, 0) ) - if want, err = r.Accumulator(); err != nil { + if want, err = e.Accumulator(); err != nil { return fmt.Errorf("error reading accumulator: %w", err) } - if td, err = r.InitialTD(); err != nil { + if td, err = e.InitialTD(); err != nil { return fmt.Errorf("error reading total difficulty: %w", err) } + it, err := era.NewIterator(e) + if err != nil { + return fmt.Errorf("error making era iterator: %w", err) + } // Starting at epoch 0, iterate through all available era1 files and // check the following: // * the block index is constructed correctly @@ -284,13 +271,14 @@ func checkAccumulator(r *era.Reader) error { // * the accumulator is correct by recomputing it locally, // which verifies the blocks are all correct (via hash) // * the receipts root matches the value in the block - for j := 0; ; j++ { - // read() walks the block index, so we're able to + for j := 0; it.Next(); j++ { + // next() walks the block index, so we're able to // implicitly verify it. - block, receipts, err := r.Read() - if err == io.EOF { - break - } else if err != nil { + if it.Error() != nil { + return fmt.Errorf("error reading block %d: %w", start+uint64(j), err) + } + block, receipts, err := it.BlockAndReceipts() + if it.Error() != nil { return fmt.Errorf("error reading block %d: %w", start+uint64(j), err) } tr := types.DeriveSha(block.Transactions(), trie.NewStackTrie(nil)) diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 5a97122dfb..7f72105fd3 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -285,20 +285,27 @@ func ImportHistory(chain *core.BlockChain, db ethdb.Database, dir string, networ buf.Reset() // Import all block data from Era1. - r, err := era.NewReader(f) + e, err := era.From(f) + if err != nil { + return fmt.Errorf("error opening era: %w", err) + } + it, err := era.NewIterator(e) if err != nil { return fmt.Errorf("error making era reader: %w", err) } - for j := 0; ; j += 1 { + for j := 0; it.Next(); j++ { n := i*era.MaxEra1Size + j - block, receipts, err := r.Read() - if err == io.EOF { - break - } else if err != nil { + block, err := it.Block() + if err != nil { return fmt.Errorf("error reading block %d: %w", n, err) - } else if block.Number().BitLen() == 0 { + } + if block.Number().BitLen() == 0 { continue // skip genesis } + receipts, err := it.Receipts() + if err != nil { + return fmt.Errorf("error reading receipts %d: %w", n, err) + } if status, err := chain.HeaderChain().InsertHeaderChain([]*types.Header{block.Header()}, start, forker); err != nil { return fmt.Errorf("error inserting header %d: %w", n, err) } else if status != core.CanonStatTy { diff --git a/cmd/utils/history_test.go b/cmd/utils/history_test.go index e24a68bbd6..544405464c 100644 --- a/cmd/utils/history_test.go +++ b/cmd/utils/history_test.go @@ -107,42 +107,54 @@ func TestHistoryImportAndExport(t *testing.T) { // Verify each Era. entries, _ := era.ReadDir(dir, "mainnet") for i, filename := range entries { - f, err := os.ReadFile(path.Join(dir, filename)) + f, err := os.Open(path.Join(dir, filename)) if err != nil { t.Fatalf("error opening era file: %v", err) } - if want, got := common.HexToHash(checksums[i]), common.Hash(sha256.Sum256(f)); want != got { + + var ( + h = sha256.New() + buf = bytes.NewBuffer(nil) + ) + if _, err := io.Copy(h, f); err != nil { + t.Fatalf("unable to recalculate checksum: %v", err) + } + if got, want := common.BytesToHash(h.Sum(buf.Bytes()[:])).Hex(), checksums[i]; got != want { t.Fatalf("checksum %d does not match: got %s, want %s", i, got, want) } - r, err := era.NewReader(bytes.NewReader(f)) + + e, err := era.From(f) + if err != nil { + t.Fatalf("error opening era: %v", err) + } + it, err := era.NewIterator(e) if err != nil { t.Fatalf("error making era reader: %v", err) } - for j := 0; ; j += 1 { - block, receipts, err := r.Read() - if err == io.EOF { - break - } else if err != nil { - t.Fatalf("error reading era file %d: %v", i, err) + for j := 0; it.Next(); j++ { + n := i*int(step) + j + if it.Error() != nil { + t.Fatalf("error reading block entry %d: %v", n, err) } - var ( - n = i*int(step) + j - want = chain.GetBlockByNumber(uint64(n)) - ) + block, receipts, err := it.BlockAndReceipts() + if err != nil { + t.Fatalf("error reading block entry %d: %v", n, err) + } + want := chain.GetBlockByNumber(uint64(n)) if want, got := uint64(n), block.NumberU64(); want != got { t.Fatalf("blocks out of order: want %d, got %d", want, got) } if want.Hash() != block.Hash() { - t.Fatalf("block hash mismatch %d: want %s, got %s", i+j, want.Hash().Hex(), block.Hash().Hex()) + t.Fatalf("block hash mismatch %d: want %s, got %s", n, want.Hash().Hex(), block.Hash().Hex()) } if got := types.DeriveSha(block.Transactions(), trie.NewStackTrie(nil)); got != want.TxHash() { - t.Fatalf("tx hash %d mismatch: want %s, got %s", i+j, want.TxHash(), got) + t.Fatalf("tx hash %d mismatch: want %s, got %s", n, want.TxHash(), got) } if got := types.CalcUncleHash(block.Uncles()); got != want.UncleHash() { - t.Fatalf("uncle hash %d mismatch: want %s, got %s", i+j, want.UncleHash(), got) + t.Fatalf("uncle hash %d mismatch: want %s, got %s", n, want.UncleHash(), got) } if got := types.DeriveSha(receipts, trie.NewStackTrie(nil)); got != want.ReceiptHash() { - t.Fatalf("receipt root %d mismatch: want %s, got %s", i+j, want.ReceiptHash(), got) + t.Fatalf("receipt root %d mismatch: want %s, got %s", n, want.ReceiptHash(), got) } } } diff --git a/internal/era/era.go b/internal/era/era.go index bbb499a45c..3affe0e0e7 100644 --- a/internal/era/era.go +++ b/internal/era/era.go @@ -26,6 +26,7 @@ import ( "path" "strconv" "strings" + "sync" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" @@ -83,197 +84,75 @@ func ReadDir(dir, network string) ([]string, error) { return eras, nil } -// Reader reads an Era1 archive. -// See Builder documentation for a detailed explanation of the Era1 format. -type Reader struct { - r io.ReaderAt - e *e2store.Reader - - buf [8]byte // buffer reading entry offsets - next uint64 // next block to read - length int64 // total length of r - metadata metadata // start, count info -} - -type ReadAtSeeker interface { +type ReadAtSeekCloser interface { io.ReaderAt io.Seeker + io.Closer } -// NewReader returns a new Reader instance. -func NewReader(r ReadAtSeeker) (*Reader, error) { - length, err := r.Seek(0, io.SeekEnd) +// Era reads and Era1 file. +type Era struct { + f ReadAtSeekCloser // backing era1 file + s *e2store.Reader // e2store reader over f + m metadata // start, count, length info + mu *sync.Mutex // lock for buf + buf [8]byte // buffer reading entry offsets +} + +// From returns an Era backed by f. +func From(f ReadAtSeekCloser) (*Era, error) { + m, err := readMetadata(f) if err != nil { return nil, err } - m, err := readMetadata(r, length) - if err != nil { - return nil, err - } - return &Reader{ - r: r, - e: e2store.NewReader(r), - next: m.start, - length: length, - metadata: m, + return &Era{ + f: f, + s: e2store.NewReader(f), + m: m, + mu: new(sync.Mutex), }, nil } -// readOffset reads a specific block's offset from the block index. The value n -// is the absolute block number desired. -func (r *Reader) readOffset(n uint64) (int64, error) { - var ( - firstIndex = -8 - int64(r.metadata.count)*8 // size of count - index entries - indexOffset = int64(n-r.metadata.start) * 8 // desired index * size of indexes - offOffset = r.length + firstIndex + indexOffset // offset of block offset - ) - r.clearBuffer() - if _, err := r.r.ReadAt(r.buf[:], offOffset); err != nil { - return 0, err - } - // Since the block offset is relative from its location + size of index - // value (8), we need to add it to it's offset to get the block's - // absolute offset. - return offOffset + 8 + int64(binary.LittleEndian.Uint64(r.buf[:])), nil -} - -// Read reads one (block, receipts) tuple from an Era1 archive. -func (r *Reader) Read() (*types.Block, types.Receipts, error) { - block, receipts, err := r.ReadBlockAndReceipts(r.next) - if err != nil { - return nil, nil, err - } - r.next += 1 - return block, receipts, nil -} - -// readBlob reads an entry of data. -func (r *Reader) readEntry(n uint64, skip int) (*e2store.Entry, error) { - if n < r.metadata.start || r.metadata.start+r.metadata.count < n { - return nil, fmt.Errorf("request out-of-bounds: want %d, start: %d, count: %d", n, r.metadata.start, r.metadata.count) - } - // Read the specified block's offset from the block index. - off, err := r.readOffset(n) - if err != nil { - return nil, fmt.Errorf("error reading block offset: %w", err) - } - // Skip to the requested entry. - for i := 0; i < skip; i++ { - if length, err := r.e.LengthAt(off); err != nil { - return nil, err - } else { - off += length - } - } - // Read entry. - var entry e2store.Entry - if _, err := r.e.ReadAt(&entry, off); err != nil { - return nil, err - } - return &entry, nil -} - -// readHeaderRLP reads the header number n RLP. -func (r *Reader) readHeaderRLP(n uint64) ([]byte, error) { - e, err := r.readEntry(n, 0) +// Open returns an Era backed by the given filename. +func Open(filename string) (*Era, error) { + f, err := os.Open(filename) if err != nil { return nil, err } - if e.Type != TypeCompressedHeader { - return nil, fmt.Errorf("expected header entry, got %x", e.Type) - } - return io.ReadAll(snappy.NewReader(bytes.NewReader(e.Value))) + return From(f) } -// readBodyRLP reads the block body number n RLP. -func (r *Reader) readBodyRLP(n uint64) ([]byte, error) { - e, err := r.readEntry(n, 1) +func (e *Era) Close() error { + return e.f.Close() +} + +func (e *Era) GetBlockByNumber(num uint64) (*types.Block, error) { + if e.m.start > num || e.m.start+e.m.count <= num { + return nil, fmt.Errorf("out-of-bounds") + } + off, err := e.readOffset(num) if err != nil { return nil, err } - if e.Type != TypeCompressedBody { - return nil, fmt.Errorf("expected body entry, got %x", e.Type) - } - return io.ReadAll(snappy.NewReader(bytes.NewReader(e.Value))) -} - -// readReceiptsRLP reads the receipts RLP associated with number n. -func (r *Reader) readReceiptsRLP(n uint64) ([]byte, error) { - e, err := r.readEntry(n, 2) - if err != nil { - return nil, err - } - if e.Type != TypeCompressedReceipts { - return nil, fmt.Errorf("expected receipts entry, got %x", e.Type) - } - return io.ReadAll(snappy.NewReader(bytes.NewReader(e.Value))) -} - -// readTotalDifficulty reads the total difficulty of block number n. -func (r *Reader) readTotalDifficulty(n uint64) (*big.Int, error) { - e, err := r.readEntry(n, 3) - if err != nil { - return nil, err - } - if e.Type != TypeTotalDifficulty { - return nil, fmt.Errorf("expected TD entry, got %x", e.Type) - } - return new(big.Int).SetBytes(reverseOrder(e.Value)), nil -} - -// ReadHeader reads the header number n. -func (r *Reader) ReadHeader(n uint64) (*types.Header, error) { - h, err := r.readHeaderRLP(n) + r, n, err := newSnappyReader(e.s, off) if err != nil { return nil, err } var header types.Header - if err := rlp.DecodeBytes(h, &header); err != nil { - return nil, err - } - return &header, nil -} - -// ReadBlock reads the block number n. -func (r *Reader) ReadBlock(n uint64) (*types.Block, error) { - header, err := r.ReadHeader(n) - if err != nil { - return nil, err - } - b, err := r.readBodyRLP(n) - if err != nil { + if err := rlp.Decode(r, &header); err != nil { return nil, err } + off += int64(n) var body types.Body - if err := rlp.DecodeBytes(b, &body); err != nil { + if err := rlp.Decode(r, &body); err != nil { return nil, err } - return types.NewBlockWithHeader(header).WithBody(body.Transactions, body.Uncles), nil -} - -// ReadBlockAndReceipts reads the block number n and associated receipts. -func (r *Reader) ReadBlockAndReceipts(n uint64) (*types.Block, types.Receipts, error) { - // Read block. - block, err := r.ReadBlock(n) - if err != nil { - return nil, nil, err - } - // Read receipts. - rr, err := r.readReceiptsRLP(n) - if err != nil { - return nil, nil, err - } - // Decode receipts. - var receipts types.Receipts - if err := rlp.DecodeBytes(rr, &receipts); err != nil { - return nil, nil, err - } - return block, receipts, err + return types.NewBlockWithHeader(&header).WithBody(body.Transactions, body.Uncles), nil } // Accumulator reads the accumulator entry in the Era1 file. -func (r *Reader) Accumulator() (common.Hash, error) { - entry, err := r.e.Find(TypeAccumulator) +func (e *Era) Accumulator() (common.Hash, error) { + entry, err := e.s.Find(TypeAccumulator) if err != nil { return common.Hash{}, err } @@ -282,53 +161,126 @@ func (r *Reader) Accumulator() (common.Hash, error) { // InitialTD returns initial total difficulty before the difficulty of the // first block of the Era1 is applied. -func (r *Reader) InitialTD() (*big.Int, error) { - h, err := r.ReadHeader(r.Start()) - if err != nil { +func (e *Era) InitialTD() (*big.Int, error) { + var ( + r io.Reader + header types.Header + rawTd []byte + n int + off int64 + err error + ) + + // Read first header. + if off, err = e.readOffset(e.m.start); err != nil { return nil, err } - // Above seek also sets reader so next TD entry will be for this block. - entry, err := r.e.Find(TypeTotalDifficulty) - if err != nil { + if r, n, err = newSnappyReader(e.s, off); err != nil { return nil, err } - td := new(big.Int).SetBytes(reverseOrder(entry.Value)) - return td.Sub(td, h.Difficulty), nil + if err := rlp.Decode(r, header); err != nil { + return nil, err + } + off += int64(n) + + // Skip over next two records. + for i := 0; i < 2; i++ { + length, err := e.s.LengthAt(off) + if err != nil { + return nil, err + } + off += length + } + + // Read total difficulty after first block. + if r, n, err = newReader(e.s, off); err != nil { + return nil, err + } + if err := rlp.Decode(r, rawTd); err != nil { + return nil, err + } + td := new(big.Int).SetBytes(reverseOrder(rawTd)) + return td.Sub(td, header.Difficulty), nil } // Start returns the listed start block. -func (r *Reader) Start() uint64 { - return r.metadata.start +func (e *Era) Start() uint64 { + return e.m.start } // Count returns the total number of blocks in the Era1. -func (r *Reader) Count() uint64 { - return r.metadata.count +func (e *Era) Count() uint64 { + return e.m.count +} + +// readOffset reads a specific block's offset from the block index. The value n +// is the absolute block number desired. +func (e *Era) readOffset(n uint64) (int64, error) { + var ( + firstIndex = -8 - int64(e.m.count)*8 // size of count - index entries + indexOffset = int64(n-e.m.start) * 8 // desired index * size of indexes + offOffset = e.m.length + firstIndex + indexOffset // offset of block offset + ) + e.mu.Lock() + defer e.mu.Unlock() + clearBuffer(e.buf[:]) + if _, err := e.f.ReadAt(e.buf[:], offOffset); err != nil { + return 0, err + } + // Since the block offset is relative from its location + size of index + // value (8), we need to add it to it's offset to get the block's + // absolute offset. + return offOffset + 8 + int64(binary.LittleEndian.Uint64(e.buf[:])), nil +} + +// newReader returns an io.Reader for the e2store entry value at off. +func newReader(e *e2store.Reader, off int64) (io.Reader, int, error) { + var ( + entry e2store.Entry + n int + err error + ) + if n, err = e.ReadAt(&entry, off); err != nil { + return nil, n, err + } + return bytes.NewReader(entry.Value), n, nil +} + +// newReader returns a snappy.Reader for the e2store entry value at off. +func newSnappyReader(e *e2store.Reader, off int64) (io.Reader, int, error) { + r, n, err := newReader(e, off) + return snappy.NewReader(r), n, err } // clearBuffer zeroes out the buffer. -func (r *Reader) clearBuffer() { - for i := 0; i < len(r.buf); i++ { - r.buf[i] = 0 +func clearBuffer(buf []byte) { + for i := 0; i < len(buf); i++ { + buf[i] = 0 } } // metadata wraps the metadata in the block index. type metadata struct { - start, count uint64 + start uint64 + count uint64 + length int64 } // readMetadata reads the metadata stored in an Era1 file's block index. -func readMetadata(r io.ReaderAt, length int64) (m metadata, err error) { +func readMetadata(f ReadAtSeekCloser) (m metadata, err error) { + // Determine length of reader. + if m.length, err = f.Seek(0, io.SeekEnd); err != nil { + return + } b := make([]byte, 16) // Read count. It's the last 8 bytes of the file. - if _, err = r.ReadAt(b[:8], length-8); err != nil { + if _, err = f.ReadAt(b[:8], m.length-8); err != nil { return } m.count = binary.LittleEndian.Uint64(b) // Read start. It's at the offset -sizeof(m.count) - // count*sizeof(indexEntry) - sizeof(m.start) - if _, err = r.ReadAt(b[8:], length-16-int64(m.count*8)); err != nil { + if _, err = f.ReadAt(b[8:], m.length-16-int64(m.count*8)); err != nil { return } m.start = binary.LittleEndian.Uint64(b[8:]) diff --git a/internal/era/era_test.go b/internal/era/era_test.go index a4fd2c3b06..ee5d9e82a0 100644 --- a/internal/era/era_test.go +++ b/internal/era/era_test.go @@ -18,6 +18,7 @@ package era import ( "bytes" + "io" "math/big" "os" "testing" @@ -71,43 +72,52 @@ func TestEra1Builder(t *testing.T) { } // Verify Era1 contents. - r, err := NewReader(f) + e, err := Open(f.Name()) if err != nil { - t.Fatalf("failed to make reader: %s", err) + t.Fatalf("failed to open era: %v", err) + } + it, err := NewRawIterator(e) + if err != nil { + t.Fatalf("failed to make iterator: %s", err) } for i := uint64(0); i < uint64(len(chain.headers)); i++ { + if !it.Next() { + t.Fatalf("expected more entries") + } + if it.Error() != nil { + t.Fatalf("unexpected error %v", it.Error()) + } // Check headers. - header, err := r.readHeaderRLP(i) + header, err := io.ReadAll(it.Header) if err != nil { - t.Fatalf("error reading from era1: %v", err) + t.Fatalf("error reading header: %v", err) } if !bytes.Equal(header, chain.headers[i]) { t.Fatalf("mismatched header: want %s, got %s", chain.headers[i], header) } - // Check bodies. - body, err := r.readBodyRLP(i) + body, err := io.ReadAll(it.Body) if err != nil { - t.Fatalf("error reading from era1: %v", err) + t.Fatalf("error reading body: %v", err) } if !bytes.Equal(body, chain.bodies[i]) { t.Fatalf("mismatched body: want %s, got %s", chain.bodies[i], body) } - // Check receipts. - receipts, err := r.readReceiptsRLP(i) + receipts, err := io.ReadAll(it.Receipts) if err != nil { - t.Fatalf("error reading from era1: %v", err) + t.Fatalf("error reading receipts: %v", err) } if !bytes.Equal(receipts, chain.receipts[i]) { - t.Fatalf("mismatched body: want %s, got %s", chain.receipts[i], receipts) + t.Fatalf("mismatched receipts: want %s, got %s", chain.receipts[i], receipts) } // Check total difficulty. - td, err := r.readTotalDifficulty(i) + rawTd, err := io.ReadAll(it.TotalDifficulty) if err != nil { - t.Fatalf("error reading from era1: %v", err) + t.Fatalf("error reading td: %v", err) } + td := new(big.Int).SetBytes(reverseOrder(rawTd)) if td.Cmp(chain.tds[i]) != 0 { t.Fatalf("mismatched tds: want %s, got %s", chain.tds[i], td) } diff --git a/internal/era/iterator.go b/internal/era/iterator.go new file mode 100644 index 0000000000..2f353e6e85 --- /dev/null +++ b/internal/era/iterator.go @@ -0,0 +1,154 @@ +// Copyright 2023 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package era + +import ( + "io" + "math/big" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" +) + +// Iterator wraps RawIterator and returns decoded Era1 entries. +type Iterator struct { + inner *RawIterator +} + +// NewIterator returns a new Iterator instance. +func NewIterator(e *Era) (*Iterator, error) { + inner, err := NewRawIterator(e) + if err != nil { + return nil, err + } + return &Iterator{inner}, nil +} + +// Next moves the iterator to the next block entry. +func (it *Iterator) Next() bool { + return it.inner.Next() +} + +// Error returns the error status of the iterator. +func (it *Iterator) Error() error { + return it.inner.Error() +} + +// Block returns the block for the iterator's current position. +func (it *Iterator) Block() (*types.Block, error) { + var ( + header types.Header + body types.Body + ) + if err := rlp.Decode(it.inner.Header, &header); err != nil { + return nil, err + } + if err := rlp.Decode(it.inner.Body, &body); err != nil { + return nil, err + } + return types.NewBlockWithHeader(&header).WithBody(body.Transactions, body.Uncles), nil +} + +// Receipts returns the receipts for the iterator's current position. +func (it *Iterator) Receipts() (types.Receipts, error) { + var receipts types.Receipts + err := rlp.Decode(it.inner.Receipts, &receipts) + return receipts, err +} + +// BlockAndReceipts returns the block and receipts for the iterator's current +// position. +func (it *Iterator) BlockAndReceipts() (*types.Block, types.Receipts, error) { + b, err := it.Block() + if err != nil { + return nil, nil, err + } + r, err := it.Receipts() + if err != nil { + return nil, nil, err + } + return b, r, nil +} + +// TotalDifficulty returns the total difficulty for the iterator's current +// position. +func (it *Iterator) TotalDifficulty() (*big.Int, error) { + var td []byte + if err := rlp.Decode(it.inner.TotalDifficulty, td); err != nil { + return nil, err + } + return new(big.Int).SetBytes(reverseOrder(td)), nil +} + +// RawIterator reads an RLP-encode Era1 entries. +type RawIterator struct { + e *Era // backing Era1 + curOffset int64 // offset of current block + next uint64 // next block to read + err error // last error + + Header io.Reader + Body io.Reader + Receipts io.Reader + TotalDifficulty io.Reader +} + +// NewRawIterator returns a new Iterator instance. +func NewRawIterator(e *Era) (*RawIterator, error) { + return &RawIterator{ + e: e, + next: e.m.start, + }, nil +} + +// Next moves the iterator to the next block entry. +func (it *RawIterator) Next() bool { + if it.e.m.start+it.e.m.count <= it.next { + return false + } + off, err := it.e.readOffset(it.next) + if err == io.EOF { + it.err = err + return false + } + var n int + if it.Header, n, it.err = newSnappyReader(it.e.s, off); it.err != nil { + return true + } + off += int64(n) + if it.Body, n, it.err = newSnappyReader(it.e.s, off); it.err != nil { + return true + } + off += int64(n) + if it.Receipts, n, it.err = newSnappyReader(it.e.s, off); it.err != nil { + return true + } + off += int64(n) + if it.TotalDifficulty, n, it.err = newReader(it.e.s, off); it.err != nil { + return true + } + it.next += 1 + return true +} + +// Error returns the error status of the iterator. +func (it *RawIterator) Error() error { + if it.err == io.EOF { + return nil + } + return it.err +}