diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index ed44a89034..734b8cca1a 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -519,11 +519,11 @@ func importHistory(ctx *cli.Context) error { format := ctx.String(utils.EraFormatFlag.Name) switch format { case "era1", "era": - if err := utils.ImportHistory(chain, dir, network); err != nil { + if err := utils.ImportHistory(chain, dir, network, utils.Era1); err != nil { return err } case "erae": - if err := utils.ImportHistoryEraE(chain, dir, network); err != nil { + if err := utils.ImportHistory(chain, dir, network, utils.EraE); err != nil { return err } default: diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 23ce48dd77..cd53c4a51e 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -44,7 +44,6 @@ import ( "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/internal/debug" - "github.com/ethereum/go-ethereum/internal/era" "github.com/ethereum/go-ethereum/internal/era2" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" @@ -58,12 +57,7 @@ const ( importBatchSize = 2500 ) -type ExportFormat int - -const ( - Era1 ExportFormat = iota - EraE -) +type EraFileFormat int // ErrImportInterrupted is returned when the user interrupts the import process. var ErrImportInterrupted = errors.New("interrupted") @@ -258,11 +252,11 @@ func readList(filename string) ([]string, error) { // ImportHistory imports Era1 files containing historical block information, // starting from genesis. The assumption is held that the provided chain // segment in Era1 file should all be canonical and verified. -func ImportHistory(chain *core.BlockChain, dir string, network string) error { +func ImportHistory(chain *core.BlockChain, dir string, network string, format Format) error { if chain.CurrentSnapBlock().Number.BitLen() != 0 { return errors.New("history import only supported when starting from genesis") } - entries, err := era.ReadDir(dir, network) + entries, err := format.ReadDir(dir, network) if err != nil { return fmt.Errorf("error reading %s: %w", dir, err) } @@ -271,42 +265,50 @@ func ImportHistory(chain *core.BlockChain, dir string, network string) error { return fmt.Errorf("unable to read checksums.txt: %w", err) } if len(checksums) != len(entries) { - return fmt.Errorf("expected equal number of checksums and entries, have: %d checksums, %d entries", len(checksums), len(entries)) + return fmt.Errorf("expected equal number of checksums and entries, have: %d checksums, %d entries", + len(checksums), len(entries)) } + var ( start = time.Now() reported = time.Now() imported = 0 h = sha256.New() - buf = bytes.NewBuffer(nil) + scratch = bytes.NewBuffer(nil) ) - for i, filename := range entries { + + for i, file := range entries { err := func() error { - f, err := os.Open(filepath.Join(dir, filename)) - if err != nil { - return fmt.Errorf("unable to open era: %w", err) - } - defer f.Close() + path := filepath.Join(dir, file) - // Validate checksum. + // validate against checksum file in directory + f, err := os.Open(path) + if err != nil { + return fmt.Errorf("open %s: %w", path, err) + } if _, err := io.Copy(h, f); err != nil { - return fmt.Errorf("unable to recalculate checksum: %w", err) - } - if have, want := common.BytesToHash(h.Sum(buf.Bytes()[:])).Hex(), checksums[i]; have != want { - return fmt.Errorf("checksum mismatch: have %s, want %s", have, want) + f.Close() + return fmt.Errorf("checksum %s: %w", path, err) } + got := common.BytesToHash(h.Sum(scratch.Bytes()[:])).Hex() + want := checksums[i] h.Reset() - buf.Reset() + scratch.Reset() - // Import all block data from Era1. - e, err := era.From(f) - if err != nil { - return fmt.Errorf("error opening era: %w", err) + if got != want { + f.Close() + return fmt.Errorf("%s checksum mismatch: have %s want %s", file, got, want) } - it, err := era.NewIterator(e) - if err != nil { - return fmt.Errorf("error making era reader: %w", err) + // rewind for reading + if _, err := f.Seek(0, io.SeekStart); err != nil { + f.Close() + return fmt.Errorf("rewind %s: %w", file, err) } + it, err := format.NewIterator(f) + if err != nil { + return fmt.Errorf("error creating iterator: %w", err) + } + for it.Next() { block, err := it.Block() if err != nil { @@ -315,30 +317,32 @@ func ImportHistory(chain *core.BlockChain, dir string, network string) error { if block.Number().BitLen() == 0 { continue // skip genesis } - receipts, err := it.Receipts() + rcpts, err := it.Receipts() if err != nil { return fmt.Errorf("error reading receipts %d: %w", it.Number(), err) } - encReceipts := types.EncodeBlockReceiptLists([]types.Receipts{receipts}) - if _, err := chain.InsertReceiptChain([]*types.Block{block}, encReceipts, math.MaxUint64); err != nil { + enc := types.EncodeBlockReceiptLists([]types.Receipts{rcpts}) + if _, err := chain.InsertReceiptChain([]*types.Block{block}, enc, math.MaxUint64); err != nil { return fmt.Errorf("error inserting body %d: %w", it.Number(), err) } - imported += 1 + imported++ - // Give the user some feedback that something is happening. if time.Since(reported) >= 8*time.Second { - log.Info("Importing Era files", "head", it.Number(), "imported", imported, "elapsed", common.PrettyDuration(time.Since(start))) + log.Info("Importing Era files", "head", it.Number(), "imported", imported, + "elapsed", common.PrettyDuration(time.Since(start))) imported = 0 reported = time.Now() } } + if err := it.Error(); err != nil { + return err + } return nil }() if err != nil { return err } } - return nil } @@ -511,8 +515,8 @@ func ExportAppendChain(blockchain *core.BlockChain, fn string, first uint64, las // ExportHistory exports blockchain history into the specified directory, // following the Era format. -func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64, f ExportFormat) error { - log.Info("Exporting blockchain history", "dir", dir, "format", f) +func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64, f Format) error { + log.Info("Exporting blockchain history", "dir", dir) if head := bc.CurrentBlock().Number.Uint64(); head < last { log.Warn("Last block beyond head, setting last = head", "head", head, "last", last) last = head @@ -525,26 +529,6 @@ func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64, f return fmt.Errorf("error creating output directory: %w", err) } - var ( - filename func(string, int, common.Hash) string - newBuilder func(io.Writer) any - add func(any, *types.Block, types.Receipts, *big.Int) error - ) - - if f == Era1 { - filename = era.Filename - newBuilder = func(w io.Writer) any { return era.NewBuilder(w) } - add = func(b any, blk *types.Block, rcpt types.Receipts, td *big.Int) error { - return b.(*era.Builder).Add(blk, rcpt, td) - } - } else { - filename = era2.Filename - newBuilder = func(w io.Writer) any { return era2.NewBuilder(w) } - add = func(b any, blk *types.Block, rcpt types.Receipts, td *big.Int) error { - return b.(*era2.Builder).Add(*blk.Header(), *blk.Body(), rcpt, td, nil) - } - } - td := new(big.Int) for n := uint64(0); n < first; n++ { td.Add(td, bc.GetHeaderByNumber(n).Difficulty) @@ -560,16 +544,16 @@ func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64, f for batch := first; batch <= last; batch += step { idx := int(batch / step) - tmpPath := filepath.Join(dir, filename(network, idx, common.Hash{})) + tmpPath := filepath.Join(dir, f.Filename(network, idx, common.Hash{})) if err := func() error { - f, err := os.Create(tmpPath) + fh, err := os.Create(tmpPath) if err != nil { return err } - defer f.Close() + defer fh.Close() - bldr := newBuilder(f) + bldr := f.NewBuilder(fh) for j := uint64(0); j < step && batch+j <= last; j++ { n := batch + j @@ -582,26 +566,27 @@ func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64, f return fmt.Errorf("receipts for #%d missing", n) } td.Add(td, blk.Difficulty()) - if err := add(bldr, blk, rcpt, new(big.Int).Set(td)); err != nil { + + if err := bldr.Add(blk, rcpt, new(big.Int).Set(td), nil); err != nil { return err } } - root, err := bldr.(interface{ Finalize() (common.Hash, error) }).Finalize() + root, err := bldr.Finalize() if err != nil { return err } - final := filepath.Join(dir, filename(network, idx, root)) + final := filepath.Join(dir, f.Filename(network, idx, root)) if err := os.Rename(tmpPath, final); err != nil { return err } - if _, err := f.Seek(0, io.SeekStart); err != nil { + if _, err := fh.Seek(0, io.SeekStart); err != nil { return err } h.Reset() buf.Reset() - if _, err := io.Copy(h, f); err != nil { + if _, err := io.Copy(h, fh); err != nil { return err } checksums = append(checksums, common.BytesToHash(h.Sum(buf.Bytes()[:])).Hex()) diff --git a/cmd/utils/era_interface.go b/cmd/utils/era_interface.go new file mode 100644 index 0000000000..de3fbde707 --- /dev/null +++ b/cmd/utils/era_interface.go @@ -0,0 +1,82 @@ +package utils + +import ( + "io" + "math/big" + "os" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/internal/era" + "github.com/ethereum/go-ethereum/internal/era2" +) + +// Iterator is the common, read‑only view returned by both Era‑1 and Era‑E. +type Iterator interface { + Next() bool + Number() uint64 + Block() (*types.Block, error) + Receipts() (types.Receipts, error) + Error() error +} + +// Builder (unchanged) – writes one archive file. +type Builder interface { + Add(blk *types.Block, rcpts types.Receipts, td *big.Int, proof era2.Proof) error + Finalize() (common.Hash, error) +} + +// Format now encapsulates *all* format‑specific behaviour, both read & write. +type Format interface { + // ----- writer side ----- + Filename(network string, epoch int, root common.Hash) string + NewBuilder(w io.Writer) Builder + + // ----- reader side ----- + ReadDir(dir, network string) ([]string, error) + NewIterator(f *os.File) (Iterator, error) +} + +type era1Builder struct{ *era.Builder } + +func (b *era1Builder) Add(blk *types.Block, rc types.Receipts, td *big.Int, proof era2.Proof) error { + return b.Builder.Add(blk, rc, td) +} + +type era1Format struct{} + +func (era1Format) Filename(n string, e int, h common.Hash) string { return era.Filename(n, e, h) } +func (era1Format) NewBuilder(w io.Writer) Builder { return &era1Builder{era.NewBuilder(w)} } +func (era1Format) ReadDir(dir, net string) ([]string, error) { return era.ReadDir(dir, net) } +func (era1Format) NewIterator(f *os.File) (Iterator, error) { + e, err := era.From(f) + if err != nil { + return nil, err + } + return era.NewIterator(e) +} + +var Era1 Format = era1Format{} + +type eraeBuilder struct{ *era2.Builder } + +func (b *eraeBuilder) Add(blk *types.Block, rc types.Receipts, td *big.Int, proof era2.Proof) error { + return b.Builder.Add(blk, rc, td, nil) // no proofs yet +} + +type eraeFormat struct{} + +func (eraeFormat) Filename(n string, e int, h common.Hash) string { return era2.Filename(n, e, h) } +func (eraeFormat) NewBuilder(w io.Writer) Builder { return &eraeBuilder{era2.NewBuilder(w)} } +func (eraeFormat) ReadDir(dir, net string) ([]string, error) { return era2.ReadDir(dir, net) } +func (eraeFormat) NewIterator(f *os.File) (Iterator, error) { + e, err := era2.From(f) + if err != nil { + return nil, err + } + return era2.NewIterator(e) +} + +var EraE Format = eraeFormat{} + +// Exported singleton diff --git a/cmd/utils/history_test.go b/cmd/utils/history_test.go index 3f0bbe7e3c..10044a3f83 100644 --- a/cmd/utils/history_test.go +++ b/cmd/utils/history_test.go @@ -32,7 +32,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/internal/era" + "github.com/ethereum/go-ethereum/internal/era2" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/triedb" @@ -89,7 +89,7 @@ func TestHistoryImportAndExport(t *testing.T) { dir := t.TempDir() // Export history to temp directory. - if err := ExportHistory(chain, dir, 0, count, step, EraE); err != nil { + if err := ExportHistory(chain, dir, 0, count, step, Era1); err != nil { t.Fatalf("error exporting history: %v", err) } @@ -101,7 +101,7 @@ func TestHistoryImportAndExport(t *testing.T) { checksums := strings.Split(string(b), "\n") // Verify each Era. - entries, _ := era.ReadDir(dir, "mainnet") + entries, _ := era2.ReadDir(dir, "mainnet") for i, filename := range entries { func() { f, err := os.Open(filepath.Join(dir, filename)) @@ -118,12 +118,12 @@ func TestHistoryImportAndExport(t *testing.T) { if got, want := common.BytesToHash(h.Sum(buf.Bytes()[:])).Hex(), checksums[i]; got != want { t.Fatalf("checksum %d does not match: got %s, want %s", i, got, want) } - e, err := era.From(f) + e, err := era2.From(f) if err != nil { t.Fatalf("error opening era: %v", err) } defer e.Close() - it, err := era.NewIterator(e) + it, err := era2.NewIterator(e) if err != nil { t.Fatalf("error making era reader: %v", err) } @@ -170,7 +170,7 @@ func TestHistoryImportAndExport(t *testing.T) { if err != nil { t.Fatalf("unable to initialize chain: %v", err) } - if err := ImportHistory(imported, dir, "mainnet"); err != nil { + if err := ImportHistory(imported, dir, "mainnet", EraE); err != nil { t.Fatalf("failed to import chain: %v", err) } if have, want := imported.CurrentHeader(), chain.CurrentHeader(); have.Hash() != want.Hash() { diff --git a/internal/era2/builder.go b/internal/era2/builder.go index 5d2029a851..740015df08 100644 --- a/internal/era2/builder.go +++ b/internal/era2/builder.go @@ -113,7 +113,9 @@ func NewBuilder(w io.Writer) *Builder { } // Add writes a block entry, its reciepts, and optionally its proofs as well into the e2store file. -func (b *Builder) Add(header types.Header, body types.Body, receipts types.Receipts, td *big.Int, proof Proof) error { +func (b *Builder) Add(block *types.Block, receipts types.Receipts, td *big.Int, proof Proof) error { + header := block.Header() + body := block.Body() if len(b.buff.headers) == 0 { // first block determines wether proofs are expected b.expectsProofs = proof != nil } else if b.expectsProofs && proof == nil { // every later block must follow this policy diff --git a/internal/era2/era.go b/internal/era2/era.go index 1f28c7a85c..cc55de5489 100644 --- a/internal/era2/era.go +++ b/internal/era2/era.go @@ -46,11 +46,11 @@ type componentType int // TypeCompressedHeader, TypeCompressedBody, TypeCompressedReceipts, TypeTotalDifficulty, and TypeProof are the different types of components that can be present in the era file. const ( - componentHeader componentType = iota - componentBody - componentReceipts - componentTD - componentProof + header componentType = iota + body + receipts + td + proof ) type ReadAtSeekCloser interface { @@ -274,11 +274,11 @@ func (e *Era) loadIndex() error { } // headerOff, bodyOff, receiptOff, tdOff, and proofOff return the offsets of the respective components for a given block number. -func (e *Era) headerOff(num uint64) (uint64, error) { return e.indexOffset(num, componentHeader) } -func (e *Era) bodyOff(num uint64) (uint64, error) { return e.indexOffset(num, componentBody) } -func (e *Era) receiptOff(num uint64) (uint64, error) { return e.indexOffset(num, componentReceipts) } -func (e *Era) tdOff(num uint64) (uint64, error) { return e.indexOffset(num, componentTD) } -func (e *Era) proofOff(num uint64) (uint64, error) { return e.indexOffset(num, componentProof) } +func (e *Era) headerOff(num uint64) (uint64, error) { return e.indexOffset(num, header) } +func (e *Era) bodyOff(num uint64) (uint64, error) { return e.indexOffset(num, body) } +func (e *Era) receiptOff(num uint64) (uint64, error) { return e.indexOffset(num, receipts) } +func (e *Era) tdOff(num uint64) (uint64, error) { return e.indexOffset(num, td) } +func (e *Era) proofOff(num uint64) (uint64, error) { return e.indexOffset(num, proof) } // indexOffset calculates offset to a certain component for a block number within a file. func (e *Era) indexOffset(n uint64, component componentType) (uint64, error) { diff --git a/internal/era2/era_test.go b/internal/era2/era_test.go index 447c7676ff..7bf7cd1688 100644 --- a/internal/era2/era_test.go +++ b/internal/era2/era_test.go @@ -65,7 +65,7 @@ func TestEra2Builder(t *testing.T) { receipts = chain.receipts[i] td = chain.tds[i] ) - if err = builder.Add(header, body, receipts, td, nil); err != nil { + if err = builder.Add(types.NewBlockWithHeader(&header).WithBody(body), receipts, td, nil); err != nil { t.Fatalf("error adding entry: %v", err) } } diff --git a/internal/era2/iterator.go b/internal/era2/iterator.go new file mode 100644 index 0000000000..316825cfd9 --- /dev/null +++ b/internal/era2/iterator.go @@ -0,0 +1,213 @@ +// Copyright 2024 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package era2 + +import ( + "errors" + "io" + "math/big" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/internal/era/e2store" + "github.com/ethereum/go-ethereum/rlp" + "github.com/klauspost/compress/snappy" +) + +type Iterator struct { + inner *RawIterator +} + +// NewIterator returns a header/body/receipt iterator over the archive. +// Call Next immediately to position on the first block. +func NewIterator(e *Era) (*Iterator, error) { + inner, err := NewRawIterator(e) + if err != nil { + return nil, err + } + return &Iterator{inner: inner}, nil +} + +// Next advances to the next block entry. +func (it *Iterator) Next() bool { return it.inner.Next() } + +// Number is the number of the block currently loaded. +func (it *Iterator) Number() uint64 { return it.inner.next - 1 } + +// Error returns any iteration error (EOF is reported as nil, identical +// to the Era‑1 iterator behaviour). +func (it *Iterator) Error() error { return it.inner.Error() } + +// Block decodes the current header+body into a *types.Block. +func (it *Iterator) Block() (*types.Block, error) { + if it.inner.Header == nil || it.inner.Body == nil { + return nil, errors.New("header and body must be non‑nil") + } + var ( + h types.Header + b types.Body + ) + if err := rlp.Decode(it.inner.Header, &h); err != nil { + return nil, err + } + if err := rlp.Decode(it.inner.Body, &b); err != nil { + return nil, err + } + return types.NewBlockWithHeader(&h).WithBody(b), nil +} + +// Receipts decodes receipts for the current block. +func (it *Iterator) Receipts() (types.Receipts, error) { + if it.inner.Receipts == nil { + return nil, errors.New("receipts must be non‑nil") + } + var r types.Receipts + return r, rlp.Decode(it.inner.Receipts, &r) +} + +// BlockAndReceipts is a convenience wrapper. +func (it *Iterator) BlockAndReceipts() (*types.Block, types.Receipts, error) { + b, err := it.Block() + if err != nil { + return nil, nil, err + } + r, err := it.Receipts() + if err != nil { + return nil, nil, err + } + return b, r, nil +} + +// TotalDifficulty returns the TD at the current position (if present). +func (it *Iterator) TotalDifficulty() (*big.Int, error) { + if it.inner.TotalDifficulty == nil { + return nil, errors.New("total‑difficulty stream is nil") + } + tdBytes, err := io.ReadAll(it.inner.TotalDifficulty) + if err != nil { + return nil, err + } + return new(big.Int).SetBytes(reverseOrder(tdBytes)), nil +} + +// ----------------------------------------------------------------------------- +// Low‑level iterator (raw TLV/offset handling, no decoding) +// ----------------------------------------------------------------------------- + +type RawIterator struct { + e *Era + next uint64 // next block to pull + err error + + Header io.Reader + Body io.Reader + Receipts io.Reader + TotalDifficulty io.Reader // nil when archive omits TD +} + +// NewRawIterator creates an iterator positioned *before* the first block. +func NewRawIterator(e *Era) (*RawIterator, error) { + return &RawIterator{e: e, next: e.m.start}, nil +} + +// Next loads the next block’s components; returns false on EOF or error. +func (it *RawIterator) Next() bool { + it.err = nil // clear previous error + + if it.next >= it.e.m.start+it.e.m.count { + it.clear() + return false + } + + headerOffset, err := it.e.headerOff(it.next) + if err != nil { + it.setErr(err) + return false + } + it.Header, _, err = newSnappyReader(it.e.s, TypeCompressedHeader, int64(headerOffset)) + if err != nil { + it.setErr(err) + return false + } + + bodyOffset, err := it.e.bodyOff(it.next) + if err != nil { + it.setErr(err) + return false + } + it.Body, _, err = newSnappyReader(it.e.s, TypeCompressedBody, int64(bodyOffset)) + if err != nil { + it.setErr(err) + return false + } + + receiptsOffset, err := it.e.receiptOff(it.next) + if err != nil { + it.setErr(err) + return false + } + it.Receipts, _, err = newSnappyReader(it.e.s, TypeCompressedReceipts, int64(receiptsOffset)) + if err != nil { + it.setErr(err) + return false + } + + if int(header)+3 < int(it.e.m.components) { + tdOffset, err := it.e.tdOff(it.next) + if err != nil { + it.setErr(err) + return false + } + it.TotalDifficulty, _, err = it.e.s.ReaderAt(TypeTotalDifficulty, int64(tdOffset)) + if err != nil { + it.setErr(err) + return false + } + } else { + it.TotalDifficulty = nil + } + + it.next++ + return true +} + +func (it *RawIterator) Number() uint64 { return it.next - 1 } + +func (it *RawIterator) Error() error { + if it.err == io.EOF { + return nil + } + return it.err +} + +func (it *RawIterator) setErr(err error) { + it.err = err + it.clear() +} + +func (it *RawIterator) clear() { + it.Header, it.Body, it.Receipts, it.TotalDifficulty = nil, nil, nil, nil +} + +// newSnappyReader behaves like era.newSnappyReader: returns a snappy.Reader +// plus the length of the underlying TLV payload so callers can advance offsets. +func newSnappyReader(r *e2store.Reader, typ uint16, off int64) (io.Reader, int64, error) { + raw, n, err := r.ReaderAt(typ, off) + if err != nil { + return nil, 0, err + } + return snappy.NewReader(raw), int64(n), nil +}