forks/go-ethereum
1
0
Fork 1
mirror of https://github.com/ethereum/go-ethereum.git synced 2026-07-21 20:26:41 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

added some abstraction

Browse source
This commit is contained in:
shantichanal 2025-07-24 02:53:32 +02:00 committed by lightclient
parent a8168195be
commit c50b4043f4
No known key found for this signature in database
GPG key ID: 657913021EF45A6A
8 changed files with 370 additions and 88 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
cmd/geth/chaincmd.go
View file

@ -519,11 +519,11 @@ func importHistory(ctx *cli.Context) error {
format := ctx.String(utils.EraFormatFlag.Name)
switch format {
case "era1", "era":
if err := utils.ImportHistory(chain, dir, network); err != nil {
if err := utils.ImportHistory(chain, dir, network, utils.Era1); err != nil {
return err
}
case "erae":
if err := utils.ImportHistoryEraE(chain, dir, network); err != nil {
if err := utils.ImportHistory(chain, dir, network, utils.EraE); err != nil {
return err
}
default:

121
cmd/utils/cmd.go
View file

@ -44,7 +44,6 @@ import (
"github.com/ethereum/go-ethereum/eth/ethconfig"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/internal/debug"
"github.com/ethereum/go-ethereum/internal/era"
"github.com/ethereum/go-ethereum/internal/era2"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/node"
@ -58,12 +57,7 @@ const (
importBatchSize = 2500
)
type ExportFormat int
const (
Era1 ExportFormat = iota
EraE
)
type EraFileFormat int
// ErrImportInterrupted is returned when the user interrupts the import process.
var ErrImportInterrupted = errors.New("interrupted")
@ -258,11 +252,11 @@ func readList(filename string) ([]string, error) {
// ImportHistory imports Era1 files containing historical block information,
// starting from genesis. The assumption is held that the provided chain
// segment in Era1 file should all be canonical and verified.
func ImportHistory(chain *core.BlockChain, dir string, network string) error {
func ImportHistory(chain *core.BlockChain, dir string, network string, format Format) error {
if chain.CurrentSnapBlock().Number.BitLen() != 0 {
return errors.New("history import only supported when starting from genesis")
}
entries, err := era.ReadDir(dir, network)
entries, err := format.ReadDir(dir, network)
if err != nil {
return fmt.Errorf("error reading %s: %w", dir, err)
}
@ -271,42 +265,50 @@ func ImportHistory(chain *core.BlockChain, dir string, network string) error {
return fmt.Errorf("unable to read checksums.txt: %w", err)
}
if len(checksums) != len(entries) {
return fmt.Errorf("expected equal number of checksums and entries, have: %d checksums, %d entries", len(checksums), len(entries))
return fmt.Errorf("expected equal number of checksums and entries, have: %d checksums, %d entries",
len(checksums), len(entries))
}
var (
start = time.Now()
reported = time.Now()
imported = 0
h = sha256.New()
buf = bytes.NewBuffer(nil)
scratch = bytes.NewBuffer(nil)
)
for i, filename := range entries {
for i, file := range entries {
err := func() error {
f, err := os.Open(filepath.Join(dir, filename))
if err != nil {
return fmt.Errorf("unable to open era: %w", err)
}
defer f.Close()
path := filepath.Join(dir, file)
// Validate checksum.
// validate against checksum file in directory
f, err := os.Open(path)
if err != nil {
return fmt.Errorf("open %s: %w", path, err)
}
if _, err := io.Copy(h, f); err != nil {
return fmt.Errorf("unable to recalculate checksum: %w", err)
}
if have, want := common.BytesToHash(h.Sum(buf.Bytes()[:])).Hex(), checksums[i]; have != want {
return fmt.Errorf("checksum mismatch: have %s, want %s", have, want)
f.Close()
return fmt.Errorf("checksum %s: %w", path, err)
}
got := common.BytesToHash(h.Sum(scratch.Bytes()[:])).Hex()
want := checksums[i]
h.Reset()
buf.Reset()
scratch.Reset()
// Import all block data from Era1.
e, err := era.From(f)
if err != nil {
return fmt.Errorf("error opening era: %w", err)
if got != want {
f.Close()
return fmt.Errorf("%s checksum mismatch: have %s want %s", file, got, want)
}
it, err := era.NewIterator(e)
if err != nil {
return fmt.Errorf("error making era reader: %w", err)
// rewind for reading
if _, err := f.Seek(0, io.SeekStart); err != nil {
f.Close()
return fmt.Errorf("rewind %s: %w", file, err)
}
it, err := format.NewIterator(f)
if err != nil {
return fmt.Errorf("error creating iterator: %w", err)
}
for it.Next() {
block, err := it.Block()
if err != nil {
@ -315,30 +317,32 @@ func ImportHistory(chain *core.BlockChain, dir string, network string) error {
if block.Number().BitLen() == 0 {
continue // skip genesis
}
receipts, err := it.Receipts()
rcpts, err := it.Receipts()
if err != nil {
return fmt.Errorf("error reading receipts %d: %w", it.Number(), err)
}
encReceipts := types.EncodeBlockReceiptLists([]types.Receipts{receipts})
if _, err := chain.InsertReceiptChain([]*types.Block{block}, encReceipts, math.MaxUint64); err != nil {
enc := types.EncodeBlockReceiptLists([]types.Receipts{rcpts})
if _, err := chain.InsertReceiptChain([]*types.Block{block}, enc, math.MaxUint64); err != nil {
return fmt.Errorf("error inserting body %d: %w", it.Number(), err)
}
imported += 1
imported++
// Give the user some feedback that something is happening.
if time.Since(reported) >= 8*time.Second {
log.Info("Importing Era files", "head", it.Number(), "imported", imported, "elapsed", common.PrettyDuration(time.Since(start)))
log.Info("Importing Era files", "head", it.Number(), "imported", imported,
"elapsed", common.PrettyDuration(time.Since(start)))
imported = 0
reported = time.Now()
}
}
if err := it.Error(); err != nil {
return err
}
return nil
}()
if err != nil {
return err
}
}
return nil
}
@ -511,8 +515,8 @@ func ExportAppendChain(blockchain *core.BlockChain, fn string, first uint64, las
// ExportHistory exports blockchain history into the specified directory,
// following the Era format.
func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64, f ExportFormat) error {
log.Info("Exporting blockchain history", "dir", dir, "format", f)
func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64, f Format) error {
log.Info("Exporting blockchain history", "dir", dir)
if head := bc.CurrentBlock().Number.Uint64(); head < last {
log.Warn("Last block beyond head, setting last = head", "head", head, "last", last)
last = head
@ -525,26 +529,6 @@ func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64, f
return fmt.Errorf("error creating output directory: %w", err)
}
var (
filename func(string, int, common.Hash) string
newBuilder func(io.Writer) any
add func(any, *types.Block, types.Receipts, *big.Int) error
)
if f == Era1 {
filename = era.Filename
newBuilder = func(w io.Writer) any { return era.NewBuilder(w) }
add = func(b any, blk *types.Block, rcpt types.Receipts, td *big.Int) error {
return b.(*era.Builder).Add(blk, rcpt, td)
}
} else {
filename = era2.Filename
newBuilder = func(w io.Writer) any { return era2.NewBuilder(w) }
add = func(b any, blk *types.Block, rcpt types.Receipts, td *big.Int) error {
return b.(*era2.Builder).Add(*blk.Header(), *blk.Body(), rcpt, td, nil)
}
}
td := new(big.Int)
for n := uint64(0); n < first; n++ {
td.Add(td, bc.GetHeaderByNumber(n).Difficulty)
@ -560,16 +544,16 @@ func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64, f
for batch := first; batch <= last; batch += step {
idx := int(batch / step)
tmpPath := filepath.Join(dir, filename(network, idx, common.Hash{}))
tmpPath := filepath.Join(dir, f.Filename(network, idx, common.Hash{}))
if err := func() error {
f, err := os.Create(tmpPath)
fh, err := os.Create(tmpPath)
if err != nil {
return err
}
defer f.Close()
defer fh.Close()
bldr := newBuilder(f)
bldr := f.NewBuilder(fh)
for j := uint64(0); j < step && batch+j <= last; j++ {
n := batch + j
@ -582,26 +566,27 @@ func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64, f
return fmt.Errorf("receipts for #%d missing", n)
}
td.Add(td, blk.Difficulty())
if err := add(bldr, blk, rcpt, new(big.Int).Set(td)); err != nil {
if err := bldr.Add(blk, rcpt, new(big.Int).Set(td), nil); err != nil {
return err
}
}
root, err := bldr.(interface{ Finalize() (common.Hash, error) }).Finalize()
root, err := bldr.Finalize()
if err != nil {
return err
}
final := filepath.Join(dir, filename(network, idx, root))
final := filepath.Join(dir, f.Filename(network, idx, root))
if err := os.Rename(tmpPath, final); err != nil {
return err
}
if _, err := f.Seek(0, io.SeekStart); err != nil {
if _, err := fh.Seek(0, io.SeekStart); err != nil {
return err
}
h.Reset()
buf.Reset()
if _, err := io.Copy(h, f); err != nil {
if _, err := io.Copy(h, fh); err != nil {
return err
}
checksums = append(checksums, common.BytesToHash(h.Sum(buf.Bytes()[:])).Hex())

82
cmd/utils/era_interface.go Normal file
View file

@ -0,0 +1,82 @@
package utils
import (
"io"
"math/big"
"os"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/internal/era"
"github.com/ethereum/go-ethereum/internal/era2"
)
// Iterator is the common, read‑only view returned by both Era‑1 and Era‑E.
type Iterator interface {
Next() bool
Number() uint64
Block() (*types.Block, error)
Receipts() (types.Receipts, error)
Error() error
}
// Builder (unchanged) – writes one archive file.
type Builder interface {
Add(blk *types.Block, rcpts types.Receipts, td *big.Int, proof era2.Proof) error
Finalize() (common.Hash, error)
}
// Format now encapsulates *all* format‑specific behaviour, both read & write.
type Format interface {
// ----- writer side -----
Filename(network string, epoch int, root common.Hash) string
NewBuilder(w io.Writer) Builder
// ----- reader side -----
ReadDir(dir, network string) ([]string, error)
NewIterator(f *os.File) (Iterator, error)
}
type era1Builder struct{ *era.Builder }
func (b *era1Builder) Add(blk *types.Block, rc types.Receipts, td *big.Int, proof era2.Proof) error {
return b.Builder.Add(blk, rc, td)
}
type era1Format struct{}
func (era1Format) Filename(n string, e int, h common.Hash) string { return era.Filename(n, e, h) }
func (era1Format) NewBuilder(w io.Writer) Builder { return &era1Builder{era.NewBuilder(w)} }
func (era1Format) ReadDir(dir, net string) ([]string, error) { return era.ReadDir(dir, net) }
func (era1Format) NewIterator(f *os.File) (Iterator, error) {
e, err := era.From(f)
if err != nil {
return nil, err
}
return era.NewIterator(e)
}
var Era1 Format = era1Format{}
type eraeBuilder struct{ *era2.Builder }
func (b *eraeBuilder) Add(blk *types.Block, rc types.Receipts, td *big.Int, proof era2.Proof) error {
return b.Builder.Add(blk, rc, td, nil) // no proofs yet
}
type eraeFormat struct{}
func (eraeFormat) Filename(n string, e int, h common.Hash) string { return era2.Filename(n, e, h) }
func (eraeFormat) NewBuilder(w io.Writer) Builder { return &eraeBuilder{era2.NewBuilder(w)} }
func (eraeFormat) ReadDir(dir, net string) ([]string, error) { return era2.ReadDir(dir, net) }
func (eraeFormat) NewIterator(f *os.File) (Iterator, error) {
e, err := era2.From(f)
if err != nil {
return nil, err
}
return era2.NewIterator(e)
}
var EraE Format = eraeFormat{}
// Exported singleton

12
cmd/utils/history_test.go
View file

@ -32,7 +32,7 @@ import (
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/internal/era"
"github.com/ethereum/go-ethereum/internal/era2"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/trie"
"github.com/ethereum/go-ethereum/triedb"
@ -89,7 +89,7 @@ func TestHistoryImportAndExport(t *testing.T) {
dir := t.TempDir()
// Export history to temp directory.
if err := ExportHistory(chain, dir, 0, count, step, EraE); err != nil {
if err := ExportHistory(chain, dir, 0, count, step, Era1); err != nil {
t.Fatalf("error exporting history: %v", err)
}
@ -101,7 +101,7 @@ func TestHistoryImportAndExport(t *testing.T) {
checksums := strings.Split(string(b), "\n")
// Verify each Era.
entries, _ := era.ReadDir(dir, "mainnet")
entries, _ := era2.ReadDir(dir, "mainnet")
for i, filename := range entries {
func() {
f, err := os.Open(filepath.Join(dir, filename))
@ -118,12 +118,12 @@ func TestHistoryImportAndExport(t *testing.T) {
if got, want := common.BytesToHash(h.Sum(buf.Bytes()[:])).Hex(), checksums[i]; got != want {
t.Fatalf("checksum %d does not match: got %s, want %s", i, got, want)
}
e, err := era.From(f)
e, err := era2.From(f)
if err != nil {
t.Fatalf("error opening era: %v", err)
}
defer e.Close()
it, err := era.NewIterator(e)
it, err := era2.NewIterator(e)
if err != nil {
t.Fatalf("error making era reader: %v", err)
}
@ -170,7 +170,7 @@ func TestHistoryImportAndExport(t *testing.T) {
if err != nil {
t.Fatalf("unable to initialize chain: %v", err)
}
if err := ImportHistory(imported, dir, "mainnet"); err != nil {
if err := ImportHistory(imported, dir, "mainnet", EraE); err != nil {
t.Fatalf("failed to import chain: %v", err)
}
if have, want := imported.CurrentHeader(), chain.CurrentHeader(); have.Hash() != want.Hash() {

4
internal/era2/builder.go
View file

@ -113,7 +113,9 @@ func NewBuilder(w io.Writer) *Builder {
}
// Add writes a block entry, its reciepts, and optionally its proofs as well into the e2store file.
func (b *Builder) Add(header types.Header, body types.Body, receipts types.Receipts, td *big.Int, proof Proof) error {
func (b *Builder) Add(block *types.Block, receipts types.Receipts, td *big.Int, proof Proof) error {
header := block.Header()
body := block.Body()
if len(b.buff.headers) == 0 { // first block determines wether proofs are expected
b.expectsProofs = proof != nil
} else if b.expectsProofs && proof == nil { // every later block must follow this policy

20
internal/era2/era.go
View file

@ -46,11 +46,11 @@ type componentType int
// TypeCompressedHeader, TypeCompressedBody, TypeCompressedReceipts, TypeTotalDifficulty, and TypeProof are the different types of components that can be present in the era file.
const (
componentHeader componentType = iota
componentBody
componentReceipts
componentTD
componentProof
header componentType = iota
body
receipts
td
proof
)
type ReadAtSeekCloser interface {
@ -274,11 +274,11 @@ func (e *Era) loadIndex() error {
}
// headerOff, bodyOff, receiptOff, tdOff, and proofOff return the offsets of the respective components for a given block number.
func (e *Era) headerOff(num uint64) (uint64, error) { return e.indexOffset(num, componentHeader) }
func (e *Era) bodyOff(num uint64) (uint64, error) { return e.indexOffset(num, componentBody) }
func (e *Era) receiptOff(num uint64) (uint64, error) { return e.indexOffset(num, componentReceipts) }
func (e *Era) tdOff(num uint64) (uint64, error) { return e.indexOffset(num, componentTD) }
func (e *Era) proofOff(num uint64) (uint64, error) { return e.indexOffset(num, componentProof) }
func (e *Era) headerOff(num uint64) (uint64, error) { return e.indexOffset(num, header) }
func (e *Era) bodyOff(num uint64) (uint64, error) { return e.indexOffset(num, body) }
func (e *Era) receiptOff(num uint64) (uint64, error) { return e.indexOffset(num, receipts) }
func (e *Era) tdOff(num uint64) (uint64, error) { return e.indexOffset(num, td) }
func (e *Era) proofOff(num uint64) (uint64, error) { return e.indexOffset(num, proof) }
// indexOffset calculates offset to a certain component for a block number within a file.
func (e *Era) indexOffset(n uint64, component componentType) (uint64, error) {

2
internal/era2/era_test.go
View file

@ -65,7 +65,7 @@ func TestEra2Builder(t *testing.T) {
receipts = chain.receipts[i]
td = chain.tds[i]
)
if err = builder.Add(header, body, receipts, td, nil); err != nil {
if err = builder.Add(types.NewBlockWithHeader(&header).WithBody(body), receipts, td, nil); err != nil {
t.Fatalf("error adding entry: %v", err)
}
}

213
internal/era2/iterator.go Normal file
View file

@ -0,0 +1,213 @@
// Copyright 2024 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package era2
import (
"errors"
"io"
"math/big"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/internal/era/e2store"
"github.com/ethereum/go-ethereum/rlp"
"github.com/klauspost/compress/snappy"
)
type Iterator struct {
inner *RawIterator
}
// NewIterator returns a header/body/receipt iterator over the archive.
// Call Next immediately to position on the first block.
func NewIterator(e *Era) (*Iterator, error) {
inner, err := NewRawIterator(e)
if err != nil {
return nil, err
}
return &Iterator{inner: inner}, nil
}
// Next advances to the next block entry.
func (it *Iterator) Next() bool { return it.inner.Next() }
// Number is the number of the block currently loaded.
func (it *Iterator) Number() uint64 { return it.inner.next - 1 }
// Error returns any iteration error (EOF is reported as nil, identical
// to the Era‑1 iterator behaviour).
func (it *Iterator) Error() error { return it.inner.Error() }
// Block decodes the current header+body into a *types.Block.
func (it *Iterator) Block() (*types.Block, error) {
if it.inner.Header == nil || it.inner.Body == nil {
return nil, errors.New("header and body must be non‑nil")
}
var (
h types.Header
b types.Body
)
if err := rlp.Decode(it.inner.Header, &h); err != nil {
return nil, err
}
if err := rlp.Decode(it.inner.Body, &b); err != nil {
return nil, err
}
return types.NewBlockWithHeader(&h).WithBody(b), nil
}
// Receipts decodes receipts for the current block.
func (it *Iterator) Receipts() (types.Receipts, error) {
if it.inner.Receipts == nil {
return nil, errors.New("receipts must be non‑nil")
}
var r types.Receipts
return r, rlp.Decode(it.inner.Receipts, &r)
}
// BlockAndReceipts is a convenience wrapper.
func (it *Iterator) BlockAndReceipts() (*types.Block, types.Receipts, error) {
b, err := it.Block()
if err != nil {
return nil, nil, err
}
r, err := it.Receipts()
if err != nil {
return nil, nil, err
}
return b, r, nil
}
// TotalDifficulty returns the TD at the current position (if present).
func (it *Iterator) TotalDifficulty() (*big.Int, error) {
if it.inner.TotalDifficulty == nil {
return nil, errors.New("total‑difficulty stream is nil")
}
tdBytes, err := io.ReadAll(it.inner.TotalDifficulty)
if err != nil {
return nil, err
}
return new(big.Int).SetBytes(reverseOrder(tdBytes)), nil
}
// -----------------------------------------------------------------------------
// Low‑level iterator (raw TLV/offset handling, no decoding)
// -----------------------------------------------------------------------------
type RawIterator struct {
e *Era
next uint64 // next block to pull
err error
Header io.Reader
Body io.Reader
Receipts io.Reader
TotalDifficulty io.Reader // nil when archive omits TD
}
// NewRawIterator creates an iterator positioned *before* the first block.
func NewRawIterator(e *Era) (*RawIterator, error) {
return &RawIterator{e: e, next: e.m.start}, nil
}
// Next loads the next block’s components; returns false on EOF or error.
func (it *RawIterator) Next() bool {
it.err = nil // clear previous error
if it.next >= it.e.m.start+it.e.m.count {
it.clear()
return false
}
headerOffset, err := it.e.headerOff(it.next)
if err != nil {
it.setErr(err)
return false
}
it.Header, _, err = newSnappyReader(it.e.s, TypeCompressedHeader, int64(headerOffset))
if err != nil {
it.setErr(err)
return false
}
bodyOffset, err := it.e.bodyOff(it.next)
if err != nil {
it.setErr(err)
return false
}
it.Body, _, err = newSnappyReader(it.e.s, TypeCompressedBody, int64(bodyOffset))
if err != nil {
it.setErr(err)
return false
}
receiptsOffset, err := it.e.receiptOff(it.next)
if err != nil {
it.setErr(err)
return false
}
it.Receipts, _, err = newSnappyReader(it.e.s, TypeCompressedReceipts, int64(receiptsOffset))
if err != nil {
it.setErr(err)
return false
}
if int(header)+3 < int(it.e.m.components) {
tdOffset, err := it.e.tdOff(it.next)
if err != nil {
it.setErr(err)
return false
}
it.TotalDifficulty, _, err = it.e.s.ReaderAt(TypeTotalDifficulty, int64(tdOffset))
if err != nil {
it.setErr(err)
return false
}
} else {
it.TotalDifficulty = nil
}
it.next++
return true
}
func (it *RawIterator) Number() uint64 { return it.next - 1 }
func (it *RawIterator) Error() error {
if it.err == io.EOF {
return nil
}
return it.err
}
func (it *RawIterator) setErr(err error) {
it.err = err
it.clear()
}
func (it *RawIterator) clear() {
it.Header, it.Body, it.Receipts, it.TotalDifficulty = nil, nil, nil, nil
}
// newSnappyReader behaves like era.newSnappyReader: returns a snappy.Reader
// plus the length of the underlying TLV payload so callers can advance offsets.
func newSnappyReader(r *e2store.Reader, typ uint16, off int64) (io.Reader, int64, error) {
raw, n, err := r.ReaderAt(typ, off)
if err != nil {
return nil, 0, err
}
return snappy.NewReader(raw), int64(n), nil
}