mirror of
https://github.com/ethereum/go-ethereum.git
synced 2026-07-21 20:26:41 +00:00
added some abstraction
This commit is contained in:
parent
a8168195be
commit
c50b4043f4
8 changed files with 370 additions and 88 deletions
|
|
@ -519,11 +519,11 @@ func importHistory(ctx *cli.Context) error {
|
||||||
format := ctx.String(utils.EraFormatFlag.Name)
|
format := ctx.String(utils.EraFormatFlag.Name)
|
||||||
switch format {
|
switch format {
|
||||||
case "era1", "era":
|
case "era1", "era":
|
||||||
if err := utils.ImportHistory(chain, dir, network); err != nil {
|
if err := utils.ImportHistory(chain, dir, network, utils.Era1); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
case "erae":
|
case "erae":
|
||||||
if err := utils.ImportHistoryEraE(chain, dir, network); err != nil {
|
if err := utils.ImportHistory(chain, dir, network, utils.EraE); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
default:
|
default:
|
||||||
|
|
|
||||||
121
cmd/utils/cmd.go
121
cmd/utils/cmd.go
|
|
@ -44,7 +44,6 @@ import (
|
||||||
"github.com/ethereum/go-ethereum/eth/ethconfig"
|
"github.com/ethereum/go-ethereum/eth/ethconfig"
|
||||||
"github.com/ethereum/go-ethereum/ethdb"
|
"github.com/ethereum/go-ethereum/ethdb"
|
||||||
"github.com/ethereum/go-ethereum/internal/debug"
|
"github.com/ethereum/go-ethereum/internal/debug"
|
||||||
"github.com/ethereum/go-ethereum/internal/era"
|
|
||||||
"github.com/ethereum/go-ethereum/internal/era2"
|
"github.com/ethereum/go-ethereum/internal/era2"
|
||||||
"github.com/ethereum/go-ethereum/log"
|
"github.com/ethereum/go-ethereum/log"
|
||||||
"github.com/ethereum/go-ethereum/node"
|
"github.com/ethereum/go-ethereum/node"
|
||||||
|
|
@ -58,12 +57,7 @@ const (
|
||||||
importBatchSize = 2500
|
importBatchSize = 2500
|
||||||
)
|
)
|
||||||
|
|
||||||
type ExportFormat int
|
type EraFileFormat int
|
||||||
|
|
||||||
const (
|
|
||||||
Era1 ExportFormat = iota
|
|
||||||
EraE
|
|
||||||
)
|
|
||||||
|
|
||||||
// ErrImportInterrupted is returned when the user interrupts the import process.
|
// ErrImportInterrupted is returned when the user interrupts the import process.
|
||||||
var ErrImportInterrupted = errors.New("interrupted")
|
var ErrImportInterrupted = errors.New("interrupted")
|
||||||
|
|
@ -258,11 +252,11 @@ func readList(filename string) ([]string, error) {
|
||||||
// ImportHistory imports Era1 files containing historical block information,
|
// ImportHistory imports Era1 files containing historical block information,
|
||||||
// starting from genesis. The assumption is held that the provided chain
|
// starting from genesis. The assumption is held that the provided chain
|
||||||
// segment in Era1 file should all be canonical and verified.
|
// segment in Era1 file should all be canonical and verified.
|
||||||
func ImportHistory(chain *core.BlockChain, dir string, network string) error {
|
func ImportHistory(chain *core.BlockChain, dir string, network string, format Format) error {
|
||||||
if chain.CurrentSnapBlock().Number.BitLen() != 0 {
|
if chain.CurrentSnapBlock().Number.BitLen() != 0 {
|
||||||
return errors.New("history import only supported when starting from genesis")
|
return errors.New("history import only supported when starting from genesis")
|
||||||
}
|
}
|
||||||
entries, err := era.ReadDir(dir, network)
|
entries, err := format.ReadDir(dir, network)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("error reading %s: %w", dir, err)
|
return fmt.Errorf("error reading %s: %w", dir, err)
|
||||||
}
|
}
|
||||||
|
|
@ -271,42 +265,50 @@ func ImportHistory(chain *core.BlockChain, dir string, network string) error {
|
||||||
return fmt.Errorf("unable to read checksums.txt: %w", err)
|
return fmt.Errorf("unable to read checksums.txt: %w", err)
|
||||||
}
|
}
|
||||||
if len(checksums) != len(entries) {
|
if len(checksums) != len(entries) {
|
||||||
return fmt.Errorf("expected equal number of checksums and entries, have: %d checksums, %d entries", len(checksums), len(entries))
|
return fmt.Errorf("expected equal number of checksums and entries, have: %d checksums, %d entries",
|
||||||
|
len(checksums), len(entries))
|
||||||
}
|
}
|
||||||
|
|
||||||
var (
|
var (
|
||||||
start = time.Now()
|
start = time.Now()
|
||||||
reported = time.Now()
|
reported = time.Now()
|
||||||
imported = 0
|
imported = 0
|
||||||
h = sha256.New()
|
h = sha256.New()
|
||||||
buf = bytes.NewBuffer(nil)
|
scratch = bytes.NewBuffer(nil)
|
||||||
)
|
)
|
||||||
for i, filename := range entries {
|
|
||||||
|
for i, file := range entries {
|
||||||
err := func() error {
|
err := func() error {
|
||||||
f, err := os.Open(filepath.Join(dir, filename))
|
path := filepath.Join(dir, file)
|
||||||
if err != nil {
|
|
||||||
return fmt.Errorf("unable to open era: %w", err)
|
|
||||||
}
|
|
||||||
defer f.Close()
|
|
||||||
|
|
||||||
// Validate checksum.
|
// validate against checksum file in directory
|
||||||
|
f, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("open %s: %w", path, err)
|
||||||
|
}
|
||||||
if _, err := io.Copy(h, f); err != nil {
|
if _, err := io.Copy(h, f); err != nil {
|
||||||
return fmt.Errorf("unable to recalculate checksum: %w", err)
|
f.Close()
|
||||||
}
|
return fmt.Errorf("checksum %s: %w", path, err)
|
||||||
if have, want := common.BytesToHash(h.Sum(buf.Bytes()[:])).Hex(), checksums[i]; have != want {
|
|
||||||
return fmt.Errorf("checksum mismatch: have %s, want %s", have, want)
|
|
||||||
}
|
}
|
||||||
|
got := common.BytesToHash(h.Sum(scratch.Bytes()[:])).Hex()
|
||||||
|
want := checksums[i]
|
||||||
h.Reset()
|
h.Reset()
|
||||||
buf.Reset()
|
scratch.Reset()
|
||||||
|
|
||||||
// Import all block data from Era1.
|
if got != want {
|
||||||
e, err := era.From(f)
|
f.Close()
|
||||||
if err != nil {
|
return fmt.Errorf("%s checksum mismatch: have %s want %s", file, got, want)
|
||||||
return fmt.Errorf("error opening era: %w", err)
|
|
||||||
}
|
}
|
||||||
it, err := era.NewIterator(e)
|
// rewind for reading
|
||||||
if err != nil {
|
if _, err := f.Seek(0, io.SeekStart); err != nil {
|
||||||
return fmt.Errorf("error making era reader: %w", err)
|
f.Close()
|
||||||
|
return fmt.Errorf("rewind %s: %w", file, err)
|
||||||
}
|
}
|
||||||
|
it, err := format.NewIterator(f)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("error creating iterator: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
for it.Next() {
|
for it.Next() {
|
||||||
block, err := it.Block()
|
block, err := it.Block()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|
@ -315,30 +317,32 @@ func ImportHistory(chain *core.BlockChain, dir string, network string) error {
|
||||||
if block.Number().BitLen() == 0 {
|
if block.Number().BitLen() == 0 {
|
||||||
continue // skip genesis
|
continue // skip genesis
|
||||||
}
|
}
|
||||||
receipts, err := it.Receipts()
|
rcpts, err := it.Receipts()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("error reading receipts %d: %w", it.Number(), err)
|
return fmt.Errorf("error reading receipts %d: %w", it.Number(), err)
|
||||||
}
|
}
|
||||||
encReceipts := types.EncodeBlockReceiptLists([]types.Receipts{receipts})
|
enc := types.EncodeBlockReceiptLists([]types.Receipts{rcpts})
|
||||||
if _, err := chain.InsertReceiptChain([]*types.Block{block}, encReceipts, math.MaxUint64); err != nil {
|
if _, err := chain.InsertReceiptChain([]*types.Block{block}, enc, math.MaxUint64); err != nil {
|
||||||
return fmt.Errorf("error inserting body %d: %w", it.Number(), err)
|
return fmt.Errorf("error inserting body %d: %w", it.Number(), err)
|
||||||
}
|
}
|
||||||
imported += 1
|
imported++
|
||||||
|
|
||||||
// Give the user some feedback that something is happening.
|
|
||||||
if time.Since(reported) >= 8*time.Second {
|
if time.Since(reported) >= 8*time.Second {
|
||||||
log.Info("Importing Era files", "head", it.Number(), "imported", imported, "elapsed", common.PrettyDuration(time.Since(start)))
|
log.Info("Importing Era files", "head", it.Number(), "imported", imported,
|
||||||
|
"elapsed", common.PrettyDuration(time.Since(start)))
|
||||||
imported = 0
|
imported = 0
|
||||||
reported = time.Now()
|
reported = time.Now()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
if err := it.Error(); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}()
|
}()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -511,8 +515,8 @@ func ExportAppendChain(blockchain *core.BlockChain, fn string, first uint64, las
|
||||||
|
|
||||||
// ExportHistory exports blockchain history into the specified directory,
|
// ExportHistory exports blockchain history into the specified directory,
|
||||||
// following the Era format.
|
// following the Era format.
|
||||||
func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64, f ExportFormat) error {
|
func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64, f Format) error {
|
||||||
log.Info("Exporting blockchain history", "dir", dir, "format", f)
|
log.Info("Exporting blockchain history", "dir", dir)
|
||||||
if head := bc.CurrentBlock().Number.Uint64(); head < last {
|
if head := bc.CurrentBlock().Number.Uint64(); head < last {
|
||||||
log.Warn("Last block beyond head, setting last = head", "head", head, "last", last)
|
log.Warn("Last block beyond head, setting last = head", "head", head, "last", last)
|
||||||
last = head
|
last = head
|
||||||
|
|
@ -525,26 +529,6 @@ func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64, f
|
||||||
return fmt.Errorf("error creating output directory: %w", err)
|
return fmt.Errorf("error creating output directory: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
var (
|
|
||||||
filename func(string, int, common.Hash) string
|
|
||||||
newBuilder func(io.Writer) any
|
|
||||||
add func(any, *types.Block, types.Receipts, *big.Int) error
|
|
||||||
)
|
|
||||||
|
|
||||||
if f == Era1 {
|
|
||||||
filename = era.Filename
|
|
||||||
newBuilder = func(w io.Writer) any { return era.NewBuilder(w) }
|
|
||||||
add = func(b any, blk *types.Block, rcpt types.Receipts, td *big.Int) error {
|
|
||||||
return b.(*era.Builder).Add(blk, rcpt, td)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
filename = era2.Filename
|
|
||||||
newBuilder = func(w io.Writer) any { return era2.NewBuilder(w) }
|
|
||||||
add = func(b any, blk *types.Block, rcpt types.Receipts, td *big.Int) error {
|
|
||||||
return b.(*era2.Builder).Add(*blk.Header(), *blk.Body(), rcpt, td, nil)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
td := new(big.Int)
|
td := new(big.Int)
|
||||||
for n := uint64(0); n < first; n++ {
|
for n := uint64(0); n < first; n++ {
|
||||||
td.Add(td, bc.GetHeaderByNumber(n).Difficulty)
|
td.Add(td, bc.GetHeaderByNumber(n).Difficulty)
|
||||||
|
|
@ -560,16 +544,16 @@ func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64, f
|
||||||
|
|
||||||
for batch := first; batch <= last; batch += step {
|
for batch := first; batch <= last; batch += step {
|
||||||
idx := int(batch / step)
|
idx := int(batch / step)
|
||||||
tmpPath := filepath.Join(dir, filename(network, idx, common.Hash{}))
|
tmpPath := filepath.Join(dir, f.Filename(network, idx, common.Hash{}))
|
||||||
|
|
||||||
if err := func() error {
|
if err := func() error {
|
||||||
f, err := os.Create(tmpPath)
|
fh, err := os.Create(tmpPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
defer f.Close()
|
defer fh.Close()
|
||||||
|
|
||||||
bldr := newBuilder(f)
|
bldr := f.NewBuilder(fh)
|
||||||
|
|
||||||
for j := uint64(0); j < step && batch+j <= last; j++ {
|
for j := uint64(0); j < step && batch+j <= last; j++ {
|
||||||
n := batch + j
|
n := batch + j
|
||||||
|
|
@ -582,26 +566,27 @@ func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64, f
|
||||||
return fmt.Errorf("receipts for #%d missing", n)
|
return fmt.Errorf("receipts for #%d missing", n)
|
||||||
}
|
}
|
||||||
td.Add(td, blk.Difficulty())
|
td.Add(td, blk.Difficulty())
|
||||||
if err := add(bldr, blk, rcpt, new(big.Int).Set(td)); err != nil {
|
|
||||||
|
if err := bldr.Add(blk, rcpt, new(big.Int).Set(td), nil); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
root, err := bldr.(interface{ Finalize() (common.Hash, error) }).Finalize()
|
root, err := bldr.Finalize()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
final := filepath.Join(dir, filename(network, idx, root))
|
final := filepath.Join(dir, f.Filename(network, idx, root))
|
||||||
if err := os.Rename(tmpPath, final); err != nil {
|
if err := os.Rename(tmpPath, final); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, err := f.Seek(0, io.SeekStart); err != nil {
|
if _, err := fh.Seek(0, io.SeekStart); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
h.Reset()
|
h.Reset()
|
||||||
buf.Reset()
|
buf.Reset()
|
||||||
if _, err := io.Copy(h, f); err != nil {
|
if _, err := io.Copy(h, fh); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
checksums = append(checksums, common.BytesToHash(h.Sum(buf.Bytes()[:])).Hex())
|
checksums = append(checksums, common.BytesToHash(h.Sum(buf.Bytes()[:])).Hex())
|
||||||
|
|
|
||||||
82
cmd/utils/era_interface.go
Normal file
82
cmd/utils/era_interface.go
Normal file
|
|
@ -0,0 +1,82 @@
|
||||||
|
package utils
|
||||||
|
|
||||||
|
import (
|
||||||
|
"io"
|
||||||
|
"math/big"
|
||||||
|
"os"
|
||||||
|
|
||||||
|
"github.com/ethereum/go-ethereum/common"
|
||||||
|
"github.com/ethereum/go-ethereum/core/types"
|
||||||
|
"github.com/ethereum/go-ethereum/internal/era"
|
||||||
|
"github.com/ethereum/go-ethereum/internal/era2"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Iterator is the common, read‑only view returned by both Era‑1 and Era‑E.
|
||||||
|
type Iterator interface {
|
||||||
|
Next() bool
|
||||||
|
Number() uint64
|
||||||
|
Block() (*types.Block, error)
|
||||||
|
Receipts() (types.Receipts, error)
|
||||||
|
Error() error
|
||||||
|
}
|
||||||
|
|
||||||
|
// Builder (unchanged) – writes one archive file.
|
||||||
|
type Builder interface {
|
||||||
|
Add(blk *types.Block, rcpts types.Receipts, td *big.Int, proof era2.Proof) error
|
||||||
|
Finalize() (common.Hash, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Format now encapsulates *all* format‑specific behaviour, both read & write.
|
||||||
|
type Format interface {
|
||||||
|
// ----- writer side -----
|
||||||
|
Filename(network string, epoch int, root common.Hash) string
|
||||||
|
NewBuilder(w io.Writer) Builder
|
||||||
|
|
||||||
|
// ----- reader side -----
|
||||||
|
ReadDir(dir, network string) ([]string, error)
|
||||||
|
NewIterator(f *os.File) (Iterator, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
type era1Builder struct{ *era.Builder }
|
||||||
|
|
||||||
|
func (b *era1Builder) Add(blk *types.Block, rc types.Receipts, td *big.Int, proof era2.Proof) error {
|
||||||
|
return b.Builder.Add(blk, rc, td)
|
||||||
|
}
|
||||||
|
|
||||||
|
type era1Format struct{}
|
||||||
|
|
||||||
|
func (era1Format) Filename(n string, e int, h common.Hash) string { return era.Filename(n, e, h) }
|
||||||
|
func (era1Format) NewBuilder(w io.Writer) Builder { return &era1Builder{era.NewBuilder(w)} }
|
||||||
|
func (era1Format) ReadDir(dir, net string) ([]string, error) { return era.ReadDir(dir, net) }
|
||||||
|
func (era1Format) NewIterator(f *os.File) (Iterator, error) {
|
||||||
|
e, err := era.From(f)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return era.NewIterator(e)
|
||||||
|
}
|
||||||
|
|
||||||
|
var Era1 Format = era1Format{}
|
||||||
|
|
||||||
|
type eraeBuilder struct{ *era2.Builder }
|
||||||
|
|
||||||
|
func (b *eraeBuilder) Add(blk *types.Block, rc types.Receipts, td *big.Int, proof era2.Proof) error {
|
||||||
|
return b.Builder.Add(blk, rc, td, nil) // no proofs yet
|
||||||
|
}
|
||||||
|
|
||||||
|
type eraeFormat struct{}
|
||||||
|
|
||||||
|
func (eraeFormat) Filename(n string, e int, h common.Hash) string { return era2.Filename(n, e, h) }
|
||||||
|
func (eraeFormat) NewBuilder(w io.Writer) Builder { return &eraeBuilder{era2.NewBuilder(w)} }
|
||||||
|
func (eraeFormat) ReadDir(dir, net string) ([]string, error) { return era2.ReadDir(dir, net) }
|
||||||
|
func (eraeFormat) NewIterator(f *os.File) (Iterator, error) {
|
||||||
|
e, err := era2.From(f)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return era2.NewIterator(e)
|
||||||
|
}
|
||||||
|
|
||||||
|
var EraE Format = eraeFormat{}
|
||||||
|
|
||||||
|
// Exported singleton
|
||||||
|
|
@ -32,7 +32,7 @@ import (
|
||||||
"github.com/ethereum/go-ethereum/core/rawdb"
|
"github.com/ethereum/go-ethereum/core/rawdb"
|
||||||
"github.com/ethereum/go-ethereum/core/types"
|
"github.com/ethereum/go-ethereum/core/types"
|
||||||
"github.com/ethereum/go-ethereum/crypto"
|
"github.com/ethereum/go-ethereum/crypto"
|
||||||
"github.com/ethereum/go-ethereum/internal/era"
|
"github.com/ethereum/go-ethereum/internal/era2"
|
||||||
"github.com/ethereum/go-ethereum/params"
|
"github.com/ethereum/go-ethereum/params"
|
||||||
"github.com/ethereum/go-ethereum/trie"
|
"github.com/ethereum/go-ethereum/trie"
|
||||||
"github.com/ethereum/go-ethereum/triedb"
|
"github.com/ethereum/go-ethereum/triedb"
|
||||||
|
|
@ -89,7 +89,7 @@ func TestHistoryImportAndExport(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
|
|
||||||
// Export history to temp directory.
|
// Export history to temp directory.
|
||||||
if err := ExportHistory(chain, dir, 0, count, step, EraE); err != nil {
|
if err := ExportHistory(chain, dir, 0, count, step, Era1); err != nil {
|
||||||
t.Fatalf("error exporting history: %v", err)
|
t.Fatalf("error exporting history: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -101,7 +101,7 @@ func TestHistoryImportAndExport(t *testing.T) {
|
||||||
checksums := strings.Split(string(b), "\n")
|
checksums := strings.Split(string(b), "\n")
|
||||||
|
|
||||||
// Verify each Era.
|
// Verify each Era.
|
||||||
entries, _ := era.ReadDir(dir, "mainnet")
|
entries, _ := era2.ReadDir(dir, "mainnet")
|
||||||
for i, filename := range entries {
|
for i, filename := range entries {
|
||||||
func() {
|
func() {
|
||||||
f, err := os.Open(filepath.Join(dir, filename))
|
f, err := os.Open(filepath.Join(dir, filename))
|
||||||
|
|
@ -118,12 +118,12 @@ func TestHistoryImportAndExport(t *testing.T) {
|
||||||
if got, want := common.BytesToHash(h.Sum(buf.Bytes()[:])).Hex(), checksums[i]; got != want {
|
if got, want := common.BytesToHash(h.Sum(buf.Bytes()[:])).Hex(), checksums[i]; got != want {
|
||||||
t.Fatalf("checksum %d does not match: got %s, want %s", i, got, want)
|
t.Fatalf("checksum %d does not match: got %s, want %s", i, got, want)
|
||||||
}
|
}
|
||||||
e, err := era.From(f)
|
e, err := era2.From(f)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("error opening era: %v", err)
|
t.Fatalf("error opening era: %v", err)
|
||||||
}
|
}
|
||||||
defer e.Close()
|
defer e.Close()
|
||||||
it, err := era.NewIterator(e)
|
it, err := era2.NewIterator(e)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("error making era reader: %v", err)
|
t.Fatalf("error making era reader: %v", err)
|
||||||
}
|
}
|
||||||
|
|
@ -170,7 +170,7 @@ func TestHistoryImportAndExport(t *testing.T) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("unable to initialize chain: %v", err)
|
t.Fatalf("unable to initialize chain: %v", err)
|
||||||
}
|
}
|
||||||
if err := ImportHistory(imported, dir, "mainnet"); err != nil {
|
if err := ImportHistory(imported, dir, "mainnet", EraE); err != nil {
|
||||||
t.Fatalf("failed to import chain: %v", err)
|
t.Fatalf("failed to import chain: %v", err)
|
||||||
}
|
}
|
||||||
if have, want := imported.CurrentHeader(), chain.CurrentHeader(); have.Hash() != want.Hash() {
|
if have, want := imported.CurrentHeader(), chain.CurrentHeader(); have.Hash() != want.Hash() {
|
||||||
|
|
|
||||||
|
|
@ -113,7 +113,9 @@ func NewBuilder(w io.Writer) *Builder {
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add writes a block entry, its reciepts, and optionally its proofs as well into the e2store file.
|
// Add writes a block entry, its reciepts, and optionally its proofs as well into the e2store file.
|
||||||
func (b *Builder) Add(header types.Header, body types.Body, receipts types.Receipts, td *big.Int, proof Proof) error {
|
func (b *Builder) Add(block *types.Block, receipts types.Receipts, td *big.Int, proof Proof) error {
|
||||||
|
header := block.Header()
|
||||||
|
body := block.Body()
|
||||||
if len(b.buff.headers) == 0 { // first block determines wether proofs are expected
|
if len(b.buff.headers) == 0 { // first block determines wether proofs are expected
|
||||||
b.expectsProofs = proof != nil
|
b.expectsProofs = proof != nil
|
||||||
} else if b.expectsProofs && proof == nil { // every later block must follow this policy
|
} else if b.expectsProofs && proof == nil { // every later block must follow this policy
|
||||||
|
|
|
||||||
|
|
@ -46,11 +46,11 @@ type componentType int
|
||||||
|
|
||||||
// TypeCompressedHeader, TypeCompressedBody, TypeCompressedReceipts, TypeTotalDifficulty, and TypeProof are the different types of components that can be present in the era file.
|
// TypeCompressedHeader, TypeCompressedBody, TypeCompressedReceipts, TypeTotalDifficulty, and TypeProof are the different types of components that can be present in the era file.
|
||||||
const (
|
const (
|
||||||
componentHeader componentType = iota
|
header componentType = iota
|
||||||
componentBody
|
body
|
||||||
componentReceipts
|
receipts
|
||||||
componentTD
|
td
|
||||||
componentProof
|
proof
|
||||||
)
|
)
|
||||||
|
|
||||||
type ReadAtSeekCloser interface {
|
type ReadAtSeekCloser interface {
|
||||||
|
|
@ -274,11 +274,11 @@ func (e *Era) loadIndex() error {
|
||||||
}
|
}
|
||||||
|
|
||||||
// headerOff, bodyOff, receiptOff, tdOff, and proofOff return the offsets of the respective components for a given block number.
|
// headerOff, bodyOff, receiptOff, tdOff, and proofOff return the offsets of the respective components for a given block number.
|
||||||
func (e *Era) headerOff(num uint64) (uint64, error) { return e.indexOffset(num, componentHeader) }
|
func (e *Era) headerOff(num uint64) (uint64, error) { return e.indexOffset(num, header) }
|
||||||
func (e *Era) bodyOff(num uint64) (uint64, error) { return e.indexOffset(num, componentBody) }
|
func (e *Era) bodyOff(num uint64) (uint64, error) { return e.indexOffset(num, body) }
|
||||||
func (e *Era) receiptOff(num uint64) (uint64, error) { return e.indexOffset(num, componentReceipts) }
|
func (e *Era) receiptOff(num uint64) (uint64, error) { return e.indexOffset(num, receipts) }
|
||||||
func (e *Era) tdOff(num uint64) (uint64, error) { return e.indexOffset(num, componentTD) }
|
func (e *Era) tdOff(num uint64) (uint64, error) { return e.indexOffset(num, td) }
|
||||||
func (e *Era) proofOff(num uint64) (uint64, error) { return e.indexOffset(num, componentProof) }
|
func (e *Era) proofOff(num uint64) (uint64, error) { return e.indexOffset(num, proof) }
|
||||||
|
|
||||||
// indexOffset calculates offset to a certain component for a block number within a file.
|
// indexOffset calculates offset to a certain component for a block number within a file.
|
||||||
func (e *Era) indexOffset(n uint64, component componentType) (uint64, error) {
|
func (e *Era) indexOffset(n uint64, component componentType) (uint64, error) {
|
||||||
|
|
|
||||||
|
|
@ -65,7 +65,7 @@ func TestEra2Builder(t *testing.T) {
|
||||||
receipts = chain.receipts[i]
|
receipts = chain.receipts[i]
|
||||||
td = chain.tds[i]
|
td = chain.tds[i]
|
||||||
)
|
)
|
||||||
if err = builder.Add(header, body, receipts, td, nil); err != nil {
|
if err = builder.Add(types.NewBlockWithHeader(&header).WithBody(body), receipts, td, nil); err != nil {
|
||||||
t.Fatalf("error adding entry: %v", err)
|
t.Fatalf("error adding entry: %v", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
213
internal/era2/iterator.go
Normal file
213
internal/era2/iterator.go
Normal file
|
|
@ -0,0 +1,213 @@
|
||||||
|
// Copyright 2024 The go-ethereum Authors
|
||||||
|
// This file is part of the go-ethereum library.
|
||||||
|
//
|
||||||
|
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Lesser General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Lesser General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Lesser General Public License
|
||||||
|
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package era2
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"io"
|
||||||
|
"math/big"
|
||||||
|
|
||||||
|
"github.com/ethereum/go-ethereum/core/types"
|
||||||
|
"github.com/ethereum/go-ethereum/internal/era/e2store"
|
||||||
|
"github.com/ethereum/go-ethereum/rlp"
|
||||||
|
"github.com/klauspost/compress/snappy"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Iterator struct {
|
||||||
|
inner *RawIterator
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewIterator returns a header/body/receipt iterator over the archive.
|
||||||
|
// Call Next immediately to position on the first block.
|
||||||
|
func NewIterator(e *Era) (*Iterator, error) {
|
||||||
|
inner, err := NewRawIterator(e)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return &Iterator{inner: inner}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Next advances to the next block entry.
|
||||||
|
func (it *Iterator) Next() bool { return it.inner.Next() }
|
||||||
|
|
||||||
|
// Number is the number of the block currently loaded.
|
||||||
|
func (it *Iterator) Number() uint64 { return it.inner.next - 1 }
|
||||||
|
|
||||||
|
// Error returns any iteration error (EOF is reported as nil, identical
|
||||||
|
// to the Era‑1 iterator behaviour).
|
||||||
|
func (it *Iterator) Error() error { return it.inner.Error() }
|
||||||
|
|
||||||
|
// Block decodes the current header+body into a *types.Block.
|
||||||
|
func (it *Iterator) Block() (*types.Block, error) {
|
||||||
|
if it.inner.Header == nil || it.inner.Body == nil {
|
||||||
|
return nil, errors.New("header and body must be non‑nil")
|
||||||
|
}
|
||||||
|
var (
|
||||||
|
h types.Header
|
||||||
|
b types.Body
|
||||||
|
)
|
||||||
|
if err := rlp.Decode(it.inner.Header, &h); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := rlp.Decode(it.inner.Body, &b); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return types.NewBlockWithHeader(&h).WithBody(b), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Receipts decodes receipts for the current block.
|
||||||
|
func (it *Iterator) Receipts() (types.Receipts, error) {
|
||||||
|
if it.inner.Receipts == nil {
|
||||||
|
return nil, errors.New("receipts must be non‑nil")
|
||||||
|
}
|
||||||
|
var r types.Receipts
|
||||||
|
return r, rlp.Decode(it.inner.Receipts, &r)
|
||||||
|
}
|
||||||
|
|
||||||
|
// BlockAndReceipts is a convenience wrapper.
|
||||||
|
func (it *Iterator) BlockAndReceipts() (*types.Block, types.Receipts, error) {
|
||||||
|
b, err := it.Block()
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
r, err := it.Receipts()
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, err
|
||||||
|
}
|
||||||
|
return b, r, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// TotalDifficulty returns the TD at the current position (if present).
|
||||||
|
func (it *Iterator) TotalDifficulty() (*big.Int, error) {
|
||||||
|
if it.inner.TotalDifficulty == nil {
|
||||||
|
return nil, errors.New("total‑difficulty stream is nil")
|
||||||
|
}
|
||||||
|
tdBytes, err := io.ReadAll(it.inner.TotalDifficulty)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return new(big.Int).SetBytes(reverseOrder(tdBytes)), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// -----------------------------------------------------------------------------
|
||||||
|
// Low‑level iterator (raw TLV/offset handling, no decoding)
|
||||||
|
// -----------------------------------------------------------------------------
|
||||||
|
|
||||||
|
type RawIterator struct {
|
||||||
|
e *Era
|
||||||
|
next uint64 // next block to pull
|
||||||
|
err error
|
||||||
|
|
||||||
|
Header io.Reader
|
||||||
|
Body io.Reader
|
||||||
|
Receipts io.Reader
|
||||||
|
TotalDifficulty io.Reader // nil when archive omits TD
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewRawIterator creates an iterator positioned *before* the first block.
|
||||||
|
func NewRawIterator(e *Era) (*RawIterator, error) {
|
||||||
|
return &RawIterator{e: e, next: e.m.start}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Next loads the next block’s components; returns false on EOF or error.
|
||||||
|
func (it *RawIterator) Next() bool {
|
||||||
|
it.err = nil // clear previous error
|
||||||
|
|
||||||
|
if it.next >= it.e.m.start+it.e.m.count {
|
||||||
|
it.clear()
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
headerOffset, err := it.e.headerOff(it.next)
|
||||||
|
if err != nil {
|
||||||
|
it.setErr(err)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
it.Header, _, err = newSnappyReader(it.e.s, TypeCompressedHeader, int64(headerOffset))
|
||||||
|
if err != nil {
|
||||||
|
it.setErr(err)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
bodyOffset, err := it.e.bodyOff(it.next)
|
||||||
|
if err != nil {
|
||||||
|
it.setErr(err)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
it.Body, _, err = newSnappyReader(it.e.s, TypeCompressedBody, int64(bodyOffset))
|
||||||
|
if err != nil {
|
||||||
|
it.setErr(err)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
receiptsOffset, err := it.e.receiptOff(it.next)
|
||||||
|
if err != nil {
|
||||||
|
it.setErr(err)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
it.Receipts, _, err = newSnappyReader(it.e.s, TypeCompressedReceipts, int64(receiptsOffset))
|
||||||
|
if err != nil {
|
||||||
|
it.setErr(err)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
if int(header)+3 < int(it.e.m.components) {
|
||||||
|
tdOffset, err := it.e.tdOff(it.next)
|
||||||
|
if err != nil {
|
||||||
|
it.setErr(err)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
it.TotalDifficulty, _, err = it.e.s.ReaderAt(TypeTotalDifficulty, int64(tdOffset))
|
||||||
|
if err != nil {
|
||||||
|
it.setErr(err)
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
it.TotalDifficulty = nil
|
||||||
|
}
|
||||||
|
|
||||||
|
it.next++
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (it *RawIterator) Number() uint64 { return it.next - 1 }
|
||||||
|
|
||||||
|
func (it *RawIterator) Error() error {
|
||||||
|
if it.err == io.EOF {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return it.err
|
||||||
|
}
|
||||||
|
|
||||||
|
func (it *RawIterator) setErr(err error) {
|
||||||
|
it.err = err
|
||||||
|
it.clear()
|
||||||
|
}
|
||||||
|
|
||||||
|
func (it *RawIterator) clear() {
|
||||||
|
it.Header, it.Body, it.Receipts, it.TotalDifficulty = nil, nil, nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// newSnappyReader behaves like era.newSnappyReader: returns a snappy.Reader
|
||||||
|
// plus the length of the underlying TLV payload so callers can advance offsets.
|
||||||
|
func newSnappyReader(r *e2store.Reader, typ uint16, off int64) (io.Reader, int64, error) {
|
||||||
|
raw, n, err := r.ReaderAt(typ, off)
|
||||||
|
if err != nil {
|
||||||
|
return nil, 0, err
|
||||||
|
}
|
||||||
|
return snappy.NewReader(raw), int64(n), nil
|
||||||
|
}
|
||||||
Loading…
Reference in a new issue