built cmd

This commit is contained in:
shantichanal 2025-07-23 02:26:00 +02:00 committed by lightclient
parent 0de0cf6925
commit a8168195be
No known key found for this signature in database
GPG key ID: 657913021EF45A6A
5 changed files with 188 additions and 17 deletions

View file

@ -43,6 +43,7 @@ import (
"github.com/ethereum/go-ethereum/internal/debug" "github.com/ethereum/go-ethereum/internal/debug"
"github.com/ethereum/go-ethereum/internal/era" "github.com/ethereum/go-ethereum/internal/era"
"github.com/ethereum/go-ethereum/internal/era/eradl" "github.com/ethereum/go-ethereum/internal/era/eradl"
"github.com/ethereum/go-ethereum/internal/era2"
"github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/internal/flags"
"github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/node"
@ -152,7 +153,7 @@ be gzipped.`,
Name: "import-history", Name: "import-history",
Usage: "Import an Era archive", Usage: "Import an Era archive",
ArgsUsage: "<dir>", ArgsUsage: "<dir>",
Flags: slices.Concat([]cli.Flag{utils.TxLookupLimitFlag, utils.TransactionHistoryFlag}, utils.DatabaseFlags, utils.NetworkFlags), Flags: slices.Concat([]cli.Flag{utils.TxLookupLimitFlag, utils.TransactionHistoryFlag, utils.EraFormatFlag}, utils.DatabaseFlags, utils.NetworkFlags),
Description: ` Description: `
The import-history command will import blocks and their corresponding receipts The import-history command will import blocks and their corresponding receipts
from Era archives. from Era archives.
@ -163,7 +164,7 @@ from Era archives.
Name: "export-history", Name: "export-history",
Usage: "Export blockchain history to Era archives", Usage: "Export blockchain history to Era archives",
ArgsUsage: "<dir> <first> <last>", ArgsUsage: "<dir> <first> <last>",
Flags: utils.DatabaseFlags, Flags: slices.Concat([]cli.Flag{utils.EraFormatFlag}, utils.DatabaseFlags),
Description: ` Description: `
The export-history command will export blocks and their corresponding receipts The export-history command will export blocks and their corresponding receipts
into Era archives. Eras are typically packaged in steps of 8192 blocks. into Era archives. Eras are typically packaged in steps of 8192 blocks.
@ -515,15 +516,25 @@ func importHistory(ctx *cli.Context) error {
network = networks[0] network = networks[0]
} }
if err := utils.ImportHistory(chain, dir, network); err != nil { format := ctx.String(utils.EraFormatFlag.Name)
return err switch format {
case "era1", "era":
if err := utils.ImportHistory(chain, dir, network); err != nil {
return err
}
case "erae":
if err := utils.ImportHistoryEraE(chain, dir, network); err != nil {
return err
}
default:
return fmt.Errorf("unknown --era.format %q (expected 'era1' or 'erae')", format)
} }
fmt.Printf("Import done in %v\n", time.Since(start)) fmt.Printf("Import done in %v\n", time.Since(start))
return nil return nil
} }
// exportHistory exports chain history in Era archives at a specified // exportHistory exports chain history in Era archives at a specified directory.
// directory.
func exportHistory(ctx *cli.Context) error { func exportHistory(ctx *cli.Context) error {
if ctx.Args().Len() != 3 { if ctx.Args().Len() != 3 {
utils.Fatalf("usage: %s", ctx.Command.ArgsUsage) utils.Fatalf("usage: %s", ctx.Command.ArgsUsage)
@ -549,10 +560,21 @@ func exportHistory(ctx *cli.Context) error {
if head := chain.CurrentSnapBlock(); uint64(last) > head.Number.Uint64() { if head := chain.CurrentSnapBlock(); uint64(last) > head.Number.Uint64() {
utils.Fatalf("Export error: block number %d larger than head block %d\n", uint64(last), head.Number.Uint64()) utils.Fatalf("Export error: block number %d larger than head block %d\n", uint64(last), head.Number.Uint64())
} }
err := utils.ExportHistory(chain, dir, uint64(first), uint64(last), uint64(era.MaxEra1Size), utils.EraE)
if err != nil { format := ctx.String(utils.EraFormatFlag.Name)
utils.Fatalf("Export error: %v\n", err) switch format {
case "era1", "era":
if err := utils.ExportHistory(chain, dir, uint64(first), uint64(last), uint64(era.MaxEra1Size), utils.Era1); err != nil {
utils.Fatalf("Export error: %v\n", err)
}
case "erae":
if err := utils.ExportHistory(chain, dir, uint64(first), uint64(last), uint64(era2.MaxEraESize), utils.EraE); err != nil {
utils.Fatalf("Export error: %v\n", err)
}
default:
return fmt.Errorf("unknown archive format %q (use 'era1' or 'erae')", format)
} }
fmt.Printf("Export done in %v\n", time.Since(start)) fmt.Printf("Export done in %v\n", time.Since(start))
return nil return nil
} }

View file

@ -342,6 +342,105 @@ func ImportHistory(chain *core.BlockChain, dir string, network string) error {
return nil return nil
} }
// ImportHistoryEraE imports Era-E files containing historical block information,
// starting from genesis. Currently this function follows the assumptions that the provided chain
// segment in Era-E file should all be canonical and verified. In the future, this function will be
// extended to support importing Era-E files with proof verification.
func ImportHistoryEraE(chain *core.BlockChain, dir, network string) error {
if chain.CurrentSnapBlock().Number.Sign() != 0 {
return errors.New("history import only supported when starting from genesis")
}
entries, err := era2.ReadDir(dir, network)
if err != nil {
return fmt.Errorf("reading %q: %w", dir, err)
}
checksums, err := readList(filepath.Join(dir, "checksums.txt"))
if err != nil {
return fmt.Errorf("reading checksums.txt: %w", err)
}
if len(entries) != len(checksums) {
return fmt.Errorf("mismatch: %d erae files, %d checksums", len(entries), len(checksums))
}
var (
start = time.Now()
reported = time.Now()
imported int
h = sha256.New()
scratch bytes.Buffer
)
for i, file := range entries {
path := filepath.Join(dir, file)
// validate against checksum file in directory
f, err := os.Open(path)
if err != nil {
return fmt.Errorf("open %s: %w", path, err)
}
if _, err := io.Copy(h, f); err != nil {
f.Close()
return fmt.Errorf("checksum %s: %w", path, err)
}
got := common.BytesToHash(h.Sum(scratch.Bytes()[:])).Hex()
want := checksums[i]
h.Reset()
scratch.Reset()
if got != want {
f.Close()
return fmt.Errorf("%s checksum mismatch: have %s want %s", file, got, want)
}
// rewind for reading
if _, err := f.Seek(0, io.SeekStart); err != nil {
f.Close()
return fmt.Errorf("rewind %s: %w", file, err)
}
// import archive
e, err := era2.From(f)
if err != nil {
f.Close()
return fmt.Errorf("open erae %s: %w", file, err)
}
blockCount := e.Count()
for j := uint64(0); j < blockCount; j++ {
hdr, err := e.GetHeader(e.Start() + j)
if err != nil {
return fmt.Errorf("header #%d: %w", hdr.Number.Uint64(), err)
}
if hdr.Number.Sign() == 0 { // skip genesis
continue
}
body, err := e.GetBody(hdr.Number.Uint64())
if err != nil {
return fmt.Errorf("body #%d: %w", hdr.Number.Uint64(), err)
}
rcpts, err := e.GetReceipts(hdr.Number.Uint64(), 1)
if err != nil {
return fmt.Errorf("receipts #%d: %w", hdr.Number.Uint64(), err)
}
blk := types.NewBlockWithHeader(hdr).WithBody(*body)
enc := types.EncodeBlockReceiptLists(rcpts)
if _, err := chain.InsertReceiptChain([]*types.Block{blk}, enc, ^uint64(0)); err != nil {
return fmt.Errorf("insert #%d: %w", hdr.Number.Uint64(), err)
}
imported++
if time.Since(reported) >= 8*time.Second {
log.Info("Importing EraE", "head", hdr.Number, "imported", imported, "elapsed", common.PrettyDuration(time.Since(start)))
reported = time.Now()
imported = 0
}
}
f.Close()
}
log.Info("EraE import complete", "duration", common.PrettyDuration(time.Since(start)))
return nil
}
func missingBlocks(chain *core.BlockChain, blocks []*types.Block) []*types.Block { func missingBlocks(chain *core.BlockChain, blocks []*types.Block) []*types.Block {
head := chain.CurrentBlock() head := chain.CurrentBlock()
for i, block := range blocks { for i, block := range blocks {

View file

@ -114,6 +114,12 @@ var (
Usage: "Root directory for era1 history (default = inside ancient/chain)", Usage: "Root directory for era1 history (default = inside ancient/chain)",
Category: flags.EthCategory, Category: flags.EthCategory,
} }
EraFormatFlag = &cli.StringFlag{
Name: "era.format",
Usage: "Archive format to import: 'era1' or 'erae'",
Value: "era1",
Category: flags.EthCategory,
}
MinFreeDiskSpaceFlag = &cli.IntFlag{ MinFreeDiskSpaceFlag = &cli.IntFlag{
Name: "datadir.minfreedisk", Name: "datadir.minfreedisk",
Usage: "Minimum free disk space in MB, once reached triggers auto shut down (default = --cache.gc converted to MB, 0 = disabled)", Usage: "Minimum free disk space in MB, once reached triggers auto shut down (default = --cache.gc converted to MB, 0 = disabled)",

View file

@ -92,7 +92,7 @@ type offsets struct {
// Builder is used to build an Era2 e2store file. It collects block entries and writes them to the underlying e2store.Writer. // Builder is used to build an Era2 e2store file. It collects block entries and writes them to the underlying e2store.Writer.
type Builder struct { type Builder struct {
w *e2store.Writer w *e2store.Writer
buf *bytes.Buffer tmp *bytes.Buffer
buff buffer buff buffer
off offsets off offsets
@ -105,10 +105,10 @@ type Builder struct {
// NewBuilder returns a new Builder instance. // NewBuilder returns a new Builder instance.
func NewBuilder(w io.Writer) *Builder { func NewBuilder(w io.Writer) *Builder {
buf := bytes.NewBuffer(nil) tmp := bytes.NewBuffer(nil)
return &Builder{ return &Builder{
w: e2store.NewWriter(w), w: e2store.NewWriter(w),
buf: buf, tmp: tmp,
} }
} }
@ -259,17 +259,17 @@ func uint256LE(v *big.Int) []byte {
// SnappyWrite compresses the input data using snappy and writes it to the e2store file. // SnappyWrite compresses the input data using snappy and writes it to the e2store file.
func (b *Builder) snappyWrite(typ uint16, in []byte) error { func (b *Builder) snappyWrite(typ uint16, in []byte) error {
var buf = b.buf var tmp = b.tmp
snappy := snappy.NewBufferedWriter(b.buf) snappy := snappy.NewBufferedWriter(b.tmp)
buf.Reset() tmp.Reset()
snappy.Reset(buf) snappy.Reset(tmp)
if _, err := snappy.Write(in); err != nil { if _, err := snappy.Write(in); err != nil {
return fmt.Errorf("error snappy encoding: %w", err) return fmt.Errorf("error snappy encoding: %w", err)
} }
if err := snappy.Flush(); err != nil { if err := snappy.Flush(); err != nil {
return fmt.Errorf("error flushing snappy encoding: %w", err) return fmt.Errorf("error flushing snappy encoding: %w", err)
} }
n, err := b.w.Write(typ, b.buf.Bytes()) n, err := b.w.Write(typ, b.tmp.Bytes())
b.written += uint64(n) b.written += uint64(n)
if err != nil { if err != nil {
return fmt.Errorf("error writing e2store entry: %w", err) return fmt.Errorf("error writing e2store entry: %w", err)

View file

@ -22,6 +22,9 @@ import (
"io" "io"
"math/big" "math/big"
"os" "os"
"path"
"strconv"
"strings"
"github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/types"
@ -92,6 +95,47 @@ func (e *Era) Close() error {
return err return err
} }
// From returns an Era backed by f.
func From(f ReadAtSeekCloser) (*Era, error) {
e := &Era{f: f, s: e2store.NewReader(f)}
if err := e.loadIndex(); err != nil {
f.Close()
return nil, err
}
return e, nil
}
// ReadDir reads all the era1 files in a directory for a given network.
// Format: <network>-<epoch>-<hexroot>.erae
func ReadDir(dir, network string) ([]string, error) {
entries, err := os.ReadDir(dir)
if err != nil {
return nil, fmt.Errorf("error reading directory %s: %w", dir, err)
}
var (
next = uint64(0)
eras []string
)
for _, entry := range entries {
if path.Ext(entry.Name()) != ".erae" {
continue
}
parts := strings.Split(entry.Name(), "-")
if len(parts) != 3 || parts[0] != network {
// Invalid era1 filename, skip.
continue
}
if epoch, err := strconv.ParseUint(parts[1], 10, 64); err != nil {
return nil, fmt.Errorf("malformed era1 filename: %s", entry.Name())
} else if epoch != next {
return nil, fmt.Errorf("missing epoch %d", next)
}
next += 1
eras = append(eras, entry.Name())
}
return eras, nil
}
// Start retrieves the starting block number. // Start retrieves the starting block number.
func (e *Era) Start() uint64 { func (e *Era) Start() uint64 {
return e.m.start return e.m.start