diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index f0a33ac3d4..ed44a89034 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -43,6 +43,7 @@ import ( "github.com/ethereum/go-ethereum/internal/debug" "github.com/ethereum/go-ethereum/internal/era" "github.com/ethereum/go-ethereum/internal/era/eradl" + "github.com/ethereum/go-ethereum/internal/era2" "github.com/ethereum/go-ethereum/internal/flags" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" @@ -152,7 +153,7 @@ be gzipped.`, Name: "import-history", Usage: "Import an Era archive", ArgsUsage: "", - Flags: slices.Concat([]cli.Flag{utils.TxLookupLimitFlag, utils.TransactionHistoryFlag}, utils.DatabaseFlags, utils.NetworkFlags), + Flags: slices.Concat([]cli.Flag{utils.TxLookupLimitFlag, utils.TransactionHistoryFlag, utils.EraFormatFlag}, utils.DatabaseFlags, utils.NetworkFlags), Description: ` The import-history command will import blocks and their corresponding receipts from Era archives. @@ -163,7 +164,7 @@ from Era archives. Name: "export-history", Usage: "Export blockchain history to Era archives", ArgsUsage: " ", - Flags: utils.DatabaseFlags, + Flags: slices.Concat([]cli.Flag{utils.EraFormatFlag}, utils.DatabaseFlags), Description: ` The export-history command will export blocks and their corresponding receipts into Era archives. Eras are typically packaged in steps of 8192 blocks. @@ -515,15 +516,25 @@ func importHistory(ctx *cli.Context) error { network = networks[0] } - if err := utils.ImportHistory(chain, dir, network); err != nil { - return err + format := ctx.String(utils.EraFormatFlag.Name) + switch format { + case "era1", "era": + if err := utils.ImportHistory(chain, dir, network); err != nil { + return err + } + case "erae": + if err := utils.ImportHistoryEraE(chain, dir, network); err != nil { + return err + } + default: + return fmt.Errorf("unknown --era.format %q (expected 'era1' or 'erae')", format) } + fmt.Printf("Import done in %v\n", time.Since(start)) return nil } -// exportHistory exports chain history in Era archives at a specified -// directory. +// exportHistory exports chain history in Era archives at a specified directory. func exportHistory(ctx *cli.Context) error { if ctx.Args().Len() != 3 { utils.Fatalf("usage: %s", ctx.Command.ArgsUsage) @@ -549,10 +560,21 @@ func exportHistory(ctx *cli.Context) error { if head := chain.CurrentSnapBlock(); uint64(last) > head.Number.Uint64() { utils.Fatalf("Export error: block number %d larger than head block %d\n", uint64(last), head.Number.Uint64()) } - err := utils.ExportHistory(chain, dir, uint64(first), uint64(last), uint64(era.MaxEra1Size), utils.EraE) - if err != nil { - utils.Fatalf("Export error: %v\n", err) + + format := ctx.String(utils.EraFormatFlag.Name) + switch format { + case "era1", "era": + if err := utils.ExportHistory(chain, dir, uint64(first), uint64(last), uint64(era.MaxEra1Size), utils.Era1); err != nil { + utils.Fatalf("Export error: %v\n", err) + } + case "erae": + if err := utils.ExportHistory(chain, dir, uint64(first), uint64(last), uint64(era2.MaxEraESize), utils.EraE); err != nil { + utils.Fatalf("Export error: %v\n", err) + } + default: + return fmt.Errorf("unknown archive format %q (use 'era1' or 'erae')", format) } + fmt.Printf("Export done in %v\n", time.Since(start)) return nil } diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 77cbf4c604..23ce48dd77 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -342,6 +342,105 @@ func ImportHistory(chain *core.BlockChain, dir string, network string) error { return nil } +// ImportHistoryEraE imports Era-E files containing historical block information, +// starting from genesis. Currently this function follows the assumptions that the provided chain +// segment in Era-E file should all be canonical and verified. In the future, this function will be +// extended to support importing Era-E files with proof verification. +func ImportHistoryEraE(chain *core.BlockChain, dir, network string) error { + if chain.CurrentSnapBlock().Number.Sign() != 0 { + return errors.New("history import only supported when starting from genesis") + } + + entries, err := era2.ReadDir(dir, network) + if err != nil { + return fmt.Errorf("reading %q: %w", dir, err) + } + checksums, err := readList(filepath.Join(dir, "checksums.txt")) + if err != nil { + return fmt.Errorf("reading checksums.txt: %w", err) + } + if len(entries) != len(checksums) { + return fmt.Errorf("mismatch: %d erae files, %d checksums", len(entries), len(checksums)) + } + + var ( + start = time.Now() + reported = time.Now() + imported int + h = sha256.New() + scratch bytes.Buffer + ) + + for i, file := range entries { + path := filepath.Join(dir, file) + + // validate against checksum file in directory + f, err := os.Open(path) + if err != nil { + return fmt.Errorf("open %s: %w", path, err) + } + if _, err := io.Copy(h, f); err != nil { + f.Close() + return fmt.Errorf("checksum %s: %w", path, err) + } + got := common.BytesToHash(h.Sum(scratch.Bytes()[:])).Hex() + want := checksums[i] + h.Reset() + scratch.Reset() + if got != want { + f.Close() + return fmt.Errorf("%s checksum mismatch: have %s want %s", file, got, want) + } + // rewind for reading + if _, err := f.Seek(0, io.SeekStart); err != nil { + f.Close() + return fmt.Errorf("rewind %s: %w", file, err) + } + // import archive + e, err := era2.From(f) + if err != nil { + f.Close() + return fmt.Errorf("open erae %s: %w", file, err) + } + blockCount := e.Count() + + for j := uint64(0); j < blockCount; j++ { + hdr, err := e.GetHeader(e.Start() + j) + if err != nil { + return fmt.Errorf("header #%d: %w", hdr.Number.Uint64(), err) + } + if hdr.Number.Sign() == 0 { // skip genesis + continue + } + body, err := e.GetBody(hdr.Number.Uint64()) + if err != nil { + return fmt.Errorf("body #%d: %w", hdr.Number.Uint64(), err) + } + rcpts, err := e.GetReceipts(hdr.Number.Uint64(), 1) + if err != nil { + return fmt.Errorf("receipts #%d: %w", hdr.Number.Uint64(), err) + } + blk := types.NewBlockWithHeader(hdr).WithBody(*body) + + enc := types.EncodeBlockReceiptLists(rcpts) + if _, err := chain.InsertReceiptChain([]*types.Block{blk}, enc, ^uint64(0)); err != nil { + return fmt.Errorf("insert #%d: %w", hdr.Number.Uint64(), err) + } + + imported++ + if time.Since(reported) >= 8*time.Second { + log.Info("Importing Era‑E", "head", hdr.Number, "imported", imported, "elapsed", common.PrettyDuration(time.Since(start))) + reported = time.Now() + imported = 0 + } + } + f.Close() + } + + log.Info("Era‑E import complete", "duration", common.PrettyDuration(time.Since(start))) + return nil +} + func missingBlocks(chain *core.BlockChain, blocks []*types.Block) []*types.Block { head := chain.CurrentBlock() for i, block := range blocks { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index fe8375454f..78bf3f3645 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -114,6 +114,12 @@ var ( Usage: "Root directory for era1 history (default = inside ancient/chain)", Category: flags.EthCategory, } + EraFormatFlag = &cli.StringFlag{ + Name: "era.format", + Usage: "Archive format to import: 'era1' or 'erae'", + Value: "era1", + Category: flags.EthCategory, + } MinFreeDiskSpaceFlag = &cli.IntFlag{ Name: "datadir.minfreedisk", Usage: "Minimum free disk space in MB, once reached triggers auto shut down (default = --cache.gc converted to MB, 0 = disabled)", diff --git a/internal/era2/builder.go b/internal/era2/builder.go index d198315277..5d2029a851 100644 --- a/internal/era2/builder.go +++ b/internal/era2/builder.go @@ -92,7 +92,7 @@ type offsets struct { // Builder is used to build an Era2 e2store file. It collects block entries and writes them to the underlying e2store.Writer. type Builder struct { w *e2store.Writer - buf *bytes.Buffer + tmp *bytes.Buffer buff buffer off offsets @@ -105,10 +105,10 @@ type Builder struct { // NewBuilder returns a new Builder instance. func NewBuilder(w io.Writer) *Builder { - buf := bytes.NewBuffer(nil) + tmp := bytes.NewBuffer(nil) return &Builder{ w: e2store.NewWriter(w), - buf: buf, + tmp: tmp, } } @@ -259,17 +259,17 @@ func uint256LE(v *big.Int) []byte { // SnappyWrite compresses the input data using snappy and writes it to the e2store file. func (b *Builder) snappyWrite(typ uint16, in []byte) error { - var buf = b.buf - snappy := snappy.NewBufferedWriter(b.buf) - buf.Reset() - snappy.Reset(buf) + var tmp = b.tmp + snappy := snappy.NewBufferedWriter(b.tmp) + tmp.Reset() + snappy.Reset(tmp) if _, err := snappy.Write(in); err != nil { return fmt.Errorf("error snappy encoding: %w", err) } if err := snappy.Flush(); err != nil { return fmt.Errorf("error flushing snappy encoding: %w", err) } - n, err := b.w.Write(typ, b.buf.Bytes()) + n, err := b.w.Write(typ, b.tmp.Bytes()) b.written += uint64(n) if err != nil { return fmt.Errorf("error writing e2store entry: %w", err) diff --git a/internal/era2/era.go b/internal/era2/era.go index e5ebb132a1..1f28c7a85c 100644 --- a/internal/era2/era.go +++ b/internal/era2/era.go @@ -22,6 +22,9 @@ import ( "io" "math/big" "os" + "path" + "strconv" + "strings" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" @@ -92,6 +95,47 @@ func (e *Era) Close() error { return err } +// From returns an Era backed by f. +func From(f ReadAtSeekCloser) (*Era, error) { + e := &Era{f: f, s: e2store.NewReader(f)} + if err := e.loadIndex(); err != nil { + f.Close() + return nil, err + } + return e, nil +} + +// ReadDir reads all the era1 files in a directory for a given network. +// Format: --.erae +func ReadDir(dir, network string) ([]string, error) { + entries, err := os.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("error reading directory %s: %w", dir, err) + } + var ( + next = uint64(0) + eras []string + ) + for _, entry := range entries { + if path.Ext(entry.Name()) != ".erae" { + continue + } + parts := strings.Split(entry.Name(), "-") + if len(parts) != 3 || parts[0] != network { + // Invalid era1 filename, skip. + continue + } + if epoch, err := strconv.ParseUint(parts[1], 10, 64); err != nil { + return nil, fmt.Errorf("malformed era1 filename: %s", entry.Name()) + } else if epoch != next { + return nil, fmt.Errorf("missing epoch %d", next) + } + next += 1 + eras = append(eras, entry.Name()) + } + return eras, nil +} + // Start retrieves the starting block number. func (e *Era) Start() uint64 { return e.m.start