From 67b48e414272cdad3d779fe9084b9701bff03c65 Mon Sep 17 00:00:00 2001 From: jeevan-sid Date: Wed, 18 Mar 2026 19:53:49 +0530 Subject: [PATCH] feat: improve import-history ux --- cmd/geth/chaincmd.go | 2 +- cmd/utils/cmd.go | 44 +++++++++++++++++++++++++++------ cmd/utils/history_test.go | 2 +- core/rawdb/accessors_indexes.go | 15 +++++++++++ core/rawdb/schema.go | 2 ++ 5 files changed, 55 insertions(+), 10 deletions(-) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 7e14ec1c60..470d6e21c5 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -538,7 +538,7 @@ func importHistory(ctx *cli.Context) error { default: return fmt.Errorf("unknown --era.format %q (expected 'era1' or 'erae')", format) } - if err := utils.ImportHistory(chain, dir, network, from); err != nil { + if err := utils.ImportHistory(chain, db, dir, network, from); err != nil { return err } diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 995724e6fc..bb385b5239 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -252,10 +252,7 @@ func readList(filename string) ([]string, error) { // ImportHistory imports Era1 files containing historical block information, // starting from genesis. The assumption is held that the provided chain // segment in Era1 file should all be canonical and verified. -func ImportHistory(chain *core.BlockChain, dir string, network string, from func(f era.ReadAtSeekCloser) (era.Era, error)) error { - if chain.CurrentSnapBlock().Number.BitLen() != 0 { - return errors.New("history import only supported when starting from genesis") - } +func ImportHistory(chain *core.BlockChain, db ethdb.Database, dir string, network string, from func(f era.ReadAtSeekCloser) (era.Era, error)) error { entries, err := era.ReadDir(dir, network) if err != nil { return fmt.Errorf("error reading %s: %w", dir, err) @@ -269,6 +266,13 @@ func ImportHistory(chain *core.BlockChain, dir string, network string, from func len(checksums), len(entries)) } + // Determine resume point from last successfully imported block + var resumeBlock uint64 + if tail := rawdb.ReadEraImportTail(db); tail != nil { + resumeBlock = *tail + log.Info("Resuming era import", "lastBlock", resumeBlock) + } + var ( start = time.Now() reported = time.Now() @@ -281,12 +285,29 @@ func ImportHistory(chain *core.BlockChain, dir string, network string, from func err := func() error { path := filepath.Join(dir, file) - // validate against checksum file in directory f, err := os.Open(path) if err != nil { return fmt.Errorf("open %s: %w", path, err) } defer f.Close() + + // Peek at era block range to see if we can skip entirely + e, err := from(f) + if err != nil { + return fmt.Errorf("error opening era: %w", err) + } + eraStart := e.Start() + eraEnd := eraStart + e.Count() - 1 + + // Skip era files fully behind resume point + if resumeBlock > 0 && eraEnd <= resumeBlock { + log.Debug("Skipping already imported Era file", "file", file, "eraEnd", eraEnd, "resumeBlock", resumeBlock) + return nil + } + + if _, err := f.Seek(0, io.SeekStart); err != nil { + return fmt.Errorf("seek %s: %w", path, err) + } if _, err := io.Copy(h, f); err != nil { return fmt.Errorf("checksum %s: %w", path, err) } @@ -294,12 +315,13 @@ func ImportHistory(chain *core.BlockChain, dir string, network string, from func want := checksums[i] h.Reset() scratch.Reset() - if got != want { return fmt.Errorf("%s checksum mismatch: have %s want %s", file, got, want) } - // Import all block data from Era1. - e, err := from(f) + if _, err := f.Seek(0, io.SeekStart); err != nil { + return fmt.Errorf("seek %s: %w", path, err) + } + e, err = from(f) if err != nil { return fmt.Errorf("error opening era: %w", err) } @@ -316,6 +338,10 @@ func ImportHistory(chain *core.BlockChain, dir string, network string, from func if block.Number().BitLen() == 0 { continue // skip genesis } + // Skip blocks already imported (mid-epoch resume) + if resumeBlock > 0 && block.Number().Uint64() <= resumeBlock { + continue + } receipts, err := it.Receipts() if err != nil { return fmt.Errorf("error reading receipts %d: %w", it.Number(), err) @@ -324,6 +350,8 @@ func ImportHistory(chain *core.BlockChain, dir string, network string, from func if _, err := chain.InsertReceiptChain([]*types.Block{block}, enc, math.MaxUint64); err != nil { return fmt.Errorf("error inserting body %d: %w", it.Number(), err) } + rawdb.WriteEraImportTail(db, block.Number().Uint64()) + resumeBlock = block.Number().Uint64() imported++ if time.Since(reported) >= 8*time.Second { diff --git a/cmd/utils/history_test.go b/cmd/utils/history_test.go index 6631946129..4b94117587 100644 --- a/cmd/utils/history_test.go +++ b/cmd/utils/history_test.go @@ -182,7 +182,7 @@ func TestHistoryImportAndExport(t *testing.T) { if err != nil { t.Fatalf("unable to initialize chain: %v", err) } - if err := ImportHistory(imported, dir, "mainnet", tt.from); err != nil { + if err := ImportHistory(imported, db2, dir, "mainnet", tt.from); err != nil { t.Fatalf("failed to import chain: %v", err) } if have, want := imported.CurrentHeader(), chain.CurrentHeader(); have.Hash() != want.Hash() { diff --git a/core/rawdb/accessors_indexes.go b/core/rawdb/accessors_indexes.go index 8c8c3ec9bb..c2426291db 100644 --- a/core/rawdb/accessors_indexes.go +++ b/core/rawdb/accessors_indexes.go @@ -32,6 +32,21 @@ import ( "github.com/ethereum/go-ethereum/rlp" ) +func ReadEraImportTail(db ethdb.KeyValueReader) *uint64 { + data, _ := db.Get(eraImportTailKey) + if len(data) == 0 { + return nil + } + tail := binary.BigEndian.Uint64(data) + return &tail +} + +func WriteEraImportTail(db ethdb.KeyValueWriter, blockNumber uint64) { + buf := make([]byte, 8) + binary.BigEndian.PutUint64(buf, blockNumber) + db.Put(eraImportTailKey, buf) +} + // DecodeTxLookupEntry decodes the supplied tx lookup data. func DecodeTxLookupEntry(data []byte, db ethdb.Reader) *uint64 { // Database v6 tx lookup just stores the block number diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index d9140c5fd6..6077700b2a 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -87,6 +87,8 @@ var ( // txIndexTailKey tracks the oldest block whose transactions have been indexed. txIndexTailKey = []byte("TransactionIndexTail") + eraImportTailKey = []byte("eraImportTail") // eraImportTailKey -> last fully imported block + // fastTxLookupLimitKey tracks the transaction lookup limit during fast sync. // This flag is deprecated, it's kept to avoid reporting errors when inspect // database.