From 67b48e414272cdad3d779fe9084b9701bff03c65 Mon Sep 17 00:00:00 2001 From: jeevan-sid Date: Wed, 18 Mar 2026 19:53:49 +0530 Subject: [PATCH 1/3] feat: improve import-history ux --- cmd/geth/chaincmd.go | 2 +- cmd/utils/cmd.go | 44 +++++++++++++++++++++++++++------ cmd/utils/history_test.go | 2 +- core/rawdb/accessors_indexes.go | 15 +++++++++++ core/rawdb/schema.go | 2 ++ 5 files changed, 55 insertions(+), 10 deletions(-) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 7e14ec1c60..470d6e21c5 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -538,7 +538,7 @@ func importHistory(ctx *cli.Context) error { default: return fmt.Errorf("unknown --era.format %q (expected 'era1' or 'erae')", format) } - if err := utils.ImportHistory(chain, dir, network, from); err != nil { + if err := utils.ImportHistory(chain, db, dir, network, from); err != nil { return err } diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 995724e6fc..bb385b5239 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -252,10 +252,7 @@ func readList(filename string) ([]string, error) { // ImportHistory imports Era1 files containing historical block information, // starting from genesis. The assumption is held that the provided chain // segment in Era1 file should all be canonical and verified. -func ImportHistory(chain *core.BlockChain, dir string, network string, from func(f era.ReadAtSeekCloser) (era.Era, error)) error { - if chain.CurrentSnapBlock().Number.BitLen() != 0 { - return errors.New("history import only supported when starting from genesis") - } +func ImportHistory(chain *core.BlockChain, db ethdb.Database, dir string, network string, from func(f era.ReadAtSeekCloser) (era.Era, error)) error { entries, err := era.ReadDir(dir, network) if err != nil { return fmt.Errorf("error reading %s: %w", dir, err) @@ -269,6 +266,13 @@ func ImportHistory(chain *core.BlockChain, dir string, network string, from func len(checksums), len(entries)) } + // Determine resume point from last successfully imported block + var resumeBlock uint64 + if tail := rawdb.ReadEraImportTail(db); tail != nil { + resumeBlock = *tail + log.Info("Resuming era import", "lastBlock", resumeBlock) + } + var ( start = time.Now() reported = time.Now() @@ -281,12 +285,29 @@ func ImportHistory(chain *core.BlockChain, dir string, network string, from func err := func() error { path := filepath.Join(dir, file) - // validate against checksum file in directory f, err := os.Open(path) if err != nil { return fmt.Errorf("open %s: %w", path, err) } defer f.Close() + + // Peek at era block range to see if we can skip entirely + e, err := from(f) + if err != nil { + return fmt.Errorf("error opening era: %w", err) + } + eraStart := e.Start() + eraEnd := eraStart + e.Count() - 1 + + // Skip era files fully behind resume point + if resumeBlock > 0 && eraEnd <= resumeBlock { + log.Debug("Skipping already imported Era file", "file", file, "eraEnd", eraEnd, "resumeBlock", resumeBlock) + return nil + } + + if _, err := f.Seek(0, io.SeekStart); err != nil { + return fmt.Errorf("seek %s: %w", path, err) + } if _, err := io.Copy(h, f); err != nil { return fmt.Errorf("checksum %s: %w", path, err) } @@ -294,12 +315,13 @@ func ImportHistory(chain *core.BlockChain, dir string, network string, from func want := checksums[i] h.Reset() scratch.Reset() - if got != want { return fmt.Errorf("%s checksum mismatch: have %s want %s", file, got, want) } - // Import all block data from Era1. - e, err := from(f) + if _, err := f.Seek(0, io.SeekStart); err != nil { + return fmt.Errorf("seek %s: %w", path, err) + } + e, err = from(f) if err != nil { return fmt.Errorf("error opening era: %w", err) } @@ -316,6 +338,10 @@ func ImportHistory(chain *core.BlockChain, dir string, network string, from func if block.Number().BitLen() == 0 { continue // skip genesis } + // Skip blocks already imported (mid-epoch resume) + if resumeBlock > 0 && block.Number().Uint64() <= resumeBlock { + continue + } receipts, err := it.Receipts() if err != nil { return fmt.Errorf("error reading receipts %d: %w", it.Number(), err) @@ -324,6 +350,8 @@ func ImportHistory(chain *core.BlockChain, dir string, network string, from func if _, err := chain.InsertReceiptChain([]*types.Block{block}, enc, math.MaxUint64); err != nil { return fmt.Errorf("error inserting body %d: %w", it.Number(), err) } + rawdb.WriteEraImportTail(db, block.Number().Uint64()) + resumeBlock = block.Number().Uint64() imported++ if time.Since(reported) >= 8*time.Second { diff --git a/cmd/utils/history_test.go b/cmd/utils/history_test.go index 6631946129..4b94117587 100644 --- a/cmd/utils/history_test.go +++ b/cmd/utils/history_test.go @@ -182,7 +182,7 @@ func TestHistoryImportAndExport(t *testing.T) { if err != nil { t.Fatalf("unable to initialize chain: %v", err) } - if err := ImportHistory(imported, dir, "mainnet", tt.from); err != nil { + if err := ImportHistory(imported, db2, dir, "mainnet", tt.from); err != nil { t.Fatalf("failed to import chain: %v", err) } if have, want := imported.CurrentHeader(), chain.CurrentHeader(); have.Hash() != want.Hash() { diff --git a/core/rawdb/accessors_indexes.go b/core/rawdb/accessors_indexes.go index 8c8c3ec9bb..c2426291db 100644 --- a/core/rawdb/accessors_indexes.go +++ b/core/rawdb/accessors_indexes.go @@ -32,6 +32,21 @@ import ( "github.com/ethereum/go-ethereum/rlp" ) +func ReadEraImportTail(db ethdb.KeyValueReader) *uint64 { + data, _ := db.Get(eraImportTailKey) + if len(data) == 0 { + return nil + } + tail := binary.BigEndian.Uint64(data) + return &tail +} + +func WriteEraImportTail(db ethdb.KeyValueWriter, blockNumber uint64) { + buf := make([]byte, 8) + binary.BigEndian.PutUint64(buf, blockNumber) + db.Put(eraImportTailKey, buf) +} + // DecodeTxLookupEntry decodes the supplied tx lookup data. func DecodeTxLookupEntry(data []byte, db ethdb.Reader) *uint64 { // Database v6 tx lookup just stores the block number diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index d9140c5fd6..6077700b2a 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -87,6 +87,8 @@ var ( // txIndexTailKey tracks the oldest block whose transactions have been indexed. txIndexTailKey = []byte("TransactionIndexTail") + eraImportTailKey = []byte("eraImportTail") // eraImportTailKey -> last fully imported block + // fastTxLookupLimitKey tracks the transaction lookup limit during fast sync. // This flag is deprecated, it's kept to avoid reporting errors when inspect // database. From bf0d1254491ff55d4befce9dd59cb55b8652eb35 Mon Sep 17 00:00:00 2001 From: 0xjvn Date: Tue, 31 Mar 2026 21:36:47 +0530 Subject: [PATCH 2/3] fix: conflicts on merge --- cmd/geth/chaincmd.go | 1 - cmd/utils/cmd.go | 24 ++++++++++++++++-------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 9dcd2caee3..030c45a2f6 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -541,7 +541,6 @@ func importHistory(ctx *cli.Context) error { if err := utils.ImportHistory(chain, db, dir, network, from); err != nil { return err } - fmt.Printf("Import done in %v\n", time.Since(start)) return nil } diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index 80916cd4eb..af1b9bc336 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -249,9 +249,9 @@ func readList(filename string) ([]string, error) { return strings.Split(string(b), "\n"), nil } -// ImportHistory imports Era1 files containing historical block information, +// ImportHistory imports Era1/Erae files containing historical block information, // starting from genesis. The assumption is held that the provided chain -// segment in Era1 file should all be canonical and verified. +// segment in Era1/Erae file should all be canonical and verified. func ImportHistory(chain *core.BlockChain, db ethdb.Database, dir string, network string, from func(f era.ReadAtSeekCloser) (era.Era, error)) error { entries, err := era.ReadDir(dir, network) if err != nil { @@ -280,6 +280,7 @@ func ImportHistory(chain *core.BlockChain, db ethdb.Database, dir string, networ h = sha256.New() buf = bytes.NewBuffer(nil) ) + for i, file := range entries { err := func() error { path := filepath.Join(dir, file) @@ -290,9 +291,10 @@ func ImportHistory(chain *core.BlockChain, db ethdb.Database, dir string, networ } defer f.Close() - // Peek at era block range to see if we can skip entirely. + // Peek at era block range to check if we can skip entirely. e, err := from(f) if err != nil { + f.Close() return fmt.Errorf("error opening era: %w", err) } eraStart := e.Start() @@ -305,7 +307,12 @@ func ImportHistory(chain *core.BlockChain, db ethdb.Database, dir string, networ return nil } - // Validate against checksum file in directory. + // reopen for checksum + import. + f, err = os.Open(path) + if err != nil { + return fmt.Errorf("open %s: %w", path, err) + } + // Validate checksum. if _, err := f.Seek(0, io.SeekStart); err != nil { return fmt.Errorf("seek %s: %w", path, err) } @@ -319,7 +326,7 @@ func ImportHistory(chain *core.BlockChain, db ethdb.Database, dir string, networ return fmt.Errorf("%s checksum mismatch: have %s want %s", file, got, checksums[i]) } - // Import all block data from Era1. + // Seek back for import. if _, err := f.Seek(0, io.SeekStart); err != nil { return fmt.Errorf("seek %s: %w", path, err) } @@ -333,6 +340,7 @@ func ImportHistory(chain *core.BlockChain, db ethdb.Database, dir string, networ if err != nil { return fmt.Errorf("error creating iterator: %w", err) } + var ( blocks = make([]*types.Block, 0, importBatchSize) receiptsList = make([]types.Receipts, 0, importBatchSize) @@ -345,15 +353,14 @@ func ImportHistory(chain *core.BlockChain, db ethdb.Database, dir string, networ return fmt.Errorf("error inserting blocks %d-%d: %w", blocks[0].NumberU64(), blocks[len(blocks)-1].NumberU64(), err) } - // Track the last successfully imported block for resume support. + // Persist tail after each successful batch. lastBlock := blocks[len(blocks)-1].NumberU64() rawdb.WriteEraImportTail(db, lastBlock) resumeBlock = lastBlock imported += len(blocks) if time.Since(reported) >= 8*time.Second { - head := blocks[len(blocks)-1].NumberU64() - log.Info("Importing Era files", "head", head, "imported", imported, + log.Info("Importing Era files", "head", lastBlock, "imported", imported, "elapsed", common.PrettyDuration(time.Since(start))) imported = 0 reported = time.Now() @@ -363,6 +370,7 @@ func ImportHistory(chain *core.BlockChain, db ethdb.Database, dir string, networ return nil } ) + for it.Next() { block, err := it.Block() if err != nil { From 134be59d2378774634f0c785c6573a6b1c9d5dd7 Mon Sep 17 00:00:00 2001 From: jeevan-sid Date: Tue, 31 Mar 2026 22:05:10 +0530 Subject: [PATCH 3/3] fix: remove redundant seek --- cmd/utils/cmd.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cmd/utils/cmd.go b/cmd/utils/cmd.go index af1b9bc336..0771e4218e 100644 --- a/cmd/utils/cmd.go +++ b/cmd/utils/cmd.go @@ -312,10 +312,6 @@ func ImportHistory(chain *core.BlockChain, db ethdb.Database, dir string, networ if err != nil { return fmt.Errorf("open %s: %w", path, err) } - // Validate checksum. - if _, err := f.Seek(0, io.SeekStart); err != nil { - return fmt.Errorf("seek %s: %w", path, err) - } if _, err := io.Copy(h, f); err != nil { return fmt.Errorf("checksum %s: %w", path, err) }