cmd/utils: don't store entire era file in-memory during import / export

cmd/utils/cmd.go

@@ -19,7 +19,6 @@ package utils
 
 import (
 	"bufio"
-	"bytes"
 	"compress/gzip"
 	"crypto/sha256"
 	"errors"
@@ -263,22 +262,29 @@ func ImportHistory(chain *core.BlockChain, db ethdb.Database, dir string, networ
 		reported = time.Now()
 		imported = 0
 		forker   = core.NewForkChoice(chain, nil)
+		h        = sha256.New()
+		buf      []byte
 	)
 	for i, filename := range entries {
-		// Read entire Era1 to memory. Max historical Era1 is around
-		// 600MB. This is a lot to load at once, but it speeds up the
-		// import substantially.
-		f, err := os.ReadFile(path.Join(dir, filename))
+		err := func() error {
+			f, err := os.Open(path.Join(dir, filename))
 			if err != nil {
 				return fmt.Errorf("unable to open era: %w", err)
 			}
+			defer f.Close()
 
-		if have, want := common.Hash(sha256.Sum256(f)).Hex(), checksums[i]; have != want {
+			// Validate checksum.
+			if _, err := io.Copy(h, f); err != nil {
+				return fmt.Errorf("unable to recalculate checksum: %w", err)
+			}
+			if have, want := common.BytesToHash(h.Sum(buf)).Hex(), checksums[i]; have != want {
 				return fmt.Errorf("checksum mismatch: have %s, want %s", have, want)
 			}
+			h.Reset()
+			buf = buf[:0]
 
 			// Import all block data from Era1.
-		r, err := era.NewReader(bytes.NewReader(f))
+			r, err := era.NewReader(f)
 			if err != nil {
 				return fmt.Errorf("error making era reader: %w", err)
 			}
@@ -309,6 +315,11 @@ func ImportHistory(chain *core.BlockChain, db ethdb.Database, dir string, networ
 					reported = time.Now()
 				}
 			}
+			return nil
+		}()
+		if err != nil {
+			return err
+		}
 	}
 
 	return nil
@@ -401,13 +412,20 @@ func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64) er
 	var (
 		start     = time.Now()
 		reported  = time.Now()
+		h         = sha256.New()
+		buf       []byte
 		checksums []string
 	)
 	for i := first; i <= last; i += step {
-		var (
-			buf = bytes.NewBuffer(nil)
-			w   = era.NewBuilder(buf)
-		)
+		err := func() error {
+			filename := path.Join(dir, era.Filename(network, int(i/step), common.Hash{}))
+			f, err := os.Create(filename)
+			if err != nil {
+				return fmt.Errorf("could not create era file: %w", err)
+			}
+			defer f.Close()
+
+			w := era.NewBuilder(f)
 			for j := uint64(0); j < step && j <= last-i; j++ {
 				var (
 					n     = i + j
@@ -432,16 +450,24 @@ func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64) er
 			if err != nil {
 				return fmt.Errorf("export failed to finalize %d: %w", step/i, err)
 			}
+			// Set correct filename with root.
+			os.Rename(filename, path.Join(dir, era.Filename(network, int(i/step), root)))
 
 			// Compute checksum of entire Era1.
-		checksums = append(checksums, common.Hash(sha256.Sum256(buf.Bytes())).Hex())
-
-		// Write Era1 to disk.
-		filename := path.Join(dir, era.Filename(network, int(i/step), root))
-		if err := os.WriteFile(filename, buf.Bytes(), os.ModePerm); err != nil {
+			if _, err := f.Seek(0, io.SeekStart); err != nil {
+				return err
+			}
+			if _, err := io.Copy(h, f); err != nil {
+				return fmt.Errorf("unable to calculate checksum: %w", err)
+			}
+			checksums = append(checksums, common.BytesToHash(h.Sum(buf)).Hex())
+			h.Reset()
+			buf = buf[:0]
+			return nil
+		}()
+		if err != nil {
 			return err
 		}
-
 		if time.Since(reported) >= 8*time.Second {
 			log.Info("Exporting blocks", "exported", i, "elapsed", common.PrettyDuration(time.Since(start)))
 			reported = time.Now()