all: dump trie journal into file (#546)

* all: load/dump trie disklayer into file

Signed-off-by: jsvisa <delweng@gmail.com>

* cmd: set cache.trie.journal=trie-disklayer.rlp as default

Signed-off-by: jsvisa <delweng@gmail.com>

* journal: directly write into file

Signed-off-by: jsvisa <delweng@gmail.com>

---------

Signed-off-by: jsvisa <delweng@gmail.com>

cmd/utils/flags.go

@@ -1680,6 +1680,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 	if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheSnapshotFlag.Name) {
 		cfg.SnapshotCache = ctx.Int(CacheFlag.Name) * ctx.Int(CacheSnapshotFlag.Name) / 100
 	}
+	if journal := ctx.String(CacheTrieJournalFlag.Name); journal != "" {
+		cfg.TrieJournal = stack.ResolvePath(journal)
+	}
 	if ctx.IsSet(CacheLogSizeFlag.Name) {
 		cfg.FilterLogCacheSize = ctx.Int(CacheLogSizeFlag.Name)
 	}
@@ -2203,6 +2206,9 @@ func MakeChain(ctx *cli.Context, stack *node.Node, readonly bool) (*core.BlockCh
 		options.Preimages = true
 		log.Info("Enabling recording of key preimages since archive mode is used")
 	}
+	if journal := ctx.String(CacheTrieJournalFlag.Name); journal != "" {
+		options.TrieJournal = stack.ResolvePath(journal)
+	}
 	if !ctx.Bool(SnapshotFlag.Name) {
 		options.SnapshotLimit = 0 // Disabled
 	} else if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheSnapshotFlag.Name) {

cmd/utils/flags_legacy.go

@@ -63,6 +63,7 @@ var (
 	CacheTrieJournalFlag = &cli.StringFlag{
 		Name:     "cache.trie.journal",
 		Usage:    "Disk journal directory for trie cache to survive node restarts",
+		Value:    "trie-disklayer.rlp",
 		Category: flags.DeprecatedCategory,
 	}
 	CacheTrieRejournalFlag = &cli.DurationFlag{

core/blockchain.go

@@ -166,6 +166,7 @@ type BlockChainConfig struct {
 	TrieDirtyLimit   int           // Memory limit (MB) at which to start flushing dirty trie nodes to disk
 	TrieTimeLimit    time.Duration // Time limit after which to flush the current in-memory trie to disk
 	TrieNoAsyncFlush bool          // Whether the asynchronous buffer flushing is disallowed
+	TrieJournal      string        // Path used to store the trie pathdb journal
 
 	Preimages    bool   // Whether to store preimage of trie key to the disk
 	StateHistory uint64 // Number of blocks from head whose state histories are reserved.

eth/backend.go

@@ -227,6 +227,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 			TrieDirtyLimit:   config.TrieDirtyCache,
 			ArchiveMode:      config.NoPruning,
 			TrieTimeLimit:    config.TrieTimeout,
+			TrieJournal:      config.TrieJournal,
 			SnapshotLimit:    config.SnapshotCache,
 			Preimages:        config.Preimages,
 			StateHistory:     config.StateHistory,

eth/ethconfig/config.go

@@ -127,6 +127,7 @@ type Config struct {
 	TrieTimeout    time.Duration
 	SnapshotCache  int
 	Preimages      bool
+	TrieJournal    string
 
 	// This is the number of blocks for which logs will be cached in the filter system.
 	FilterLogCacheSize int

triedb/pathdb/journal.go

@@ -21,6 +21,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"os"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -51,11 +52,24 @@ const journalVersion uint64 = 3
 
 // loadJournal tries to parse the layer journal from the disk.
 func (db *Database) loadJournal(diskRoot common.Hash) (layer, error) {
-	journal := rawdb.ReadTrieJournal(db.diskdb)
+	var reader io.Reader
-	if len(journal) == 0 {
+	if db.config.Journal != "" && common.FileExist(db.config.Journal) {
-		return nil, errMissJournal
+		log.Info("Load pathdb disklayer journal", "path", db.config.Journal)
+		// If a journal file is specified, read it from there
+		f, err := os.OpenFile(db.config.Journal, os.O_RDONLY, 0644)
+		if err != nil {
+			return nil, fmt.Errorf("failed to read journal file %s: %w", db.config.Journal, err)
+		}
+		defer f.Close()
+		reader = f
+	} else {
+		journal := rawdb.ReadTrieJournal(db.diskdb)
+		if len(journal) == 0 {
+			return nil, errMissJournal
+		}
+		reader = bytes.NewReader(journal)
 	}
-	r := rlp.NewStream(bytes.NewReader(journal), 0)
+	r := rlp.NewStream(reader, 0)
 
 	// Firstly, resolve the first element as the journal version
 	version, err := r.Uint64()
@@ -316,8 +330,21 @@ func (db *Database) Journal(root common.Hash) error {
 	if db.readOnly {
 		return errDatabaseReadOnly
 	}
+
+	var journal io.Writer
+	// Store the journal into the database and return
+	if db.config.Journal != "" {
+		file, err := os.OpenFile(db.config.Journal, os.O_WRONLY|os.O_CREATE, 0644)
+		if err != nil {
+			return fmt.Errorf("failed to open journal file %s: %w", db.config.Journal, err)
+		}
+		defer file.Close()
+		journal = file
+	} else {
+		journal = new(bytes.Buffer)
+	}
+
 	// Firstly write out the metadata of journal
-	journal := new(bytes.Buffer)
 	if err := rlp.Encode(journal, journalVersion); err != nil {
 		return err
 	}
@@ -334,11 +361,20 @@ func (db *Database) Journal(root common.Hash) error {
 	if err := l.journal(journal); err != nil {
 		return err
 	}
+
 	// Store the journal into the database and return
-	rawdb.WriteTrieJournal(db.diskdb, journal.Bytes())
+	var size int
+	if db.config.Journal == "" {
+		data := journal.(*bytes.Buffer)
+		size = data.Len()
+		rawdb.WriteTrieJournal(db.diskdb, data.Bytes())
+	} else {
+		stat, _ := journal.(*os.File).Stat()
+		size = int(stat.Size())
+	}
 
 	// Set the db in read only mode to reject all following mutations
 	db.readOnly = true
-	log.Info("Persisted dirty state to disk", "size", common.StorageSize(journal.Len()), "elapsed", common.PrettyDuration(time.Since(start)))
+	log.Info("Persisted dirty state to disk", "size", common.StorageSize(size), "elapsed", common.PrettyDuration(time.Since(start)))
 	return nil
 }