feat: support separate database for state data

cmd/geth/chaincmd.go

@@ -229,6 +229,15 @@ func initGenesis(ctx *cli.Context) error {
 		}
 		defer chaindb.Close()
 
+		// if the trie data dir has been set, new trie db with a new state database
+		if ctx.IsSet(utils.SeparateDBFlag.Name) {
+			statediskdb, dbErr := stack.OpenDatabaseWithFreezer(name+"/state", 0, 0, "", "", false)
+			if dbErr != nil {
+				utils.Fatalf("Failed to open separate trie database: %v", dbErr)
+			}
+			chaindb.SetStateStore(statediskdb)
+		}
+
 		triedb := utils.MakeTrieDatabase(ctx, chaindb, ctx.Bool(utils.CachePreimagesFlag.Name), false, genesis.IsVerkle())
 		defer triedb.Close()
 

cmd/geth/config.go

@@ -179,6 +179,9 @@ func makeFullNode(ctx *cli.Context) *node.Node {
 		cfg.Eth.OverrideVerkle = &v
 	}
 
+	if ctx.IsSet(utils.SeparateDBFlag.Name) && !stack.IsSeparatedDB() {
+		utils.Fatalf("Failed to locate separate database subdirectory when separatedb parameter has been set")
+	}
 	backend, eth := utils.RegisterEthService(stack, &cfg.Eth)
 
 	// Create gauge with geth system and build information

cmd/geth/dbcmd.go

@@ -428,6 +428,11 @@ func dbStats(ctx *cli.Context) error {
 	defer db.Close()
 
 	showLeveldbStats(db)
+	if db.StateStore() != nil {
+		fmt.Println("show stats of state store")
+		showLeveldbStats(db.StateStore())
+	}
+
 	return nil
 }
 
@@ -441,13 +446,31 @@ func dbCompact(ctx *cli.Context) error {
 	log.Info("Stats before compaction")
 	showLeveldbStats(db)
 
+	statediskdb := db.StateStore()
+	if statediskdb != nil {
+		fmt.Println("show stats of state store")
+		showLeveldbStats(statediskdb)
+	}
+
 	log.Info("Triggering compaction")
 	if err := db.Compact(nil, nil); err != nil {
-		log.Info("Compact err", "error", err)
+		log.Error("Compact err", "error", err)
 		return err
 	}
+
+	if statediskdb != nil {
+		if err := statediskdb.Compact(nil, nil); err != nil {
+			log.Error("Compact err", "error", err)
+			return err
+		}
+	}
+
 	log.Info("Stats after compaction")
 	showLeveldbStats(db)
+	if statediskdb != nil {
+		fmt.Println("show stats of state store after compaction")
+		showLeveldbStats(statediskdb)
+	}
 	return nil
 }
 
@@ -468,8 +491,17 @@ func dbGet(ctx *cli.Context) error {
 		return err
 	}
 
+	statediskdb := db.StateStore()
 	data, err := db.Get(key)
 	if err != nil {
+		// if separate trie db exist, try to get it from separate db
+		if statediskdb != nil {
+			statedata, dberr := statediskdb.Get(key)
+			if dberr == nil {
+				fmt.Printf("key %#x: %#x\n", key, statedata)
+				return nil
+			}
+		}
 		log.Info("Get operation failed", "key", fmt.Sprintf("%#x", key), "error", err)
 		return err
 	}

cmd/geth/snapshot.go

@@ -174,9 +174,6 @@ func pruneState(ctx *cli.Context) error {
 	chaindb := utils.MakeChainDatabase(ctx, stack, false)
 	defer chaindb.Close()
 
-	if rawdb.ReadStateScheme(chaindb) != rawdb.HashScheme {
-		log.Crit("Offline pruning is not required for path scheme")
-	}
 	prunerconfig := pruner.Config{
 		Datadir:   stack.ResolvePath(""),
 		BloomSize: ctx.Uint64(utils.BloomFilterSizeFlag.Name),

cmd/utils/flags.go

@@ -94,6 +94,12 @@ var (
 		Value:    flags.DirectoryString(node.DefaultDataDir()),
 		Category: flags.EthCategory,
 	}
+	SeparateDBFlag = &cli.BoolFlag{
+		Name: "separatedb",
+		Usage: "Enable a separated trie database, it will be created within a subdirectory called state, " +
+			"Users can copy this state directory to another directory or disk, and then create a symbolic link to the state directory under the chaindata",
+		Category: flags.EthCategory,
+	}
 	RemoteDBFlag = &cli.StringFlag{
 		Name:     "remotedb",
 		Usage:    "URL for remote database",
@@ -974,6 +980,7 @@ var (
 		DBEngineFlag,
 		StateSchemeFlag,
 		HttpHeaderFlag,
+		SeparateDBFlag,
 	}
 )
 
@@ -2070,6 +2077,11 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly bool) ethdb.
 		chainDb, err = stack.OpenDatabase("lightchaindata", cache, handles, "", readonly)
 	default:
 		chainDb, err = stack.OpenDatabaseWithFreezer("chaindata", cache, handles, ctx.String(AncientFlag.Name), "", readonly)
+		// set the separate state database
+		if stack.IsSeparatedDB() && err == nil {
+			stateDiskDb := MakeStateDataBase(ctx, stack, readonly, false)
+			chainDb.SetStateStore(stateDiskDb)
+		}
 	}
 	if err != nil {
 		Fatalf("Could not open database: %v", err)
@@ -2077,6 +2089,17 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly bool) ethdb.
 	return chainDb
 }
 
+// MakeStateDataBase open a separate state database using the flags passed to the client and will hard crash if it fails.
+func MakeStateDataBase(ctx *cli.Context, stack *node.Node, readonly, disableFreeze bool) ethdb.Database {
+	cache := ctx.Int(CacheFlag.Name) * ctx.Int(CacheDatabaseFlag.Name) / 100
+	handles := MakeDatabaseHandles(ctx.Int(FDLimitFlag.Name)) / 2
+	statediskdb, err := stack.OpenDatabaseWithFreezer("chaindata/state", cache, handles, "", "", readonly)
+	if err != nil {
+		Fatalf("Failed to open separate trie database: %v", err)
+	}
+	return statediskdb
+}
+
 // tryMakeReadOnlyDatabase try to open the chain database in read-only mode,
 // or fallback to write mode if the database is not initialized.
 func tryMakeReadOnlyDatabase(ctx *cli.Context, stack *node.Node) ethdb.Database {

core/rawdb/accessors_trie.go

@@ -247,12 +247,12 @@ func DeleteTrieNode(db ethdb.KeyValueWriter, owner common.Hash, path []byte, has
 // if the state is not present in database.
 func ReadStateScheme(db ethdb.Reader) string {
 	// Check if state in path-based scheme is present.
-	if HasAccountTrieNode(db, nil) {
+	if HasAccountTrieNode(db.StateStoreReader(), nil) {
 		return PathScheme
 	}
 	// The root node might be deleted during the initial snap sync, check
 	// the persistent state id then.
-	if id := ReadPersistentStateID(db); id != 0 {
+	if id := ReadPersistentStateID(db.StateStoreReader()); id != 0 {
 		return PathScheme
 	}
 	// In a hash-based scheme, the genesis state is consistently stored
@@ -262,7 +262,7 @@ func ReadStateScheme(db ethdb.Reader) string {
 	if header == nil {
 		return "" // empty datadir
 	}
-	if !HasLegacyTrieNode(db, header.Root) {
+	if !HasLegacyTrieNode(db.StateStoreReader(), header.Root) {
 		return "" // no state in disk
 	}
 	return HashScheme

core/rawdb/ancient_utils.go

@@ -89,6 +89,9 @@ func inspectFreezers(db ethdb.Database) ([]freezerInfo, error) {
 			infos = append(infos, info)
 
 		case StateFreezerName:
+			if db.StateStore() != nil {
+				continue
+			}
 			datadir, err := db.AncientDatadir()
 			if err != nil {
 				return nil, err

core/rawdb/database.go

@@ -41,6 +41,14 @@ type freezerdb struct {
 
 	readOnly    bool
 	ancientRoot string
+	stateStore  ethdb.Database
+}
+
+func (frdb *freezerdb) StateStoreReader() ethdb.Reader {
+	if frdb.stateStore == nil {
+		return frdb
+	}
+	return frdb.stateStore
 }
 
 // AncientDatadir returns the path of root ancient directory.
@@ -58,12 +66,29 @@ func (frdb *freezerdb) Close() error {
 	if err := frdb.KeyValueStore.Close(); err != nil {
 		errs = append(errs, err)
 	}
+
+	if frdb.stateStore != nil {
+		if err := frdb.stateStore.Close(); err != nil {
+			errs = append(errs, err)
+		}
+	}
 	if len(errs) != 0 {
 		return fmt.Errorf("%v", errs)
 	}
 	return nil
 }
 
+func (frdb *freezerdb) StateStore() ethdb.Database {
+	return frdb.stateStore
+}
+
+func (frdb *freezerdb) SetStateStore(state ethdb.Database) {
+	if frdb.stateStore != nil {
+		frdb.stateStore.Close()
+	}
+	frdb.stateStore = state
+}
+
 // Freeze is a helper method used for external testing to trigger and block until
 // a freeze cycle completes, without having to sleep for a minute to trigger the
 // automatic background run.
@@ -81,6 +106,7 @@ func (frdb *freezerdb) Freeze() error {
 // nofreezedb is a database wrapper that disables freezer data retrievals.
 type nofreezedb struct {
 	ethdb.KeyValueStore
+	stateStore ethdb.Database
 }
 
 // HasAncient returns an error as we don't have a backing chain freezer.
@@ -133,6 +159,21 @@ func (db *nofreezedb) Sync() error {
 	return errNotSupported
 }
 
+func (db *nofreezedb) StateStore() ethdb.Database {
+	return db.stateStore
+}
+
+func (db *nofreezedb) SetStateStore(state ethdb.Database) {
+	db.stateStore = state
+}
+
+func (db *nofreezedb) StateStoreReader() ethdb.Reader {
+	if db.stateStore != nil {
+		return db.stateStore
+	}
+	return db
+}
+
 func (db *nofreezedb) ReadAncients(fn func(reader ethdb.AncientReaderOp) error) (err error) {
 	// Unlike other ancient-related methods, this method does not return
 	// errNotSupported when invoked.
@@ -456,6 +497,11 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
 	it := db.NewIterator(keyPrefix, keyStart)
 	defer it.Release()
 
+	var trieIter ethdb.Iterator
+	if db.StateStore() != nil {
+		trieIter = db.StateStore().NewIterator(keyPrefix, nil)
+		defer trieIter.Release()
+	}
 	var (
 		count  int64
 		start  = time.Now()
@@ -506,14 +552,14 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
 			bodies.Add(size)
 		case bytes.HasPrefix(key, blockReceiptsPrefix) && len(key) == (len(blockReceiptsPrefix)+8+common.HashLength):
 			receipts.Add(size)
+		case IsLegacyTrieNode(key, it.Value()):
+			legacyTries.Add(size)
 		case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerTDSuffix):
 			tds.Add(size)
 		case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerHashSuffix):
 			numHashPairings.Add(size)
 		case bytes.HasPrefix(key, headerNumberPrefix) && len(key) == (len(headerNumberPrefix)+common.HashLength):
 			hashNumPairings.Add(size)
-		case IsLegacyTrieNode(key, it.Value()):
-			legacyTries.Add(size)
 		case bytes.HasPrefix(key, stateIDPrefix) && len(key) == len(stateIDPrefix)+common.HashLength:
 			stateLookups.Add(size)
 		case IsAccountTrieNode(key):
@@ -575,6 +621,46 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
 			logged = time.Now()
 		}
 	}
+	// inspect separate trie db
+	if trieIter != nil {
+		count = 0
+		logged = time.Now()
+		for trieIter.Next() {
+			var (
+				key   = trieIter.Key()
+				value = trieIter.Value()
+				size  = common.StorageSize(len(key) + len(value))
+			)
+
+			switch {
+			case IsLegacyTrieNode(key, value):
+				legacyTries.Add(size)
+			case bytes.HasPrefix(key, stateIDPrefix) && len(key) == len(stateIDPrefix)+common.HashLength:
+				stateLookups.Add(size)
+			case IsAccountTrieNode(key):
+				accountTries.Add(size)
+			case IsStorageTrieNode(key):
+				storageTries.Add(size)
+			default:
+				var accounted bool
+				for _, meta := range [][]byte{
+					fastTrieProgressKey, persistentStateIDKey, trieJournalKey} {
+					if bytes.Equal(key, meta) {
+						metadata.Add(size)
+						break
+					}
+				}
+				if !accounted {
+					unaccounted.Add(size)
+				}
+			}
+			count++
+			if count%1000 == 0 && time.Since(logged) > 8*time.Second {
+				log.Info("Inspecting separate state database", "count", count, "elapsed", common.PrettyDuration(time.Since(start)))
+				logged = time.Now()
+			}
+		}
+	}
 	// Display the database statistic of key-value store.
 	stats := [][]string{
 		{"Key-Value store", "Headers", headers.Size(), headers.Count()},
@@ -615,6 +701,28 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
 		}
 		total += ancient.size()
 	}
+
+	// inspect ancient state in separate trie db if exist
+	if trieIter != nil {
+		stateAncients, err := inspectFreezers(db.StateStore())
+		if err != nil {
+			return err
+		}
+		for _, ancient := range stateAncients {
+			for _, table := range ancient.sizes {
+				if ancient.name == "chain" {
+					break
+				}
+				stats = append(stats, []string{
+					fmt.Sprintf("Ancient store (%s)", strings.Title(ancient.name)),
+					strings.Title(table.name),
+					table.size.String(),
+					fmt.Sprintf("%d", ancient.count()),
+				})
+			}
+			total += ancient.size()
+		}
+	}
 	table := tablewriter.NewWriter(os.Stdout)
 	table.SetHeader([]string{"Database", "Category", "Size", "Items"})
 	table.SetFooter([]string{"", "Total", total.String(), " "})

core/rawdb/table.go

@@ -195,6 +195,18 @@ func (t *table) NewBatch() ethdb.Batch {
 	return &tableBatch{t.db.NewBatch(), t.prefix}
 }
 
+func (t *table) StateStore() ethdb.Database {
+	return nil
+}
+
+func (t *table) SetStateStore(state ethdb.Database) {
+	panic("not implement")
+}
+
+func (t *table) StateStoreReader() ethdb.Reader {
+	return nil
+}
+
 // NewBatchWithSize creates a write-only database batch with pre-allocated buffer.
 func (t *table) NewBatchWithSize(size int) ethdb.Batch {
 	return &tableBatch{t.db.NewBatchWithSize(size), t.prefix}

core/state/pruner/pruner.go

@@ -125,13 +125,19 @@ func prune(snaptree *snapshot.Tree, root common.Hash, maindb ethdb.Database, sta
 	// that the false-positive is low enough(~0.05%). The probability of the
 	// dangling node is the state root is super low. So the dangling nodes in
 	// theory will never ever be visited again.
+	var pruneDB ethdb.Database
+	if maindb != nil && maindb.StateStore() != nil {
+		pruneDB = maindb.StateStore()
+	} else {
+		pruneDB = maindb
+	}
 	var (
 		skipped, count int
 		size           common.StorageSize
 		pstart         = time.Now()
 		logged         = time.Now()
-		batch          = maindb.NewBatch()
+		batch          = pruneDB.NewBatch()
-		iter           = maindb.NewIterator(nil, nil)
+		iter           = pruneDB.NewIterator(nil, nil)
 	)
 	for iter.Next() {
 		key := iter.Key()
@@ -178,7 +184,7 @@ func prune(snaptree *snapshot.Tree, root common.Hash, maindb ethdb.Database, sta
 				batch.Reset()
 
 				iter.Release()
-				iter = maindb.NewIterator(nil, key)
+				iter = pruneDB.NewIterator(nil, key)
 			}
 		}
 	}
@@ -221,7 +227,7 @@ func prune(snaptree *snapshot.Tree, root common.Hash, maindb ethdb.Database, sta
 				end = nil
 			}
 			log.Info("Compacting database", "range", fmt.Sprintf("%#x-%#x", start, end), "elapsed", common.PrettyDuration(time.Since(cstart)))
-			if err := maindb.Compact(start, end); err != nil {
+			if err := pruneDB.Compact(start, end); err != nil {
 				log.Error("Database compaction failed", "error", err)
 				return err
 			}
@@ -266,10 +272,17 @@ func (p *Pruner) Prune(root common.Hash) error {
 		// Use the bottom-most diff layer as the target
 		root = layers[len(layers)-1].Root()
 	}
+	// if the separated state db has been set, use this db to prune data
+	var trienodedb ethdb.Database
+	if p.db != nil && p.db.StateStore() != nil {
+		trienodedb = p.db.StateStore()
+	} else {
+		trienodedb = p.db
+	}
 	// Ensure the root is really present. The weak assumption
 	// is the presence of root can indicate the presence of the
 	// entire trie.
-	if !rawdb.HasLegacyTrieNode(p.db, root) {
+	if !rawdb.HasLegacyTrieNode(trienodedb, root) {
 		// The special case is for clique based networks(goerli
 		// and some other private networks), it's possible that two
 		// consecutive blocks will have same root. In this case snapshot
@@ -283,7 +296,7 @@ func (p *Pruner) Prune(root common.Hash) error {
 		// as the pruning target.
 		var found bool
 		for i := len(layers) - 2; i >= 2; i-- {
-			if rawdb.HasLegacyTrieNode(p.db, layers[i].Root()) {
+			if rawdb.HasLegacyTrieNode(trienodedb, layers[i].Root()) {
 				root = layers[i].Root()
 				found = true
 				log.Info("Selecting middle-layer as the pruning target", "root", root, "depth", i)

eth/backend.go

@@ -59,6 +59,10 @@ import (
 	"github.com/ethereum/go-ethereum/rpc"
 )
 
+const (
+	ChainDBNamespace = "eth/db/chaindata/"
+)
+
 // Config contains the configuration options of the ETH protocol.
 // Deprecated: use ethconfig.Config instead.
 type Config = ethconfig.Config
@@ -127,7 +131,8 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 	log.Info("Allocated trie memory caches", "clean", common.StorageSize(config.TrieCleanCache)*1024*1024, "dirty", common.StorageSize(config.TrieDirtyCache)*1024*1024)
 
 	// Assemble the Ethereum object
-	chainDb, err := stack.OpenDatabaseWithFreezer("chaindata", config.DatabaseCache, config.DatabaseHandles, config.DatabaseFreezer, "eth/db/chaindata/", false)
+	chainDb, err := stack.OpenAndMergeDatabase("chaindata", config.DatabaseCache, config.DatabaseHandles,
+		config.DatabaseFreezer, ChainDBNamespace, false)
 	if err != nil {
 		return nil, err
 	}

ethdb/database.go

@@ -153,11 +153,16 @@ type AncientStater interface {
 	AncientDatadir() (string, error)
 }
 
+type StateStoreReader interface {
+	StateStoreReader() Reader
+}
+
 // Reader contains the methods required to read data from both key-value as well as
 // immutable ancient data.
 type Reader interface {
 	KeyValueReader
 	AncientReader
+	StateStoreReader
 }
 
 // Writer contains the methods required to write data to both key-value as well as
@@ -190,11 +195,17 @@ type ResettableAncientStore interface {
 	Reset() error
 }
 
+type StateStore interface {
+	StateStore() Database
+	SetStateStore(state Database)
+}
+
 // Database contains all the methods required by the high level database to not
 // only access the key-value data store but also the ancient chain store.
 type Database interface {
 	Reader
 	Writer
+	StateStore
 	Batcher
 	Iteratee
 	Stater

ethdb/remotedb/remotedb.go

@@ -82,6 +82,18 @@ func (db *Database) AncientSize(kind string) (uint64, error) {
 	panic("not supported")
 }
 
+func (db *Database) StateStore() ethdb.Database {
+	panic("not supported")
+}
+
+func (db *Database) SetStateStore(state ethdb.Database) {
+	panic("not supported")
+}
+
+func (db *Database) StateStoreReader() ethdb.Reader {
+	return db
+}
+
 func (db *Database) ReadAncients(fn func(op ethdb.AncientReaderOp) error) (err error) {
 	return fn(db)
 }

node/node.go

@@ -739,6 +739,36 @@ func (n *Node) OpenDatabase(name string, cache, handles int, namespace string, r
 	return db, err
 }
 
+func (n *Node) OpenAndMergeDatabase(name string, cache, handles int, ancient, namespace string, readonly bool) (ethdb.Database, error) {
+	chainDataHandles := handles
+
+	var statediskdb ethdb.Database
+	var err error
+	// Open the separated state database if the state directory exists
+	if n.IsSeparatedDB() {
+		// Allocate half of the  handles and cache to this separate state data database
+		statediskdb, err = n.OpenDatabaseWithFreezer(name+"/state", cache/2, chainDataHandles/2, "", "eth/db/statedata/", readonly)
+		if err != nil {
+			return nil, err
+		}
+
+		// Reduce the handles and cache to this separate database because it is not a complete database with no trie data storing in it.
+		cache = int(float64(cache) * 0.6)
+		chainDataHandles = int(float64(chainDataHandles) * 0.6)
+	}
+
+	chainDB, err := n.OpenDatabaseWithFreezer(name, cache, chainDataHandles, ancient, namespace, readonly)
+	if err != nil {
+		return nil, err
+	}
+
+	if statediskdb != nil {
+		chainDB.SetStateStore(statediskdb)
+	}
+
+	return chainDB, nil
+}
+
 // OpenDatabaseWithFreezer opens an existing database with the given name (or
 // creates one if no previous can be found) from within the node's data directory,
 // also attaching a chain freezer to it that moves ancient chain data from the
@@ -772,6 +802,16 @@ func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, ancient
 	return db, err
 }
 
+// IsSeparatedDB check the state subdirectory of db, if subdirectory exists, return true
+func (n *Node) IsSeparatedDB() bool {
+	separateDir := filepath.Join(n.ResolvePath("chaindata"), "state")
+	fileInfo, err := os.Stat(separateDir)
+	if os.IsNotExist(err) {
+		return false
+	}
+	return fileInfo.IsDir()
+}
+
 // ResolvePath returns the absolute path of a resource in the instance directory.
 func (n *Node) ResolvePath(x string) string {
 	return n.config.ResolvePath(x)

triedb/database.go

@@ -90,16 +90,23 @@ type Database struct {
 // the legacy hash-based scheme is used by default.
 func NewDatabase(diskdb ethdb.Database, config *Config) *Database {
 	// Sanitize the config and use the default one if it's not specified.
+	var triediskdb ethdb.Database
+	if diskdb != nil && diskdb.StateStore() != nil {
+		triediskdb = diskdb.StateStore()
+	} else {
+		triediskdb = diskdb
+	}
+
 	if config == nil {
 		config = HashDefaults
 	}
 	var preimages *preimageStore
 	if config.Preimages {
-		preimages = newPreimageStore(diskdb)
+		preimages = newPreimageStore(triediskdb)
 	}
 	db := &Database{
 		config:    config,
-		diskdb:    diskdb,
+		diskdb:    triediskdb,
 		preimages: preimages,
 	}
 	if config.HashDB != nil && config.PathDB != nil {