diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index d787f340a3..2536d42b31 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -229,6 +229,15 @@ func initGenesis(ctx *cli.Context) error { } defer chaindb.Close() + // if the trie data dir has been set, new trie db with a new state database + if ctx.IsSet(utils.SeparateDBFlag.Name) { + statediskdb, dbErr := stack.OpenDatabaseWithFreezer(name+"/state", 0, 0, "", "", false) + if dbErr != nil { + utils.Fatalf("Failed to open separate trie database: %v", dbErr) + } + chaindb.SetStateStore(statediskdb) + } + triedb := utils.MakeTrieDatabase(ctx, chaindb, ctx.Bool(utils.CachePreimagesFlag.Name), false, genesis.IsVerkle()) defer triedb.Close() diff --git a/cmd/geth/config.go b/cmd/geth/config.go index 522e5e22f2..c82e062ae4 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -179,6 +179,9 @@ func makeFullNode(ctx *cli.Context) *node.Node { cfg.Eth.OverrideVerkle = &v } + if ctx.IsSet(utils.SeparateDBFlag.Name) && !stack.IsSeparatedDB() { + utils.Fatalf("Failed to locate separate database subdirectory when separatedb parameter has been set") + } backend, eth := utils.RegisterEthService(stack, &cfg.Eth) // Create gauge with geth system and build information diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index 742eadd5f3..79db88b0d7 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -428,6 +428,11 @@ func dbStats(ctx *cli.Context) error { defer db.Close() showLeveldbStats(db) + if db.StateStore() != nil { + fmt.Println("show stats of state store") + showLeveldbStats(db.StateStore()) + } + return nil } @@ -441,13 +446,31 @@ func dbCompact(ctx *cli.Context) error { log.Info("Stats before compaction") showLeveldbStats(db) + statediskdb := db.StateStore() + if statediskdb != nil { + fmt.Println("show stats of state store") + showLeveldbStats(statediskdb) + } + log.Info("Triggering compaction") if err := db.Compact(nil, nil); err != nil { - log.Info("Compact err", "error", err) + log.Error("Compact err", "error", err) return err } + + if statediskdb != nil { + if err := statediskdb.Compact(nil, nil); err != nil { + log.Error("Compact err", "error", err) + return err + } + } + log.Info("Stats after compaction") showLeveldbStats(db) + if statediskdb != nil { + fmt.Println("show stats of state store after compaction") + showLeveldbStats(statediskdb) + } return nil } @@ -468,8 +491,17 @@ func dbGet(ctx *cli.Context) error { return err } + statediskdb := db.StateStore() data, err := db.Get(key) if err != nil { + // if separate trie db exist, try to get it from separate db + if statediskdb != nil { + statedata, dberr := statediskdb.Get(key) + if dberr == nil { + fmt.Printf("key %#x: %#x\n", key, statedata) + return nil + } + } log.Info("Get operation failed", "key", fmt.Sprintf("%#x", key), "error", err) return err } diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 192c850868..08a56dc00c 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -174,9 +174,6 @@ func pruneState(ctx *cli.Context) error { chaindb := utils.MakeChainDatabase(ctx, stack, false) defer chaindb.Close() - if rawdb.ReadStateScheme(chaindb) != rawdb.HashScheme { - log.Crit("Offline pruning is not required for path scheme") - } prunerconfig := pruner.Config{ Datadir: stack.ResolvePath(""), BloomSize: ctx.Uint64(utils.BloomFilterSizeFlag.Name), diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index ecf6acc186..ff4c4e226b 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -94,6 +94,12 @@ var ( Value: flags.DirectoryString(node.DefaultDataDir()), Category: flags.EthCategory, } + SeparateDBFlag = &cli.BoolFlag{ + Name: "separatedb", + Usage: "Enable a separated trie database, it will be created within a subdirectory called state, " + + "Users can copy this state directory to another directory or disk, and then create a symbolic link to the state directory under the chaindata", + Category: flags.EthCategory, + } RemoteDBFlag = &cli.StringFlag{ Name: "remotedb", Usage: "URL for remote database", @@ -974,6 +980,7 @@ var ( DBEngineFlag, StateSchemeFlag, HttpHeaderFlag, + SeparateDBFlag, } ) @@ -2070,6 +2077,11 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly bool) ethdb. chainDb, err = stack.OpenDatabase("lightchaindata", cache, handles, "", readonly) default: chainDb, err = stack.OpenDatabaseWithFreezer("chaindata", cache, handles, ctx.String(AncientFlag.Name), "", readonly) + // set the separate state database + if stack.IsSeparatedDB() && err == nil { + stateDiskDb := MakeStateDataBase(ctx, stack, readonly, false) + chainDb.SetStateStore(stateDiskDb) + } } if err != nil { Fatalf("Could not open database: %v", err) @@ -2077,6 +2089,17 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly bool) ethdb. return chainDb } +// MakeStateDataBase open a separate state database using the flags passed to the client and will hard crash if it fails. +func MakeStateDataBase(ctx *cli.Context, stack *node.Node, readonly, disableFreeze bool) ethdb.Database { + cache := ctx.Int(CacheFlag.Name) * ctx.Int(CacheDatabaseFlag.Name) / 100 + handles := MakeDatabaseHandles(ctx.Int(FDLimitFlag.Name)) / 2 + statediskdb, err := stack.OpenDatabaseWithFreezer("chaindata/state", cache, handles, "", "", readonly) + if err != nil { + Fatalf("Failed to open separate trie database: %v", err) + } + return statediskdb +} + // tryMakeReadOnlyDatabase try to open the chain database in read-only mode, // or fallback to write mode if the database is not initialized. func tryMakeReadOnlyDatabase(ctx *cli.Context, stack *node.Node) ethdb.Database { diff --git a/core/rawdb/accessors_trie.go b/core/rawdb/accessors_trie.go index 44eb715d04..2d55fd8ae9 100644 --- a/core/rawdb/accessors_trie.go +++ b/core/rawdb/accessors_trie.go @@ -247,12 +247,12 @@ func DeleteTrieNode(db ethdb.KeyValueWriter, owner common.Hash, path []byte, has // if the state is not present in database. func ReadStateScheme(db ethdb.Reader) string { // Check if state in path-based scheme is present. - if HasAccountTrieNode(db, nil) { + if HasAccountTrieNode(db.StateStoreReader(), nil) { return PathScheme } // The root node might be deleted during the initial snap sync, check // the persistent state id then. - if id := ReadPersistentStateID(db); id != 0 { + if id := ReadPersistentStateID(db.StateStoreReader()); id != 0 { return PathScheme } // In a hash-based scheme, the genesis state is consistently stored @@ -262,7 +262,7 @@ func ReadStateScheme(db ethdb.Reader) string { if header == nil { return "" // empty datadir } - if !HasLegacyTrieNode(db, header.Root) { + if !HasLegacyTrieNode(db.StateStoreReader(), header.Root) { return "" // no state in disk } return HashScheme diff --git a/core/rawdb/ancient_utils.go b/core/rawdb/ancient_utils.go index 1c69639c9d..6ee8909987 100644 --- a/core/rawdb/ancient_utils.go +++ b/core/rawdb/ancient_utils.go @@ -89,6 +89,9 @@ func inspectFreezers(db ethdb.Database) ([]freezerInfo, error) { infos = append(infos, info) case StateFreezerName: + if db.StateStore() != nil { + continue + } datadir, err := db.AncientDatadir() if err != nil { return nil, err diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 3436958de7..3a50a0d33d 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -41,6 +41,14 @@ type freezerdb struct { readOnly bool ancientRoot string + stateStore ethdb.Database +} + +func (frdb *freezerdb) StateStoreReader() ethdb.Reader { + if frdb.stateStore == nil { + return frdb + } + return frdb.stateStore } // AncientDatadir returns the path of root ancient directory. @@ -58,12 +66,29 @@ func (frdb *freezerdb) Close() error { if err := frdb.KeyValueStore.Close(); err != nil { errs = append(errs, err) } + + if frdb.stateStore != nil { + if err := frdb.stateStore.Close(); err != nil { + errs = append(errs, err) + } + } if len(errs) != 0 { return fmt.Errorf("%v", errs) } return nil } +func (frdb *freezerdb) StateStore() ethdb.Database { + return frdb.stateStore +} + +func (frdb *freezerdb) SetStateStore(state ethdb.Database) { + if frdb.stateStore != nil { + frdb.stateStore.Close() + } + frdb.stateStore = state +} + // Freeze is a helper method used for external testing to trigger and block until // a freeze cycle completes, without having to sleep for a minute to trigger the // automatic background run. @@ -81,6 +106,7 @@ func (frdb *freezerdb) Freeze() error { // nofreezedb is a database wrapper that disables freezer data retrievals. type nofreezedb struct { ethdb.KeyValueStore + stateStore ethdb.Database } // HasAncient returns an error as we don't have a backing chain freezer. @@ -133,6 +159,21 @@ func (db *nofreezedb) Sync() error { return errNotSupported } +func (db *nofreezedb) StateStore() ethdb.Database { + return db.stateStore +} + +func (db *nofreezedb) SetStateStore(state ethdb.Database) { + db.stateStore = state +} + +func (db *nofreezedb) StateStoreReader() ethdb.Reader { + if db.stateStore != nil { + return db.stateStore + } + return db +} + func (db *nofreezedb) ReadAncients(fn func(reader ethdb.AncientReaderOp) error) (err error) { // Unlike other ancient-related methods, this method does not return // errNotSupported when invoked. @@ -456,6 +497,11 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { it := db.NewIterator(keyPrefix, keyStart) defer it.Release() + var trieIter ethdb.Iterator + if db.StateStore() != nil { + trieIter = db.StateStore().NewIterator(keyPrefix, nil) + defer trieIter.Release() + } var ( count int64 start = time.Now() @@ -506,14 +552,14 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { bodies.Add(size) case bytes.HasPrefix(key, blockReceiptsPrefix) && len(key) == (len(blockReceiptsPrefix)+8+common.HashLength): receipts.Add(size) + case IsLegacyTrieNode(key, it.Value()): + legacyTries.Add(size) case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerTDSuffix): tds.Add(size) case bytes.HasPrefix(key, headerPrefix) && bytes.HasSuffix(key, headerHashSuffix): numHashPairings.Add(size) case bytes.HasPrefix(key, headerNumberPrefix) && len(key) == (len(headerNumberPrefix)+common.HashLength): hashNumPairings.Add(size) - case IsLegacyTrieNode(key, it.Value()): - legacyTries.Add(size) case bytes.HasPrefix(key, stateIDPrefix) && len(key) == len(stateIDPrefix)+common.HashLength: stateLookups.Add(size) case IsAccountTrieNode(key): @@ -575,6 +621,46 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { logged = time.Now() } } + // inspect separate trie db + if trieIter != nil { + count = 0 + logged = time.Now() + for trieIter.Next() { + var ( + key = trieIter.Key() + value = trieIter.Value() + size = common.StorageSize(len(key) + len(value)) + ) + + switch { + case IsLegacyTrieNode(key, value): + legacyTries.Add(size) + case bytes.HasPrefix(key, stateIDPrefix) && len(key) == len(stateIDPrefix)+common.HashLength: + stateLookups.Add(size) + case IsAccountTrieNode(key): + accountTries.Add(size) + case IsStorageTrieNode(key): + storageTries.Add(size) + default: + var accounted bool + for _, meta := range [][]byte{ + fastTrieProgressKey, persistentStateIDKey, trieJournalKey} { + if bytes.Equal(key, meta) { + metadata.Add(size) + break + } + } + if !accounted { + unaccounted.Add(size) + } + } + count++ + if count%1000 == 0 && time.Since(logged) > 8*time.Second { + log.Info("Inspecting separate state database", "count", count, "elapsed", common.PrettyDuration(time.Since(start))) + logged = time.Now() + } + } + } // Display the database statistic of key-value store. stats := [][]string{ {"Key-Value store", "Headers", headers.Size(), headers.Count()}, @@ -615,6 +701,28 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { } total += ancient.size() } + + // inspect ancient state in separate trie db if exist + if trieIter != nil { + stateAncients, err := inspectFreezers(db.StateStore()) + if err != nil { + return err + } + for _, ancient := range stateAncients { + for _, table := range ancient.sizes { + if ancient.name == "chain" { + break + } + stats = append(stats, []string{ + fmt.Sprintf("Ancient store (%s)", strings.Title(ancient.name)), + strings.Title(table.name), + table.size.String(), + fmt.Sprintf("%d", ancient.count()), + }) + } + total += ancient.size() + } + } table := tablewriter.NewWriter(os.Stdout) table.SetHeader([]string{"Database", "Category", "Size", "Items"}) table.SetFooter([]string{"", "Total", total.String(), " "}) diff --git a/core/rawdb/table.go b/core/rawdb/table.go index 19e4ed5b5c..a945c3e905 100644 --- a/core/rawdb/table.go +++ b/core/rawdb/table.go @@ -195,6 +195,18 @@ func (t *table) NewBatch() ethdb.Batch { return &tableBatch{t.db.NewBatch(), t.prefix} } +func (t *table) StateStore() ethdb.Database { + return nil +} + +func (t *table) SetStateStore(state ethdb.Database) { + panic("not implement") +} + +func (t *table) StateStoreReader() ethdb.Reader { + return nil +} + // NewBatchWithSize creates a write-only database batch with pre-allocated buffer. func (t *table) NewBatchWithSize(size int) ethdb.Batch { return &tableBatch{t.db.NewBatchWithSize(size), t.prefix} diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index 59c580daca..53df00f654 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -125,13 +125,19 @@ func prune(snaptree *snapshot.Tree, root common.Hash, maindb ethdb.Database, sta // that the false-positive is low enough(~0.05%). The probability of the // dangling node is the state root is super low. So the dangling nodes in // theory will never ever be visited again. + var pruneDB ethdb.Database + if maindb != nil && maindb.StateStore() != nil { + pruneDB = maindb.StateStore() + } else { + pruneDB = maindb + } var ( skipped, count int size common.StorageSize pstart = time.Now() logged = time.Now() - batch = maindb.NewBatch() - iter = maindb.NewIterator(nil, nil) + batch = pruneDB.NewBatch() + iter = pruneDB.NewIterator(nil, nil) ) for iter.Next() { key := iter.Key() @@ -178,7 +184,7 @@ func prune(snaptree *snapshot.Tree, root common.Hash, maindb ethdb.Database, sta batch.Reset() iter.Release() - iter = maindb.NewIterator(nil, key) + iter = pruneDB.NewIterator(nil, key) } } } @@ -221,7 +227,7 @@ func prune(snaptree *snapshot.Tree, root common.Hash, maindb ethdb.Database, sta end = nil } log.Info("Compacting database", "range", fmt.Sprintf("%#x-%#x", start, end), "elapsed", common.PrettyDuration(time.Since(cstart))) - if err := maindb.Compact(start, end); err != nil { + if err := pruneDB.Compact(start, end); err != nil { log.Error("Database compaction failed", "error", err) return err } @@ -266,10 +272,17 @@ func (p *Pruner) Prune(root common.Hash) error { // Use the bottom-most diff layer as the target root = layers[len(layers)-1].Root() } + // if the separated state db has been set, use this db to prune data + var trienodedb ethdb.Database + if p.db != nil && p.db.StateStore() != nil { + trienodedb = p.db.StateStore() + } else { + trienodedb = p.db + } // Ensure the root is really present. The weak assumption // is the presence of root can indicate the presence of the // entire trie. - if !rawdb.HasLegacyTrieNode(p.db, root) { + if !rawdb.HasLegacyTrieNode(trienodedb, root) { // The special case is for clique based networks(goerli // and some other private networks), it's possible that two // consecutive blocks will have same root. In this case snapshot @@ -283,7 +296,7 @@ func (p *Pruner) Prune(root common.Hash) error { // as the pruning target. var found bool for i := len(layers) - 2; i >= 2; i-- { - if rawdb.HasLegacyTrieNode(p.db, layers[i].Root()) { + if rawdb.HasLegacyTrieNode(trienodedb, layers[i].Root()) { root = layers[i].Root() found = true log.Info("Selecting middle-layer as the pruning target", "root", root, "depth", i) diff --git a/eth/backend.go b/eth/backend.go index e616b5f2f1..d751bca0f3 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -59,6 +59,10 @@ import ( "github.com/ethereum/go-ethereum/rpc" ) +const ( + ChainDBNamespace = "eth/db/chaindata/" +) + // Config contains the configuration options of the ETH protocol. // Deprecated: use ethconfig.Config instead. type Config = ethconfig.Config @@ -127,7 +131,8 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { log.Info("Allocated trie memory caches", "clean", common.StorageSize(config.TrieCleanCache)*1024*1024, "dirty", common.StorageSize(config.TrieDirtyCache)*1024*1024) // Assemble the Ethereum object - chainDb, err := stack.OpenDatabaseWithFreezer("chaindata", config.DatabaseCache, config.DatabaseHandles, config.DatabaseFreezer, "eth/db/chaindata/", false) + chainDb, err := stack.OpenAndMergeDatabase("chaindata", config.DatabaseCache, config.DatabaseHandles, + config.DatabaseFreezer, ChainDBNamespace, false) if err != nil { return nil, err } diff --git a/ethdb/database.go b/ethdb/database.go index 3ec1f70e3b..7fea031bf3 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -153,11 +153,16 @@ type AncientStater interface { AncientDatadir() (string, error) } +type StateStoreReader interface { + StateStoreReader() Reader +} + // Reader contains the methods required to read data from both key-value as well as // immutable ancient data. type Reader interface { KeyValueReader AncientReader + StateStoreReader } // Writer contains the methods required to write data to both key-value as well as @@ -190,11 +195,17 @@ type ResettableAncientStore interface { Reset() error } +type StateStore interface { + StateStore() Database + SetStateStore(state Database) +} + // Database contains all the methods required by the high level database to not // only access the key-value data store but also the ancient chain store. type Database interface { Reader Writer + StateStore Batcher Iteratee Stater diff --git a/ethdb/remotedb/remotedb.go b/ethdb/remotedb/remotedb.go index c1c803caf2..e4df060108 100644 --- a/ethdb/remotedb/remotedb.go +++ b/ethdb/remotedb/remotedb.go @@ -82,6 +82,18 @@ func (db *Database) AncientSize(kind string) (uint64, error) { panic("not supported") } +func (db *Database) StateStore() ethdb.Database { + panic("not supported") +} + +func (db *Database) SetStateStore(state ethdb.Database) { + panic("not supported") +} + +func (db *Database) StateStoreReader() ethdb.Reader { + return db +} + func (db *Database) ReadAncients(fn func(op ethdb.AncientReaderOp) error) (err error) { return fn(db) } diff --git a/node/node.go b/node/node.go index 633f88f058..7b0e945fae 100644 --- a/node/node.go +++ b/node/node.go @@ -739,6 +739,36 @@ func (n *Node) OpenDatabase(name string, cache, handles int, namespace string, r return db, err } +func (n *Node) OpenAndMergeDatabase(name string, cache, handles int, ancient, namespace string, readonly bool) (ethdb.Database, error) { + chainDataHandles := handles + + var statediskdb ethdb.Database + var err error + // Open the separated state database if the state directory exists + if n.IsSeparatedDB() { + // Allocate half of the handles and cache to this separate state data database + statediskdb, err = n.OpenDatabaseWithFreezer(name+"/state", cache/2, chainDataHandles/2, "", "eth/db/statedata/", readonly) + if err != nil { + return nil, err + } + + // Reduce the handles and cache to this separate database because it is not a complete database with no trie data storing in it. + cache = int(float64(cache) * 0.6) + chainDataHandles = int(float64(chainDataHandles) * 0.6) + } + + chainDB, err := n.OpenDatabaseWithFreezer(name, cache, chainDataHandles, ancient, namespace, readonly) + if err != nil { + return nil, err + } + + if statediskdb != nil { + chainDB.SetStateStore(statediskdb) + } + + return chainDB, nil +} + // OpenDatabaseWithFreezer opens an existing database with the given name (or // creates one if no previous can be found) from within the node's data directory, // also attaching a chain freezer to it that moves ancient chain data from the @@ -772,6 +802,16 @@ func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, ancient return db, err } +// IsSeparatedDB check the state subdirectory of db, if subdirectory exists, return true +func (n *Node) IsSeparatedDB() bool { + separateDir := filepath.Join(n.ResolvePath("chaindata"), "state") + fileInfo, err := os.Stat(separateDir) + if os.IsNotExist(err) { + return false + } + return fileInfo.IsDir() +} + // ResolvePath returns the absolute path of a resource in the instance directory. func (n *Node) ResolvePath(x string) string { return n.config.ResolvePath(x) diff --git a/triedb/database.go b/triedb/database.go index 10f77982f3..9fd7a49630 100644 --- a/triedb/database.go +++ b/triedb/database.go @@ -90,16 +90,23 @@ type Database struct { // the legacy hash-based scheme is used by default. func NewDatabase(diskdb ethdb.Database, config *Config) *Database { // Sanitize the config and use the default one if it's not specified. + var triediskdb ethdb.Database + if diskdb != nil && diskdb.StateStore() != nil { + triediskdb = diskdb.StateStore() + } else { + triediskdb = diskdb + } + if config == nil { config = HashDefaults } var preimages *preimageStore if config.Preimages { - preimages = newPreimageStore(diskdb) + preimages = newPreimageStore(triediskdb) } db := &Database{ config: config, - diskdb: diskdb, + diskdb: triediskdb, preimages: preimages, } if config.HashDB != nil && config.PathDB != nil {