diff --git a/cmd/geth/main.go b/cmd/geth/main.go index f2bb275520..abf0303484 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -291,6 +291,7 @@ JavaScript API. See https://github.com/ethereum/go-ethereum/wiki/Javascipt-Conso utils.FastSyncFlag, utils.CacheFlag, utils.LightKDFFlag, + utils.StateRetentionFlag, utils.JSpathFlag, utils.ListenPortFlag, utils.MaxPeersFlag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 7a6ff704c4..7c02544f41 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -69,6 +69,7 @@ var AppHelpFlagGroups = []flagGroup{ utils.GenesisFileFlag, utils.IdentityFlag, utils.FastSyncFlag, + utils.StateRetentionFlag, utils.LightKDFFlag, utils.CacheFlag, utils.BlockchainVersionFlag, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 63efa08ee3..6246d35b99 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -163,6 +163,11 @@ var ( Name: "lightkdf", Usage: "Reduce key-derivation RAM & CPU usage at some expense of KDF strength", } + StateRetentionFlag = cli.IntFlag{ + Name: "retain", + Usage: "Number of recent state tries to retain (0 = keep all history)", + Value: 5000, + } // Miner settings // TODO: refactor CPU vs GPU mining flags MiningEnabledFlag = cli.BoolFlag{ @@ -613,6 +618,7 @@ func MakeSystemNode(name, version string, extra []byte, ctx *cli.Context) *node. ethConf := ð.Config{ Genesis: MakeGenesisBlock(ctx), FastSync: ctx.GlobalBool(FastSyncFlag.Name), + StateRetention: ctx.GlobalInt(StateRetentionFlag.Name), BlockChainVersion: ctx.GlobalInt(BlockchainVersionFlag.Name), DatabaseCache: ctx.GlobalInt(CacheFlag.Name), NetworkId: ctx.GlobalInt(NetworkIdFlag.Name), diff --git a/eth/backend.go b/eth/backend.go index abd1214caa..6e130a9c89 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -33,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/common/compiler" "github.com/ethereum/go-ethereum/common/httpclient" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/filters" @@ -45,6 +46,7 @@ import ( "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/rlp" rpc "github.com/ethereum/go-ethereum/rpc/v2" + "github.com/ethereum/go-ethereum/trie" ) const ( @@ -61,9 +63,10 @@ var ( ) type Config struct { - NetworkId int // Network ID to use for selecting peers to connect to - Genesis string // Genesis JSON to seed the chain database with - FastSync bool // Enables the state download based fast synchronisation algorithm + NetworkId int // Network ID to use for selecting peers to connect to + Genesis string // Genesis JSON to seed the chain database with + FastSync bool // Enables the state download based fast synchronisation algorithm + StateRetention int // Number of recent state tries to retain pruning BlockChainVersion int SkipBcVersionCheck bool // e.g. blockchain export @@ -136,6 +139,10 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { const dbCount = 3 ethdb.OpenFileLimit = 128 / (dbCount + 1) + // Sanity check that the user doesn't throw away more data than what would hurt the network + if config.StateRetention != 0 && config.StateRetention < downloader.FsStateRetention { + return nil, fmt.Errorf("configured state retention (%d) too low (< %d) for network security", config.StateRetention, downloader.FsStateRetention) + } // Open the chain database and perform any upgrades needed chainDb, err := ctx.OpenDatabase("chaindata", config.DatabaseCache) if err != nil { @@ -150,7 +157,9 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { if err := addMipmapBloomBins(chainDb); err != nil { return nil, err } - + if err := indexStateDatabase(chainDb, uint64(config.StateRetention)); err != nil { + return nil, err + } dappDb, err := ctx.OpenDatabase("dapp", config.DatabaseCache) if err != nil { return nil, err @@ -578,3 +587,63 @@ func addMipmapBloomBins(db ethdb.Database) (err error) { glog.V(logger.Info).Infoln("upgrade completed in", time.Since(tstart)) return nil } + +// indexStateDatabase iterates over the state tries of the topmost blocks in the +// chain and inserts all child-parent references into the trie index to facilitate +// state trie pruning afterwards. +func indexStateDatabase(db ethdb.Database, retain uint64) error { + // Short circuit if the head block is already indexed + head := core.GetBlock(db, core.GetHeadBlockHash(db)) + if head == nil { + return nil // Empty database, don't die on it + } + if _, err := db.Get(trie.ParentReferenceIndexKey(head.Hash().Bytes(), head.Root().Bytes())); err == nil { + return nil // Head already indexed, assume up to date database + } + // Iterate over the top blocks that are being retained and index all the state tries + glog.V(logger.Info).Infoln("Indexing database for state pruning (first block takes a few minutes, please be patient)...") + + from, till := uint64(0), head.NumberU64() + if from+retain < till { + from = till - retain + } + for num := from; num <= till; num++ { + glog.V(logger.Info).Infof("%.2f%%: Indexing state of block #%d...", 100*float64(num-from)/float64(till-from+1), num) + + // Load the block and ensure any database corruption is signaled + header := core.GetHeader(db, core.GetCanonicalHash(db, num)) + if header == nil { + return fmt.Errorf("block #%d missing", num) + } + // Short circuit if the block has already been indexed previously + if _, err := db.Get(trie.ParentReferenceIndexKey(header.Hash().Bytes(), header.Root.Bytes())); err == nil { + continue + } + // Otherwise iterate the entire state trie and insert the indices + stateDb, err := state.New(header.Root, db) + if err != nil { + return err + } + it := state.NewNodeIterator(stateDb) + it.PreOrderHook = func(hash, parent common.Hash) bool { + // If we've already indexed this branch, skip + _, err := db.Get(trie.ParentReferenceIndexKey(parent.Bytes(), hash.Bytes())) + if err == nil { + return false + } + return true + } + for it.Next() { + if it.Hash != (common.Hash{}) && it.Parent != (common.Hash{}) { + if err := db.Put(trie.ParentReferenceIndexKey(it.Parent.Bytes(), it.Hash.Bytes()), nil); err != nil { + return err + } + } + } + if err := db.Put(trie.ParentReferenceIndexKey(header.Hash().Bytes(), header.Root.Bytes()), nil); err != nil { + return err + } + } + glog.V(logger.Info).Infof("Database successfully indexed") + return nil +} diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 1639947304..4ddd128777 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -67,6 +67,8 @@ var ( fsHeaderForceVerify = 24 // Number of headers to verify before and after the pivot to accept it fsPivotInterval = 512 // Number of headers out of which to randomize the pivot point fsMinFullBlocks = 1024 // Number of blocks to retrieve fully even in fast sync + + FsStateRetention = fsMinFullBlocks + fsPivotInterval + 1 // Number of state tries needed to reliably fast sync ) var ( diff --git a/trie/iterator.go b/trie/iterator.go index df9efece7f..4f7bf924cc 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -243,18 +243,18 @@ func (it *NodeIterator) step() { } parent.child++ it.stack = append(it.stack, &nodeIteratorState{node: node.Val, parent: ancestor, child: -1}) - } else if hash, ok := parent.node.(hashNode); ok { + } else if hashNode, ok := parent.node.(hashNode); ok { // Hash node, resolve the hash child from the database, then the node itself if parent.child >= 0 { break } parent.child++ - node, err := it.trie.resolveHash(hash, nil, nil) - if err != nil { - panic(err) - } - if hash := common.BytesToHash(hash); it.PreOrderHook == nil || it.PreOrderHook(hash, ancestor) { + if hash := common.BytesToHash(hashNode); it.PreOrderHook == nil || it.PreOrderHook(hash, ancestor) { + node, err := it.trie.resolveHash(hashNode, nil, nil) + if err != nil { + panic(err) + } it.stack = append(it.stack, &nodeIteratorState{hash: hash, node: node, parent: ancestor, child: -1}) } } else {