cmd, eth, trie: implement state database upgrade-indexing

cmd/geth/main.go

@@ -291,6 +291,7 @@ JavaScript API. See https://github.com/ethereum/go-ethereum/wiki/Javascipt-Conso
 		utils.FastSyncFlag,
 		utils.CacheFlag,
 		utils.LightKDFFlag,
+		utils.StateRetentionFlag,
 		utils.JSpathFlag,
 		utils.ListenPortFlag,
 		utils.MaxPeersFlag,

cmd/geth/usage.go

@@ -69,6 +69,7 @@ var AppHelpFlagGroups = []flagGroup{
 			utils.GenesisFileFlag,
 			utils.IdentityFlag,
 			utils.FastSyncFlag,
+			utils.StateRetentionFlag,
 			utils.LightKDFFlag,
 			utils.CacheFlag,
 			utils.BlockchainVersionFlag,

cmd/utils/flags.go

@@ -163,6 +163,11 @@ var (
 		Name:  "lightkdf",
 		Usage: "Reduce key-derivation RAM & CPU usage at some expense of KDF strength",
 	}
+	StateRetentionFlag = cli.IntFlag{
+		Name:  "retain",
+		Usage: "Number of recent state tries to retain (0 = keep all history)",
+		Value: 5000,
+	}
 	// Miner settings
 	// TODO: refactor CPU vs GPU mining flags
 	MiningEnabledFlag = cli.BoolFlag{
@@ -613,6 +618,7 @@ func MakeSystemNode(name, version string, extra []byte, ctx *cli.Context) *node.
 	ethConf := &eth.Config{
 		Genesis:                 MakeGenesisBlock(ctx),
 		FastSync:                ctx.GlobalBool(FastSyncFlag.Name),
+		StateRetention:          ctx.GlobalInt(StateRetentionFlag.Name),
 		BlockChainVersion:       ctx.GlobalInt(BlockchainVersionFlag.Name),
 		DatabaseCache:           ctx.GlobalInt(CacheFlag.Name),
 		NetworkId:               ctx.GlobalInt(NetworkIdFlag.Name),

eth/backend.go

@@ -33,6 +33,7 @@ import (
 	"github.com/ethereum/go-ethereum/common/compiler"
 	"github.com/ethereum/go-ethereum/common/httpclient"
 	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/eth/downloader"
 	"github.com/ethereum/go-ethereum/eth/filters"
@@ -45,6 +46,7 @@ import (
 	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/rlp"
 	rpc "github.com/ethereum/go-ethereum/rpc/v2"
+	"github.com/ethereum/go-ethereum/trie"
 )
 
 const (
@@ -64,6 +66,7 @@ type Config struct {
 	NetworkId      int    // Network ID to use for selecting peers to connect to
 	Genesis        string // Genesis JSON to seed the chain database with
 	FastSync       bool   // Enables the state download based fast synchronisation algorithm
+	StateRetention int    // Number of recent state tries to retain pruning
 
 	BlockChainVersion  int
 	SkipBcVersionCheck bool // e.g. blockchain export
@@ -136,6 +139,10 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) {
 	const dbCount = 3
 	ethdb.OpenFileLimit = 128 / (dbCount + 1)
 
+	// Sanity check that the user doesn't throw away more data than what would hurt the network
+	if config.StateRetention != 0 && config.StateRetention < downloader.FsStateRetention {
+		return nil, fmt.Errorf("configured state retention (%d) too low (< %d) for network security", config.StateRetention, downloader.FsStateRetention)
+	}
 	// Open the chain database and perform any upgrades needed
 	chainDb, err := ctx.OpenDatabase("chaindata", config.DatabaseCache)
 	if err != nil {
@@ -150,7 +157,9 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) {
 	if err := addMipmapBloomBins(chainDb); err != nil {
 		return nil, err
 	}
-
+	if err := indexStateDatabase(chainDb, uint64(config.StateRetention)); err != nil {
+		return nil, err
+	}
 	dappDb, err := ctx.OpenDatabase("dapp", config.DatabaseCache)
 	if err != nil {
 		return nil, err
@@ -578,3 +587,63 @@ func addMipmapBloomBins(db ethdb.Database) (err error) {
 	glog.V(logger.Info).Infoln("upgrade completed in", time.Since(tstart))
 	return nil
 }
+
+// indexStateDatabase iterates over the state tries of the topmost blocks in the
+// chain and inserts all child-parent references into the trie index to facilitate
+// state trie pruning afterwards.
+func indexStateDatabase(db ethdb.Database, retain uint64) error {
+	// Short circuit if the head block is already indexed
+	head := core.GetBlock(db, core.GetHeadBlockHash(db))
+	if head == nil {
+		return nil // Empty database, don't die on it
+	}
+	if _, err := db.Get(trie.ParentReferenceIndexKey(head.Hash().Bytes(), head.Root().Bytes())); err == nil {
+		return nil // Head already indexed, assume up to date database
+	}
+	// Iterate over the top blocks that are being retained and index all the state tries
+	glog.V(logger.Info).Infoln("Indexing database for state pruning (first block takes a few minutes, please be patient)...")
+
+	from, till := uint64(0), head.NumberU64()
+	if from+retain < till {
+		from = till - retain
+	}
+	for num := from; num <= till; num++ {
+		glog.V(logger.Info).Infof("%.2f%%: Indexing state of block #%d...", 100*float64(num-from)/float64(till-from+1), num)
+
+		// Load the block and ensure any database corruption is signaled
+		header := core.GetHeader(db, core.GetCanonicalHash(db, num))
+		if header == nil {
+			return fmt.Errorf("block #%d missing", num)
+		}
+		// Short circuit if the block has already been indexed previously
+		if _, err := db.Get(trie.ParentReferenceIndexKey(header.Hash().Bytes(), header.Root.Bytes())); err == nil {
+			continue
+		}
+		// Otherwise iterate the entire state trie and insert the indices
+		stateDb, err := state.New(header.Root, db)
+		if err != nil {
+			return err
+		}
+		it := state.NewNodeIterator(stateDb)
+		it.PreOrderHook = func(hash, parent common.Hash) bool {
+			// If we've already indexed this branch, skip
+			_, err := db.Get(trie.ParentReferenceIndexKey(parent.Bytes(), hash.Bytes()))
+			if err == nil {
+				return false
+			}
+			return true
+		}
+		for it.Next() {
+			if it.Hash != (common.Hash{}) && it.Parent != (common.Hash{}) {
+				if err := db.Put(trie.ParentReferenceIndexKey(it.Parent.Bytes(), it.Hash.Bytes()), nil); err != nil {
+					return err
+				}
+			}
+		}
+		if err := db.Put(trie.ParentReferenceIndexKey(header.Hash().Bytes(), header.Root.Bytes()), nil); err != nil {
+			return err
+		}
+	}
+	glog.V(logger.Info).Infof("Database successfully indexed")
+	return nil
+}

eth/downloader/downloader.go

@@ -67,6 +67,8 @@ var (
 	fsHeaderForceVerify    = 24   // Number of headers to verify before and after the pivot to accept it
 	fsPivotInterval        = 512  // Number of headers out of which to randomize the pivot point
 	fsMinFullBlocks        = 1024 // Number of blocks to retrieve fully even in fast sync
+
+	FsStateRetention = fsMinFullBlocks + fsPivotInterval + 1 // Number of state tries needed to reliably fast sync
 )
 
 var (

trie/iterator.go

@@ -243,18 +243,18 @@ func (it *NodeIterator) step() {
 			}
 			parent.child++
 			it.stack = append(it.stack, &nodeIteratorState{node: node.Val, parent: ancestor, child: -1})
-		} else if hash, ok := parent.node.(hashNode); ok {
+		} else if hashNode, ok := parent.node.(hashNode); ok {
 			// Hash node, resolve the hash child from the database, then the node itself
 			if parent.child >= 0 {
 				break
 			}
 			parent.child++
 
-			node, err := it.trie.resolveHash(hash, nil, nil)
+			if hash := common.BytesToHash(hashNode); it.PreOrderHook == nil || it.PreOrderHook(hash, ancestor) {
+				node, err := it.trie.resolveHash(hashNode, nil, nil)
 				if err != nil {
 					panic(err)
 				}
-			if hash := common.BytesToHash(hash); it.PreOrderHook == nil || it.PreOrderHook(hash, ancestor) {
 				it.stack = append(it.stack, &nodeIteratorState{hash: hash, node: node, parent: ancestor, child: -1})
 			}
 		} else {