Implement ancient db prunning

.gitignore

@@ -47,3 +47,6 @@ profile.cov
 /dashboard/assets/package-lock.json
 
 **/yarn-error.log
+
+# vim swap files
+*.swp

cmd/geth/chaincmd.go

@@ -192,7 +192,7 @@ func initGenesis(ctx *cli.Context) error {
 	stack, _ := makeConfigNode(ctx)
 	defer stack.Close()
 	for _, name := range []string{"chaindata", "lightchaindata"} {
-		chaindb, err := stack.OpenDatabaseWithFreezer(name, 0, 0, ctx.GlobalString(utils.AncientFlag.Name), "", false)
+		chaindb, err := stack.OpenDatabaseWithFreezer(name, 0, 0, ctx.GlobalString(utils.AncientFlag.Name), "", false, ctx.GlobalUint64(utils.AncientRecentLimitFlag.Name))
 		if err != nil {
 			utils.Fatalf("Failed to open database: %v", err)
 		}

cmd/utils/flags.go

@@ -122,6 +122,11 @@ var (
 		Name:  "datadir.ancient",
 		Usage: "Data directory for ancient chain segments (default = inside chaindata)",
 	}
+	AncientRecentLimitFlag = cli.Uint64Flag{
+		Name:  "ancient.recentlimit",
+		Usage: "Keep only the specified amount of recent ancient blocks and won't do ancient related checks on startup (default = 0, means keep all)",
+		Value: 0,
+	}
 	MinFreeDiskSpaceFlag = DirectoryFlag{
 		Name:  "datadir.minfreedisk",
 		Usage: "Minimum free disk space in MB, once reached triggers auto shut down (default = --cache.gc converted to MB, 0 = disabled)",
@@ -849,6 +854,7 @@ var (
 	DatabasePathFlags = []cli.Flag{
 		DataDirFlag,
 		AncientFlag,
+		AncientRecentLimitFlag,
 		RemoteDBFlag,
 	}
 )
@@ -1637,7 +1643,6 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 	if ctx.GlobalIsSet(AncientFlag.Name) {
 		cfg.DatabaseFreezer = ctx.GlobalString(AncientFlag.Name)
 	}
-
 	if gcmode := ctx.GlobalString(GCModeFlag.Name); gcmode != "full" && gcmode != "archive" {
 		Fatalf("--%s must be either 'full' or 'archive'", GCModeFlag.Name)
 	}
@@ -1656,6 +1661,14 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) {
 	if ctx.GlobalIsSet(TxLookupLimitFlag.Name) {
 		cfg.TxLookupLimit = ctx.GlobalUint64(TxLookupLimitFlag.Name)
 	}
+	if ctx.GlobalIsSet(AncientRecentLimitFlag.Name) {
+		cfg.AncientRecentLimit = ctx.GlobalUint64(AncientRecentLimitFlag.Name)
+		if cfg.TxLookupLimit == 0 || cfg.TxLookupLimit > cfg.AncientRecentLimit+params.FullImmutabilityThreshold {
+			cfg.TxLookupLimit = cfg.AncientRecentLimit + params.FullImmutabilityThreshold
+			log.Warn("Reducing TxLookupLimit to meet ancient db purging, as it's insane to lookup txs in purged blocks")
+		}
+	}
+
 	if ctx.GlobalIsSet(CacheFlag.Name) || ctx.GlobalIsSet(CacheTrieFlag.Name) {
 		cfg.TrieCleanCache = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheTrieFlag.Name) / 100
 	}
@@ -1979,7 +1992,7 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly bool) ethdb.
 	case ctx.GlobalString(SyncModeFlag.Name) == "light":
 		chainDb, err = stack.OpenDatabase("lightchaindata", cache, handles, "", readonly)
 	default:
-		chainDb, err = stack.OpenDatabaseWithFreezer("chaindata", cache, handles, ctx.GlobalString(AncientFlag.Name), "", readonly)
+		chainDb, err = stack.OpenDatabaseWithFreezer("chaindata", cache, handles, ctx.GlobalString(AncientFlag.Name), "", readonly, ctx.GlobalUint64(AncientRecentLimitFlag.Name))
 	}
 	if err != nil {
 		Fatalf("Could not open database: %v", err)

core/rawdb/chain_freezer.go

@@ -37,6 +37,10 @@ const (
 	// freezerBatchLimit is the maximum number of blocks to freeze in one batch
 	// before doing an fsync and deleting it from the key-value store.
 	freezerBatchLimit = 30000
+
+	// cleanerRecheckInterval is the frequency to check the freezer database to
+	// find the bloks that might be pruned
+	cleanerRecheckInterval = 1 * time.Minute
 )
 
 // chainFreezer is a wrapper of freezer with additional chain freezing feature.
@@ -48,6 +52,10 @@ type chainFreezer struct {
 	// so take advantage of that (https://golang.org/pkg/sync/atomic/#pkg-note-BUG).
 	threshold uint64 // Number of recent blocks not to freeze (params.FullImmutabilityThreshold apart from tests)
 
+	// ancientRecentLimit Number of recent blocks to keep in ancient db
+	// if not 0, blocks older than `HEAD - ancientRecentLimit` will be purged from ancient db
+	ancientRecentLimit uint64
+
 	*Freezer
 	quit    chan struct{}
 	wg      sync.WaitGroup
@@ -55,16 +63,17 @@ type chainFreezer struct {
 }
 
 // newChainFreezer initializes the freezer for ancient chain data.
-func newChainFreezer(datadir string, namespace string, readonly bool, maxTableSize uint32, tables map[string]bool) (*chainFreezer, error) {
+func newChainFreezer(datadir string, namespace string, readonly bool, maxTableSize uint32, tables map[string]bool, ancientRecentLimit uint64) (*chainFreezer, error) {
 	freezer, err := NewFreezer(datadir, namespace, readonly, maxTableSize, tables)
 	if err != nil {
 		return nil, err
 	}
 	return &chainFreezer{
-		Freezer:   freezer,
-		threshold: params.FullImmutabilityThreshold,
-		quit:      make(chan struct{}),
-		trigger:   make(chan chan struct{}),
+		Freezer:            freezer,
+		ancientRecentLimit: ancientRecentLimit,
+		threshold:          params.FullImmutabilityThreshold,
+		quit:               make(chan struct{}),
+		trigger:            make(chan chan struct{}),
 	}, nil
 }
 
@@ -250,6 +259,57 @@ func (f *chainFreezer) freeze(db ethdb.KeyValueStore) {
 	}
 }
 
+// clean is a background thread that periodically checks the blockchain for any
+// import progress and cleans ancient data from the freezer.
+//
+// it calculates the highest block(named `cleanTo`) that can be cleaned, then starts
+// cleaning from the genesis+1 block in batches, until reaches the `cleanTo` block.
+func (f *chainFreezer) clean(db ethdb.KeyValueStore) {
+	nfdb := &nofreezedb{KeyValueStore: db}
+
+	for {
+		select {
+		case <-time.NewTimer(cleanerRecheckInterval).C:
+		case <-f.quit:
+			log.Info("Freezer cleaner shutting down")
+			return
+		}
+
+		// we won't start the prunning until chain has initialised itself
+		hash := ReadHeadBlockHash(nfdb)
+		log.Info("Read head block hash", hash.Hex())
+		if hash == (common.Hash{}) {
+			log.Info("current full block hash unavailable, schedule next round") // new chain, empty database
+			continue
+		}
+
+		// Calculates the cleanTo block
+		number := ReadHeaderNumber(nfdb, hash)
+		if number == nil {
+			log.Info("Current full block number unavailable, schedule next round", "hash", hash)
+			continue
+		}
+
+		start := time.Now()
+		tail, _ := f.Tail()
+		frozen, _ := f.Ancients()
+		if frozen < f.ancientRecentLimit || frozen < tail {
+			log.Error("ancient clean error target block", "frozen", frozen, "tail", tail, "ancientRecentLimit", f.ancientRecentLimit)
+			continue
+		}
+
+		cleanTo := frozen - f.ancientRecentLimit
+		// truncate
+		f.TruncateTail(cleanTo)
+
+		// Log something friendly for the user
+		context := []interface{}{
+			"blocks", cleanTo - tail, "frozen", frozen, "ancientRecentLimit", f.ancientRecentLimit, "elapsed", common.PrettyDuration(time.Since(start)), "cleaned to", cleanTo,
+		}
+		log.Info("cleaned chain segment", context...)
+	}
+}
+
 func (f *chainFreezer) freezeRange(nfdb *nofreezedb, number, limit uint64) (hashes []common.Hash, err error) {
 	hashes = make([]common.Hash, 0, limit-number)
 

core/rawdb/database.go

@@ -165,9 +165,9 @@ func NewDatabase(db ethdb.KeyValueStore) ethdb.Database {
 // NewDatabaseWithFreezer creates a high level database on top of a given key-
 // value data store with a freezer moving immutable chain segments into cold
 // storage.
-func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace string, readonly bool) (ethdb.Database, error) {
+func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace string, readonly bool, ancientRecentLimit uint64) (ethdb.Database, error) {
 	// Create the idle freezer instance
-	frdb, err := newChainFreezer(freezer, namespace, readonly, freezerTableSize, FreezerNoSnappy)
+	frdb, err := newChainFreezer(freezer, namespace, readonly, freezerTableSize, FreezerNoSnappy, ancientRecentLimit)
 	if err != nil {
 		return nil, err
 	}
@@ -193,7 +193,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace st
 	// If the genesis hash is empty, we have a new key-value store, so nothing to
 	// validate in this method. If, however, the genesis hash is not nil, compare
 	// it to the freezer content.
-	if kvgenesis, _ := db.Get(headerHashKey(0)); len(kvgenesis) > 0 {
+	if kvgenesis, _ := db.Get(headerHashKey(0)); len(kvgenesis) > 0 && frdb.ancientRecentLimit == 0 {
 		if frozen, _ := frdb.Ancients(); frozen > 0 {
 			// If the freezer already contains something, ensure that the genesis blocks
 			// match, otherwise we might mix up freezers across chains and destroy both
@@ -242,6 +242,14 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace st
 			frdb.freeze(db)
 			frdb.wg.Done()
 		}()
+
+		if frdb.ancientRecentLimit != 0 {
+			frdb.wg.Add(1)
+			go func() {
+				frdb.clean(db)
+				frdb.wg.Done()
+			}()
+		}
 	}
 	return &freezerdb{
 		KeyValueStore: db,
@@ -274,12 +282,12 @@ func NewLevelDBDatabase(file string, cache int, handles int, namespace string, r
 
 // NewLevelDBDatabaseWithFreezer creates a persistent key-value database with a
 // freezer moving immutable chain segments into cold storage.
-func NewLevelDBDatabaseWithFreezer(file string, cache int, handles int, freezer string, namespace string, readonly bool) (ethdb.Database, error) {
+func NewLevelDBDatabaseWithFreezer(file string, cache int, handles int, freezer string, namespace string, readonly bool, ancientRecentLimit uint64) (ethdb.Database, error) {
 	kvdb, err := leveldb.New(file, cache, handles, namespace, readonly)
 	if err != nil {
 		return nil, err
 	}
-	frdb, err := NewDatabaseWithFreezer(kvdb, freezer, namespace, readonly)
+	frdb, err := NewDatabaseWithFreezer(kvdb, freezer, namespace, readonly, ancientRecentLimit)
 	if err != nil {
 		kvdb.Close()
 		return nil, err

core/rawdb/freezer.go

@@ -52,7 +52,7 @@ var (
 )
 
 // freezerTableSize defines the maximum size of freezer data files.
-const freezerTableSize = 2 * 1000 * 1000 * 1000
+const freezerTableSize = 2 * 1000 * 1000
 
 // Freezer is a memory mapped append-only database to store immutable ordered
 // data into flat files:

eth/backend.go

@@ -133,7 +133,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) {
 	ethashConfig.NotifyFull = config.Miner.NotifyFull
 
 	// Assemble the Ethereum object
-	chainDb, err := stack.OpenDatabaseWithFreezer("chaindata", config.DatabaseCache, config.DatabaseHandles, config.DatabaseFreezer, "eth/db/chaindata/", false)
+	chainDb, err := stack.OpenDatabaseWithFreezer("chaindata", config.DatabaseCache, config.DatabaseHandles, config.DatabaseFreezer, "eth/db/chaindata/", false, config.AncientRecentLimit)
 	if err != nil {
 		return nil, err
 	}

eth/ethconfig/config.go

@@ -163,6 +163,7 @@ type Config struct {
 	DatabaseHandles    int  `toml:"-"`
 	DatabaseCache      int
 	DatabaseFreezer    string
+	AncientRecentLimit uint64
 
 	TrieCleanCache          int
 	TrieCleanCacheJournal   string        `toml:",omitempty"` // Disk journal directory for trie cache to survive node restarts

node/node.go

@@ -717,7 +717,7 @@ func (n *Node) OpenDatabase(name string, cache, handles int, namespace string, r
 // also attaching a chain freezer to it that moves ancient chain data from the
 // database to immutable append-only files. If the node is an ephemeral one, a
 // memory database is returned.
-func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, freezer, namespace string, readonly bool) (ethdb.Database, error) {
+func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, freezer, namespace string, readonly bool, ancientRecentLimit uint64) (ethdb.Database, error) {
 	n.lock.Lock()
 	defer n.lock.Unlock()
 	if n.state == closedState {
@@ -736,7 +736,7 @@ func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, freezer,
 		case !filepath.IsAbs(freezer):
 			freezer = n.ResolvePath(freezer)
 		}
-		db, err = rawdb.NewLevelDBDatabaseWithFreezer(root, cache, handles, freezer, namespace, readonly)
+		db, err = rawdb.NewLevelDBDatabaseWithFreezer(root, cache, handles, freezer, namespace, readonly, ancientRecentLimit)
 	}
 
 	if err == nil {