ethdb: print a warning log if write delay exceeds threshold

ethdb/database.go

@@ -34,7 +34,10 @@ import (
 	gometrics "github.com/rcrowley/go-metrics"
 )
 
-var OpenFileLimit = 64
+const (
+	writeDelayNThreshold = 250
+	writeDelayThreshold  = 500 * time.Millisecond
+)
 
 type LDBDatabase struct {
 	fn string      // filename for reporting
@@ -178,11 +181,9 @@ func (db *LDBDatabase) LDB() *leveldb.DB {
 
 // Meter configures the database metrics collectors and
 func (db *LDBDatabase) Meter(prefix string) {
-	// Short circuit metering if the metrics system is disabled
+	if metrics.Enabled {
-	if !metrics.Enabled {
+		// Initialize all database related metrics collector at the requested prefix
-		return
+		// if metric is enable.
-	}
-	// Initialize all the metrics collector at the requested prefix
 		db.getTimer = metrics.NewTimer(prefix + "user/gets")
 		db.putTimer = metrics.NewTimer(prefix + "user/puts")
 		db.delTimer = metrics.NewTimer(prefix + "user/dels")
@@ -190,11 +191,13 @@ func (db *LDBDatabase) Meter(prefix string) {
 		db.readMeter = metrics.NewMeter(prefix + "user/reads")
 		db.writeMeter = metrics.NewMeter(prefix + "user/writes")
 
-	db.writeDelayMeter = metrics.NewMeter(prefix + "compact/writedelay/duration")
-	db.writeDelayNMeter = metrics.NewMeter(prefix + "compact/writedelay/counter")
 		db.compTimeMeter = metrics.NewMeter(prefix + "compact/time")
 		db.compReadMeter = metrics.NewMeter(prefix + "compact/input")
 		db.compWriteMeter = metrics.NewMeter(prefix + "compact/output")
+	}
+	// Initialize essential write delay metrics no matter metric is enable or not.
+	db.writeDelayMeter = metrics.NewMeter(prefix + "compact/writedelay/duration")
+	db.writeDelayNMeter = metrics.NewMeter(prefix + "compact/writedelay/counter")
 
 	// Create a quit channel for the periodic collector and run it
 	db.quitLock.Lock()
@@ -223,6 +226,7 @@ func (db *LDBDatabase) meter(refresh time.Duration) {
 	}
 	// Iterate ad infinitum and collect the stats
 	for i := 1; ; i++ {
+		if metrics.Enabled {
 			// Retrieve the database stats
 			stats, err := db.db.GetProperty("leveldb.stats")
 			if err != nil {
@@ -268,8 +272,9 @@ func (db *LDBDatabase) meter(refresh time.Duration) {
 			if db.compWriteMeter != nil {
 				db.compWriteMeter.Mark(int64((counters[i%2][2] - counters[(i-1)%2][2]) * 1024 * 1024))
 			}
+		}
 
-		// Stat write delay.
+		// Gather write delay statistic
 		writeDelay, err := db.db.GetProperty("leveldb.writedelay")
 		if err != nil {
 			db.log.Error("Failed to read database write delay statistic", "err", err)
@@ -294,11 +299,22 @@ func (db *LDBDatabase) meter(refresh time.Duration) {
 
 		if db.writeDelayNMeter != nil {
 			db.writeDelayNMeter.Mark(int64((counters[i%2][3] - counters[(i-1)%2][3])))
+			// If the write delay number been collected in the last minute exceeds the predefined threshold,
+			// print a warning log here.
+			if int(db.writeDelayNMeter.Rate1()) > writeDelayNThreshold {
+				db.log.Warn("Write delay number exceeds the threshold (250) in the last minute")
+			}
 		}
 
 		if db.writeDelayMeter != nil {
 			db.writeDelayMeter.Mark(int64((counters[i%2][4] - counters[(i-1)%2][4])))
+			// If the write delay duration been collected in the last minute exceeds the predefined threshold,
+			// print a warning log here.
+			if int64(db.writeDelayMeter.Rate1()) > writeDelayThreshold.Nanoseconds() {
+				db.log.Warn("Write delay duration exceeds the threshold (0.5sec) in the last minute")
 			}
+		}
+
 		// Sleep a bit, then repeat the stats collection
 		select {
 		case errc := <-db.quitChan: