ethdb: print a warning log if write delay exceeds threshold

This commit is contained in:
rjl493456442 2017-12-19 16:02:28 +08:00
parent 522e0d4424
commit c3bb251d33

View file

@ -34,7 +34,10 @@ import (
gometrics "github.com/rcrowley/go-metrics"
)
var OpenFileLimit = 64
const (
writeDelayNThreshold = 250
writeDelayThreshold = 500 * time.Millisecond
)
type LDBDatabase struct {
fn string // filename for reporting
@ -178,11 +181,9 @@ func (db *LDBDatabase) LDB() *leveldb.DB {
// Meter configures the database metrics collectors and
func (db *LDBDatabase) Meter(prefix string) {
// Short circuit metering if the metrics system is disabled
if !metrics.Enabled {
return
}
// Initialize all the metrics collector at the requested prefix
if metrics.Enabled {
// Initialize all database related metrics collector at the requested prefix
// if metric is enable.
db.getTimer = metrics.NewTimer(prefix + "user/gets")
db.putTimer = metrics.NewTimer(prefix + "user/puts")
db.delTimer = metrics.NewTimer(prefix + "user/dels")
@ -190,11 +191,13 @@ func (db *LDBDatabase) Meter(prefix string) {
db.readMeter = metrics.NewMeter(prefix + "user/reads")
db.writeMeter = metrics.NewMeter(prefix + "user/writes")
db.writeDelayMeter = metrics.NewMeter(prefix + "compact/writedelay/duration")
db.writeDelayNMeter = metrics.NewMeter(prefix + "compact/writedelay/counter")
db.compTimeMeter = metrics.NewMeter(prefix + "compact/time")
db.compReadMeter = metrics.NewMeter(prefix + "compact/input")
db.compWriteMeter = metrics.NewMeter(prefix + "compact/output")
}
// Initialize essential write delay metrics no matter metric is enable or not.
db.writeDelayMeter = metrics.NewMeter(prefix + "compact/writedelay/duration")
db.writeDelayNMeter = metrics.NewMeter(prefix + "compact/writedelay/counter")
// Create a quit channel for the periodic collector and run it
db.quitLock.Lock()
@ -223,6 +226,7 @@ func (db *LDBDatabase) meter(refresh time.Duration) {
}
// Iterate ad infinitum and collect the stats
for i := 1; ; i++ {
if metrics.Enabled {
// Retrieve the database stats
stats, err := db.db.GetProperty("leveldb.stats")
if err != nil {
@ -268,8 +272,9 @@ func (db *LDBDatabase) meter(refresh time.Duration) {
if db.compWriteMeter != nil {
db.compWriteMeter.Mark(int64((counters[i%2][2] - counters[(i-1)%2][2]) * 1024 * 1024))
}
}
// Stat write delay.
// Gather write delay statistic
writeDelay, err := db.db.GetProperty("leveldb.writedelay")
if err != nil {
db.log.Error("Failed to read database write delay statistic", "err", err)
@ -294,11 +299,22 @@ func (db *LDBDatabase) meter(refresh time.Duration) {
if db.writeDelayNMeter != nil {
db.writeDelayNMeter.Mark(int64((counters[i%2][3] - counters[(i-1)%2][3])))
// If the write delay number been collected in the last minute exceeds the predefined threshold,
// print a warning log here.
if int(db.writeDelayNMeter.Rate1()) > writeDelayNThreshold {
db.log.Warn("Write delay number exceeds the threshold (250) in the last minute")
}
}
if db.writeDelayMeter != nil {
db.writeDelayMeter.Mark(int64((counters[i%2][4] - counters[(i-1)%2][4])))
// If the write delay duration been collected in the last minute exceeds the predefined threshold,
// print a warning log here.
if int64(db.writeDelayMeter.Rate1()) > writeDelayThreshold.Nanoseconds() {
db.log.Warn("Write delay duration exceeds the threshold (0.5sec) in the last minute")
}
}
// Sleep a bit, then repeat the stats collection
select {
case errc := <-db.quitChan: