mirror of
https://github.com/ethereum/go-ethereum.git
synced 2026-07-27 15:16:43 +00:00
change log levels and extract duplicate logic
This commit is contained in:
Rafael Sampaio
Chen Kai
parent
775d71541d
commit
9c4f3b2f8d
5 changed files with 110 additions and 138 deletions
|
|
@ -419,6 +419,7 @@ func initBeacon(config Config, server *rpc.Server, conn discover.UDPConn, localN
|
|||
DB: sqlDb,
|
||||
NodeId: localNode.ID(),
|
||||
Spec: configs.Mainnet,
|
||||
NetworkName: portalwire.Beacon.Name(),
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
|
|||
|
|
@ -1,9 +1,12 @@
|
|||
package metrics
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"errors"
|
||||
"os"
|
||||
"path"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/ethereum/go-ethereum/log"
|
||||
|
|
@ -17,6 +20,17 @@ type networkFileMetric struct {
|
|||
network string
|
||||
}
|
||||
|
||||
type PortalStorageMetrics struct {
|
||||
RadiusRatio GaugeFloat64
|
||||
EntriesCount Gauge
|
||||
ContentStorageUsage Gauge
|
||||
}
|
||||
|
||||
const (
|
||||
countEntrySql = "SELECT COUNT(1) FROM kvstore;"
|
||||
contentStorageUsageSql = "SELECT SUM( length(value) ) FROM kvstore;"
|
||||
)
|
||||
|
||||
// CollectPortalMetrics periodically collects various metrics about system entities.
|
||||
func CollectPortalMetrics(refresh time.Duration, networks []string, dataDir string) {
|
||||
// Short circuit if the metrics system is disabled
|
||||
|
|
@ -80,3 +94,60 @@ func CollectPortalMetrics(refresh time.Duration, networks []string, dataDir stri
|
|||
time.Sleep(refresh)
|
||||
}
|
||||
}
|
||||
|
||||
func NewPortalStorageMetrics(network string, db *sql.DB) (*PortalStorageMetrics, error) {
|
||||
if !Enabled {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if network != portalwire.History.Name() && network != portalwire.Beacon.Name() && network != portalwire.State.Name() {
|
||||
log.Debug("Unknow network for metrics", "network", network)
|
||||
return nil, errors.New("unknow network for metrics")
|
||||
}
|
||||
|
||||
var countSql string
|
||||
var contentSql string
|
||||
if network == portalwire.Beacon.Name() {
|
||||
countSql = strings.Replace(countEntrySql, "kvstore", "beacon", 1)
|
||||
contentSql = strings.Replace(contentStorageUsageSql, "kvstore", "beacon", 1)
|
||||
contentSql = strings.Replace(contentSql, "value", "content_value", 1)
|
||||
} else {
|
||||
countSql = countEntrySql
|
||||
contentSql = contentStorageUsageSql
|
||||
}
|
||||
|
||||
storageMetrics := &PortalStorageMetrics{}
|
||||
|
||||
storageMetrics.RadiusRatio = NewRegisteredGaugeFloat64("portal/"+network+"/radius_ratio", nil)
|
||||
storageMetrics.RadiusRatio.Update(1)
|
||||
|
||||
storageMetrics.EntriesCount = NewRegisteredGauge("portal/"+network+"/entry_count", nil)
|
||||
log.Debug("Counting entities in " + network + " storage for metrics")
|
||||
var res *int64 = new(int64)
|
||||
q := db.QueryRow(countSql)
|
||||
if q.Err() == sql.ErrNoRows {
|
||||
storageMetrics.EntriesCount.Update(0)
|
||||
} else if q.Err() != nil {
|
||||
log.Error("Querry execution error", "network", network, "metric", "entry_count", "err", q.Err())
|
||||
return nil, q.Err()
|
||||
} else {
|
||||
q.Scan(res)
|
||||
storageMetrics.EntriesCount.Update(*res)
|
||||
}
|
||||
|
||||
storageMetrics.ContentStorageUsage = NewRegisteredGauge("portal/"+network+"/content_storage", nil)
|
||||
log.Debug("Counting storage usage (bytes) in " + network + " for metrics")
|
||||
var res2 *int64 = new(int64)
|
||||
q2 := db.QueryRow(contentSql)
|
||||
if q2.Err() == sql.ErrNoRows {
|
||||
storageMetrics.ContentStorageUsage.Update(0)
|
||||
} else if q2.Err() != nil {
|
||||
log.Error("Querry execution error", "network", network, "metric", "entry_count", "err", q2.Err())
|
||||
return nil, q2.Err()
|
||||
} else {
|
||||
q2.Scan(res2)
|
||||
storageMetrics.ContentStorageUsage.Update(*res2)
|
||||
}
|
||||
|
||||
return storageMetrics, nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,17 +16,6 @@ import (
|
|||
|
||||
const BytesInMB uint64 = 1000 * 1000
|
||||
|
||||
var (
|
||||
radiusRatio metrics.GaugeFloat64
|
||||
entriesCount metrics.Gauge
|
||||
contentStorageUsage metrics.Gauge
|
||||
)
|
||||
|
||||
const (
|
||||
countEntrySql = "SELECT COUNT(1) FROM beacon;"
|
||||
contentStorageUsageSql = "SELECT SUM( length(content_value) ) FROM beacon;"
|
||||
)
|
||||
|
||||
type BeaconStorage struct {
|
||||
storageCapacityInBytes uint64
|
||||
db *sql.DB
|
||||
|
|
@ -35,6 +24,8 @@ type BeaconStorage struct {
|
|||
cache *beaconStorageCache
|
||||
}
|
||||
|
||||
var portalStorageMetrics *metrics.PortalStorageMetrics
|
||||
|
||||
type beaconStorageCache struct {
|
||||
OptimisticUpdate []byte
|
||||
FinalityUpdate []byte
|
||||
|
|
@ -53,44 +44,13 @@ func NewBeaconStorage(config storage.PortalStorageConfig) (storage.ContentStorag
|
|||
if err := bs.setup(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if metrics.Enabled {
|
||||
radiusRatio = metrics.NewRegisteredGaugeFloat64("portal/beacon/radius_ratio", nil)
|
||||
radiusRatio.Update(1)
|
||||
|
||||
entriesCount = metrics.NewRegisteredGauge("portal/beacon/entry_count", nil)
|
||||
log.Info("Counting entities in beacon storage for metrics")
|
||||
count, err := config.DB.Prepare(countEntrySql)
|
||||
if err != nil {
|
||||
log.Error("Querry preparation error", "network", config.NetworkName, "metric", "entry_count", "err", err)
|
||||
return nil, err
|
||||
}
|
||||
var res *int64 = new(int64)
|
||||
q := count.QueryRow()
|
||||
if q.Err() != nil {
|
||||
log.Error("Querry execution error", "network", config.NetworkName, "metric", "entry_count", "err", err)
|
||||
return nil, err
|
||||
} else {
|
||||
q.Scan(res)
|
||||
}
|
||||
entriesCount.Update(*res)
|
||||
|
||||
contentStorageUsage = metrics.NewRegisteredGauge("portal/beacon/content_storage", nil)
|
||||
log.Info("Counting storage usage (bytes) in beacon for metrics")
|
||||
str, err := config.DB.Prepare(contentStorageUsageSql)
|
||||
if err != nil {
|
||||
log.Error("Querry preparation error", "network", config.NetworkName, "metric", "content_storage", "err", err)
|
||||
return nil, err
|
||||
}
|
||||
var resStr *int64 = new(int64)
|
||||
q = str.QueryRow()
|
||||
if q.Err() != nil {
|
||||
log.Error("Querry execution error", "network", config.NetworkName, "metric", "content_storage", "err", err)
|
||||
return nil, err
|
||||
} else {
|
||||
q.Scan(resStr)
|
||||
}
|
||||
contentStorageUsage.Update(*resStr)
|
||||
var err error
|
||||
portalStorageMetrics, err = metrics.NewPortalStorageMetrics(config.NetworkName, config.DB)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return bs, nil
|
||||
}
|
||||
|
||||
|
|
@ -226,8 +186,8 @@ func (bs *BeaconStorage) putContentValue(contentId, contentKey, value []byte) er
|
|||
length := 32 + len(contentKey) + len(value)
|
||||
_, err := bs.db.ExecContext(context.Background(), InsertQueryBeacon, contentId, contentKey, value, length)
|
||||
if metrics.Enabled && err != nil {
|
||||
entriesCount.Inc(1)
|
||||
contentStorageUsage.Inc(int64(len(value)))
|
||||
portalStorageMetrics.EntriesCount.Inc(1)
|
||||
portalStorageMetrics.ContentStorageUsage.Inc(int64(len(value)))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
|
@ -235,8 +195,8 @@ func (bs *BeaconStorage) putContentValue(contentId, contentKey, value []byte) er
|
|||
func (bs *BeaconStorage) putLcUpdate(period uint64, value []byte) error {
|
||||
_, err := bs.db.ExecContext(context.Background(), InsertLCUpdateQuery, period, value, 0, len(value))
|
||||
if metrics.Enabled && err != nil {
|
||||
entriesCount.Inc(1)
|
||||
contentStorageUsage.Inc(int64(len(value)))
|
||||
portalStorageMetrics.EntriesCount.Inc(1)
|
||||
portalStorageMetrics.ContentStorageUsage.Inc(int64(len(value)))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -21,12 +21,6 @@ import (
|
|||
"github.com/mattn/go-sqlite3"
|
||||
)
|
||||
|
||||
var (
|
||||
radiusRatio metrics.GaugeFloat64
|
||||
entriesCount metrics.Gauge
|
||||
contentStorageUsage metrics.Gauge
|
||||
)
|
||||
|
||||
const (
|
||||
sqliteName = "history.sqlite"
|
||||
contentDeletionFraction = 0.05 // 5% of the content will be deleted when the storage capacity is hit and radius gets adjusted.
|
||||
|
|
@ -62,6 +56,8 @@ type ContentStorage struct {
|
|||
log log.Logger
|
||||
}
|
||||
|
||||
var portalStorageMetrics *metrics.PortalStorageMetrics
|
||||
|
||||
func xor(contentId, nodeId []byte) []byte {
|
||||
// length of contentId maybe not 32bytes
|
||||
padding := make([]byte, 32)
|
||||
|
|
@ -125,25 +121,11 @@ func NewHistoryStorage(config storage.PortalStorageConfig) (storage.ContentStora
|
|||
// Check whether we already have data, and use it to set radius
|
||||
|
||||
// necessary to test NetworkName==history because state also initialize HistoryStorage
|
||||
if metrics.Enabled && strings.ToLower(config.NetworkName) == "history" {
|
||||
radiusRatio = metrics.NewRegisteredGaugeFloat64("portal/history/radius_ratio", nil)
|
||||
radiusRatio.Update(1)
|
||||
|
||||
entriesCount = metrics.NewRegisteredGauge("portal/history/entry_count", nil)
|
||||
log.Info("Counting entities in history storage for metrics")
|
||||
count, err := hs.ContentCount()
|
||||
if strings.ToLower(config.NetworkName) == "history" {
|
||||
portalStorageMetrics, err = metrics.NewPortalStorageMetrics(config.NetworkName, config.DB)
|
||||
if err != nil {
|
||||
log.Debug("Querry execution error", "network", config.NetworkName, "metric", "entry_count", "err", err)
|
||||
return nil, err
|
||||
}
|
||||
entriesCount.Update(int64(count))
|
||||
|
||||
contentStorageUsage = metrics.NewRegisteredGauge("portal/history/content_storage", nil)
|
||||
log.Info("Counting storage usage (bytes) in history for metrics")
|
||||
str, err := hs.ContentSize()
|
||||
if err != nil {
|
||||
log.Debug("Querry execution error", "network", config.NetworkName, "metric", "content_storage", "err", err)
|
||||
}
|
||||
contentStorageUsage.Update(int64(str))
|
||||
}
|
||||
|
||||
return hs, err
|
||||
|
|
@ -217,8 +199,8 @@ func (p *ContentStorage) put(contentId []byte, content []byte) PutResult {
|
|||
}
|
||||
|
||||
if metrics.Enabled {
|
||||
entriesCount.Inc(1)
|
||||
contentStorageUsage.Inc(int64(len(content)))
|
||||
portalStorageMetrics.EntriesCount.Inc(1)
|
||||
portalStorageMetrics.ContentStorageUsage.Inc(int64(len(content)))
|
||||
}
|
||||
return PutResult{}
|
||||
}
|
||||
|
|
@ -387,7 +369,7 @@ func (p *ContentStorage) EstimateNewRadius(currentRadius *uint256.Int) (*uint256
|
|||
newRadius := new(uint256.Int).Div(currentRadius, uint256.MustFromBig(bigFormat))
|
||||
newRadius.Mul(newRadius, uint256.NewInt(100))
|
||||
newRadius.Mod(newRadius, storage.MaxDistance)
|
||||
radiusRatio.Update(newRadius.Float64() / 100)
|
||||
portalStorageMetrics.RadiusRatio.Update(newRadius.Float64() / 100)
|
||||
}
|
||||
return new(uint256.Int).Div(currentRadius, uint256.MustFromBig(bigFormat)), nil
|
||||
}
|
||||
|
|
@ -451,7 +433,7 @@ func (p *ContentStorage) deleteContentFraction(fraction float64) (deleteCount in
|
|||
if metrics.Enabled {
|
||||
dis.Mul(dis, uint256.NewInt(100))
|
||||
dis.Mod(dis, storage.MaxDistance)
|
||||
radiusRatio.Update(dis.Float64() / 100)
|
||||
portalStorageMetrics.RadiusRatio.Update(dis.Float64() / 100)
|
||||
}
|
||||
}
|
||||
// row must close first, or database is locked
|
||||
|
|
@ -475,8 +457,8 @@ func (p *ContentStorage) del(contentId []byte) error {
|
|||
}
|
||||
_, err = p.delStmt.Exec(contentId)
|
||||
if metrics.Enabled && err != nil {
|
||||
entriesCount.Dec(1)
|
||||
contentStorageUsage.Dec(int64(sizeDel))
|
||||
portalStorageMetrics.EntriesCount.Dec(1)
|
||||
portalStorageMetrics.ContentStorageUsage.Dec(int64(sizeDel))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
|
@ -499,8 +481,8 @@ func (p *ContentStorage) batchDel(ids [][]byte) error {
|
|||
// delete items
|
||||
_, err = p.sqliteDB.Exec(query, args...)
|
||||
if metrics.Enabled && err != nil {
|
||||
entriesCount.Dec(int64(len(args)))
|
||||
contentStorageUsage.Dec(int64(sizeDel))
|
||||
portalStorageMetrics.EntriesCount.Dec(int64(len(args)))
|
||||
portalStorageMetrics.ContentStorageUsage.Dec(int64(sizeDel))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
|
@ -528,8 +510,8 @@ func (p *ContentStorage) deleteContentOutOfRadius(radius *uint256.Int) error {
|
|||
count, err := res.RowsAffected()
|
||||
p.log.Trace("delete items", "count", count)
|
||||
if metrics.Enabled && err != nil {
|
||||
entriesCount.Dec(count)
|
||||
contentStorageUsage.Dec(int64(sizeDel))
|
||||
portalStorageMetrics.EntriesCount.Dec(count)
|
||||
portalStorageMetrics.ContentStorageUsage.Dec(int64(sizeDel))
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,17 +14,6 @@ import (
|
|||
"github.com/protolambda/ztyp/codec"
|
||||
)
|
||||
|
||||
var (
|
||||
radiusRatio metrics.GaugeFloat64
|
||||
entriesCount metrics.Gauge
|
||||
contentStorageUsage metrics.Gauge
|
||||
)
|
||||
|
||||
const (
|
||||
countEntrySql = "SELECT COUNT(1) FROM kvstore;"
|
||||
conentStorageUsageSql = "SELECT SUM( length(value) ) FROM kvstore;"
|
||||
)
|
||||
|
||||
func defaultContentIdFunc(contentKey []byte) []byte {
|
||||
digest := sha256.Sum256(contentKey)
|
||||
return digest[:]
|
||||
|
|
@ -38,6 +27,8 @@ type StateStorage struct {
|
|||
log log.Logger
|
||||
}
|
||||
|
||||
var portalStorageMetrics *metrics.PortalStorageMetrics
|
||||
|
||||
func NewStateStorage(store storage.ContentStorage, db *sql.DB) *StateStorage {
|
||||
storage := &StateStorage{
|
||||
store: store,
|
||||
|
|
@ -45,43 +36,10 @@ func NewStateStorage(store storage.ContentStorage, db *sql.DB) *StateStorage {
|
|||
log: log.New("storage", "state"),
|
||||
}
|
||||
|
||||
if metrics.Enabled {
|
||||
radiusRatio = metrics.NewRegisteredGaugeFloat64("portal/state/radius_ratio", nil)
|
||||
radiusRatio.Update(1)
|
||||
|
||||
entriesCount = metrics.NewRegisteredGauge("portal/state/entry_count", nil)
|
||||
log.Info("Counting entities in state storage for metrics")
|
||||
count, err := db.Prepare(countEntrySql)
|
||||
if err != nil {
|
||||
log.Error("Querry preparation error", "network", "state", "metric", "entry_count", "err", err)
|
||||
return nil
|
||||
}
|
||||
var res *int64 = new(int64)
|
||||
q := count.QueryRow()
|
||||
if q.Err() != nil {
|
||||
log.Error("Querry execution error", "network", "state", "metric", "entry_count", "err", err)
|
||||
return nil
|
||||
} else {
|
||||
q.Scan(&res)
|
||||
}
|
||||
entriesCount.Update(*res)
|
||||
|
||||
contentStorageUsage = metrics.NewRegisteredGauge("portal/state/content_storage", nil)
|
||||
log.Info("Counting storage usage (bytes) in state for metrics")
|
||||
str, err := db.Prepare(conentStorageUsageSql)
|
||||
if err != nil {
|
||||
log.Error("Querry preparation error", "network", "state", "metric", "content_storage", "err", err)
|
||||
return nil
|
||||
}
|
||||
var resStr *int64 = new(int64)
|
||||
q = str.QueryRow()
|
||||
if q.Err() != nil {
|
||||
log.Error("Querry execution error", "network", "state", "metric", "content_storage", "err", err)
|
||||
return nil
|
||||
} else {
|
||||
q.Scan(resStr)
|
||||
}
|
||||
contentStorageUsage.Update(*resStr)
|
||||
var err error
|
||||
portalStorageMetrics, err = metrics.NewPortalStorageMetrics("state", db)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
return storage
|
||||
|
|
@ -141,8 +99,8 @@ func (s *StateStorage) putAccountTrieNode(contentKey []byte, contentId []byte, c
|
|||
if err != nil {
|
||||
s.log.Error("failed to save data after validate", "type", contentKey[0], "key", contentKey[1:], "value", content)
|
||||
} else if metrics.Enabled {
|
||||
entriesCount.Inc(1)
|
||||
contentStorageUsage.Inc(int64(len(content)))
|
||||
portalStorageMetrics.EntriesCount.Inc(1)
|
||||
portalStorageMetrics.ContentStorageUsage.Inc(int64(len(content)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -178,8 +136,8 @@ func (s *StateStorage) putContractStorageTrieNode(contentKey []byte, contentId [
|
|||
if err != nil {
|
||||
s.log.Error("failed to save data after validate", "type", contentKey[0], "key", contentKey[1:], "value", content)
|
||||
} else if metrics.Enabled {
|
||||
entriesCount.Inc(1)
|
||||
contentStorageUsage.Inc(int64(len(content)))
|
||||
portalStorageMetrics.EntriesCount.Inc(1)
|
||||
portalStorageMetrics.ContentStorageUsage.Inc(int64(len(content)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
@ -211,8 +169,8 @@ func (s *StateStorage) putContractBytecode(contentKey []byte, contentId []byte,
|
|||
if err != nil {
|
||||
s.log.Error("failed to save data after validate", "type", contentKey[0], "key", contentKey[1:], "value", content)
|
||||
} else if metrics.Enabled {
|
||||
entriesCount.Inc(1)
|
||||
contentStorageUsage.Inc(int64(len(content)))
|
||||
portalStorageMetrics.EntriesCount.Inc(1)
|
||||
portalStorageMetrics.ContentStorageUsage.Inc(int64(len(content)))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue