mirror of
https://github.com/ethereum/go-ethereum.git
synced 2026-08-20 10:52:25 +00:00
metrics: split read/write + improve runtime histogram speed
This commit is contained in:
Martin Holst Swende
parent
90be12cab2
commit
c8e1767e33
2 changed files with 130 additions and 67 deletions
|
|
@ -17,13 +17,13 @@ func getOrRegisterRuntimeHistogram(name string, scale float64, r Registry) *runt
|
|||
|
||||
// runtimeHistogram wraps a runtime/metrics histogram.
|
||||
type runtimeHistogram struct {
|
||||
v atomic.Value
|
||||
v atomic.Value // v is a pointer to a metrics.Float64Histogram
|
||||
scaleFactor float64
|
||||
}
|
||||
|
||||
func newRuntimeHistogram(scale float64) *runtimeHistogram {
|
||||
h := &runtimeHistogram{scaleFactor: scale}
|
||||
h.update(&metrics.Float64Histogram{})
|
||||
h.update(new(metrics.Float64Histogram))
|
||||
return h
|
||||
}
|
||||
|
||||
|
|
@ -41,22 +41,17 @@ func (h *runtimeHistogram) update(mh *metrics.Float64Histogram) {
|
|||
return
|
||||
}
|
||||
|
||||
s := runtimeHistogramSnapshot{
|
||||
s := metrics.Float64Histogram{
|
||||
Counts: make([]uint64, len(mh.Counts)),
|
||||
Buckets: make([]float64, len(mh.Buckets)),
|
||||
}
|
||||
copy(s.Counts, mh.Counts)
|
||||
copy(s.Buckets, mh.Buckets)
|
||||
for i, b := range s.Buckets {
|
||||
for i, b := range mh.Buckets {
|
||||
s.Buckets[i] = b * h.scaleFactor
|
||||
}
|
||||
h.v.Store(&s)
|
||||
}
|
||||
|
||||
func (h *runtimeHistogram) load() *runtimeHistogramSnapshot {
|
||||
return h.v.Load().(*runtimeHistogramSnapshot)
|
||||
}
|
||||
|
||||
func (h *runtimeHistogram) Clear() {
|
||||
panic("runtimeHistogram does not support Clear")
|
||||
}
|
||||
|
|
@ -64,54 +59,94 @@ func (h *runtimeHistogram) Update(int64) {
|
|||
panic("runtimeHistogram does not support Update")
|
||||
}
|
||||
|
||||
// Snapshot returns a non-changing cop of the histogram.
|
||||
// Snapshot returns a non-changing copy of the histogram.
|
||||
func (h *runtimeHistogram) Snapshot() HistogramSnapshot {
|
||||
return h.load()
|
||||
hist := h.v.Load().(*metrics.Float64Histogram)
|
||||
return newRuntimeHistogramSnapshot(hist)
|
||||
}
|
||||
|
||||
type runtimeHistogramSnapshot metrics.Float64Histogram
|
||||
type runtimeHistogramSnapshot struct {
|
||||
internal *metrics.Float64Histogram
|
||||
calculated bool
|
||||
// The following fields are (lazily) calculated based on 'internal'
|
||||
mean float64
|
||||
count int64
|
||||
min int64 // min is the lowest sample value.
|
||||
max int64 // max is the highest sample value.
|
||||
variance float64
|
||||
}
|
||||
|
||||
func (h *runtimeHistogramSnapshot) Snapshot() HistogramSnapshot {
|
||||
return h
|
||||
func newRuntimeHistogramSnapshot(h *metrics.Float64Histogram) *runtimeHistogramSnapshot {
|
||||
return &runtimeHistogramSnapshot{
|
||||
internal: h,
|
||||
}
|
||||
}
|
||||
|
||||
// calc calculates the values for the snapshot. This method is not threadsafe.
|
||||
func (h *runtimeHistogramSnapshot) calc() {
|
||||
h.calculated = true
|
||||
var (
|
||||
count int64 // number of samples
|
||||
sum float64 // approx sum of all sample values
|
||||
minSet = false
|
||||
min int64
|
||||
max float64
|
||||
)
|
||||
if len(h.internal.Counts) == 0 {
|
||||
return
|
||||
}
|
||||
for i, c := range h.internal.Counts {
|
||||
count += int64(c)
|
||||
midpoint := h.midpoint(i)
|
||||
sum += midpoint * float64(c)
|
||||
if c == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
if c > 0 && !minSet {
|
||||
minSet = true
|
||||
min = int64(math.Floor(h.internal.Buckets[i]))
|
||||
}
|
||||
if count > 0 {
|
||||
edge := h.internal.Buckets[i+1]
|
||||
if math.IsInf(edge, 1) {
|
||||
edge = h.internal.Buckets[i]
|
||||
}
|
||||
if edge > max {
|
||||
max = edge
|
||||
}
|
||||
}
|
||||
}
|
||||
h.min = min
|
||||
h.max = int64(max)
|
||||
h.mean = sum / float64(count)
|
||||
h.count = count
|
||||
}
|
||||
|
||||
// Count returns the sample count.
|
||||
func (h *runtimeHistogramSnapshot) Count() int64 {
|
||||
var count int64
|
||||
for _, c := range h.Counts {
|
||||
count += int64(c)
|
||||
if !h.calculated {
|
||||
h.calc()
|
||||
}
|
||||
return count
|
||||
return h.count
|
||||
}
|
||||
|
||||
// Size returns the size of the sample at the time the snapshot was taken.
|
||||
func (h *runtimeHistogramSnapshot) Size() int {
|
||||
return len(h.Counts)
|
||||
return len(h.internal.Counts)
|
||||
}
|
||||
|
||||
// Mean returns an approximation of the mean.
|
||||
func (h *runtimeHistogramSnapshot) Mean() float64 {
|
||||
if len(h.Counts) == 0 {
|
||||
return 0
|
||||
if !h.calculated {
|
||||
h.calc()
|
||||
}
|
||||
mean, _ := h.mean()
|
||||
return mean
|
||||
}
|
||||
|
||||
// mean computes the mean and also the total sample count.
|
||||
func (h *runtimeHistogramSnapshot) mean() (mean, totalCount float64) {
|
||||
var sum float64
|
||||
for i, c := range h.Counts {
|
||||
midpoint := h.midpoint(i)
|
||||
sum += midpoint * float64(c)
|
||||
totalCount += float64(c)
|
||||
}
|
||||
return sum / totalCount, totalCount
|
||||
return h.mean
|
||||
}
|
||||
|
||||
func (h *runtimeHistogramSnapshot) midpoint(bucket int) float64 {
|
||||
high := h.Buckets[bucket+1]
|
||||
low := h.Buckets[bucket]
|
||||
high := h.internal.Buckets[bucket+1]
|
||||
low := h.internal.Buckets[bucket]
|
||||
if math.IsInf(high, 1) {
|
||||
// The edge of the highest bucket can be +Inf, and it's supposed to mean that this
|
||||
// bucket contains all remaining samples > low. We can't get the middle of an
|
||||
|
|
@ -133,23 +168,31 @@ func (h *runtimeHistogramSnapshot) StdDev() float64 {
|
|||
|
||||
// Variance approximates the variance of the histogram.
|
||||
func (h *runtimeHistogramSnapshot) Variance() float64 {
|
||||
if len(h.Counts) == 0 {
|
||||
if len(h.internal.Counts) == 0 {
|
||||
return 0
|
||||
}
|
||||
|
||||
mean, totalCount := h.mean()
|
||||
if totalCount <= 1 {
|
||||
if !h.calculated {
|
||||
h.calc()
|
||||
}
|
||||
if h.count <= 1 {
|
||||
// There is no variance when there are zero or one items.
|
||||
return 0
|
||||
}
|
||||
|
||||
// Variance is not calculated in 'calc', because it requires a second iteration.
|
||||
// Therefore we calculate it lazily in this method, triggered either by
|
||||
// a direct call to Variance or via StdDev.
|
||||
if h.variance != 0.0 {
|
||||
return h.variance
|
||||
}
|
||||
var sum float64
|
||||
for i, c := range h.Counts {
|
||||
|
||||
for i, c := range h.internal.Counts {
|
||||
midpoint := h.midpoint(i)
|
||||
d := midpoint - mean
|
||||
d := midpoint - h.mean
|
||||
sum += float64(c) * (d * d)
|
||||
}
|
||||
return sum / (totalCount - 1)
|
||||
h.variance = sum / float64(h.count-1)
|
||||
return h.variance
|
||||
}
|
||||
|
||||
// Percentile computes the p'th percentile value.
|
||||
|
|
@ -184,11 +227,11 @@ func (h *runtimeHistogramSnapshot) Percentiles(ps []float64) []float64 {
|
|||
|
||||
func (h *runtimeHistogramSnapshot) computePercentiles(thresh []float64) {
|
||||
var totalCount float64
|
||||
for i, count := range h.Counts {
|
||||
for i, count := range h.internal.Counts {
|
||||
totalCount += float64(count)
|
||||
|
||||
for len(thresh) > 0 && thresh[0] < totalCount {
|
||||
thresh[0] = h.Buckets[i]
|
||||
thresh[0] = h.internal.Buckets[i]
|
||||
thresh = thresh[1:]
|
||||
}
|
||||
if len(thresh) == 0 {
|
||||
|
|
@ -203,34 +246,25 @@ func (h *runtimeHistogramSnapshot) computePercentiles(thresh []float64) {
|
|||
|
||||
// Max returns the highest sample value.
|
||||
func (h *runtimeHistogramSnapshot) Max() int64 {
|
||||
for i := len(h.Counts) - 1; i >= 0; i-- {
|
||||
count := h.Counts[i]
|
||||
if count > 0 {
|
||||
edge := h.Buckets[i+1]
|
||||
if math.IsInf(edge, 1) {
|
||||
edge = h.Buckets[i]
|
||||
}
|
||||
return int64(math.Ceil(edge))
|
||||
}
|
||||
if !h.calculated {
|
||||
h.calc()
|
||||
}
|
||||
return 0
|
||||
return h.max
|
||||
}
|
||||
|
||||
// Min returns the lowest sample value.
|
||||
func (h *runtimeHistogramSnapshot) Min() int64 {
|
||||
for i, count := range h.Counts {
|
||||
if count > 0 {
|
||||
return int64(math.Floor(h.Buckets[i]))
|
||||
}
|
||||
if !h.calculated {
|
||||
h.calc()
|
||||
}
|
||||
return 0
|
||||
return h.min
|
||||
}
|
||||
|
||||
// Sum returns the sum of all sample values.
|
||||
func (h *runtimeHistogramSnapshot) Sum() int64 {
|
||||
var sum float64
|
||||
for i := range h.Counts {
|
||||
sum += h.Buckets[i] * float64(h.Counts[i])
|
||||
for i := range h.internal.Counts {
|
||||
sum += h.internal.Buckets[i] * float64(h.internal.Counts[i])
|
||||
}
|
||||
return int64(math.Ceil(sum))
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,11 +1,14 @@
|
|||
package metrics
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/gob"
|
||||
"fmt"
|
||||
"math"
|
||||
"reflect"
|
||||
"runtime/metrics"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
var _ Histogram = (*runtimeHistogram)(nil)
|
||||
|
|
@ -74,7 +77,7 @@ func TestRuntimeHistogramStats(t *testing.T) {
|
|||
|
||||
for i, test := range tests {
|
||||
t.Run(fmt.Sprint(i), func(t *testing.T) {
|
||||
s := runtimeHistogramSnapshot(test.h)
|
||||
s := RuntimeHistogramFromData(1.0, &test.h).Snapshot()
|
||||
|
||||
if v := s.Count(); v != test.Count {
|
||||
t.Errorf("Count() = %v, want %v", v, test.Count)
|
||||
|
|
@ -121,13 +124,39 @@ func approxEqual(x, y, ε float64) bool {
|
|||
// This test verifies that requesting Percentiles in unsorted order
|
||||
// returns them in the requested order.
|
||||
func TestRuntimeHistogramStatsPercentileOrder(t *testing.T) {
|
||||
p := runtimeHistogramSnapshot{
|
||||
s := RuntimeHistogramFromData(1.0, &metrics.Float64Histogram{
|
||||
Counts: []uint64{1, 1, 1, 1, 1, 1, 1, 1, 1, 1},
|
||||
Buckets: []float64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10},
|
||||
}
|
||||
result := p.Percentiles([]float64{1, 0.2, 0.5, 0.1, 0.2})
|
||||
}).Snapshot()
|
||||
result := s.Percentiles([]float64{1, 0.2, 0.5, 0.1, 0.2})
|
||||
expected := []float64{10, 2, 5, 1, 2}
|
||||
if !reflect.DeepEqual(result, expected) {
|
||||
t.Fatal("wrong result:", result)
|
||||
}
|
||||
}
|
||||
|
||||
func BenchmarkRuntimeHistogramSnapshotRead(b *testing.B) {
|
||||
var sLatency = "7\xff\x81\x03\x01\x01\x10Float64Histogram\x01\xff\x82\x00\x01\x02\x01\x06Counts\x01\xff\x84\x00\x01\aBuckets\x01\xff\x86\x00\x00\x00\x16\xff\x83\x02\x01\x01\b[]uint64\x01\xff\x84\x00\x01\x06\x00\x00\x17\xff\x85\x02\x01\x01\t[]float64\x01\xff\x86\x00\x01\b\x00\x00\xfe\x06T\xff\x82\x01\xff\xa2\x00\xfe\r\xef\x00\x01\x02\x02\x04\x05\x04\b\x15\x17 B?6.L;$!2) \x1a? \x190aH7FY6#\x190\x1d\x14\x10\x1b\r\t\x04\x03\x01\x01\x00\x03\x02\x00\x03\x05\x05\x02\x02\x06\x04\v\x06\n\x15\x18\x13'&.\x12=H/L&\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01\xff\xa3\xfe\xf0\xff\x00\xf8\x95\xd6&\xe8\v.q>\xf8\x95\xd6&\xe8\v.\x81>\xf8\xdfA:\xdc\x11ʼn>\xf8\x95\xd6&\xe8\v.\x91>\xf8:\x8c0\xe2\x8ey\x95>\xf8\xdfA:\xdc\x11ř>\xf8\x84\xf7C֔\x10\x9e>\xf8\x95\xd6&\xe8\v.\xa1>\xf8:\x8c0\xe2\x8ey\xa5>\xf8\xdfA:\xdc\x11ũ>\xf8\x84\xf7C֔\x10\xae>\xf8\x95\xd6&\xe8\v.\xb1>\xf8:\x8c0\xe2\x8ey\xb5>\xf8\xdfA:\xdc\x11Ź>\xf8\x84\xf7C֔\x10\xbe>\xf8\x95\xd6&\xe8\v.\xc1>\xf8:\x8c0\xe2\x8ey\xc5>\xf8\xdfA:\xdc\x11\xc5\xc9>\xf8\x84\xf7C֔\x10\xce>\xf8\x95\xd6&\xe8\v.\xd1>\xf8:\x8c0\xe2\x8ey\xd5>\xf8\xdfA:\xdc\x11\xc5\xd9>\xf8\x84\xf7C֔\x10\xde>\xf8\x95\xd6&\xe8\v.\xe1>\xf8:\x8c0\xe2\x8ey\xe5>\xf8\xdfA:\xdc\x11\xc5\xe9>\xf8\x84\xf7C֔\x10\xee>\xf8\x95\xd6&\xe8\v.\xf1>\xf8:\x8c0\xe2\x8ey\xf5>\xf8\xdfA:\xdc\x11\xc5\xf9>\xf8\x84\xf7C֔\x10\xfe>\xf8\x95\xd6&\xe8\v.\x01?\xf8:\x8c0\xe2\x8ey\x05?\xf8\xdfA:\xdc\x11\xc5\t?\xf8\x84\xf7C֔\x10\x0e?\xf8\x95\xd6&\xe8\v.\x11?\xf8:\x8c0\xe2\x8ey\x15?\xf8\xdfA:\xdc\x11\xc5\x19?\xf8\x84\xf7C֔\x10\x1e?\xf8\x95\xd6&\xe8\v.!?\xf8:\x8c0\xe2\x8ey%?\xf8\xdfA:\xdc\x11\xc5)?\xf8\x84\xf7C֔\x10.?\xf8\x95\xd6&\xe8\v.1?\xf8:\x8c0\xe2\x8ey5?\xf8\xdfA:\xdc\x11\xc59?\xf8\x84\xf7C֔\x10>?\xf8\x95\xd6&\xe8\v.A?\xf8:\x8c0\xe2\x8eyE?\xf8\xdfA:\xdc\x11\xc5I?\xf8\x84\xf7C֔\x10N?\xf8\x95\xd6&\xe8\v.Q?\xf8:\x8c0\xe2\x8eyU?\xf8\xdfA:\xdc\x11\xc5Y?\xf8\x84\xf7C֔\x10^?\xf8\x95\xd6&\xe8\v.a?\xf8:\x8c0\xe2\x8eye?\xf8\xdfA:\xdc\x11\xc5i?\xf8\x84\xf7C֔\x10n?\xf8\x95\xd6&\xe8\v.q?\xf8:\x8c0\xe2\x8eyu?\xf8\xdfA:\xdc\x11\xc5y?\xf8\x84\xf7C֔\x10~?\xf8\x95\xd6&\xe8\v.\x81?\xf8:\x8c0\xe2\x8ey\x85?\xf8\xdfA:\xdc\x11ʼn?\xf8\x84\xf7C֔\x10\x8e?\xf8\x95\xd6&\xe8\v.\x91?\xf8:\x8c0\xe2\x8ey\x95?\xf8\xdfA:\xdc\x11ř?\xf8\x84\xf7C֔\x10\x9e?\xf8\x95\xd6&\xe8\v.\xa1?\xf8:\x8c0\xe2\x8ey\xa5?\xf8\xdfA:\xdc\x11ũ?\xf8\x84\xf7C֔\x10\xae?\xf8\x95\xd6&\xe8\v.\xb1?\xf8:\x8c0\xe2\x8ey\xb5?\xf8\xdfA:\xdc\x11Ź?\xf8\x84\xf7C֔\x10\xbe?\xf8\x95\xd6&\xe8\v.\xc1?\xf8:\x8c0\xe2\x8ey\xc5?\xf8\xdfA:\xdc\x11\xc5\xc9?\xf8\x84\xf7C֔\x10\xce?\xf8\x95\xd6&\xe8\v.\xd1?\xf8:\x8c0\xe2\x8ey\xd5?\xf8\xdfA:\xdc\x11\xc5\xd9?\xf8\x84\xf7C֔\x10\xde?\xf8\x95\xd6&\xe8\v.\xe1?\xf8:\x8c0\xe2\x8ey\xe5?\xf8\xdfA:\xdc\x11\xc5\xe9?\xf8\x84\xf7C֔\x10\xee?\xf8\x95\xd6&\xe8\v.\xf1?\xf8:\x8c0\xe2\x8ey\xf5?\xf8\xdfA:\xdc\x11\xc5\xf9?\xf8\x84\xf7C֔\x10\xfe?\xf8\x95\xd6&\xe8\v.\x01@\xf8:\x8c0\xe2\x8ey\x05@\xf8\xdfA:\xdc\x11\xc5\t@\xf8\x84\xf7C֔\x10\x0e@\xf8\x95\xd6&\xe8\v.\x11@\xf8:\x8c0\xe2\x8ey\x15@\xf8\xdfA:\xdc\x11\xc5\x19@\xf8\x84\xf7C֔\x10\x1e@\xf8\x95\xd6&\xe8\v.!@\xf8:\x8c0\xe2\x8ey%@\xf8\xdfA:\xdc\x11\xc5)@\xf8\x84\xf7C֔\x10.@\xf8\x95\xd6&\xe8\v.1@\xf8:\x8c0\xe2\x8ey5@\xf8\xdfA:\xdc\x11\xc59@\xf8\x84\xf7C֔\x10>@\xf8\x95\xd6&\xe8\v.A@\xf8:\x8c0\xe2\x8eyE@\xf8\xdfA:\xdc\x11\xc5I@\xf8\x84\xf7C֔\x10N@\xf8\x95\xd6&\xe8\v.Q@\xf8:\x8c0\xe2\x8eyU@\xf8\xdfA:\xdc\x11\xc5Y@\xf8\x84\xf7C֔\x10^@\xf8\x95\xd6&\xe8\v.a@\xf8:\x8c0\xe2\x8eye@\xf8\xdfA:\xdc\x11\xc5i@\xf8\x84\xf7C֔\x10n@\xf8\x95\xd6&\xe8\v.q@\xf8:\x8c0\xe2\x8eyu@\xf8\xdfA:\xdc\x11\xc5y@\xf8\x84\xf7C֔\x10~@\xf8\x95\xd6&\xe8\v.\x81@\xf8:\x8c0\xe2\x8ey\x85@\xf8\xdfA:\xdc\x11ʼn@\xf8\x84\xf7C֔\x10\x8e@\xf8\x95\xd6&\xe8\v.\x91@\xf8:\x8c0\xe2\x8ey\x95@\xf8\xdfA:\xdc\x11ř@\xf8\x84\xf7C֔\x10\x9e@\xf8\x95\xd6&\xe8\v.\xa1@\xf8:\x8c0\xe2\x8ey\xa5@\xf8\xdfA:\xdc\x11ũ@\xf8\x84\xf7C֔\x10\xae@\xf8\x95\xd6&\xe8\v.\xb1@\xf8:\x8c0\xe2\x8ey\xb5@\xf8\xdfA:\xdc\x11Ź@\xf8\x84\xf7C֔\x10\xbe@\xf8\x95\xd6&\xe8\v.\xc1@\xf8:\x8c0\xe2\x8ey\xc5@\xf8\xdfA:\xdc\x11\xc5\xc9@\xf8\x84\xf7C֔\x10\xce@\xf8\x95\xd6&\xe8\v.\xd1@\xf8:\x8c0\xe2\x8ey\xd5@\xf8\xdfA:\xdc\x11\xc5\xd9@\xf8\x84\xf7C֔\x10\xde@\xf8\x95\xd6&\xe8\v.\xe1@\xf8:\x8c0\xe2\x8ey\xe5@\xf8\xdfA:\xdc\x11\xc5\xe9@\xf8\x84\xf7C֔\x10\xee@\xf8\x95\xd6&\xe8\v.\xf1@\xf8:\x8c0\xe2\x8ey\xf5@\xf8\xdfA:\xdc\x11\xc5\xf9@\xf8\x84\xf7C֔\x10\xfe@\xf8\x95\xd6&\xe8\v.\x01A\xfe\xf0\x7f\x00"
|
||||
|
||||
dserialize := func(data string) *metrics.Float64Histogram {
|
||||
var res metrics.Float64Histogram
|
||||
if err := gob.NewDecoder(bytes.NewReader([]byte(data))).Decode(&res); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return &res
|
||||
}
|
||||
latency := RuntimeHistogramFromData(float64(time.Second), dserialize(sLatency))
|
||||
b.ResetTimer()
|
||||
b.ReportAllocs()
|
||||
for i := 0; i < b.N; i++ {
|
||||
snap := latency.Snapshot()
|
||||
// These are the fields that influxdb accesses
|
||||
_ = snap.Count()
|
||||
_ = snap.Max()
|
||||
_ = snap.Mean()
|
||||
_ = snap.Min()
|
||||
_ = snap.StdDev()
|
||||
_ = snap.Variance()
|
||||
_ = snap.Percentiles([]float64{0.25, 0.5, 0.75, 0.95, 0.99, 0.999, 0.9999})
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue