From d58f6291a2b16350799b6bb31103fc74bf36113a Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 20 Jan 2026 10:33:09 +0100 Subject: [PATCH] internal/debug: add integration with Grafana Pyroscope (#33623) This adds support for Grafana Pyroscope, a continuous profiling solution. The client is configured similarly to metrics, i.e. run geth --pyroscope --pyroscope.server=https://... This commit is a resubmit of #33261 with some changes. --------- Co-authored-by: Carlos Bermudez Porto --- cmd/keeper/go.sum | 4 +- go.mod | 4 +- go.sum | 10 ++- internal/debug/flags.go | 14 ++++ internal/debug/pyroscope.go | 134 ++++++++++++++++++++++++++++++++++++ 5 files changed, 161 insertions(+), 5 deletions(-) create mode 100644 internal/debug/pyroscope.go diff --git a/cmd/keeper/go.sum b/cmd/keeper/go.sum index b93969cc60..62f10968e2 100644 --- a/cmd/keeper/go.sum +++ b/cmd/keeper/go.sum @@ -63,8 +63,8 @@ github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZ github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= github.com/holiman/uint256 v1.3.2 h1:a9EgMPSC1AAaj1SZL5zIQD3WbwTuHrMGOerLjGmM/TA= github.com/holiman/uint256 v1.3.2/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= -github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4= -github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= +github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= diff --git a/go.mod b/go.mod index 7bfb6d25d7..306b08ff1a 100644 --- a/go.mod +++ b/go.mod @@ -121,10 +121,12 @@ require ( github.com/golang/protobuf v1.5.4 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/pprof v0.0.0-20230207041349-798e818bf904 // indirect + github.com/grafana/pyroscope-go v1.2.7 + github.com/grafana/pyroscope-go/godeltaprof v0.1.9 // indirect github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kilic/bls12-381 v0.1.0 // indirect - github.com/klauspost/compress v1.16.0 // indirect + github.com/klauspost/compress v1.17.8 // indirect github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect diff --git a/go.sum b/go.sum index c9978a3d9e..dad819e09d 100644 --- a/go.sum +++ b/go.sum @@ -188,6 +188,10 @@ github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grafana/pyroscope-go v1.2.7 h1:VWBBlqxjyR0Cwk2W6UrE8CdcdD80GOFNutj0Kb1T8ac= +github.com/grafana/pyroscope-go v1.2.7/go.mod h1:o/bpSLiJYYP6HQtvcoVKiE9s5RiNgjYTj1DhiddP2Pc= +github.com/grafana/pyroscope-go/godeltaprof v0.1.9 h1:c1Us8i6eSmkW+Ez05d3co8kasnuOY813tbMN8i/a3Og= +github.com/grafana/pyroscope-go/godeltaprof v0.1.9/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU= github.com/graph-gophers/graphql-go v1.3.0 h1:Eb9x/q6MFpCLz7jBCiP/WTxjSDrYLR1QY41SORZyNJ0= github.com/graph-gophers/graphql-go v1.3.0/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE= @@ -223,8 +227,8 @@ github.com/kilic/bls12-381 v0.1.0 h1:encrdjqKMEvabVQ7qYOKu1OvhqpK4s47wDYtNiPtlp4 github.com/kilic/bls12-381 v0.1.0/go.mod h1:vDTTHJONJ6G+P2R74EhnyotQDTliQDnFEwhdmfzw1ig= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4= -github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= +github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= @@ -340,6 +344,8 @@ github.com/status-im/keycard-go v0.2.0/go.mod h1:wlp8ZLbsmrF6g6WjugPAx+IzoLrkdf9 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= diff --git a/internal/debug/flags.go b/internal/debug/flags.go index 30b0ddb3be..b4e55c46c1 100644 --- a/internal/debug/flags.go +++ b/internal/debug/flags.go @@ -162,6 +162,11 @@ var Flags = []cli.Flag{ blockprofilerateFlag, cpuprofileFlag, traceFlag, + pyroscopeFlag, + pyroscopeServerFlag, + pyroscopeAuthUsernameFlag, + pyroscopeAuthPasswordFlag, + pyroscopeTagsFlag, } var ( @@ -298,6 +303,14 @@ func Setup(ctx *cli.Context) error { // It cannot be imported because it will cause a cyclical dependency. StartPProf(address, !ctx.IsSet("metrics.addr")) } + + // Pyroscope profiling + if ctx.Bool(pyroscopeFlag.Name) { + if err := startPyroscope(ctx); err != nil { + return err + } + } + if len(logFile) > 0 || rotation { log.Info("Logging configured", context...) } @@ -321,6 +334,7 @@ func StartPProf(address string, withMetrics bool) { // Exit stops all running profiles, flushing their output to the // respective file. func Exit() { + stopPyroscope() Handler.StopCPUProfile() Handler.StopGoTrace() if logOutputFile != nil { diff --git a/internal/debug/pyroscope.go b/internal/debug/pyroscope.go new file mode 100644 index 0000000000..d0804cb891 --- /dev/null +++ b/internal/debug/pyroscope.go @@ -0,0 +1,134 @@ +// Copyright 2026 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package debug + +import ( + "fmt" + "strings" + + "github.com/ethereum/go-ethereum/internal/flags" + "github.com/ethereum/go-ethereum/log" + "github.com/grafana/pyroscope-go" + "github.com/urfave/cli/v2" +) + +var ( + pyroscopeFlag = &cli.BoolFlag{ + Name: "pyroscope", + Usage: "Enable Pyroscope profiling", + Value: false, + Category: flags.LoggingCategory, + } + pyroscopeServerFlag = &cli.StringFlag{ + Name: "pyroscope.server", + Usage: "Pyroscope server URL to push profiling data to", + Value: "http://localhost:4040", + Category: flags.LoggingCategory, + } + pyroscopeAuthUsernameFlag = &cli.StringFlag{ + Name: "pyroscope.username", + Usage: "Pyroscope basic authentication username", + Value: "", + Category: flags.LoggingCategory, + } + pyroscopeAuthPasswordFlag = &cli.StringFlag{ + Name: "pyroscope.password", + Usage: "Pyroscope basic authentication password", + Value: "", + Category: flags.LoggingCategory, + } + pyroscopeTagsFlag = &cli.StringFlag{ + Name: "pyroscope.tags", + Usage: "Comma separated list of key=value tags to add to profiling data", + Value: "", + Category: flags.LoggingCategory, + } +) + +// This holds the globally-configured Pyroscope instance. +var pyroscopeProfiler *pyroscope.Profiler + +func startPyroscope(ctx *cli.Context) error { + server := ctx.String(pyroscopeServerFlag.Name) + authUsername := ctx.String(pyroscopeAuthUsernameFlag.Name) + authPassword := ctx.String(pyroscopeAuthPasswordFlag.Name) + + rawTags := ctx.String(pyroscopeTagsFlag.Name) + tags := make(map[string]string) + for tag := range strings.SplitSeq(rawTags, ",") { + tag = strings.TrimSpace(tag) + if tag == "" { + continue + } + k, v, _ := strings.Cut(tag, "=") + tags[k] = v + } + + config := pyroscope.Config{ + ApplicationName: "geth", + ServerAddress: server, + BasicAuthUser: authUsername, + BasicAuthPassword: authPassword, + Logger: &pyroscopeLogger{Logger: log.Root()}, + Tags: tags, + ProfileTypes: []pyroscope.ProfileType{ + // Enabling all profile types + pyroscope.ProfileCPU, + pyroscope.ProfileAllocObjects, + pyroscope.ProfileAllocSpace, + pyroscope.ProfileInuseObjects, + pyroscope.ProfileInuseSpace, + pyroscope.ProfileGoroutines, + pyroscope.ProfileMutexCount, + pyroscope.ProfileMutexDuration, + pyroscope.ProfileBlockCount, + pyroscope.ProfileBlockDuration, + }, + } + + profiler, err := pyroscope.Start(config) + if err != nil { + return err + } + pyroscopeProfiler = profiler + log.Info("Enabled Pyroscope") + return nil +} + +func stopPyroscope() { + if pyroscopeProfiler != nil { + pyroscopeProfiler.Stop() + pyroscopeProfiler = nil + } +} + +// Small wrapper for log.Logger to satisfy pyroscope.Logger interface +type pyroscopeLogger struct { + Logger log.Logger +} + +func (l *pyroscopeLogger) Infof(format string, v ...any) { + l.Logger.Info(fmt.Sprintf("Pyroscope: "+format, v...)) +} + +func (l *pyroscopeLogger) Debugf(format string, v ...any) { + l.Logger.Debug(fmt.Sprintf("Pyroscope: "+format, v...)) +} + +func (l *pyroscopeLogger) Errorf(format string, v ...any) { + l.Logger.Error(fmt.Sprintf("Pyroscope: "+format, v...)) +}