diff --git a/go.mod b/go.mod index 7bfb6d25d7..8a22c1a4c4 100644 --- a/go.mod +++ b/go.mod @@ -121,10 +121,12 @@ require ( github.com/golang/protobuf v1.5.4 // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/pprof v0.0.0-20230207041349-798e818bf904 // indirect + github.com/grafana/pyroscope-go v1.2.7 // indirect + github.com/grafana/pyroscope-go/godeltaprof v0.1.9 // indirect github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kilic/bls12-381 v0.1.0 // indirect - github.com/klauspost/compress v1.16.0 // indirect + github.com/klauspost/compress v1.17.8 // indirect github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect diff --git a/go.sum b/go.sum index c9978a3d9e..7967c81d22 100644 --- a/go.sum +++ b/go.sum @@ -188,6 +188,10 @@ github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grafana/pyroscope-go v1.2.7 h1:VWBBlqxjyR0Cwk2W6UrE8CdcdD80GOFNutj0Kb1T8ac= +github.com/grafana/pyroscope-go v1.2.7/go.mod h1:o/bpSLiJYYP6HQtvcoVKiE9s5RiNgjYTj1DhiddP2Pc= +github.com/grafana/pyroscope-go/godeltaprof v0.1.9 h1:c1Us8i6eSmkW+Ez05d3co8kasnuOY813tbMN8i/a3Og= +github.com/grafana/pyroscope-go/godeltaprof v0.1.9/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU= github.com/graph-gophers/graphql-go v1.3.0 h1:Eb9x/q6MFpCLz7jBCiP/WTxjSDrYLR1QY41SORZyNJ0= github.com/graph-gophers/graphql-go v1.3.0/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE= @@ -225,6 +229,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4= github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU= +github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= diff --git a/internal/debug/api.go b/internal/debug/api.go index 5a2781cc77..14dbe71c7a 100644 --- a/internal/debug/api.go +++ b/internal/debug/api.go @@ -23,6 +23,7 @@ package debug import ( "bytes" "errors" + "fmt" "io" "os" "os/user" @@ -37,6 +38,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/log" + "github.com/grafana/pyroscope-go" "github.com/hashicorp/go-bexpr" ) @@ -52,6 +54,7 @@ type HandlerT struct { cpuFile string traceW io.WriteCloser traceFile string + profiler *pyroscope.Profiler } // Verbosity sets the log verbosity ceiling. The verbosity of individual packages @@ -127,6 +130,73 @@ func (h *HandlerT) StopCPUProfile() error { return nil } +// Small wrapper for log.Logger to satisfy pyroscope.Logger interface +type pyroscopeLogger struct { + Logger log.Logger +} + +func (l *pyroscopeLogger) Infof(format string, v ...interface{}) { + l.Logger.Info(fmt.Sprintf(format, v...)) +} + +func (l *pyroscopeLogger) Debugf(format string, v ...interface{}) { + l.Logger.Debug(fmt.Sprintf(format, v...)) +} + +func (l *pyroscopeLogger) Errorf(format string, v ...interface{}) { + l.Logger.Error(fmt.Sprintf(format, v...)) +} + +// StartPyroscopeProfiler starts the Pyroscope profiler for later use +func (h *HandlerT) StartPyroscopeProfiler(server string) error { + h.mu.Lock() + defer h.mu.Unlock() + if h.profiler != nil { + log.Info("Pyroscope profiling already started") + return nil + } + profiler, err := pyroscope.Start( + pyroscope.Config{ + ApplicationName: "geth", + ServerAddress: server, + Logger: &pyroscopeLogger{Logger: log.Root().With("module", "pyroscope")}, + // Tags: nil, + ProfileTypes: []pyroscope.ProfileType{ + // Enabling all profile types + pyroscope.ProfileCPU, + pyroscope.ProfileAllocObjects, + pyroscope.ProfileAllocSpace, + pyroscope.ProfileInuseObjects, + pyroscope.ProfileInuseSpace, + pyroscope.ProfileGoroutines, + pyroscope.ProfileMutexCount, + pyroscope.ProfileMutexDuration, + pyroscope.ProfileBlockCount, + pyroscope.ProfileBlockDuration, + }, + }, + ) + if err != nil { + return err + } + h.profiler = profiler + log.Info("Pyroscope profiling started") + return nil +} + +// StopPyroscopeProfiler stops the Pyroscope profiler and returns the error +func (h *HandlerT) StopPyroscopeProfiler() error { + h.mu.Lock() + defer h.mu.Unlock() + if h.profiler != nil { + err := h.profiler.Stop() + h.profiler = nil + return err + } + log.Info("Pyroscope profiling stopped") + return nil +} + // GoTrace turns on tracing for nsec seconds and writes // trace data to file. func (h *HandlerT) GoTrace(file string, nsec uint) error { diff --git a/internal/debug/flags.go b/internal/debug/flags.go index 30b0ddb3be..98de5565c2 100644 --- a/internal/debug/flags.go +++ b/internal/debug/flags.go @@ -140,6 +140,18 @@ var ( Usage: "Write Go execution trace to the given file", Category: flags.LoggingCategory, } + pyroscopeFlag = &cli.BoolFlag{ + Name: "pyroscope", + Usage: "Enable Pyroscope profiling", + Value: false, + Category: flags.LoggingCategory, + } + pyroscopeServerFlag = &cli.StringFlag{ + Name: "pyroscope.server", + Usage: "Pyroscope server URL to push profiling data to", + Value: "http://localhost:4040", + Category: flags.LoggingCategory, + } ) // Flags holds all command-line flags required for debugging. @@ -162,6 +174,8 @@ var Flags = []cli.Flag{ blockprofilerateFlag, cpuprofileFlag, traceFlag, + pyroscopeFlag, + pyroscopeServerFlag, } var ( @@ -298,6 +312,14 @@ func Setup(ctx *cli.Context) error { // It cannot be imported because it will cause a cyclical dependency. StartPProf(address, !ctx.IsSet("metrics.addr")) } + + // Pyroscope profiling + if ctx.Bool(pyroscopeFlag.Name) { + pyroscopeServer := ctx.String(pyroscopeServerFlag.Name) + + Handler.StartPyroscopeProfiler(pyroscopeServer) + } + if len(logFile) > 0 || rotation { log.Info("Logging configured", context...) } @@ -321,6 +343,7 @@ func StartPProf(address string, withMetrics bool) { // Exit stops all running profiles, flushing their output to the // respective file. func Exit() { + Handler.StopPyroscopeProfiler() Handler.StopCPUProfile() Handler.StopGoTrace() if logOutputFile != nil {