Skip to content

Commit

Permalink
Adds grafana pyroscope to worker and backend (#3144)
Browse files Browse the repository at this point in the history
  • Loading branch information
nickzelei authored Jan 14, 2025
1 parent 92e78e6 commit 4e9c45b
Show file tree
Hide file tree
Showing 6 changed files with 141 additions and 0 deletions.
14 changes: 14 additions & 0 deletions backend/internal/cmds/mgmt/serve/connect/cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,14 @@ import (
"connectrpc.com/otelconnect"
"github.com/auth0/go-jwt-middleware/v2/validator"
"github.com/go-logr/logr"
"github.com/grafana/pyroscope-go"
"github.com/jackc/pgx/v5/stdlib"
db_queries "github.com/nucleuscloud/neosync/backend/gen/go/db"
"github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1/mgmtv1alpha1connect"
connectionmanager "github.com/nucleuscloud/neosync/internal/connection-manager"
"github.com/nucleuscloud/neosync/internal/connectrpc/validate"
http_client "github.com/nucleuscloud/neosync/internal/http/client"
pyroscope_env "github.com/nucleuscloud/neosync/internal/pyroscope"
"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
"go.opentelemetry.io/otel/exporters/stdout/stdoutmetric"
Expand Down Expand Up @@ -129,6 +131,18 @@ func serve(ctx context.Context) error {
}
slogger.Debug(fmt.Sprintf("neosync cloud enabled: %t", ncloudlicense.IsValid()))

pyroscopeConfig, isPyroscopeEnabled, err := pyroscope_env.NewFromEnv("neosync-api", slogger)
if err != nil {
return fmt.Errorf("unable to initialize pyroscope from env: %w", err)
}
if isPyroscopeEnabled {
profiler, err := pyroscope.Start(*pyroscopeConfig)
if err != nil {
return fmt.Errorf("unable to start pyroscope profiler: %w", err)
}
defer profiler.Stop() //nolint:errcheck
}

cascadelicense := license.NewCascadeLicense(
ncloudlicense,
eelicense,
Expand Down
2 changes: 2 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ require (
github.com/go-sql-driver/mysql v1.8.1
github.com/golang-migrate/migrate/v4 v4.18.1
github.com/google/uuid v1.6.0
github.com/grafana/pyroscope-go v1.2.0
github.com/itchyny/gojq v0.12.17
github.com/jackc/pgx/v5 v5.7.2
github.com/lib/pq v1.10.9
Expand Down Expand Up @@ -265,6 +266,7 @@ require (
github.com/gosimple/slug v1.13.1 // indirect
github.com/gosimple/unidecode v1.0.1 // indirect
github.com/govalues/decimal v0.1.32 // indirect
github.com/grafana/pyroscope-go/godeltaprof v0.1.8 // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 // indirect
github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect
Expand Down
4 changes: 4 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -1501,6 +1501,10 @@ github.com/gosimple/unidecode v1.0.1 h1:hZzFTMMqSswvf0LBJZCZgThIZrpDHFXux9KeGmn6
github.com/gosimple/unidecode v1.0.1/go.mod h1:CP0Cr1Y1kogOtx0bJblKzsVWrqYaqfNOnHzpgWw4Awc=
github.com/govalues/decimal v0.1.32 h1:jsZHwjLKteAlG5nGjlqvhtkGBq7/4SKkk6yGTluwPk0=
github.com/govalues/decimal v0.1.32/go.mod h1:Ee7eI3Llf7hfqDZtpj8Q6NCIgJy1iY3kH1pSwDrNqlM=
github.com/grafana/pyroscope-go v1.2.0 h1:aILLKjTj8CS8f/24OPMGPewQSYlhmdQMBmol1d3KGj8=
github.com/grafana/pyroscope-go v1.2.0/go.mod h1:2GHr28Nr05bg2pElS+dDsc98f3JTUh2f6Fz1hWXrqwk=
github.com/grafana/pyroscope-go/godeltaprof v0.1.8 h1:iwOtYXeeVSAeYefJNaxDytgjKtUuKQbJqgAIjlnicKg=
github.com/grafana/pyroscope-go/godeltaprof v0.1.8/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU=
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI=
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8=
github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
Expand Down
42 changes: 42 additions & 0 deletions internal/pyroscope/logger/logger.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
package pyroscope_logger

import (
"fmt"
"log/slog"

"github.com/grafana/pyroscope-go"
)

type PyroscopeLogger struct {
logger *slog.Logger
}

var _ pyroscope.Logger = (*PyroscopeLogger)(nil)

func (p *PyroscopeLogger) Debugf(format string, args ...any) {
p.logger.Debug(fmt.Sprintf(format, args...))
}

func (p *PyroscopeLogger) Infof(format string, args ...any) {
p.logger.Info(fmt.Sprintf(format, args...))
}

func (p *PyroscopeLogger) Errorf(format string, args ...any) {
p.logger.Error(fmt.Sprintf(format, args...))
}

func New(logger *slog.Logger) *PyroscopeLogger {
return &PyroscopeLogger{logger: logger}
}

type noopLogger struct{}

var _ pyroscope.Logger = (*noopLogger)(nil)

func (n *noopLogger) Debugf(_ string, _ ...any) {}
func (n *noopLogger) Infof(_ string, _ ...any) {}
func (n *noopLogger) Errorf(_ string, _ ...any) {}

func NewNoop() *noopLogger {
return &noopLogger{}
}
64 changes: 64 additions & 0 deletions internal/pyroscope/pyroscope.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
package pyroscope_env

import (
"errors"
"log/slog"
"runtime"

"github.com/grafana/pyroscope-go"
pyroscope_logger "github.com/nucleuscloud/neosync/internal/pyroscope/logger"
"github.com/spf13/viper"
)

func NewFromEnv(applicationName string, logger *slog.Logger) (*pyroscope.Config, bool, error) {
isPyroscopeEnabled := viper.GetBool("PYROSCOPE_ENABLED")
if !isPyroscopeEnabled {
return nil, false, nil
}

logger.Debug("pyroscope is enabled")
serverAddress := viper.GetString("PYROSCOPE_SERVER_ADDRESS")
if serverAddress == "" {
return nil, false, errors.New("PYROSCOPE_SERVER_ADDRESS is required")
}
basicAuthUser := viper.GetString("PYROSCOPE_BASIC_AUTH_USER")
basicAuthPassword := viper.GetString("PYROSCOPE_BASIC_AUTH_PASSWORD")

pyroscopeTags := map[string]string{}

isLoggerEnabled := viper.GetBool("PYROSCOPE_LOGGER_ENABLED")
var pyroscopeLogger pyroscope.Logger
if isLoggerEnabled {
pyroscopeLogger = pyroscope_logger.New(logger)
} else {
pyroscopeLogger = pyroscope_logger.NewNoop()
}

runtime.SetMutexProfileFraction(5)
runtime.SetBlockProfileRate(5)

pyroscopeConfig := &pyroscope.Config{
ApplicationName: applicationName,
ServerAddress: serverAddress,
Logger: pyroscopeLogger,
BasicAuthUser: basicAuthUser,
BasicAuthPassword: basicAuthPassword,
Tags: pyroscopeTags,
ProfileTypes: []pyroscope.ProfileType{
pyroscope.ProfileCPU,
pyroscope.ProfileAllocObjects,
pyroscope.ProfileAllocSpace,
pyroscope.ProfileInuseObjects,
pyroscope.ProfileInuseSpace,

pyroscope.ProfileGoroutines,
pyroscope.ProfileMutexCount,
pyroscope.ProfileMutexDuration,
pyroscope.ProfileBlockCount,
pyroscope.ProfileBlockDuration,
},
DisableGCRuns: true,
}

return pyroscopeConfig, true, nil
}
15 changes: 15 additions & 0 deletions worker/internal/cmds/worker/serve/serve.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
cloudlicense "github.com/nucleuscloud/neosync/internal/ee/cloud-license"
"github.com/nucleuscloud/neosync/internal/ee/license"
neosyncotel "github.com/nucleuscloud/neosync/internal/otel"
pyroscope_env "github.com/nucleuscloud/neosync/internal/pyroscope"
neosync_redis "github.com/nucleuscloud/neosync/worker/internal/redis"
accountstatus_activity "github.com/nucleuscloud/neosync/worker/pkg/workflows/datasync/activities/account-status"
genbenthosconfigs_activity "github.com/nucleuscloud/neosync/worker/pkg/workflows/datasync/activities/gen-benthos-configs"
Expand Down Expand Up @@ -55,6 +56,8 @@ import (
"golang.org/x/net/http2/h2c"

"net/http/pprof"

"github.com/grafana/pyroscope-go"
)

func NewCmd() *cobra.Command {
Expand Down Expand Up @@ -84,6 +87,18 @@ func serve(ctx context.Context) error {
}
logger.Debug(fmt.Sprintf("neosync cloud enabled: %t", ncloudlicense.IsValid()))

pyroscopeConfig, isPyroscopeEnabled, err := pyroscope_env.NewFromEnv("neosync-worker", logger)
if err != nil {
return fmt.Errorf("unable to initialize pyroscope from env: %w", err)
}
if isPyroscopeEnabled {
profiler, err := pyroscope.Start(*pyroscopeConfig)
if err != nil {
return fmt.Errorf("unable to start pyroscope profiler: %w", err)
}
defer profiler.Stop() //nolint:errcheck
}

var syncActivityMeter metric.Meter
temporalClientInterceptors := []interceptor.ClientInterceptor{}
var temopralMeterHandler client.MetricsHandler
Expand Down

0 comments on commit 4e9c45b

Please sign in to comment.