diff --git a/backend/internal/cmds/mgmt/serve/connect/cmd.go b/backend/internal/cmds/mgmt/serve/connect/cmd.go index a40b16fb56..7bddb5e889 100644 --- a/backend/internal/cmds/mgmt/serve/connect/cmd.go +++ b/backend/internal/cmds/mgmt/serve/connect/cmd.go @@ -17,12 +17,14 @@ import ( "connectrpc.com/otelconnect" "github.com/auth0/go-jwt-middleware/v2/validator" "github.com/go-logr/logr" + "github.com/grafana/pyroscope-go" "github.com/jackc/pgx/v5/stdlib" db_queries "github.com/nucleuscloud/neosync/backend/gen/go/db" "github.com/nucleuscloud/neosync/backend/gen/go/protos/mgmt/v1alpha1/mgmtv1alpha1connect" connectionmanager "github.com/nucleuscloud/neosync/internal/connection-manager" "github.com/nucleuscloud/neosync/internal/connectrpc/validate" http_client "github.com/nucleuscloud/neosync/internal/http/client" + pyroscope_env "github.com/nucleuscloud/neosync/internal/pyroscope" "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc" "go.opentelemetry.io/otel/exporters/stdout/stdoutmetric" @@ -129,6 +131,18 @@ func serve(ctx context.Context) error { } slogger.Debug(fmt.Sprintf("neosync cloud enabled: %t", ncloudlicense.IsValid())) + pyroscopeConfig, isPyroscopeEnabled, err := pyroscope_env.NewFromEnv("neosync-api", slogger) + if err != nil { + return fmt.Errorf("unable to initialize pyroscope from env: %w", err) + } + if isPyroscopeEnabled { + profiler, err := pyroscope.Start(*pyroscopeConfig) + if err != nil { + return fmt.Errorf("unable to start pyroscope profiler: %w", err) + } + defer profiler.Stop() //nolint:errcheck + } + cascadelicense := license.NewCascadeLicense( ncloudlicense, eelicense, diff --git a/go.mod b/go.mod index 73c2f00d51..687bf611e3 100644 --- a/go.mod +++ b/go.mod @@ -44,6 +44,7 @@ require ( github.com/go-sql-driver/mysql v1.8.1 github.com/golang-migrate/migrate/v4 v4.18.1 github.com/google/uuid v1.6.0 + github.com/grafana/pyroscope-go v1.2.0 github.com/itchyny/gojq v0.12.17 github.com/jackc/pgx/v5 v5.7.2 github.com/lib/pq v1.10.9 @@ -265,6 +266,7 @@ require ( github.com/gosimple/slug v1.13.1 // indirect github.com/gosimple/unidecode v1.0.1 // indirect github.com/govalues/decimal v0.1.32 // indirect + github.com/grafana/pyroscope-go/godeltaprof v0.1.8 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 // indirect github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect diff --git a/go.sum b/go.sum index a78bd2d045..50b509b029 100644 --- a/go.sum +++ b/go.sum @@ -1501,6 +1501,10 @@ github.com/gosimple/unidecode v1.0.1 h1:hZzFTMMqSswvf0LBJZCZgThIZrpDHFXux9KeGmn6 github.com/gosimple/unidecode v1.0.1/go.mod h1:CP0Cr1Y1kogOtx0bJblKzsVWrqYaqfNOnHzpgWw4Awc= github.com/govalues/decimal v0.1.32 h1:jsZHwjLKteAlG5nGjlqvhtkGBq7/4SKkk6yGTluwPk0= github.com/govalues/decimal v0.1.32/go.mod h1:Ee7eI3Llf7hfqDZtpj8Q6NCIgJy1iY3kH1pSwDrNqlM= +github.com/grafana/pyroscope-go v1.2.0 h1:aILLKjTj8CS8f/24OPMGPewQSYlhmdQMBmol1d3KGj8= +github.com/grafana/pyroscope-go v1.2.0/go.mod h1:2GHr28Nr05bg2pElS+dDsc98f3JTUh2f6Fz1hWXrqwk= +github.com/grafana/pyroscope-go/godeltaprof v0.1.8 h1:iwOtYXeeVSAeYefJNaxDytgjKtUuKQbJqgAIjlnicKg= +github.com/grafana/pyroscope-go/godeltaprof v0.1.8/go.mod h1:2+l7K7twW49Ct4wFluZD3tZ6e0SjanjcUUBPVD/UuGU= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= diff --git a/internal/pyroscope/logger/logger.go b/internal/pyroscope/logger/logger.go new file mode 100644 index 0000000000..0eb640b5bc --- /dev/null +++ b/internal/pyroscope/logger/logger.go @@ -0,0 +1,42 @@ +package pyroscope_logger + +import ( + "fmt" + "log/slog" + + "github.com/grafana/pyroscope-go" +) + +type PyroscopeLogger struct { + logger *slog.Logger +} + +var _ pyroscope.Logger = (*PyroscopeLogger)(nil) + +func (p *PyroscopeLogger) Debugf(format string, args ...any) { + p.logger.Debug(fmt.Sprintf(format, args...)) +} + +func (p *PyroscopeLogger) Infof(format string, args ...any) { + p.logger.Info(fmt.Sprintf(format, args...)) +} + +func (p *PyroscopeLogger) Errorf(format string, args ...any) { + p.logger.Error(fmt.Sprintf(format, args...)) +} + +func New(logger *slog.Logger) *PyroscopeLogger { + return &PyroscopeLogger{logger: logger} +} + +type noopLogger struct{} + +var _ pyroscope.Logger = (*noopLogger)(nil) + +func (n *noopLogger) Debugf(_ string, _ ...any) {} +func (n *noopLogger) Infof(_ string, _ ...any) {} +func (n *noopLogger) Errorf(_ string, _ ...any) {} + +func NewNoop() *noopLogger { + return &noopLogger{} +} diff --git a/internal/pyroscope/pyroscope.go b/internal/pyroscope/pyroscope.go new file mode 100644 index 0000000000..2230bf22c3 --- /dev/null +++ b/internal/pyroscope/pyroscope.go @@ -0,0 +1,64 @@ +package pyroscope_env + +import ( + "errors" + "log/slog" + "runtime" + + "github.com/grafana/pyroscope-go" + pyroscope_logger "github.com/nucleuscloud/neosync/internal/pyroscope/logger" + "github.com/spf13/viper" +) + +func NewFromEnv(applicationName string, logger *slog.Logger) (*pyroscope.Config, bool, error) { + isPyroscopeEnabled := viper.GetBool("PYROSCOPE_ENABLED") + if !isPyroscopeEnabled { + return nil, false, nil + } + + logger.Debug("pyroscope is enabled") + serverAddress := viper.GetString("PYROSCOPE_SERVER_ADDRESS") + if serverAddress == "" { + return nil, false, errors.New("PYROSCOPE_SERVER_ADDRESS is required") + } + basicAuthUser := viper.GetString("PYROSCOPE_BASIC_AUTH_USER") + basicAuthPassword := viper.GetString("PYROSCOPE_BASIC_AUTH_PASSWORD") + + pyroscopeTags := map[string]string{} + + isLoggerEnabled := viper.GetBool("PYROSCOPE_LOGGER_ENABLED") + var pyroscopeLogger pyroscope.Logger + if isLoggerEnabled { + pyroscopeLogger = pyroscope_logger.New(logger) + } else { + pyroscopeLogger = pyroscope_logger.NewNoop() + } + + runtime.SetMutexProfileFraction(5) + runtime.SetBlockProfileRate(5) + + pyroscopeConfig := &pyroscope.Config{ + ApplicationName: applicationName, + ServerAddress: serverAddress, + Logger: pyroscopeLogger, + BasicAuthUser: basicAuthUser, + BasicAuthPassword: basicAuthPassword, + Tags: pyroscopeTags, + ProfileTypes: []pyroscope.ProfileType{ + pyroscope.ProfileCPU, + pyroscope.ProfileAllocObjects, + pyroscope.ProfileAllocSpace, + pyroscope.ProfileInuseObjects, + pyroscope.ProfileInuseSpace, + + pyroscope.ProfileGoroutines, + pyroscope.ProfileMutexCount, + pyroscope.ProfileMutexDuration, + pyroscope.ProfileBlockCount, + pyroscope.ProfileBlockDuration, + }, + DisableGCRuns: true, + } + + return pyroscopeConfig, true, nil +} diff --git a/worker/internal/cmds/worker/serve/serve.go b/worker/internal/cmds/worker/serve/serve.go index 3811434449..df2b30741c 100644 --- a/worker/internal/cmds/worker/serve/serve.go +++ b/worker/internal/cmds/worker/serve/serve.go @@ -27,6 +27,7 @@ import ( cloudlicense "github.com/nucleuscloud/neosync/internal/ee/cloud-license" "github.com/nucleuscloud/neosync/internal/ee/license" neosyncotel "github.com/nucleuscloud/neosync/internal/otel" + pyroscope_env "github.com/nucleuscloud/neosync/internal/pyroscope" neosync_redis "github.com/nucleuscloud/neosync/worker/internal/redis" accountstatus_activity "github.com/nucleuscloud/neosync/worker/pkg/workflows/datasync/activities/account-status" genbenthosconfigs_activity "github.com/nucleuscloud/neosync/worker/pkg/workflows/datasync/activities/gen-benthos-configs" @@ -55,6 +56,8 @@ import ( "golang.org/x/net/http2/h2c" "net/http/pprof" + + "github.com/grafana/pyroscope-go" ) func NewCmd() *cobra.Command { @@ -84,6 +87,18 @@ func serve(ctx context.Context) error { } logger.Debug(fmt.Sprintf("neosync cloud enabled: %t", ncloudlicense.IsValid())) + pyroscopeConfig, isPyroscopeEnabled, err := pyroscope_env.NewFromEnv("neosync-worker", logger) + if err != nil { + return fmt.Errorf("unable to initialize pyroscope from env: %w", err) + } + if isPyroscopeEnabled { + profiler, err := pyroscope.Start(*pyroscopeConfig) + if err != nil { + return fmt.Errorf("unable to start pyroscope profiler: %w", err) + } + defer profiler.Stop() //nolint:errcheck + } + var syncActivityMeter metric.Meter temporalClientInterceptors := []interceptor.ClientInterceptor{} var temopralMeterHandler client.MetricsHandler