From 5c4d2b58a5c6fef3f8037b1629ad9762efd2ff4a Mon Sep 17 00:00:00 2001 From: Eliott Bouhana <47679741+eliottness@users.noreply.github.com> Date: Mon, 6 Jan 2025 16:34:04 +0100 Subject: [PATCH 1/5] fix: workaround for otel agent dependency causes orchestrion failure (#3066) --- internal/agent-otel-workaround.go | 18 ++++++++++++++++++ internal/agent.go | 8 -------- 2 files changed, 18 insertions(+), 8 deletions(-) create mode 100644 internal/agent-otel-workaround.go diff --git a/internal/agent-otel-workaround.go b/internal/agent-otel-workaround.go new file mode 100644 index 0000000000..f7b02edb0b --- /dev/null +++ b/internal/agent-otel-workaround.go @@ -0,0 +1,18 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2022 Datadog, Inc. + +//go:build otel_workaround + +package internal + +import ( + // OTel did a breaking change to the module go.opentelemetry.io/collector/pdata which is imported by the agent + // and go.opentelemetry.io/collector/pdata/pprofile depends on it and is breaking because of it + // For some reason the dependency closure won't let use upgrade this module past the point where it does not break anymore + // So we are forced to add a blank import of this module to give us back the control over its version + // + // TODO: remove this once github.com/datadog-agent/pkg/trace has upgraded both modules past the breaking change + _ "go.opentelemetry.io/collector/pdata/pprofile" +) diff --git a/internal/agent.go b/internal/agent.go index 8b0023274a..f4bcdce8b0 100644 --- a/internal/agent.go +++ b/internal/agent.go @@ -11,14 +11,6 @@ import ( "os" "gopkg.in/DataDog/dd-trace-go.v1/internal/log" - - // OTel did a breaking change to the module go.opentelemetry.io/collector/pdata which is imported by the agent - // and go.opentelemetry.io/collector/pdata/pprofile depends on it and is breaking because of it - // For some reason the dependency closure won't let use upgrade this module past the point where it does not break anymore - // So we are forced to add a blank import of this module to give us back the control over its version - // - // TODO: remove this once github.com/datadog-agent/pkg/trace has upgraded both modules past the breaking change - _ "go.opentelemetry.io/collector/pdata/pprofile" ) const ( From 414cb33eaf3a120eeb72fa5e46099ef63423bc9b Mon Sep 17 00:00:00 2001 From: Flavien Darche <11708575+e-n-0@users.noreply.github.com> Date: Mon, 6 Jan 2025 16:29:58 +0000 Subject: [PATCH 2/5] contrib/envoyproxy: fix resource name (#3047) --- contrib/envoyproxy/go-control-plane/envoy.go | 2 ++ contrib/envoyproxy/go-control-plane/envoy_test.go | 6 +++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/contrib/envoyproxy/go-control-plane/envoy.go b/contrib/envoyproxy/go-control-plane/envoy.go index 52279e0138..a38b5a5e3c 100644 --- a/contrib/envoyproxy/go-control-plane/envoy.go +++ b/contrib/envoyproxy/go-control-plane/envoy.go @@ -11,6 +11,7 @@ import ( "io" "math" "net/http" + "path" "strings" "gopkg.in/DataDog/dd-trace-go.v1/contrib/internal/httptrace" @@ -204,6 +205,7 @@ func processRequestHeaders(ctx context.Context, req *envoyextproc.ProcessingRequ var blocked bool fakeResponseWriter := newFakeResponseWriter() wrappedResponseWriter, request, afterHandle, blocked := httptrace.BeforeHandle(&httptrace.ServeConfig{ + Resource: request.Method + " " + path.Clean(request.URL.Path), SpanOpts: []ddtrace.StartSpanOption{ tracer.Tag(ext.SpanKind, ext.SpanKindServer), tracer.Tag(ext.Component, componentName), diff --git a/contrib/envoyproxy/go-control-plane/envoy_test.go b/contrib/envoyproxy/go-control-plane/envoy_test.go index 8af05eaab3..257cf83538 100644 --- a/contrib/envoyproxy/go-control-plane/envoy_test.go +++ b/contrib/envoyproxy/go-control-plane/envoy_test.go @@ -254,7 +254,7 @@ func TestGeneratedSpan(t *testing.T) { stream, err := client.Process(ctx) require.NoError(t, err) - end2EndStreamRequest(t, stream, "/resource-span", "GET", map[string]string{"user-agent": "Mistake Not...", "test-key": "test-value"}, map[string]string{"response-test-key": "response-test-value"}, false) + end2EndStreamRequest(t, stream, "/../../../resource-span/.?id=test", "GET", map[string]string{"user-agent": "Mistake Not...", "test-key": "test-value"}, map[string]string{"response-test-key": "response-test-value"}, false) err = stream.CloseSend() require.NoError(t, err) @@ -266,10 +266,10 @@ func TestGeneratedSpan(t *testing.T) { // Check for tags span := finished[0] require.Equal(t, "http.request", span.OperationName()) - require.Equal(t, "https://datadoghq.com/resource-span", span.Tag("http.url")) + require.Equal(t, "https://datadoghq.com/../../../resource-span/.?id=test", span.Tag("http.url")) require.Equal(t, "GET", span.Tag("http.method")) require.Equal(t, "datadoghq.com", span.Tag("http.host")) - // require.Equal(t, "GET /resource-span", span.Tag("resource.name")) + require.Equal(t, "GET /resource-span", span.Tag("resource.name")) require.Equal(t, "server", span.Tag("span.kind")) require.Equal(t, "Mistake Not...", span.Tag("http.useragent")) }) From 98feb67c7a068d2e0ee7c8580a90c6464c5ba2e3 Mon Sep 17 00:00:00 2001 From: Flavien Darche <11708575+e-n-0@users.noreply.github.com> Date: Mon, 6 Jan 2025 16:41:27 +0000 Subject: [PATCH 3/5] appsec: Service Extension callout (#2965) --- .../workflows/service-extensions-publish.yml | 130 ++++++++++++++ .../cmd/serviceextensions/.gitignore | 1 + .../cmd/serviceextensions/Dockerfile | 35 ++++ .../cmd/serviceextensions/README.md | 42 +++++ .../cmd/serviceextensions/main.go | 164 ++++++++++++++++++ .../internal/telemetrytest/telemetry_test.go | 2 +- ddtrace/tracer/option_test.go | 2 +- go.mod | 2 +- internal/env.go | 17 ++ 9 files changed, 392 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/service-extensions-publish.yml create mode 100644 contrib/envoyproxy/go-control-plane/cmd/serviceextensions/.gitignore create mode 100644 contrib/envoyproxy/go-control-plane/cmd/serviceextensions/Dockerfile create mode 100644 contrib/envoyproxy/go-control-plane/cmd/serviceextensions/README.md create mode 100644 contrib/envoyproxy/go-control-plane/cmd/serviceextensions/main.go diff --git a/.github/workflows/service-extensions-publish.yml b/.github/workflows/service-extensions-publish.yml new file mode 100644 index 0000000000..bc1d34c8f5 --- /dev/null +++ b/.github/workflows/service-extensions-publish.yml @@ -0,0 +1,130 @@ +name: Publish Service Extensions Callout images packages + +on: + push: + tags: + - 'v*.*' + workflow_dispatch: + inputs: + tag_name: + description: 'Docker image tag to use for the package' + required: true + default: 'dev' + commit_sha: + description: 'Commit SHA to checkout' + required: true + set_as_latest: + description: 'Set the tag as latest' + required: false + default: 'false' + +permissions: + contents: read + packages: write + +env: + TAG_NAME: ${{ github.ref_name || github.event.inputs.tag_name }} + REF_NAME: ${{ github.ref || github.event.inputs.commit_sha }} + COMMIT_SHA: ${{ github.sha || github.event.inputs.commit_sha }} + PUSH_LATEST: ${{ github.event.inputs.set_as_latest || 'true' }} + REGISTRY_IMAGE: ghcr.io/datadog/dd-trace-go/service-extensions-callout + +jobs: + build-service-extensions: + runs-on: ${{ matrix.platform == 'linux/amd64' && 'ubuntu-latest' || 'arm-4core-linux' }} + strategy: + matrix: + platform: [ linux/amd64, linux/arm64 ] + + steps: + - name: Prepare + run: | + platform=${{ matrix.platform }} + echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV + + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + ref: ${{ env.REF_NAME }} + + - name: Install Docker (only arm64) + if: matrix.platform == 'linux/arm64' + run: | + sudo apt-get update + sudo apt-get install -y docker.io + sudo systemctl start docker + sudo systemctl enable docker + sudo usermod -aG docker $USER + newgrp docker + sudo chmod 666 /var/run/docker.sock + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.8.0 + + - name: Login to Docker + shell: bash + run: docker login -u publisher -p ${{ secrets.GITHUB_TOKEN }} ghcr.io + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY_IMAGE }} + + - name: Build and push by digest + id: build + uses: docker/build-push-action@v6 + with: + context: . + file: ./contrib/envoyproxy/go-control-plane/cmd/serviceextensions/Dockerfile + platforms: ${{ matrix.platform }} + labels: ${{ steps.meta.outputs.labels }} + outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true + + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digests-${{ env.PLATFORM_PAIR }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + publish-service-extensions: + runs-on: ubuntu-latest + needs: + - build-service-extensions + + steps: + - name: Download digests + uses: actions/download-artifact@v4 + with: + path: /tmp/digests + pattern: digests-* + merge-multiple: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.8.0 + + - name: Login to Docker + shell: bash + run: docker login -u publisher -p ${{ secrets.GITHUB_TOKEN }} ghcr.io + + - name: Create tags + id: tags + run: | + tagname=${TAG_NAME//\//-} # remove slashes from tag name + echo "tags=-t ghcr.io/datadog/dd-trace-go/service-extensions-callout:${tagname} \ + -t ghcr.io/datadog/dd-trace-go/service-extensions-callout:${{ env.COMMIT_SHA }} \ + ${{ env.PUSH_LATEST == 'true' && '-t ghcr.io/datadog/dd-trace-go/service-extensions-callout:latest' }}" >> $GITHUB_OUTPUT + + - name: Create manifest list and push + working-directory: /tmp/digests + run: | + docker buildx imagetools create ${{ steps.tags.outputs.tags }} \ + $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *) diff --git a/contrib/envoyproxy/go-control-plane/cmd/serviceextensions/.gitignore b/contrib/envoyproxy/go-control-plane/cmd/serviceextensions/.gitignore new file mode 100644 index 0000000000..68295c4a55 --- /dev/null +++ b/contrib/envoyproxy/go-control-plane/cmd/serviceextensions/.gitignore @@ -0,0 +1 @@ +serviceextensions \ No newline at end of file diff --git a/contrib/envoyproxy/go-control-plane/cmd/serviceextensions/Dockerfile b/contrib/envoyproxy/go-control-plane/cmd/serviceextensions/Dockerfile new file mode 100644 index 0000000000..871572d07c --- /dev/null +++ b/contrib/envoyproxy/go-control-plane/cmd/serviceextensions/Dockerfile @@ -0,0 +1,35 @@ +# Build stage +FROM golang:1.23-alpine AS builder +ENV CGO_ENABLED=1 + +WORKDIR /app +COPY . . + +RUN apk add --no-cache --update git build-base openssl + +# Generate SSL self-signed localhost certificate +RUN openssl genrsa -out localhost.key 3072 +RUN openssl req -new \ + -key localhost.key \ + -subj "/C=US/ST=New York/O=Datadog/OU=gRPC/CN=localhost" \ + -out request.csr +RUN openssl x509 -req -days 3660 \ + -in request.csr \ + -signkey localhost.key \ + -out localhost.crt + +# Build the serviceextensions binary +RUN go build -tags=appsec -o ./contrib/envoyproxy/go-control-plane/cmd/serviceextensions/serviceextensions ./contrib/envoyproxy/go-control-plane/cmd/serviceextensions + +# Runtime stage +FROM alpine:3.20.3 +RUN apk --no-cache add ca-certificates tzdata libc6-compat libgcc libstdc++ +WORKDIR /app +COPY --from=builder /app/contrib/envoyproxy/go-control-plane/cmd/serviceextensions/serviceextensions /app/serviceextensions +COPY --from=builder /app/localhost.crt /app/localhost.crt +COPY --from=builder /app/localhost.key /app/localhost.key + +EXPOSE 80 +EXPOSE 443 + +CMD ["./serviceextensions"] diff --git a/contrib/envoyproxy/go-control-plane/cmd/serviceextensions/README.md b/contrib/envoyproxy/go-control-plane/cmd/serviceextensions/README.md new file mode 100644 index 0000000000..b2b5b8fab6 --- /dev/null +++ b/contrib/envoyproxy/go-control-plane/cmd/serviceextensions/README.md @@ -0,0 +1,42 @@ +# ASM Service Extension + +[GCP Services Extensions](https://cloud.google.com/service-extensions/docs/overview) enable Google Cloud users to provide programmability and extensibility on Cloud Load Balancing data paths and at the edge. + +## Installation + +### From Release + +This package provides a docker image to be used with Google Cloud Service Extensions. +The images are published at each release of the tracer and can be found in [the repo registry](https://github.com/DataDog/dd-trace-go/pkgs/container/dd-trace-go%2Fservice-extensions-callout). + +### Build image + +The docker image can be build locally using docker. Start by cloning the `dd-trace-go` repo, `cd` inside it and run that command: +```sh +docker build --build-arg -f contrib/envoyproxy/go-control-plane/cmd/serviceextensions/Dockerfile -t datadog/dd-trace-go/service-extensions-callout:local . +``` + +## Configuration + +The ASM Service Extension expose some configuration. The configuration can be tweaked if the Service Extension is only used as an External Processor for Envoy that is not operated by GCP. + +>**GCP requires that the default configuration for the Service Extension should not change.** + +| Environment variable | Default value | Description | +|---|---|---| +| `DD_SERVICE_EXTENSION_HOST` | `0.0.0.0` | Host on where the gRPC and HTTP server should listen to. | +| `DD_SERVICE_EXTENSION_PORT` | `443` | Port used by the gRPC Server.
Envoy Google backend’s is only using secure connection to Service Extension. | +| `DD_SERVICE_EXTENSION_HEALTHCHECK_PORT` | `80` | Port used for the HTTP server for the health check. | + +> The Service Extension need to be connected to a deployed [Datadog agent](https://docs.datadoghq.com/agent). + +| Environment variable | Default value | Description | +|---|---|---| +| `DD_AGENT_HOST` | `N/A` | Host of a running Datadog Agent. | +| `DD_TRACE_AGENT_PORT` | `8126` | Port of a running Datadog Agent. | + +### SSL Configuration + +The Envoy of GCP is configured to communicate to the Service Extension with TLS. + +`localhost` self signed certificates are generated and bundled into the ASM Service Extension docker image and loaded at the start of the gRPC server. \ No newline at end of file diff --git a/contrib/envoyproxy/go-control-plane/cmd/serviceextensions/main.go b/contrib/envoyproxy/go-control-plane/cmd/serviceextensions/main.go new file mode 100644 index 0000000000..3186ae76b4 --- /dev/null +++ b/contrib/envoyproxy/go-control-plane/cmd/serviceextensions/main.go @@ -0,0 +1,164 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016 Datadog, Inc. + +package main + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "net" + "net/http" + "os" + "os/signal" + "strconv" + "syscall" + "time" + + "golang.org/x/sync/errgroup" + + gocontrolplane "gopkg.in/DataDog/dd-trace-go.v1/contrib/envoyproxy/go-control-plane" + "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" + "gopkg.in/DataDog/dd-trace-go.v1/internal" + "gopkg.in/DataDog/dd-trace-go.v1/internal/log" + "gopkg.in/DataDog/dd-trace-go.v1/internal/version" + + extproc "github.com/envoyproxy/go-control-plane/envoy/service/ext_proc/v3" + "github.com/gorilla/mux" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" +) + +// AppsecCalloutExtensionService defines the struct that follows the ExternalProcessorServer interface. +type AppsecCalloutExtensionService struct { + extproc.ExternalProcessorServer +} + +type serviceExtensionConfig struct { + extensionPort string + extensionHost string + healthcheckPort string +} + +func loadConfig() serviceExtensionConfig { + extensionPortInt := internal.IntEnv("DD_SERVICE_EXTENSION_PORT", 443) + healthcheckPortInt := internal.IntEnv("DD_SERVICE_EXTENSION_HEALTHCHECK_PORT", 80) + extensionHostStr := internal.IpEnv("DD_SERVICE_EXTENSION_HOST", net.IP{0, 0, 0, 0}).String() + + extensionPortStr := strconv.FormatInt(int64(extensionPortInt), 10) + healthcheckPortStr := strconv.FormatInt(int64(healthcheckPortInt), 10) + + return serviceExtensionConfig{ + extensionPort: extensionPortStr, + extensionHost: extensionHostStr, + healthcheckPort: healthcheckPortStr, + } +} + +func main() { + // Set the DD_VERSION to the current tracer version if not set + if os.Getenv("DD_VERSION") == "" { + if err := os.Setenv("DD_VERSION", version.Tag); err != nil { + log.Error("service_extension: failed to set DD_VERSION environment variable: %v\n", err) + } + } + + config := loadConfig() + + if err := startService(config); err != nil { + log.Error("service_extension: %v\n", err) + log.Flush() + os.Exit(1) + } + + log.Info("service_extension: shutting down\n") +} + +func startService(config serviceExtensionConfig) error { + var extensionService AppsecCalloutExtensionService + + tracer.Start(tracer.WithAppSecEnabled(true)) + defer tracer.Stop() + // TODO: Enable ASM standalone mode when it is developed (should be done for Q4 2024) + + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + g, ctx := errgroup.WithContext(ctx) + + g.Go(func() error { + return startGPRCSsl(ctx, &extensionService, config) + }) + + g.Go(func() error { + return startHealthCheck(ctx, config) + }) + + if err := g.Wait(); err != nil { + return err + } + + return nil +} + +func startHealthCheck(ctx context.Context, config serviceExtensionConfig) error { + muxServer := mux.NewRouter() + muxServer.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + w.Write([]byte(`{"status": "ok", "library": {"language": "golang", "version": "` + version.Tag + `"}}`)) + }) + + server := &http.Server{ + Addr: config.extensionHost + ":" + config.healthcheckPort, + Handler: muxServer, + } + + go func() { + <-ctx.Done() + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := server.Shutdown(shutdownCtx); err != nil { + log.Error("service_extension: health check server shutdown: %v\n", err) + } + }() + + log.Info("service_extension: health check server started on %s:%s\n", config.extensionHost, config.healthcheckPort) + if err := server.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + return fmt.Errorf("health check http server: %v", err) + } + + return nil +} + +func startGPRCSsl(ctx context.Context, service extproc.ExternalProcessorServer, config serviceExtensionConfig) error { + cert, err := tls.LoadX509KeyPair("localhost.crt", "localhost.key") + if err != nil { + return fmt.Errorf("failed to load key pair: %v", err) + } + + lis, err := net.Listen("tcp", config.extensionHost+":"+config.extensionPort) + if err != nil { + return fmt.Errorf("gRPC server: %v", err) + } + + grpcCredentials := credentials.NewServerTLSFromCert(&cert) + grpcServer := grpc.NewServer(grpc.Creds(grpcCredentials)) + + appsecEnvoyExternalProcessorServer := gocontrolplane.AppsecEnvoyExternalProcessorServer(service) + + go func() { + <-ctx.Done() + grpcServer.GracefulStop() + }() + + extproc.RegisterExternalProcessorServer(grpcServer, appsecEnvoyExternalProcessorServer) + log.Info("service_extension: callout gRPC server started on %s:%s\n", config.extensionHost, config.extensionPort) + if err := grpcServer.Serve(lis); err != nil { + return fmt.Errorf("error starting gRPC server: %v", err) + } + + return nil +} diff --git a/contrib/internal/telemetrytest/telemetry_test.go b/contrib/internal/telemetrytest/telemetry_test.go index a203228fc2..7e94ca420d 100644 --- a/contrib/internal/telemetrytest/telemetry_test.go +++ b/contrib/internal/telemetrytest/telemetry_test.go @@ -83,7 +83,7 @@ func TestTelemetryEnabled(t *testing.T) { packages = append(packages, out) } for _, pkg := range packages { - if strings.Contains(pkg.ImportPath, "/test") || strings.Contains(pkg.ImportPath, "/internal") { + if strings.Contains(pkg.ImportPath, "/test") || strings.Contains(pkg.ImportPath, "/internal") || strings.Contains(pkg.ImportPath, "/cmd") { continue } if !pkg.hasTelemetryImport(t) { diff --git a/ddtrace/tracer/option_test.go b/ddtrace/tracer/option_test.go index 36c3475801..16588ce95a 100644 --- a/ddtrace/tracer/option_test.go +++ b/ddtrace/tracer/option_test.go @@ -343,7 +343,7 @@ func TestIntegrationEnabled(t *testing.T) { packages = append(packages, out) } for _, pkg := range packages { - if strings.Contains(pkg.ImportPath, "/test") || strings.Contains(pkg.ImportPath, "/internal") { + if strings.Contains(pkg.ImportPath, "/test") || strings.Contains(pkg.ImportPath, "/internal") || strings.Contains(pkg.ImportPath, "/cmd") { continue } p := strings.Replace(pkg.Dir, pkg.Root, "../..", 1) diff --git a/go.mod b/go.mod index bbe71fe323..f78b187447 100644 --- a/go.mod +++ b/go.mod @@ -101,6 +101,7 @@ require ( go.uber.org/goleak v1.3.0 golang.org/x/mod v0.20.0 golang.org/x/oauth2 v0.18.0 + golang.org/x/sync v0.10.0 golang.org/x/sys v0.28.0 golang.org/x/time v0.6.0 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 @@ -295,7 +296,6 @@ require ( golang.org/x/crypto v0.31.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect golang.org/x/net v0.33.0 // indirect - golang.org/x/sync v0.10.0 // indirect golang.org/x/term v0.27.0 // indirect golang.org/x/text v0.21.0 // indirect golang.org/x/tools v0.24.0 // indirect diff --git a/internal/env.go b/internal/env.go index 62704b7792..2e760526ac 100644 --- a/internal/env.go +++ b/internal/env.go @@ -6,6 +6,7 @@ package internal import ( + "net" "os" "strconv" "strings" @@ -59,6 +60,22 @@ func DurationEnv(key string, def time.Duration) time.Duration { return v } +// IpEnv returns the valid IP value of an environment variable, or def otherwise. +func IpEnv(key string, def net.IP) net.IP { + vv, ok := os.LookupEnv(key) + if !ok { + return def + } + + ip := net.ParseIP(vv) + if ip == nil { + log.Warn("Non-IP value for env var %s, defaulting to %s", key, def.String()) + return def + } + + return ip +} + // ForEachStringTag runs fn on every key val pair encountered in str. // str may contain multiple key val pairs separated by either space // or comma (but not a mixture of both), and each key val pair is separated by a delimiter. From e460de6797ff71ce63ef49efaa9235ba7f6dbd2d Mon Sep 17 00:00:00 2001 From: Eliott Bouhana <47679741+eliottness@users.noreply.github.com> Date: Tue, 7 Jan 2025 19:54:28 +0100 Subject: [PATCH 4/5] internal/remoteconfig: support for DD_REMOTE_CONFIGURATION_ENABLED (#3063) Signed-off-by: Eliott Bouhana --- internal/remoteconfig/remoteconfig.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/remoteconfig/remoteconfig.go b/internal/remoteconfig/remoteconfig.go index 4076c2ad34..27230cb3eb 100644 --- a/internal/remoteconfig/remoteconfig.go +++ b/internal/remoteconfig/remoteconfig.go @@ -20,6 +20,7 @@ import ( "sync" "time" + "gopkg.in/DataDog/dd-trace-go.v1/internal" "gopkg.in/DataDog/dd-trace-go.v1/internal/log" rc "github.com/DataDog/datadog-agent/pkg/remoteconfig/state" @@ -187,6 +188,10 @@ func Start(config ClientConfig) error { if err != nil { return } + if !internal.BoolEnv("DD_REMOTE_CONFIGURATION_ENABLED", true) { + // Don't start polling if the feature is disabled explicitly + return + } go func() { ticker := time.NewTicker(client.PollInterval) defer ticker.Stop() From 36479447b3c2da2952ea7ba929249c5d60054443 Mon Sep 17 00:00:00 2001 From: Eliott Bouhana <47679741+eliottness@users.noreply.github.com> Date: Wed, 8 Jan 2025 16:49:16 +0100 Subject: [PATCH 5/5] internal/appsec: add fingerprint tests (#3068) Signed-off-by: Eliott Bouhana --- internal/appsec/testdata/fp.json | 54 +++++++++++++++++++++++++- internal/appsec/waf_test.go | 66 +++++++++++++++++++++++--------- 2 files changed, 99 insertions(+), 21 deletions(-) diff --git a/internal/appsec/testdata/fp.json b/internal/appsec/testdata/fp.json index 52af47f36c..c7f630caec 100644 --- a/internal/appsec/testdata/fp.json +++ b/internal/appsec/testdata/fp.json @@ -15,6 +15,7 @@ }, "conditions": [ { + "operator": "phrase_match", "parameters": { "inputs": [ { @@ -24,13 +25,62 @@ "list": [ "$globals" ] - }, - "operator": "phrase_match" + } } ], "transformers": [ "lowercase" ] + }, + { + "id": "9d50832c-200d-4b21-a050-61379f1a9af8", + "name": "Track users.login.failure on id-auth - /id/auth/v1/login", + "tags": { + "category": "business_logic", + "custom": "1", + "type": "users.login.failure" + }, + "conditions": [ + { + "operator": "phrase_match", + "parameters": { + "inputs": [ + { + "address": "server.request.uri.raw" + } + ], + "list": [ + "/id/auth/v1/login" + ] + } + }, + { + "operator": "phrase_match", + "parameters": { + "inputs": [ + { + "address": "server.request.method" + } + ], + "list": [ + "POST" + ] + } + }, + { + "operator": "phrase_match", + "parameters": { + "inputs": [ + { + "address": "server.response.status" + } + ], + "list": [ + "401" + ] + } + } + ] } ], "processors": [ diff --git a/internal/appsec/waf_test.go b/internal/appsec/waf_test.go index 7e169ffb7a..347d4f802c 100644 --- a/internal/appsec/waf_test.go +++ b/internal/appsec/waf_test.go @@ -31,6 +31,7 @@ import ( httptrace "gopkg.in/DataDog/dd-trace-go.v1/contrib/net/http" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/mocktracer" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" + "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec" "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/config" "gopkg.in/DataDog/dd-trace-go.v1/internal/appsec/dyngo" @@ -945,7 +946,7 @@ func TestAttackerFingerprinting(t *testing.T) { // Start and trace an HTTP server mux := httptrace.NewServeMux() - mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + mux.HandleFunc("/test", func(w http.ResponseWriter, r *http.Request) { pAppsec.TrackUserLoginSuccessEvent( r.Context(), "toto", @@ -956,31 +957,58 @@ func TestAttackerFingerprinting(t *testing.T) { w.Write([]byte("Hello World!\n")) }) + + mux.HandleFunc("/id/auth/v1/login", func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(401) + }) + srv := httptest.NewServer(mux) defer srv.Close() - mt := mocktracer.Start() - defer mt.Stop() - req, err := http.NewRequest("POST", srv.URL+"/test?x=1", nil) - require.NoError(t, err) - req.AddCookie(&http.Cookie{Name: "cookie", Value: "value"}) - resp, err := srv.Client().Do(req) - require.NoError(t, err) - defer resp.Body.Close() + for _, tc := range []struct { + name string + url string + }{ + { + name: "SDK", + url: "/test?x=1", + }, + { + name: "WAF", + url: "/?x=$globals", + }, + { + name: "CustomRule", + url: "/id/auth/v1/login", + }, + } { + t.Run(tc.name, func(t *testing.T) { + mt := mocktracer.Start() + defer mt.Stop() - require.Len(t, mt.FinishedSpans(), 1) + req, err := http.NewRequest("POST", srv.URL+tc.url, nil) + require.NoError(t, err) + req.AddCookie(&http.Cookie{Name: "cookie", Value: "value"}) + resp, err := srv.Client().Do(req) + require.NoError(t, err) + defer resp.Body.Close() + + require.Len(t, mt.FinishedSpans(), 1) - tags := mt.FinishedSpans()[0].Tags() + tags := mt.FinishedSpans()[0].Tags() - require.Contains(t, tags, "_dd.appsec.fp.http.header") - require.Contains(t, tags, "_dd.appsec.fp.http.endpoint") - require.Contains(t, tags, "_dd.appsec.fp.http.network") - require.Contains(t, tags, "_dd.appsec.fp.session") + require.Contains(t, tags, "_dd.appsec.fp.http.header") + require.Contains(t, tags, "_dd.appsec.fp.http.endpoint") + require.Contains(t, tags, "_dd.appsec.fp.http.network") + require.Contains(t, tags, "_dd.appsec.fp.session") - require.Regexp(t, `^hdr-`, tags["_dd.appsec.fp.http.header"]) - require.Regexp(t, `^http-`, tags["_dd.appsec.fp.http.endpoint"]) - require.Regexp(t, `^ssn-`, tags["_dd.appsec.fp.session"]) - require.Regexp(t, `^net-`, tags["_dd.appsec.fp.http.network"]) + require.Regexp(t, `^hdr-`, tags["_dd.appsec.fp.http.header"]) + require.Regexp(t, `^http-`, tags["_dd.appsec.fp.http.endpoint"]) + require.Regexp(t, `^ssn-`, tags["_dd.appsec.fp.session"]) + require.Regexp(t, `^net-`, tags["_dd.appsec.fp.http.network"]) + }) + + } } func init() {