feat(otel): metrics

.github/workflows/test_examples.yaml

@@ -37,7 +37,8 @@ jobs:
           # - ingesthackernews
           - logrus
           - otelinstrument
-          - oteltraces
+          - otelmetric
+          - oteltrace
           - query
           - querylegacy
           # HINT(lukasmalkmus): This test would require Go 1.21 (but uses Go
@@ -67,7 +68,10 @@ jobs:
           - example: otelinstrument
             verify: |
               axiom dataset info $AXIOM_DATASET -f=json | jq -e 'any( .numEvents ; . >= 1 )'
-          - example: oteltraces
+          - example: otelmetric
+            verify: |
+              axiom dataset info $AXIOM_DATASET -f=json | jq -e 'any( .numEvents ; . > 3 )'
+          - example: oteltrace
             verify: |
               axiom dataset info $AXIOM_DATASET -f=json | jq -e 'any( .numEvents ; . == 2 )'
           - example: query
@@ -111,6 +115,10 @@ jobs:
         run: ${{ matrix.setup }}
       - name: Run example
         run: go run ./examples/${{ matrix.example }}/main.go
+        timeout-minutes: 5
+        # We have some long running examples so cancel the step after a while.
+        # We still validate the example output in the next step.
+        continue-on-error: true
       - name: Verify example
         if: matrix.verify
         run: ${{ matrix.verify }}

axiom/otel/config.go

@@ -0,0 +1,112 @@
+package otel
+
+import (
+	"time"
+
+	"github.com/axiomhq/axiom-go/internal/config"
+)
+
+const (
+	defaultMetricAPIEndpoint = "/v1/metrics"
+	defaultTraceAPIEndpoint  = "/v1/traces"
+)
+
+// Config is the configuration for OpenTelemetry components initialized by this
+// helper package. This type is exported for convenience but an [Option] is
+// naturally applied by one or more "Set"-prefixed functions.
+type Config struct {
+	config.Config
+
+	// APIEndpoint is the endpoint to use for an exporter.
+	APIEndpoint string
+	// Timeout is the timeout for an exporters underlying [http.Client].
+	Timeout time.Duration
+	// NoEnv disables the use of "AXIOM_*" environment variables.
+	NoEnv bool
+}
+
+func defaultMetricConfig() Config {
+	return Config{
+		Config:      config.Default(),
+		APIEndpoint: defaultMetricAPIEndpoint,
+	}
+}
+
+func defaultTraceConfig() Config {
+	return Config{
+		Config:      config.Default(),
+		APIEndpoint: defaultTraceAPIEndpoint,
+	}
+}
+
+// An Option modifies the behaviour of OpenTelemetry exporters. Nonetheless,
+// the official "OTEL_*" environment variables are preferred over the options or
+// "AXIOM_*" environment variables.
+type Option func(c *Config) error
+
+// SetURL sets the base URL used by the client.
+//
+// Can also be specified using the "AXIOM_URL" environment variable.
+func SetURL(baseURL string) Option {
+	return func(c *Config) error { return c.Options(config.SetURL(baseURL)) }
+}
+
+// SetToken specifies the authentication token used by the client.
+//
+// Can also be specified using the "AXIOM_TOKEN" environment variable.
+func SetToken(token string) Option {
+	return func(c *Config) error { return c.Options(config.SetToken(token)) }
+}
+
+// SetOrganizationID specifies the organization ID used by the client.
+//
+// Can also be specified using the "AXIOM_ORG_ID" environment variable.
+func SetOrganizationID(organizationID string) Option {
+	return func(c *Config) error { return c.Options(config.SetOrganizationID(organizationID)) }
+}
+
+// SetAPIEndpoint specifies the api endpoint used by the client.
+func SetAPIEndpoint(path string) Option {
+	return func(c *Config) error {
+		c.APIEndpoint = path
+		return nil
+	}
+}
+
+// SetTimeout specifies the http timeout used by the client.
+func SetTimeout(timeout time.Duration) Option {
+	return func(c *Config) error {
+		c.Timeout = timeout
+		return nil
+	}
+}
+
+// SetNoEnv prevents the client from deriving its configuration from the
+// environment (by auto reading "AXIOM_*" environment variables).
+func SetNoEnv() Option {
+	return func(c *Config) error {
+		c.NoEnv = true
+		return nil
+	}
+}
+
+func populateAndValidateConfig(base *Config, options ...Option) error {
+	// Apply supplied options.
+	for _, option := range options {
+		if option == nil {
+			continue
+		} else if err := option(base); err != nil {
+			return err
+		}
+	}
+
+	// Make sure to populate remaining fields from the environment, if not
+	// explicitly disabled.
+	if !base.NoEnv {
+		if err := base.IncorporateEnvironment(); err != nil {
+			return err
+		}
+	}
+
+	return base.Validate()
+}

axiom/otel/doc.go

@@ -4,23 +4,25 @@
 //
 //	import "github.com/axiomhq/axiom-go/axiom/otel"
 //
-// Different levels of helpers are available, from just setting up tracing to
-// getting access to lower level components to costumize tracing or integrate
-// with existing OpenTelemetry setups:
-//
-//   - [InitTracing]: Initializes OpenTelemetry and sets the global tracer
-//     prodiver so the official OpenTelemetry Go SDK can be used to get a tracer
-//     and instrument code. Sane defaults for the tracer provider are applied.
-//   - [TracerProvider]: Configures and returns a new OpenTelemetry tracer
-//     provider but does not set it as the global tracer provider.
-//   - [TraceExporter]: Configures and returns a new OpenTelemetry trace
-//     exporter. This sets up the exporter that sends traces to Axiom but allows
-//     for a more advanced setup of the tracer provider.
+// Different levels of helpers are available, from just setting up
+// instrumentation to getting access to lower level components to costumize
+// instrumentation or integrate with existing OpenTelemetry setups:
+//
+//   - [InitMetrics]/[InitTracing]: Initializes OpenTelemetry and sets the
+//     global meter/tracer prodiver so the official OpenTelemetry Go SDK can be
+//     used to get a meter/tracer and instrument code. Sane defaults for the
+//     providers are applied.
+//   - [MeterProvider]/[TracerProvider]: Configures and returns a new
+//     OpenTelemetry meter/tracer provider but does not set it as the global
+//     meter/tracer provider.
+//   - [MetricExporter]/[TraceExporter]: Configures and returns a new
+//     OpenTelemetry metric/trace exporter. This sets up the exporter that sends
+//     metrics/traces to Axiom but allows for a more advanced setup of the
+//     meter/tracer provider.
 //
 // If you wish for traces to propagate beyond the current process, you need to
 // set the global propagator to the OpenTelemetry trace context propagator. This
-// can be done
-// by calling:
+// can be done by calling:
 //
 //	import (
 //	    "go.opentelemetry.io/otel"

axiom/otel/metric.go

@@ -0,0 +1,108 @@
+package otel
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp"
+	"go.opentelemetry.io/otel/sdk/metric"
+	"go.opentelemetry.io/otel/sdk/resource"
+	semconv "go.opentelemetry.io/otel/semconv/v1.21.0"
+)
+
+// MetricExporter configures and returns a new exporter for OpenTelemetry spans.
+func MetricExporter(ctx context.Context, dataset string, options ...Option) (metric.Exporter, error) {
+	config := defaultMetricConfig()
+
+	if err := populateAndValidateConfig(&config, options...); err != nil {
+		return nil, err
+	}
+
+	u, err := config.BaseURL().Parse(config.APIEndpoint)
+	if err != nil {
+		return nil, fmt.Errorf("parse exporter url: %w", err)
+	}
+
+	opts := []otlpmetrichttp.Option{
+		otlpmetrichttp.WithEndpoint(u.Host),
+	}
+	if u.Path != "" {
+		opts = append(opts, otlpmetrichttp.WithURLPath(u.Path))
+	}
+	if u.Scheme == "http" {
+		opts = append(opts, otlpmetrichttp.WithInsecure())
+	}
+	if config.Timeout > 0 {
+		opts = append(opts, otlpmetrichttp.WithTimeout(config.Timeout))
+	}
+
+	headers := make(map[string]string)
+	if config.Token() != "" {
+		headers["Authorization"] = "Bearer " + config.Token()
+	}
+	if config.OrganizationID() != "" {
+		headers["X-Axiom-Org-Id"] = config.OrganizationID()
+	}
+	if dataset != "" {
+		headers["X-Axiom-Dataset"] = dataset
+	}
+	if len(headers) > 0 {
+		opts = append(opts, otlpmetrichttp.WithHeaders(headers))
+	}
+
+	return otlpmetrichttp.New(ctx, opts...)
+}
+
+// MeterProvider configures and returns a new OpenTelemetry meter provider.
+func MeterProvider(ctx context.Context, dataset, serviceName, serviceVersion string, options ...Option) (*metric.MeterProvider, error) {
+	exporter, err := MetricExporter(ctx, dataset, options...)
+	if err != nil {
+		return nil, err
+	}
+
+	rs, err := resource.Merge(resource.Default(), resource.NewWithAttributes(
+		semconv.SchemaURL,
+		semconv.ServiceNameKey.String(serviceName),
+		semconv.ServiceVersionKey.String(serviceVersion),
+		UserAgentAttribute(),
+	))
+	if err != nil {
+		return nil, err
+	}
+
+	opts := []metric.Option{
+		metric.WithReader(metric.NewPeriodicReader(
+			exporter,
+			metric.WithInterval(time.Second*5), // FIXME(lukasmalkmus): Just for testing!
+			metric.WithTimeout(time.Second*5),  // FIXME(lukasmalkmus): Just for testing!
+		)),
+		metric.WithResource(rs),
+	}
+
+	return metric.NewMeterProvider(opts...), nil
+}
+
+// InitMetrics initializes OpenTelemetry metrics with the given service name,
+// version and options. If initialization succeeds, the returned cleanup
+// function must be called to shut down the meter provider and flush any
+// remaining datapoints. The error returned by the cleanup function must be
+// checked, as well.
+func InitMetrics(ctx context.Context, dataset, serviceName, serviceVersion string, options ...Option) (func() error, error) {
+	meterProvider, err := MeterProvider(ctx, dataset, serviceName, serviceVersion, options...)
+	if err != nil {
+		return nil, err
+	}
+
+	otel.SetMeterProvider(meterProvider)
+
+	closeFunc := func() error {
+		ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+		defer cancel()
+
+		return meterProvider.Shutdown(ctx)
+	}
+
+	return closeFunc, nil
+}

axiom/otel/metric_integration_test.go

@@ -0,0 +1,53 @@
+//go:build integration
+
+package otel_test
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"go.opentelemetry.io/otel"
+
+	"github.com/axiomhq/axiom-go/axiom"
+	axiotel "github.com/axiomhq/axiom-go/axiom/otel"
+)
+
+func TestMetricsIntegration(t *testing.T) {
+	ctx := context.Background()
+
+	datasetSuffix := os.Getenv("AXIOM_DATASET_SUFFIX")
+	if datasetSuffix == "" {
+		datasetSuffix = "local"
+	}
+	dataset := fmt.Sprintf("test-axiom-go-otel-metric-%s", datasetSuffix)
+
+	client, err := axiom.NewClient()
+	require.NoError(t, err)
+
+	_, err = client.Datasets.Create(ctx, axiom.DatasetCreateRequest{
+		Name:        dataset,
+		Description: "This is a test dataset for otel metric integration tests.",
+	})
+	require.NoError(t, err)
+
+	t.Cleanup(func() {
+		err = client.Datasets.Delete(ctx, dataset)
+		require.NoError(t, err)
+	})
+
+	stop, err := axiotel.InitMetrics(ctx, dataset, "axiom-go-otel-test-metric", "v1.0.0")
+	require.NoError(t, err)
+	require.NotNil(t, stop)
+
+	t.Cleanup(func() { require.NoError(t, stop()) })
+
+	meter := otel.Meter("main")
+
+	counter, err := meter.Int64Counter("test")
+	require.NoError(t, err)
+
+	counter.Add(ctx, 1)
+}