feat: traces catching errors (#396)

tigrisdata-archive · Aug 3, 2022 · d9f53fc · d9f53fc
1 parent 6937b0c
commit d9f53fc
Show file tree

Hide file tree

Showing 4 changed files with 39 additions and 5 deletions.
diff --git a/server/metrics/tracing.go b/server/metrics/tracing.go
@@ -16,9 +16,13 @@ package metrics
 
 import (
 	"context"
-
+	"errors"
+	"github.com/apple/foundationdb/bindings/go/src/fdb"
+	api "github.com/tigrisdata/tigris/api/server/v1"
 	"github.com/tigrisdata/tigris/server/config"
+	"google.golang.org/grpc/status"
 	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
+	"strconv"
 )
 
 const (
@@ -32,6 +36,7 @@ type SpanMeta struct {
 	resourceName string
 	spanType     string
 	tags         map[string]string
+	span         tracer.Span
 }
 
 func NewSpanMeta(serviceName string, resourceName string, spanType string, tags map[string]string) *SpanMeta {
@@ -61,12 +66,33 @@ func (s *SpanMeta) StartTracing(ctx context.Context, childOnly bool) (context.Co
 		// There is no parent span, no need to start tracing here
 		return ctx, func() {}
 	}
-	span := tracer.StartSpan(TraceServiceName, spanOpts...)
+	s.span = tracer.StartSpan(TraceServiceName, spanOpts...)
 	for k, v := range s.tags {
-		span.SetTag(k, v)
+		s.span.SetTag(k, v)
 	}
-	ctx = tracer.ContextWithSpan(ctx, span)
+	ctx = tracer.ContextWithSpan(ctx, s.span)
 	return ctx, func() {
-		span.Finish()
+		s.span.Finish()
+	}
+}
+
+func (s *SpanMeta) FinishWithError(err error) {
+	if s.span == nil {
+		return
+	}
+	errCode := status.Code(err)
+	s.span.SetTag("grpc.code", errCode.String())
+	var tigrisErr *api.TigrisError
+	var fdbErr fdb.Error
+	if errors.As(err, &fdbErr) {
+		s.span.SetTag("error_source", "fdb")
+		s.span.SetTag("fdb_error_code", strconv.Itoa(fdbErr.Code))
+	}
+	if errors.As(err, &tigrisErr) {
+		s.span.SetTag("error_source", "tigris_server")
+		s.span.SetTag("tigris_server_error", tigrisErr.Code.String())
 	}
+	// TODO: handle known search errors
+	finishOptions := []tracer.FinishOption{tracer.WithError(err)}
+	s.span.Finish(finishOptions...)
 }
diff --git a/server/middleware/tracing.go b/server/middleware/tracing.go
@@ -35,6 +35,9 @@ func traceUnary() func(ctx context.Context, req interface{}, info *grpc.UnarySer
 		ctx, finisher = spanMeta.StartTracing(ctx, false)
 		defer finisher()
 		resp, err := handler(ctx, req)
+		if err != nil {
+			spanMeta.FinishWithError(err)
+		}
 		return resp, err
 	}
 }
@@ -50,6 +53,9 @@ func traceStream() grpc.StreamServerInterceptor {
 		defer finisher()
 		wrapper := &recvWrapper{wrapped}
 		err := handler(srv, wrapper)
+		if err != nil {
+			spanMeta.FinishWithError(err)
+		}
 		return err
 	}
 }
diff --git a/store/kv/kv.go b/store/kv/kv.go
@@ -125,6 +125,7 @@ func measureLow(ctx context.Context, name string, f func() error) {
 			metrics.FdbErrorRequests.Tagged(tags).Counter("unknown").Inc(1)
 		}
 	}
+	spanMeta.FinishWithError(err)
 }
 
 func (m *KeyValueStoreImplWithMetrics) measure(ctx context.Context, name string, f func() error) {

diff --git a/store/search/ts.go b/store/search/ts.go
@@ -59,6 +59,7 @@ func (m *storeImplWithMetrics) measure(ctx context.Context, name string, f func(
 	if config.DefaultConfig.Metrics.Search.Counters {
 		metrics.SearchErrorRequests.Tagged(tags).Counter("unknown").Inc(1)
 	}
+	spanMeta.FinishWithError(err)
 }
 
 func (m *storeImplWithMetrics) CreateCollection(ctx context.Context, schema *tsApi.CollectionSchema) (err error) {
-Original file line number
+Diff line change
@@ Expand Up @@
     			metrics.FdbErrorRequests.Tagged(tags).Counter("unknown").Inc(1)
     		}
     	}
+    	spanMeta.FinishWithError(err)
     }
     func (m *KeyValueStoreImplWithMetrics) measure(ctx context.Context, name string, f func() error) {
@@ Expand Down @@