feat: add more error info to traces (#833)

This is mostly for better Datadog compatibility. Datadog does not
natively implement OpenTelemetry and its conventions, so there is a
certain amount of impedance mismatching. Datadog also does not document
properly where certain error info needs to be stored within a span to
show up correctly in their UI. This is my best guess from the docs
scattered around their website.

We might have to iterate on this.

otelx/withspan.go

@@ -7,9 +7,12 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"reflect"
 
+	pkgerrors "github.com/pkg/errors"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/codes"
+	semconv "go.opentelemetry.io/otel/semconv/v1.27.0"
 	"go.opentelemetry.io/otel/trace"
 )
 
@@ -41,7 +44,7 @@ func WithSpan(ctx context.Context, name string, f func(context.Context) error, o
 // Usage:
 //
 //	func Divide(ctx context.Context, numerator, denominator int) (ratio int, err error) {
-//		ctx, span := tracer.Start(ctx, "my-operation")
+//		ctx, span := tracer.Start(ctx, "Divide")
 //		defer otelx.End(span, &err)
 //		if denominator == 0 {
 //			return 0, errors.New("cannot divide by zero")
@@ -62,6 +65,10 @@ func End(span trace.Span, err *error) {
 }
 
 func setErrorStatusPanic(span trace.Span, recovered any) {
+	span.SetAttributes(semconv.ExceptionEscaped(true))
+	if t := reflect.TypeOf(recovered); t != nil {
+		span.SetAttributes(semconv.ExceptionType(t.String()))
+	}
 	switch e := recovered.(type) {
 	case error:
 		span.SetStatus(codes.Error, "panic: "+e.Error())
@@ -76,6 +83,14 @@ func setErrorStatusPanic(span trace.Span, recovered any) {
 }
 
 func setErrorTags(span trace.Span, err error) {
+	span.SetAttributes(
+		attribute.String("error", err.Error()),
+		attribute.String("error.message", err.Error()),                        // compat
+		attribute.String("error.type", fmt.Sprintf("%T", errors.Unwrap(err))), // the innermost error type is the most useful here
+	)
+	if e := interface{ StackTrace() pkgerrors.StackTrace }(nil); errors.As(err, &e) {
+		span.SetAttributes(attribute.String("error.stack", fmt.Sprintf("%+v", e.StackTrace())))
+	}
 	if e := interface{ Reason() string }(nil); errors.As(err, &e) {
 		span.SetAttributes(attribute.String("error.reason", e.Reason()))
 	}
@@ -90,5 +105,4 @@ func setErrorTags(span trace.Span, err error) {
 			span.SetAttributes(attribute.String("error.details."+k, fmt.Sprintf("%v", v)))
 		}
 	}
-	span.SetAttributes(attribute.String("error", err.Error()))
 }

otelx/withspan_test.go

@@ -7,8 +7,10 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"slices"
 	"testing"
 
+	pkgerrors "github.com/pkg/errors"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.opentelemetry.io/otel/attribute"
@@ -69,6 +71,12 @@ func returnsError(ctx context.Context) (err error) {
 	return fmt.Errorf("wrapped: %w", &errWithReason{errors.New("error from returnsError()")})
 }
 
+func returnsStackTracer(ctx context.Context) (err error) {
+	_, span := trace.SpanFromContext(ctx).TracerProvider().Tracer("").Start(ctx, "returnsStackTracer")
+	defer End(span, &err)
+	return pkgerrors.WithStack(errors.New("error from returnsStackTracer()"))
+}
+
 func returnsNamedError(ctx context.Context) (err error) {
 	_, span := trace.SpanFromContext(ctx).TracerProvider().Tracer("").Start(ctx, "returnsNamedError")
 	defer End(span, &err)
@@ -105,6 +113,14 @@ func TestEnd(t *testing.T) {
 	assert.Equal(t, last(recorder).Status(), sdktrace.Status{codes.Error, "err2 message"})
 	assert.Contains(t, last(recorder).Attributes(), attribute.String("error.debug", "verbose debugging information"))
 
+	assert.Errorf(t, returnsStackTracer(ctx), "error from returnsStackTracer()")
+	require.NotEmpty(t, recorder.Ended())
+	assert.Equal(t, last(recorder).Name(), "returnsStackTracer")
+	assert.Equal(t, last(recorder).Status(), sdktrace.Status{codes.Error, "error from returnsStackTracer()"})
+	stackIdx := slices.IndexFunc(last(recorder).Attributes(), func(kv attribute.KeyValue) bool { return kv.Key == "error.stack" })
+	require.GreaterOrEqual(t, stackIdx, 0)
+	assert.Contains(t, last(recorder).Attributes()[stackIdx].Value.AsString(), "github.com/ory/x/otelx.returnsStackTracer")
+
 	assert.PanicsWithError(t, "panic from panics()", func() { panics(ctx) })
 	require.NotEmpty(t, recorder.Ended())
 	assert.Equal(t, last(recorder).Name(), "panics")