[exporter/datadog] Mark unrecoverable errors as permanent

exporter/datadogexporter/internal/clientutil/api.go

@@ -46,7 +46,7 @@ func CreateAPIClient(buildInfo component.BuildInfo, endpoint string, settings ex
 func ValidateAPIKey(ctx context.Context, apiKey string, logger *zap.Logger, apiClient *datadog.APIClient) error {
 	logger.Info("Validating API key.")
 	authAPI := datadogV1.NewAuthenticationApi(apiClient)
-	resp, _, err := authAPI.Validate(GetRequestContext(ctx, apiKey))
+	resp, httpresp, err := authAPI.Validate(GetRequestContext(ctx, apiKey))
 	if err == nil && resp.Valid != nil && *resp.Valid {
 		logger.Info("API key validation successful.")
 		return nil
@@ -56,7 +56,7 @@ func ValidateAPIKey(ctx context.Context, apiKey string, logger *zap.Logger, apiC
 		return nil
 	}
 	logger.Warn(ErrInvalidAPI.Error())
-	return ErrInvalidAPI
+	return WrapError(ErrInvalidAPI, httpresp)
 }
 
 // GetRequestContext creates a new context with API key for DatadogV2 requests

exporter/datadogexporter/internal/clientutil/error_converter.go

@@ -0,0 +1,33 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package clientutil // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter/internal/clientutil"
+
+import (
+	"net/http"
+
+	"go.opentelemetry.io/collector/consumer/consumererror"
+)
+
+// WrapError wraps an error to a permanent consumer error that won't be retried if the http response code is non-retriable.
+func WrapError(err error, resp *http.Response) error {
+	if err == nil || !isNonRetriable(resp) {
+		return err
+	}
+	return consumererror.NewPermanent(err)
+}
+
+func isNonRetriable(resp *http.Response) bool {
+	return resp.StatusCode == 400 || resp.StatusCode == 404 || resp.StatusCode == 413 || resp.StatusCode == 403
+}

exporter/datadogexporter/internal/clientutil/error_converter_test.go

@@ -0,0 +1,35 @@
+// Copyright The OpenTelemetry Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package clientutil // import "github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datadogexporter/internal/clientutil"
+
+import (
+	"fmt"
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"go.opentelemetry.io/collector/consumer/consumererror"
+)
+
+func TestWrapError(t *testing.T) {
+	respOK := http.Response{StatusCode: 200}
+	respRetriable := http.Response{StatusCode: 402}
+	respNonRetriable := http.Response{StatusCode: 404}
+	err := fmt.Errorf("Test error")
+	assert.False(t, consumererror.IsPermanent(WrapError(err, &respOK)))
+	assert.False(t, consumererror.IsPermanent(WrapError(err, &respRetriable)))
+	assert.True(t, consumererror.IsPermanent(WrapError(err, &respNonRetriable)))
+	assert.False(t, consumererror.IsPermanent(WrapError(nil, &respNonRetriable)))
+}

exporter/datadogexporter/internal/clientutil/retrier.go

@@ -31,13 +31,15 @@ type Retrier struct {
 	cfg      exporterhelper.RetrySettings
 	logger   *zap.Logger
 	scrubber scrub.Scrubber
+	retryNum int64
 }
 
 func NewRetrier(logger *zap.Logger, settings exporterhelper.RetrySettings, scrubber scrub.Scrubber) *Retrier {
 	return &Retrier{
 		cfg:      settings,
 		logger:   logger,
 		scrubber: scrubber,
+		retryNum: int64(0),
 	}
 }
 
@@ -60,7 +62,7 @@ func (r *Retrier) DoWithRetries(ctx context.Context, fn func(context.Context) er
 		Clock:               backoff.SystemClock,
 	}
 	expBackoff.Reset()
-	retryNum := int64(0)
+	r.retryNum = int64(0)
 	for {
 		err := fn(ctx)
 		if err == nil {
@@ -85,7 +87,7 @@ func (r *Retrier) DoWithRetries(ctx context.Context, fn func(context.Context) er
 			zap.Error(err),
 			zap.String("interval", backoffDelayStr),
 		)
-		retryNum++
+		r.retryNum++
 
 		// back-off, but get interrupted when shutting down or request is cancelled or timed out.
 		select {

exporter/datadogexporter/internal/clientutil/retrier_test.go

@@ -17,9 +17,12 @@ package clientutil // import "github.com/open-telemetry/opentelemetry-collector-
 import (
 	"context"
 	"errors"
+	"fmt"
+	"net/http"
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.opentelemetry.io/collector/exporter/exporterhelper"
 	"go.uber.org/zap"
@@ -46,4 +49,18 @@ func TestDoWithRetries(t *testing.T) {
 	)
 	err = retrier.DoWithRetries(ctx, func(context.Context) error { return errors.New("action failed") })
 	require.Error(t, err)
+	assert.Greater(t, retrier.retryNum, int64(0))
+}
+
+func TestNoRetriesOnPermanentError(t *testing.T) {
+	scrubber := scrub.NewScrubber()
+	retrier := NewRetrier(zap.NewNop(), exporterhelper.NewDefaultRetrySettings(), scrubber)
+	ctx := context.Background()
+	respNonRetriable := http.Response{StatusCode: 404}
+
+	err := retrier.DoWithRetries(ctx, func(context.Context) error {
+		return WrapError(fmt.Errorf("test"), &respNonRetriable)
+	})
+	require.Error(t, err)
+	assert.Equal(t, retrier.retryNum, int64(0))
 }

exporter/datadogexporter/metrics_exporter.go

@@ -168,12 +168,12 @@ func (exp *metricsExporter) pushSketches(ctx context.Context, sl sketches.Sketch
 	}
 
 	if err != nil {
-		return fmt.Errorf("failed to do sketches HTTP request: %w", err)
+		return clientutil.WrapError(fmt.Errorf("failed to do sketches HTTP request: %w", err), resp)
 	}
 	defer resp.Body.Close()
 
 	if resp.StatusCode >= 400 {
-		return fmt.Errorf("error when sending payload to %s: %s", sketches.SketchSeriesEndpoint, resp.Status)
+		return clientutil.WrapError(fmt.Errorf("error when sending payload to %s: %s", sketches.SketchSeriesEndpoint, resp.Status), resp)
 	}
 	return nil
 }
@@ -227,8 +227,8 @@ func (exp *metricsExporter) PushMetricsData(ctx context.Context, md pmetric.Metr
 				err,
 				exp.retrier.DoWithRetries(ctx, func(context.Context) error {
 					ctx = clientutil.GetRequestContext(ctx, string(exp.cfg.API.Key))
-					_, _, merr := exp.metricsAPI.SubmitMetrics(ctx, datadogV2.MetricPayload{Series: ms})
-					return merr
+					_, httpresp, merr := exp.metricsAPI.SubmitMetrics(ctx, datadogV2.MetricPayload{Series: ms})
+					return clientutil.WrapError(merr, httpresp)
 				}),
 			)
 		}