Merge pull request open-telemetry#58 from sethAmazon/fix-imds-retry

Fix imds retry

.github/workflows/build-and-test.yml

@@ -213,6 +213,7 @@ jobs:
           git diff --exit-code '.github/ISSUE_TEMPLATE' || (echo 'Dropdowns in issue templates are out of date, please run "make generate-gh-issue-templates" and commit the changes in this PR.' && exit 1)
   unittest-matrix:
     strategy:
+      fail-fast: false
       matrix:
         go-version: ["1.20", 1.19] # 1.20 is interpreted as 1.2 without quotes
         group:

cmd/configschema/go.mod

@@ -277,7 +277,7 @@ require (
 	github.com/alecthomas/participle/v2 v2.0.0 // indirect
 	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
 	github.com/aliyun/aliyun-log-go-sdk v0.1.44 // indirect
-	github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230712194013-e85dd5f17ce4 // indirect
+	github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230719154634-7dfcb28377c2 // indirect
 	github.com/andybalholm/brotli v1.0.5 // indirect
 	github.com/antonmedv/expr v1.12.5 // indirect
 	github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40 // indirect

cmd/otelcontribcol/exporters_test.go

@@ -25,6 +25,7 @@ import (
 	"runtime"
 	"testing"
 
+	override "github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.opentelemetry.io/collector/component"
@@ -496,6 +497,7 @@ func TestDefaultExporters(t *testing.T) {
 		expectedExporters[k] = struct{}{}
 	}
 	for _, tt := range tests {
+		override.IMDSRetryer = nil
 		_, ok := expFactories[tt.exporter]
 		if !ok {
 			// not part of the distro, skipping.

cmd/otelcontribcol/go.mod

@@ -5,6 +5,7 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/cmd/otelcontrib
 go 1.19
 
 require (
+	github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230719154634-7dfcb28377c2
 	github.com/open-telemetry/opentelemetry-collector-contrib/connector/countconnector v0.77.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/connector/servicegraphconnector v0.77.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/connector/spanmetricsconnector v0.77.0
@@ -269,7 +270,6 @@ require (
 	github.com/alecthomas/participle/v2 v2.0.0 // indirect
 	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
 	github.com/aliyun/aliyun-log-go-sdk v0.1.44 // indirect
-	github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230712194013-e85dd5f17ce4 // indirect
 	github.com/andybalholm/brotli v1.0.5 // indirect
 	github.com/antonmedv/expr v1.12.5 // indirect
 	github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40 // indirect

exporter/awscloudwatchlogsexporter/go.mod

@@ -19,7 +19,7 @@ require (
 )
 
 require (
-	github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230712194013-e85dd5f17ce4 // indirect
+	github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230719154634-7dfcb28377c2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect

exporter/awsemfexporter/go.mod

@@ -24,7 +24,7 @@ require (
 )
 
 require (
-	github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230712194013-e85dd5f17ce4 // indirect
+	github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230719154634-7dfcb28377c2 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect

exporter/awsxrayexporter/go.mod

@@ -18,7 +18,7 @@ require (
 )
 
 require (
-	github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230712194013-e85dd5f17ce4 // indirect
+	github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230719154634-7dfcb28377c2 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect

exporter/datadogexporter/config_test.go

@@ -17,12 +17,13 @@ package datadogexporter
 import (
 	"testing"
 
+	override "github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws"
 	"github.com/stretchr/testify/assert"
 	"go.opentelemetry.io/collector/confmap"
 )
 
 func TestValidate(t *testing.T) {
-
+	override.IMDSRetryer = nil
 	tests := []struct {
 		name string
 		cfg  *Config

exporter/datadogexporter/go.mod

@@ -2,6 +2,8 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/exporter/datado
 
 go 1.19
 
+replace github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws => ../../override/aws
+
 require (
 	github.com/DataDog/agent-payload/v5 v5.0.83
 	github.com/DataDog/datadog-agent/pkg/trace v0.45.0-rc.4
@@ -12,7 +14,8 @@ require (
 	github.com/DataDog/opentelemetry-mapping-go/pkg/quantile v0.2.0
 	github.com/DataDog/sketches-go v1.4.2
 	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.13.1
-	github.com/aws/aws-sdk-go v1.44.255
+	github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230719154634-7dfcb28377c2
+	github.com/aws/aws-sdk-go v1.44.293
 	github.com/cenkalti/backoff/v4 v4.2.1
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/ecsutil v0.77.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.77.0

exporter/datadogexporter/internal/hostmetadata/internal/ec2/ec2.go

@@ -22,8 +22,8 @@ import (
 	"sync"
 
 	"github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes/source"
+	override "github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws"
 	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/client"
 	"github.com/aws/aws-sdk-go/aws/ec2metadata"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/ec2"
@@ -64,17 +64,20 @@ func GetHostInfo(logger *zap.Logger) (hostInfo *HostInfo) {
 		return
 	}
 
+	ctx, cancel := context.WithTimeout(context.Background(), 2*override.TimePerCall)
+	defer cancel()
+
 	meta := ec2metadata.New(sess, &aws.Config{
-		Retryer: client.DefaultRetryer{NumMaxRetries: 5},
+		Retryer: override.IMDSRetryer,
 	})
 
-	if idDoc, err := meta.GetInstanceIdentityDocument(); err == nil {
+	if idDoc, err := meta.GetInstanceIdentityDocumentWithContext(ctx); err == nil {
 		hostInfo.InstanceID = idDoc.InstanceID
 	} else {
 		logger.Warn("Failed to get EC2 instance id document", zap.Error(err))
 	}
 
-	if ec2Hostname, err := meta.GetMetadata("hostname"); err == nil {
+	if ec2Hostname, err := meta.GetMetadataWithContext(ctx, "hostname"); err == nil {
 		hostInfo.EC2Hostname = ec2Hostname
 	} else {
 		logger.Warn("Failed to get EC2 hostname", zap.Error(err))

exporter/datadogexporter/internal/hostmetadata/metadata_test.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes"
 	"github.com/DataDog/opentelemetry-mapping-go/pkg/otlp/attributes/azure"
+	override "github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.opentelemetry.io/collector/component"
@@ -64,6 +65,7 @@ var (
 )
 
 func TestFillHostMetadata(t *testing.T) {
+	override.IMDSRetryer = nil
 	params := exportertest.NewNopCreateSettings()
 	params.BuildInfo = mockBuildInfo
 

extension/awsproxy/go.mod

@@ -13,7 +13,8 @@ require (
 )
 
 require (
-	github.com/aws/aws-sdk-go v1.44.255 // indirect
+	github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230719154634-7dfcb28377c2 // indirect
+	github.com/aws/aws-sdk-go v1.44.293 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect

go.mod

@@ -257,7 +257,7 @@ require (
 	github.com/alecthomas/participle/v2 v2.0.0 // indirect
 	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
 	github.com/aliyun/aliyun-log-go-sdk v0.1.44 // indirect
-	github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230712194013-e85dd5f17ce4 // indirect
+	github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230719154634-7dfcb28377c2 // indirect
 	github.com/andybalholm/brotli v1.0.5 // indirect
 	github.com/antonmedv/expr v1.12.5 // indirect
 	github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40 // indirect

internal/aws/awsutil/conn.go

@@ -16,6 +16,7 @@
 package awsutil // import "github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/awsutil"
 
 import (
+	"context"
 	"crypto/tls"
 	"errors"
 	"net/http"
@@ -26,7 +27,6 @@ import (
 	override "github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/awserr"
-	"github.com/aws/aws-sdk-go/aws/client"
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
 	"github.com/aws/aws-sdk-go/aws/defaults"
@@ -47,9 +47,11 @@ type ConnAttr interface {
 type Conn struct{}
 
 func (c *Conn) getEC2Region(s *session.Session) (string, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), override.TimePerCall)
+	defer cancel()
 	return ec2metadata.New(s, &aws.Config{
-		Retryer: client.DefaultRetryer{NumMaxRetries: 5},
-	}).Region()
+		Retryer: override.IMDSRetryer,
+	}).RegionWithContext(ctx)
 }
 
 // AWS STS endpoint constants

internal/aws/awsutil/go.mod

@@ -3,7 +3,7 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/aw
 go 1.19
 
 require (
-	github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230712194013-e85dd5f17ce4
+	github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230719154634-7dfcb28377c2
 	github.com/aws/aws-sdk-go v1.44.293
 	github.com/stretchr/testify v1.8.2
 	go.uber.org/zap v1.24.0

internal/aws/proxy/conn.go

@@ -15,6 +15,7 @@
 package proxy // import "github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/proxy"
 
 import (
+	"context"
 	"crypto/tls"
 	"encoding/json"
 	"errors"
@@ -25,10 +26,10 @@ import (
 	"strings"
 	"time"
 
+	override "github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/arn"
 	"github.com/aws/aws-sdk-go/aws/awserr"
-	"github.com/aws/aws-sdk-go/aws/client"
 	"github.com/aws/aws-sdk-go/aws/credentials"
 	"github.com/aws/aws-sdk-go/aws/credentials/stscreds"
 	"github.com/aws/aws-sdk-go/aws/ec2metadata"
@@ -80,9 +81,11 @@ var newAWSSession = func(roleArn string, region string, log *zap.Logger) (*sessi
 }
 
 var getEC2Region = func(s *session.Session) (string, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), override.TimePerCall)
+	defer cancel()
 	return ec2metadata.New(s, &aws.Config{
-		Retryer: client.DefaultRetryer{NumMaxRetries: 5},
-	}).Region()
+		Retryer: override.IMDSRetryer,
+	}).RegionWithContext(ctx)
 }
 
 func getAWSConfigSession(c *Config, logger *zap.Logger) (*aws.Config, *session.Session, error) {

internal/aws/proxy/go.mod

@@ -2,8 +2,11 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/pr
 
 go 1.19
 
+replace github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws => ../../../override/aws
+
 require (
-	github.com/aws/aws-sdk-go v1.44.255
+	github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230719154634-7dfcb28377c2
+	github.com/aws/aws-sdk-go v1.44.293
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/common v0.77.0
 	github.com/stretchr/testify v1.8.2
 	go.opentelemetry.io/collector v0.77.0

internal/aws/xray/go.mod

@@ -3,6 +3,7 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/xr
 go 1.19
 
 require (
+	github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230719154634-7dfcb28377c2
 	github.com/aws/aws-sdk-go v1.44.293
 	github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/awsutil v0.77.0
 	github.com/stretchr/testify v1.8.2
@@ -11,7 +12,6 @@ require (
 )
 
 require (
-	github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230712194013-e85dd5f17ce4 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect

internal/aws/xray/telemetry/sender.go

@@ -15,12 +15,13 @@
 package telemetry // import "github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/xray/telemetry"
 
 import (
+	"context"
 	"os"
 	"sync"
 	"time"
 
+	override "github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws"
 	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/client"
 	"github.com/aws/aws-sdk-go/aws/ec2metadata"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/xray"
@@ -169,7 +170,9 @@ type ec2MetadataProvider struct {
 
 func (p ec2MetadataProvider) get() string {
 	var metadata string
-	if result, err := p.client.GetMetadata(p.metadataKey); err == nil {
+	ctx, cancel := context.WithTimeout(context.Background(), override.TimePerCall)
+	defer cancel()
+	if result, err := p.client.GetMetadataWithContext(ctx, p.metadataKey); err == nil {
 		metadata = result
 	}
 	return metadata
@@ -190,7 +193,7 @@ func ToOptions(cfg Config, sess *session.Session, settings *awsutil.AWSSessionSe
 	}
 	if !settings.LocalMode {
 		metadataClient := ec2metadata.New(sess, &aws.Config{
-			Retryer: client.DefaultRetryer{NumMaxRetries: 5},
+			Retryer: override.IMDSRetryer,
 		})
 		hostnameProviders = append(hostnameProviders, ec2MetadataProvider{
 			client:      metadataClient,

internal/metadataproviders/aws/ec2/metadata.go

@@ -17,8 +17,8 @@ package ec2 // import "github.com/open-telemetry/opentelemetry-collector-contrib
 import (
 	"context"
 
+	override "github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws"
 	"github.com/aws/aws-sdk-go/aws"
-	"github.com/aws/aws-sdk-go/aws/client"
 	"github.com/aws/aws-sdk-go/aws/ec2metadata"
 	"github.com/aws/aws-sdk-go/aws/session"
 )
@@ -38,19 +38,25 @@ var _ Provider = (*metadataClient)(nil)
 func NewProvider(sess *session.Session) Provider {
 	return &metadataClient{
 		metadata: ec2metadata.New(sess, &aws.Config{
-			Retryer: client.DefaultRetryer{NumMaxRetries: 5},
+			Retryer: override.IMDSRetryer,
 		}),
 	}
 }
 
 func (c *metadataClient) InstanceID(ctx context.Context) (string, error) {
-	return c.metadata.GetMetadataWithContext(ctx, "instance-id")
+	childCtx, cancel := context.WithTimeout(ctx, override.TimePerCall)
+	defer cancel()
+	return c.metadata.GetMetadataWithContext(childCtx, "instance-id")
 }
 
 func (c *metadataClient) Hostname(ctx context.Context) (string, error) {
-	return c.metadata.GetMetadataWithContext(ctx, "hostname")
+	childCtx, cancel := context.WithTimeout(ctx, override.TimePerCall)
+	defer cancel()
+	return c.metadata.GetMetadataWithContext(childCtx, "hostname")
 }
 
 func (c *metadataClient) Get(ctx context.Context) (ec2metadata.EC2InstanceIdentityDocument, error) {
-	return c.metadata.GetInstanceIdentityDocumentWithContext(ctx)
+	childCtx, cancel := context.WithTimeout(ctx, override.TimePerCall)
+	defer cancel()
+	return c.metadata.GetInstanceIdentityDocumentWithContext(childCtx)
 }