Skip to content

Commit

Permalink
Add Cancel To Context For IMDS Retry To Prevent ECS Hanging Issue
Browse files Browse the repository at this point in the history 
R
  • Loading branch information
@sethAmazon
sethAmazon committed Jul 19, 2023
1 parent 05cd916 commit 8ed5eea
Show file tree
Hide file tree
Showing 24 changed files with 52 additions and 28 deletions.
2 changes: 1 addition & 1 deletion cmd/configschema/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -277,7 +277,7 @@ require (
github.com/alecthomas/participle/v2 v2.0.0 // indirect
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
github.com/aliyun/aliyun-log-go-sdk v0.1.44 // indirect
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230714201444-77612b3cbeab // indirect
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230719154634-7dfcb28377c2 // indirect
github.com/andybalholm/brotli v1.0.5 // indirect
github.com/antonmedv/expr v1.12.5 // indirect
github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40 // indirect
Expand Down
2 changes: 1 addition & 1 deletion cmd/otelcontribcol/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/cmd/otelcontrib
go 1.19

require (
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230714201444-77612b3cbeab
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230719154634-7dfcb28377c2
github.com/open-telemetry/opentelemetry-collector-contrib/connector/countconnector v0.77.0
github.com/open-telemetry/opentelemetry-collector-contrib/connector/servicegraphconnector v0.77.0
github.com/open-telemetry/opentelemetry-collector-contrib/connector/spanmetricsconnector v0.77.0
Expand Down
2 changes: 1 addition & 1 deletion exporter/awscloudwatchlogsexporter/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ require (
)

require (
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230714201444-77612b3cbeab // indirect
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230719154634-7dfcb28377c2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/protobuf v1.5.3 // indirect
Expand Down
2 changes: 1 addition & 1 deletion exporter/awsemfexporter/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ require (
)

require (
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230714201444-77612b3cbeab // indirect
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230719154634-7dfcb28377c2 // indirect
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
Expand Down
2 changes: 1 addition & 1 deletion exporter/awsxrayexporter/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ require (
)

require (
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230714201444-77612b3cbeab // indirect
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230719154634-7dfcb28377c2 // indirect
github.com/cenkalti/backoff/v4 v4.2.1 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
Expand Down
2 changes: 1 addition & 1 deletion exporter/datadogexporter/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ require (
github.com/DataDog/opentelemetry-mapping-go/pkg/quantile v0.2.0
github.com/DataDog/sketches-go v1.4.2
github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.13.1
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230714201444-77612b3cbeab
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230719154634-7dfcb28377c2
github.com/aws/aws-sdk-go v1.44.293
github.com/cenkalti/backoff/v4 v4.2.1
github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/ecsutil v0.77.0
Expand Down
7 changes: 5 additions & 2 deletions exporter/datadogexporter/internal/hostmetadata/internal/ec2/ec2.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,17 +64,20 @@ func GetHostInfo(logger *zap.Logger) (hostInfo *HostInfo) {
return
}

ctx, cancel := context.WithTimeout(context.Background(), 2*override.TimePerCall)
defer cancel()

meta := ec2metadata.New(sess, &aws.Config{
Retryer: override.IMDSRetryer,
})

if idDoc, err := meta.GetInstanceIdentityDocument(); err == nil {
if idDoc, err := meta.GetInstanceIdentityDocumentWithContext(ctx); err == nil {
hostInfo.InstanceID = idDoc.InstanceID
} else {
logger.Warn("Failed to get EC2 instance id document", zap.Error(err))
}

if ec2Hostname, err := meta.GetMetadata("hostname"); err == nil {
if ec2Hostname, err := meta.GetMetadataWithContext(ctx, "hostname"); err == nil {
hostInfo.EC2Hostname = ec2Hostname
} else {
logger.Warn("Failed to get EC2 hostname", zap.Error(err))
Expand Down
2 changes: 1 addition & 1 deletion extension/awsproxy/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ require (
)

require (
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230714201444-77612b3cbeab // indirect
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230719154634-7dfcb28377c2 // indirect
github.com/aws/aws-sdk-go v1.44.293 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
Expand Down
4 changes: 2 additions & 2 deletions extension/awsproxy/go.sum
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -257,7 +257,7 @@ require (
github.com/alecthomas/participle/v2 v2.0.0 // indirect
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
github.com/aliyun/aliyun-log-go-sdk v0.1.44 // indirect
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230714201444-77612b3cbeab // indirect
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230719154634-7dfcb28377c2 // indirect
github.com/andybalholm/brotli v1.0.5 // indirect
github.com/antonmedv/expr v1.12.5 // indirect
github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40 // indirect
Expand Down
5 changes: 4 additions & 1 deletion internal/aws/awsutil/conn.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
package awsutil // import "github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/awsutil"

import (
"context"
"crypto/tls"
"errors"
"net/http"
Expand Down Expand Up @@ -46,9 +47,11 @@ type ConnAttr interface {
type Conn struct{}

func (c *Conn) getEC2Region(s *session.Session) (string, error) {
ctx, cancel := context.WithTimeout(context.Background(), override.TimePerCall)
defer cancel()
return ec2metadata.New(s, &aws.Config{
Retryer: override.IMDSRetryer,
}).Region()
}).RegionWithContext(ctx)
}

// AWS STS endpoint constants
Expand Down
2 changes: 1 addition & 1 deletion internal/aws/awsutil/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/aw
go 1.19

require (
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230714201444-77612b3cbeab
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230719154634-7dfcb28377c2
github.com/aws/aws-sdk-go v1.44.293
github.com/stretchr/testify v1.8.2
go.uber.org/zap v1.24.0
Expand Down
5 changes: 4 additions & 1 deletion internal/aws/proxy/conn.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
package proxy // import "github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/proxy"

import (
"context"
"crypto/tls"
"encoding/json"
"errors"
Expand Down Expand Up @@ -80,9 +81,11 @@ var newAWSSession = func(roleArn string, region string, log *zap.Logger) (*sessi
}

var getEC2Region = func(s *session.Session) (string, error) {
ctx, cancel := context.WithTimeout(context.Background(), override.TimePerCall)
defer cancel()
return ec2metadata.New(s, &aws.Config{
Retryer: override.IMDSRetryer,
}).Region()
}).RegionWithContext(ctx)
}

func getAWSConfigSession(c *Config, logger *zap.Logger) (*aws.Config, *session.Session, error) {
Expand Down
2 changes: 1 addition & 1 deletion internal/aws/proxy/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@ go 1.19
replace github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws => ../../../override/aws

require (
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230714201444-77612b3cbeab
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230719154634-7dfcb28377c2
github.com/aws/aws-sdk-go v1.44.293
github.com/open-telemetry/opentelemetry-collector-contrib/internal/common v0.77.0
github.com/stretchr/testify v1.8.2
Expand Down
2 changes: 1 addition & 1 deletion internal/aws/xray/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/xr
go 1.19

require (
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230714201444-77612b3cbeab
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230719154634-7dfcb28377c2
github.com/aws/aws-sdk-go v1.44.293
github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/awsutil v0.77.0
github.com/stretchr/testify v1.8.2
Expand Down
5 changes: 4 additions & 1 deletion internal/aws/xray/telemetry/sender.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
package telemetry // import "github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/xray/telemetry"

import (
"context"
"os"
"sync"
"time"
Expand Down Expand Up @@ -169,7 +170,9 @@ type ec2MetadataProvider struct {

func (p ec2MetadataProvider) get() string {
var metadata string
if result, err := p.client.GetMetadata(p.metadataKey); err == nil {
ctx, cancel := context.WithTimeout(context.Background(), override.TimePerCall)
defer cancel()
if result, err := p.client.GetMetadataWithContext(ctx, p.metadataKey); err == nil {
metadata = result
}
return metadata
Expand Down
12 changes: 9 additions & 3 deletions internal/metadataproviders/aws/ec2/metadata.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,13 +44,19 @@ func NewProvider(sess *session.Session) Provider {
}

func (c *metadataClient) InstanceID(ctx context.Context) (string, error) {
return c.metadata.GetMetadataWithContext(ctx, "instance-id")
childCtx, cancel := context.WithTimeout(ctx, override.TimePerCall)
defer cancel()
return c.metadata.GetMetadataWithContext(childCtx, "instance-id")
}

func (c *metadataClient) Hostname(ctx context.Context) (string, error) {
return c.metadata.GetMetadataWithContext(ctx, "hostname")
childCtx, cancel := context.WithTimeout(ctx, override.TimePerCall)
defer cancel()
return c.metadata.GetMetadataWithContext(childCtx, "hostname")
}

func (c *metadataClient) Get(ctx context.Context) (ec2metadata.EC2InstanceIdentityDocument, error) {
return c.metadata.GetInstanceIdentityDocumentWithContext(ctx)
childCtx, cancel := context.WithTimeout(ctx, override.TimePerCall)
defer cancel()
return c.metadata.GetInstanceIdentityDocumentWithContext(childCtx)
}
2 changes: 1 addition & 1 deletion internal/metadataproviders/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ replace github.com/amazon-contributing/opentelemetry-collector-contrib/override/

require (
github.com/Showmax/go-fqdn v1.0.0
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230714201444-77612b3cbeab
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230719154634-7dfcb28377c2
github.com/aws/aws-sdk-go v1.44.293
github.com/docker/docker v23.0.5+incompatible
github.com/hashicorp/consul/api v1.20.0
Expand Down
4 changes: 4 additions & 0 deletions override/aws/imdsretryer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,10 @@ import (
// currently we set the var to nil in tests to mock
var IMDSRetryer request.Retryer = newIMDSRetryer()

const (
TimePerCall = 2 * time.Minute
)

type iMDSRetryer struct {
client.DefaultRetryer
}
Expand Down
2 changes: 1 addition & 1 deletion processor/resourcedetectionprocessor/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ require (
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
github.com/Microsoft/go-winio v0.5.2 // indirect
github.com/Showmax/go-fqdn v1.0.0 // indirect
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230714201444-77612b3cbeab // indirect
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230719154634-7dfcb28377c2 // indirect
github.com/armon/go-metrics v0.3.10 // indirect
github.com/benbjohnson/clock v1.3.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
Expand Down
4 changes: 2 additions & 2 deletions processor/resourcedetectionprocessor/go.sum
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion receiver/awscontainerinsightreceiver/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awscon
go 1.19

require (
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230714201444-77612b3cbeab
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230719154634-7dfcb28377c2
github.com/aws/aws-sdk-go v1.44.293
github.com/go-kit/log v0.2.1
github.com/google/cadvisor v0.47.1
Expand Down
4 changes: 3 additions & 1 deletion receiver/awscontainerinsightreceiver/internal/host/ec2metadata.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,9 @@ func (emd *ec2Metadata) refresh(ctx context.Context) {
}
emd.logger.Info("Fetch instance id and type from ec2 metadata")

doc, err := emd.client.GetInstanceIdentityDocumentWithContext(ctx)
childCtx, cancel := context.WithTimeout(ctx, override.TimePerCall)
defer cancel()
doc, err := emd.client.GetInstanceIdentityDocumentWithContext(childCtx)
if err != nil {
emd.logger.Error("Failed to get ec2 metadata", zap.Error(err))
return
Expand Down
2 changes: 1 addition & 1 deletion receiver/awsxrayreceiver/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ require (

require (
contrib.go.opencensus.io/exporter/prometheus v0.4.2 // indirect
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230714201444-77612b3cbeab // indirect
github.com/amazon-contributing/opentelemetry-collector-contrib/override/aws v0.0.0-20230719154634-7dfcb28377c2 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
Expand Down

0 comments on commit 8ed5eea

Please sign in to comment.