diff --git a/go.mod b/go.mod index 0f3273612c0..d222a3b61e3 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/aws/amazon-vpc-cni-k8s go 1.13 require ( - github.com/aws/aws-sdk-go v1.31.4 + github.com/aws/aws-sdk-go v1.33.14 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect github.com/containernetworking/cni v0.7.1 github.com/containernetworking/plugins v0.8.6 diff --git a/go.sum b/go.sum index 30825c83ae7..6130cdeea0a 100644 --- a/go.sum +++ b/go.sum @@ -9,6 +9,8 @@ github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdko github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= github.com/aws/aws-sdk-go v1.31.4 h1:YZ0uEYIWeanGuAomElHmRWMAbXVqrQixxgf2vtIjO6M= github.com/aws/aws-sdk-go v1.31.4/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= +github.com/aws/aws-sdk-go v1.33.14 h1:ucjyVEvtIdtn4acf+RKsgk6ybAYeMLXpGZeqoVvi7Kk= +github.com/aws/aws-sdk-go v1.33.14/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= diff --git a/pkg/awsutils/awsutils.go b/pkg/awsutils/awsutils.go index 1c84ecc7187..99f576bb6a1 100644 --- a/pkg/awsutils/awsutils.go +++ b/pkg/awsutils/awsutils.go @@ -66,6 +66,7 @@ const ( maxENIs = 128 clusterNameEnvVar = "CLUSTER_NAME" eniNodeTagKey = "node.k8s.amazonaws.com/instance_id" + eniCreatedAtTagKey = "node.k8s.amazonaws.com/createdAt" eniClusterTagKey = "cluster.k8s.amazonaws.com/name" additionalEniTagsEnvVar = "ADDITIONAL_ENI_TAGS" reservedTagKeyPrefix = "k8s.amazonaws.com" @@ -737,10 +738,25 @@ func (cache *EC2InstanceMetadataCache) attachENI(eniID string) (string, error) { // return ENI id, error func (cache *EC2InstanceMetadataCache) createENI(useCustomCfg bool, sg []*string, subnet string) (string, error) { eniDescription := eniDescriptionPrefix + cache.instanceID + //Tag on create with create TS + tags := []*ec2.Tag{ + { + Key: aws.String(eniCreatedAtTagKey), + Value: aws.String(time.Now().Format(time.RFC3339)), + }, + } + resourceType := "network-interface" + tagspec := []*ec2.TagSpecification{ + { + ResourceType: &resourceType, + Tags: tags, + }, + } input := &ec2.CreateNetworkInterfaceInput{ - Description: aws.String(eniDescription), - Groups: aws.StringSlice(cache.securityGroups.SortedList()), - SubnetId: aws.String(cache.subnetID), + Description: aws.String(eniDescription), + Groups: aws.StringSlice(cache.securityGroups.SortedList()), + SubnetId: aws.String(cache.subnetID), + TagSpecifications: tagspec, } if useCustomCfg { @@ -1087,14 +1103,7 @@ func (cache *EC2InstanceMetadataCache) DescribeAllENIs() ([]ENIMetadata, map[str eniMetadata := eniMap[eniID] // Check IPv4 addresses logOutOfSyncState(eniID, eniMetadata.IPv4Addresses, ec2res.PrivateIpAddresses) - tags := make(map[string]string, len(ec2res.TagSet)) - for _, tag := range ec2res.TagSet { - if tag.Key == nil || tag.Value == nil { - log.Errorf("nil tag on ENI: %v", eniMetadata.ENIID) - continue - } - tags[*tag.Key] = *tag.Value - } + tags := getTags(ec2res, eniMetadata.ENIID) if len(tags) > 0 { tagMap[eniMetadata.ENIID] = tags } @@ -1102,6 +1111,19 @@ func (cache *EC2InstanceMetadataCache) DescribeAllENIs() ([]ENIMetadata, map[str return verifiedENIs, tagMap, nil } +// getTags collects tags from an EC2 DescribeNetworkInterfaces call +func getTags(ec2res *ec2.NetworkInterface, eniID string) map[string]string { + tags := make(map[string]string, len(ec2res.TagSet)) + for _, tag := range ec2res.TagSet { + if tag.Key == nil || tag.Value == nil { + log.Errorf("nil tag on ENI: %v", eniID) + continue + } + tags[*tag.Key] = *tag.Value + } + return tags +} + var eniErrorMessageRegex = regexp.MustCompile("'([a-zA-Z0-9-]+)'") func badENIID(errMsg string) string { @@ -1296,6 +1318,38 @@ func (cache *EC2InstanceMetadataCache) cleanUpLeakedENIs() { } } +func (cache *EC2InstanceMetadataCache) tagENIcreateTS(eniID string, maxBackoffDelay time.Duration) { + // Tag the ENI with "node.k8s.amazonaws.com/createdAt=currentTime" + tags := []*ec2.Tag{ + { + Key: aws.String(eniCreatedAtTagKey), + Value: aws.String(time.Now().Format(time.RFC3339)), + }, + } + + log.Debugf("Tag untagged ENI %s: key=%s, value=%s", eniID, aws.StringValue(tags[0].Key), aws.StringValue(tags[0].Value)) + + input := &ec2.CreateTagsInput{ + Resources: []*string{ + aws.String(eniID), + }, + Tags: tags, + } + + _ = retry.RetryNWithBackoff(retry.NewSimpleBackoff(500*time.Millisecond, maxBackoffDelay, 0.3, 2), 5, func() error { + start := time.Now() + _, err := cache.ec2SVC.CreateTagsWithContext(context.Background(), input, userAgent) + awsAPILatency.WithLabelValues("CreateTags", fmt.Sprint(err != nil)).Observe(msSince(start)) + if err != nil { + awsAPIErrInc("CreateTags", err) + log.Warnf("Failed to tag the newly created ENI %s:", eniID) + return err + } + log.Debugf("Successfully tagged ENI: %s", eniID) + return nil + }) +} + // getFilteredListOfNetworkInterfaces calls DescribeNetworkInterfaces to get all available ENIs that were allocated by // the AWS CNI plugin, but were not deleted. func (cache *EC2InstanceMetadataCache) getFilteredListOfNetworkInterfaces() ([]*ec2.NetworkInterface, error) { @@ -1325,9 +1379,28 @@ func (cache *EC2InstanceMetadataCache) getFilteredListOfNetworkInterfaces() ([]* networkInterfaces := make([]*ec2.NetworkInterface, 0) for _, networkInterface := range result.NetworkInterfaces { // Verify the description starts with "aws-K8S-" - if strings.HasPrefix(aws.StringValue(networkInterface.Description), eniDescriptionPrefix) { - networkInterfaces = append(networkInterfaces, networkInterface) + if !strings.HasPrefix(aws.StringValue(networkInterface.Description), eniDescriptionPrefix) { + continue + } + // Check that it's not a newly created ENI + tags := getTags(networkInterface, aws.StringValue(networkInterface.NetworkInterfaceId)) + + if value, ok := tags[eniCreatedAtTagKey]; ok { + parsedTime, err := time.Parse(time.RFC3339, value) + if err != nil { + + } + if time.Since(parsedTime) < (5 * time.Minute) { + log.Infof("Found ENI created less than 5 mins so not cleaning up") + continue + } + log.Debugf("%v", value) + } else { + //Set time if we didn't have one. This is to catch the v1.5.x or earlier CNI versions. + cache.tagENIcreateTS(aws.StringValue(networkInterface.NetworkInterfaceId), maxENIBackoffDelay) + continue } + networkInterfaces = append(networkInterfaces, networkInterface) } if len(networkInterfaces) < 1 { diff --git a/pkg/awsutils/awsutils_test.go b/pkg/awsutils/awsutils_test.go index daef69afb47..3ca25743fe3 100644 --- a/pkg/awsutils/awsutils_test.go +++ b/pkg/awsutils/awsutils_test.go @@ -764,11 +764,25 @@ func TestEC2InstanceMetadataCache_getFilteredListOfNetworkInterfaces_OneResult(t attachmentID := eniAttachID description := eniDescriptionPrefix + "test" status := "available" - tagKey := eniNodeTagKey - tag := ec2.Tag{Key: &tagKey} + + tag := []*ec2.Tag{ + { + Key: aws.String(eniNodeTagKey), + Value: aws.String("test"), + }, + } + + timein := time.Now().Local().Add(time.Minute * time.Duration(-10)) + + tag = append(tag, &ec2.Tag{ + Key: aws.String(eniCreatedAtTagKey), + Value: aws.String(timein.Format(time.RFC3339)), + }) attachment := &ec2.NetworkInterfaceAttachment{AttachmentId: &attachmentID} + cureniID := eniID + result := &ec2.DescribeNetworkInterfacesOutput{ - NetworkInterfaces: []*ec2.NetworkInterface{{Attachment: attachment, Status: &status, TagSet: []*ec2.Tag{&tag}, Description: &description}}} + NetworkInterfaces: []*ec2.NetworkInterface{{Attachment: attachment, Status: &status, TagSet: tag, Description: &description, NetworkInterfaceId: &cureniID}}} mockEC2.EXPECT().DescribeNetworkInterfacesWithContext(gomock.Any(), gomock.Any(), gomock.Any()).Return(result, nil) ins := &EC2InstanceMetadataCache{ec2SVC: mockEC2}