Tag ENI with creation timestamp and avoid cleanup if created in last …

…5 mins

Fix unit test cases

go.mod

@@ -3,7 +3,7 @@ module github.com/aws/amazon-vpc-cni-k8s
 go 1.13
 
 require (
-	github.com/aws/aws-sdk-go v1.31.4
+	github.com/aws/aws-sdk-go v1.33.14
 	github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect
 	github.com/containernetworking/cni v0.7.1
 	github.com/containernetworking/plugins v0.8.6

go.sum

@@ -9,6 +9,8 @@ github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdko
 github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0=
 github.com/aws/aws-sdk-go v1.31.4 h1:YZ0uEYIWeanGuAomElHmRWMAbXVqrQixxgf2vtIjO6M=
 github.com/aws/aws-sdk-go v1.31.4/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
+github.com/aws/aws-sdk-go v1.33.14 h1:ucjyVEvtIdtn4acf+RKsgk6ybAYeMLXpGZeqoVvi7Kk=
+github.com/aws/aws-sdk-go v1.33.14/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=

pkg/awsutils/awsutils.go

@@ -66,6 +66,7 @@ const (
 	maxENIs                 = 128
 	clusterNameEnvVar       = "CLUSTER_NAME"
 	eniNodeTagKey           = "node.k8s.amazonaws.com/instance_id"
+	eniCreatedAtTagKey      = "node.k8s.amazonaws.com/createdAt"
 	eniClusterTagKey        = "cluster.k8s.amazonaws.com/name"
 	additionalEniTagsEnvVar = "ADDITIONAL_ENI_TAGS"
 	reservedTagKeyPrefix    = "k8s.amazonaws.com"
@@ -737,10 +738,25 @@ func (cache *EC2InstanceMetadataCache) attachENI(eniID string) (string, error) {
 // return ENI id, error
 func (cache *EC2InstanceMetadataCache) createENI(useCustomCfg bool, sg []*string, subnet string) (string, error) {
 	eniDescription := eniDescriptionPrefix + cache.instanceID
+	//Tag on create with create TS
+	tags := []*ec2.Tag{
+		{
+			Key:   aws.String(eniCreatedAtTagKey),
+			Value: aws.String(time.Now().Format(time.RFC3339)),
+		},
+	}
+	resourceType := "network-interface"
+	tagspec := []*ec2.TagSpecification{
+		{
+			ResourceType: &resourceType,
+			Tags:         tags,
+		},
+	}
 	input := &ec2.CreateNetworkInterfaceInput{
-		Description: aws.String(eniDescription),
-		Groups:      aws.StringSlice(cache.securityGroups.SortedList()),
-		SubnetId:    aws.String(cache.subnetID),
+		Description:       aws.String(eniDescription),
+		Groups:            aws.StringSlice(cache.securityGroups.SortedList()),
+		SubnetId:          aws.String(cache.subnetID),
+		TagSpecifications: tagspec,
 	}
 
 	if useCustomCfg {
@@ -1087,21 +1103,27 @@ func (cache *EC2InstanceMetadataCache) DescribeAllENIs() ([]ENIMetadata, map[str
 		eniMetadata := eniMap[eniID]
 		// Check IPv4 addresses
 		logOutOfSyncState(eniID, eniMetadata.IPv4Addresses, ec2res.PrivateIpAddresses)
-		tags := make(map[string]string, len(ec2res.TagSet))
-		for _, tag := range ec2res.TagSet {
-			if tag.Key == nil || tag.Value == nil {
-				log.Errorf("nil tag on ENI: %v", eniMetadata.ENIID)
-				continue
-			}
-			tags[*tag.Key] = *tag.Value
-		}
+		tags := getTags(ec2res, eniMetadata.ENIID)
 		if len(tags) > 0 {
 			tagMap[eniMetadata.ENIID] = tags
 		}
 	}
 	return verifiedENIs, tagMap, nil
 }
 
+// getTags collects tags from an EC2 DescribeNetworkInterfaces call
+func getTags(ec2res *ec2.NetworkInterface, eniID string) map[string]string {
+	tags := make(map[string]string, len(ec2res.TagSet))
+	for _, tag := range ec2res.TagSet {
+		if tag.Key == nil || tag.Value == nil {
+			log.Errorf("nil tag on ENI: %v", eniID)
+			continue
+		}
+		tags[*tag.Key] = *tag.Value
+	}
+	return tags
+}
+
 var eniErrorMessageRegex = regexp.MustCompile("'([a-zA-Z0-9-]+)'")
 
 func badENIID(errMsg string) string {
@@ -1296,6 +1318,38 @@ func (cache *EC2InstanceMetadataCache) cleanUpLeakedENIs() {
 	}
 }
 
+func (cache *EC2InstanceMetadataCache) tagENIcreateTS(eniID string, maxBackoffDelay time.Duration) {
+	// Tag the ENI with "node.k8s.amazonaws.com/createdAt=currentTime"
+	tags := []*ec2.Tag{
+		{
+			Key:   aws.String(eniCreatedAtTagKey),
+			Value: aws.String(time.Now().Format(time.RFC3339)),
+		},
+	}
+
+	log.Debugf("Tag untagged ENI %s: key=%s, value=%s", eniID, aws.StringValue(tags[0].Key), aws.StringValue(tags[0].Value))
+
+	input := &ec2.CreateTagsInput{
+		Resources: []*string{
+			aws.String(eniID),
+		},
+		Tags: tags,
+	}
+
+	_ = retry.RetryNWithBackoff(retry.NewSimpleBackoff(500*time.Millisecond, maxBackoffDelay, 0.3, 2), 5, func() error {
+		start := time.Now()
+		_, err := cache.ec2SVC.CreateTagsWithContext(context.Background(), input, userAgent)
+		awsAPILatency.WithLabelValues("CreateTags", fmt.Sprint(err != nil)).Observe(msSince(start))
+		if err != nil {
+			awsAPIErrInc("CreateTags", err)
+			log.Warnf("Failed to tag the newly created ENI %s:", eniID)
+			return err
+		}
+		log.Debugf("Successfully tagged ENI: %s", eniID)
+		return nil
+	})
+}
+
 // getFilteredListOfNetworkInterfaces calls DescribeNetworkInterfaces to get all available ENIs that were allocated by
 // the AWS CNI plugin, but were not deleted.
 func (cache *EC2InstanceMetadataCache) getFilteredListOfNetworkInterfaces() ([]*ec2.NetworkInterface, error) {
@@ -1325,9 +1379,28 @@ func (cache *EC2InstanceMetadataCache) getFilteredListOfNetworkInterfaces() ([]*
 	networkInterfaces := make([]*ec2.NetworkInterface, 0)
 	for _, networkInterface := range result.NetworkInterfaces {
 		// Verify the description starts with "aws-K8S-"
-		if strings.HasPrefix(aws.StringValue(networkInterface.Description), eniDescriptionPrefix) {
-			networkInterfaces = append(networkInterfaces, networkInterface)
+		if !strings.HasPrefix(aws.StringValue(networkInterface.Description), eniDescriptionPrefix) {
+			continue
+		}
+		// Check that it's not a newly created ENI
+		tags := getTags(networkInterface, aws.StringValue(networkInterface.NetworkInterfaceId))
+
+		if value, ok := tags[eniCreatedAtTagKey]; ok {
+			parsedTime, err := time.Parse(time.RFC3339, value)
+			if err != nil {
+
+			}
+			if time.Since(parsedTime) < (5 * time.Minute) {
+				log.Infof("Found ENI created less than 5 mins so not cleaning up")
+				continue
+			}
+			log.Debugf("%v", value)
+		} else {
+			//Set time if we didn't have one. This is to catch the v1.5.x or earlier CNI versions.
+			cache.tagENIcreateTS(aws.StringValue(networkInterface.NetworkInterfaceId), maxENIBackoffDelay)
+			continue
 		}
+		networkInterfaces = append(networkInterfaces, networkInterface)
 	}
 
 	if len(networkInterfaces) < 1 {

pkg/awsutils/awsutils_test.go

@@ -764,11 +764,25 @@ func TestEC2InstanceMetadataCache_getFilteredListOfNetworkInterfaces_OneResult(t
 	attachmentID := eniAttachID
 	description := eniDescriptionPrefix + "test"
 	status := "available"
-	tagKey := eniNodeTagKey
-	tag := ec2.Tag{Key: &tagKey}
+
+	tag := []*ec2.Tag{
+		{
+			Key:   aws.String(eniNodeTagKey),
+			Value: aws.String("test"),
+		},
+	}
+
+	timein := time.Now().Local().Add(time.Minute * time.Duration(-10))
+
+	tag = append(tag, &ec2.Tag{
+		Key:   aws.String(eniCreatedAtTagKey),
+		Value: aws.String(timein.Format(time.RFC3339)),
+	})
 	attachment := &ec2.NetworkInterfaceAttachment{AttachmentId: &attachmentID}
+	cureniID := eniID
+
 	result := &ec2.DescribeNetworkInterfacesOutput{
-		NetworkInterfaces: []*ec2.NetworkInterface{{Attachment: attachment, Status: &status, TagSet: []*ec2.Tag{&tag}, Description: &description}}}
+		NetworkInterfaces: []*ec2.NetworkInterface{{Attachment: attachment, Status: &status, TagSet: tag, Description: &description, NetworkInterfaceId: &cureniID}}}
 	mockEC2.EXPECT().DescribeNetworkInterfacesWithContext(gomock.Any(), gomock.Any(), gomock.Any()).Return(result, nil)
 
 	ins := &EC2InstanceMetadataCache{ec2SVC: mockEC2}