From 324b0607d26123e2c420f90784d1dcd1d138011e Mon Sep 17 00:00:00 2001
From: Jayanth Varavani <1111446+jayanthvn@users.noreply.github.com>
Date: Wed, 19 May 2021 09:56:16 -0700
Subject: [PATCH] Gracefully handle failed ENI SG update (#1341)

* gracefully handle stale ENI's SG updates

* Added counter for IMDS out of sync

Fixed merge conflict

* Added the counter for IMDS out of sync
---
 pkg/awsutils/awsutils.go | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/pkg/awsutils/awsutils.go b/pkg/awsutils/awsutils.go
index b92875247b..703b348d60 100644
--- a/pkg/awsutils/awsutils.go
+++ b/pkg/awsutils/awsutils.go
@@ -454,7 +454,7 @@ func (cache *EC2InstanceMetadataCache) RefreshSGIDs(mac string) error {
 		for _, eni := range allENIs {
 			eniIDs = append(eniIDs, eni.ENIID)
 		}
-
+		
 		newENIs := StringSet{}
 		newENIs.Set(eniIDs)
 
@@ -474,8 +474,16 @@ func (cache *EC2InstanceMetadataCache) RefreshSGIDs(mac string) error {
 			_, err = cache.ec2SVC.ModifyNetworkInterfaceAttributeWithContext(context.Background(), attributeInput)
 			awsAPILatency.WithLabelValues("ModifyNetworkInterfaceAttribute", fmt.Sprint(err != nil), awsReqStatus(err)).Observe(msSince(start))
 			if err != nil {
+				if aerr, ok := err.(awserr.Error); ok {
+					if aerr.Code() == "InvalidNetworkInterfaceID.NotFound" {
+						awsAPIErrInc("IMDSMetaDataOutOfSync", err)
+					}
+				}
 				awsAPIErrInc("ModifyNetworkInterfaceAttribute", err)
-				return errors.Wrap(err, "refreshSGIDs: unable to update the ENI's SG")
+				//No need to return error here since retry will happen in 30seconds and also
+				//If update failed due to stale ENI then returning error will prevent updating SG 
+				//for following ENIs since the list is sorted 
+			    log.Debugf("refreshSGIDs: unable to update the ENI %s SG - %v", eniID, err)
 			}
 		}
 	}
@@ -1007,6 +1015,7 @@ func (cache *EC2InstanceMetadataCache) DescribeAllENIs() (DescribeAllENIsResult,
 			if aerr.Code() == "InvalidNetworkInterfaceID.NotFound" {
 				badENIID := badENIID(aerr.Message())
 				log.Debugf("Could not find interface: %s, ID: %s", aerr.Message(), badENIID)
+				awsAPIErrInc("IMDSMetaDataOutOfSync", err)
 				// Remove this ENI from the map
 				delete(eniMap, badENIID)
 				// Remove the failing ENI ID from the EC2 API request and try again