From 64f2853e714f37219d7e0906e95d852d2f191518 Mon Sep 17 00:00:00 2001
From: Claes Mogren <mogren@amazon.com>
Date: Thu, 13 Aug 2020 08:30:19 -0700
Subject: [PATCH] Wait for ENI and secondary IPs

When a newly created ENI gets attached, we should wait for the
secondary IPv4 addresses to be assigned to the ENI before continuing.
---
 pkg/awsutils/awsutils.go |  7 ++++---
 pkg/ipamd/ipamd.go       | 15 ++++++++++++---
 2 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/pkg/awsutils/awsutils.go b/pkg/awsutils/awsutils.go
index 24ac975ff4..d9839c8723 100644
--- a/pkg/awsutils/awsutils.go
+++ b/pkg/awsutils/awsutils.go
@@ -1224,12 +1224,13 @@ func (cache *EC2InstanceMetadataCache) AllocIPAddresses(eniID string, numIPs int
 	output, err := cache.ec2SVC.AssignPrivateIpAddressesWithContext(context.Background(), input, userAgent)
 	awsAPILatency.WithLabelValues("AssignPrivateIpAddresses", fmt.Sprint(err != nil)).Observe(msSince(start))
 	if err != nil {
-		log.Errorf("Failed to allocate a private IP addresses on ENI %v: %v", eniID, err)
-		awsAPIErrInc("AssignPrivateIpAddresses", err)
 		if containsPrivateIPAddressLimitExceededError(err) {
-			log.Debug("AssignPrivateIpAddresses returned PrivateIpAddressLimitExceeded")
+			log.Debug("AssignPrivateIpAddresses returned PrivateIpAddressLimitExceeded, but this is usually not true." +
+				"Returning nil, since we will check again by calling EC2 to get what addresses were assigned to the ENI.")
 			return nil
 		}
+		log.Errorf("Failed to allocate a private IP addresses on ENI %v: %v", eniID, err)
+		awsAPIErrInc("AssignPrivateIpAddresses", err)
 		return errors.Wrap(err, "allocate IP address: failed to allocate a private IP address")
 	}
 	if output != nil {
diff --git a/pkg/ipamd/ipamd.go b/pkg/ipamd/ipamd.go
index 05643e6bee..d8de6400c7 100644
--- a/pkg/ipamd/ipamd.go
+++ b/pkg/ipamd/ipamd.go
@@ -708,7 +708,7 @@ func (c *IPAMContext) tryAllocateENI() error {
 		ipamdErrInc("increaseIPPoolAllocIPAddressesFailed")
 	}
 
-	eniMetadata, err := c.waitENIAttached(eni)
+	eniMetadata, err := c.waitENIAttached(eni, ipsToAllocate)
 	if err != nil {
 		ipamdErrInc("increaseIPPoolwaitENIAttachedFailed")
 		log.Errorf("Failed to increase pool size: Unable to discover attached ENI from metadata service %v", err)
@@ -802,7 +802,7 @@ func (c *IPAMContext) addENIaddressesToDataStore(ec2Addrs []*ec2.NetworkInterfac
 	return primaryIP
 }
 
-func (c *IPAMContext) waitENIAttached(eni string) (awsutils.ENIMetadata, error) {
+func (c *IPAMContext) waitENIAttached(eni string, ipsToAllocate int) (awsutils.ENIMetadata, error) {
 	// Wait until the ENI shows up in the instance metadata service
 	retry := 0
 	for {
@@ -813,7 +813,16 @@ func (c *IPAMContext) waitENIAttached(eni string) (awsutils.ENIMetadata, error)
 			// Verify that the ENI we are waiting for is in the returned list
 			for _, returnedENI := range enis {
 				if eni == returnedENI.ENIID {
-					return returnedENI, nil
+					// Check number of Secondary IPs attached by comparing to allocate
+					eniIPCount := len(returnedENI.IPv4Addresses)
+					if eniIPCount <= 1 {
+						log.Debug("No secondary IPv4 addresses available yet")
+						continue
+					}
+					if eniIPCount == ipsToAllocate {
+						return returnedENI, nil
+					}
+					// TODO: If we have at least 1 Secondary IP, return without an error
 				}
 			}
 			log.Debugf("Not able to find the right ENI yet (attempt %d/%d)", retry, maxRetryCheckENI)