Introduce a minimum target for ENI IPs (#612)

* Introduce a minimum target for ENI IPs

Adds a MINIMUM_IP_TARGET environment variable to inform the AWS CNI that a
particular number of total IPs is anticipated for use with a particular node.
This is useful to ensure a sufficient supply of IPs on a node up-front without
the 2x IP usage overhead of setting WARM_IP_TARGET to the same value.

* Fix multiple incorrect references to MINIMUM_IP_TARGET

Co-Authored-By: Ed Morley <501702+edmorley@users.noreply.github.com>

README.md

@@ -234,8 +234,27 @@ Specifies the number of free IP addresses that the `ipamD` daemon should attempt
 For example, if `WARM_IP_TARGET` is set to 10, then `ipamD` attempts to keep 10 free IP addresses available at all times. If the
 elastic network interfaces on the node are unable to provide these free addresses, `ipamD` attempts to allocate more interfaces
 until `WARM_IP_TARGET` free IP addresses are available.
+If both `WARM_IP_TARGET` and `MINIMUM_IP_TARGET` are set, `ipamD` will attempt to meet both constraints.
 This environment variable overrides `WARM_ENI_TARGET` behavior.
 
+`MINIMUM_IP_TARGET`
+Type: Integer
+Default: None
+
+Specifies the number of total IP addresses that the `ipamD` daemon should attempt to allocate for pod assignment on the node.
+`MINIMUM_IP_TARGET` behaves identically to `WARM_IP_TARGET` except that instead of setting a target number of free IP
+addresses to keep available at all times, it sets a target number for a floor on how many total IP addresses are allocated.
+
+`MINIMUM_IP_TARGET` is for pre-scaling, `WARM_IP_TARGET` is for dynamic scaling. For example, suppose a cluster has an
+expected pod density of approximately 30 pods per node. If `WARM_IP_TARGET` is set to 30 to ensure there are enough IPs
+allocated up front by the CNI, then 30 pods are deployed to the node, the CNI will allocate an additional 30 IPs, for
+a total of 60, accelerating IP exhaustion in the relevant subnets. If instead `MINIMUM_IP_TARGET` is set to 30 and
+`WARM_IP_TARGET` to 2, after the 30 pods are deployed the CNI would allocate an additional 2 IPs. This still provides
+elasticity, but uses roughly half as many IPs as using WARM_IP_TARGET alone (32 IPs vs 60 IPs).
+
+This also improves reliability of the EKS cluster by reducing the number of calls necessary to allocate or deallocate
+private IPs, which may be throttled, especially at scaling-related times.
+
 ---
 
 `MAX_ENI`

ipamd/datastore/data_store.go

@@ -324,7 +324,19 @@ func (ds *DataStore) isRequiredForWarmIPTarget(warmIPTarget int, eni *ENIIPPool)
 	return otherWarmIPs < warmIPTarget
 }
 
-func (ds *DataStore) getDeletableENI(warmIPTarget int) *ENIIPPool {
+// IsRequiredForMinimumIPTarget determines if this ENI is necessary to fulfill whatever MINIMUM_IP_TARGET is
+// set to.
+func (ds *DataStore) isRequiredForMinimumIPTarget(minimumIPTarget int, eni *ENIIPPool) bool {
+	otherIPs := 0
+	for _, other := range ds.eniIPPools {
+		if other.ID != eni.ID {
+			otherIPs += len(other.IPv4Addresses)
+		}
+	}
+	return otherIPs < minimumIPTarget
+}
+
+func (ds *DataStore) getDeletableENI(warmIPTarget int, minimumIPTarget int) *ENIIPPool {
 	for _, eni := range ds.eniIPPools {
 		if eni.IsPrimary {
 			log.Debugf("ENI %s cannot be deleted because it is primary", eni.ID)
@@ -351,6 +363,11 @@ func (ds *DataStore) getDeletableENI(warmIPTarget int) *ENIIPPool {
 			continue
 		}
 
+		if minimumIPTarget != 0 && ds.isRequiredForMinimumIPTarget(minimumIPTarget, eni) {
+			log.Debugf("ENI %s cannot be deleted because it is required for MINIMUM_IP_TARGET: %d", eni.ID, minimumIPTarget)
+			continue
+		}
+
 		log.Debugf("getDeletableENI: found a deletable ENI %s", eni.ID)
 		return eni
 	}
@@ -391,11 +408,11 @@ func (ds *DataStore) GetENINeedsIP(maxIPperENI int, skipPrimary bool) *ENIIPPool
 // RemoveUnusedENIFromStore removes a deletable ENI from the data store.
 // It returns the name of the ENI which has been removed from the data store and needs to be deleted,
 // or empty string if no ENI could be removed.
-func (ds *DataStore) RemoveUnusedENIFromStore(warmIPTarget int) string {
+func (ds *DataStore) RemoveUnusedENIFromStore(warmIPTarget int, minimumIPTarget int) string {
 	ds.lock.Lock()
 	defer ds.lock.Unlock()
 
-	deletableENI := ds.getDeletableENI(warmIPTarget)
+	deletableENI := ds.getDeletableENI(warmIPTarget, minimumIPTarget)
 	if deletableENI == nil {
 		return ""
 	}

ipamd/datastore/data_store_test.go

@@ -288,16 +288,73 @@ func TestPodIPv4Address(t *testing.T) {
 	assert.Equal(t, ds.eniIPPools["eni-2"].AssignedIPv4Addresses, 0)
 
 	noWarmIPTarget := 0
+	noMinimumIPTarget := 0
 
 	// Should not be able to free this ENI
-	eni := ds.RemoveUnusedENIFromStore(noWarmIPTarget)
+	eni := ds.RemoveUnusedENIFromStore(noWarmIPTarget, noMinimumIPTarget)
 	assert.True(t, eni == "")
 
 	ds.eniIPPools["eni-2"].createTime = time.Time{}
 	ds.eniIPPools["eni-2"].lastUnassignedTime = time.Time{}
-	eni = ds.RemoveUnusedENIFromStore(noWarmIPTarget)
+	eni = ds.RemoveUnusedENIFromStore(noWarmIPTarget, noMinimumIPTarget)
 	assert.Equal(t, eni, "eni-2")
 
 	assert.Equal(t, ds.total, 2)
 	assert.Equal(t, ds.assigned, 2)
 }
+
+func TestWarmENIInteractions(t *testing.T) {
+	ds := NewDataStore()
+
+	ds.AddENI("eni-1", 1, true)
+	ds.AddENI("eni-2", 2, false)
+	ds.AddENI("eni-3", 3, false)
+	ds.AddIPv4AddressToStore("eni-1", "1.1.1.1")
+	ds.AddIPv4AddressToStore("eni-1", "1.1.1.2")
+	ds.AddIPv4AddressToStore("eni-2", "1.1.2.1")
+	ds.AddIPv4AddressToStore("eni-2", "1.1.2.2")
+	ds.AddIPv4AddressToStore("eni-3", "1.1.3.1")
+
+	podInfo := k8sapi.K8SPodInfo{
+		Name:      "pod-1",
+		Namespace: "ns-1",
+		IP:        "1.1.1.1",
+	}
+	_, _, err := ds.AssignPodIPv4Address(&podInfo)
+	assert.NoError(t, err)
+
+	podInfo = k8sapi.K8SPodInfo{
+		Name:      "pod-2",
+		Namespace: "ns-2",
+		IP:        "1.1.1.2",
+	}
+	_, _, err = ds.AssignPodIPv4Address(&podInfo)
+	assert.NoError(t, err)
+
+	noWarmIPTarget := 0
+
+	ds.eniIPPools["eni-2"].createTime = time.Time{}
+	ds.eniIPPools["eni-2"].lastUnassignedTime = time.Time{}
+	ds.eniIPPools["eni-3"].createTime = time.Time{}
+	ds.eniIPPools["eni-3"].lastUnassignedTime = time.Time{}
+
+	// We have three ENIs, 5 IPs and two pods on ENI 1. Each ENI can handle two pods.
+	// We should not be able to remove any ENIs if either warmIPTarget >= 3 or minimumWarmIPTarget >= 5
+	eni := ds.RemoveUnusedENIFromStore(3, 1)
+	assert.Equal(t, "", eni)
+	// Should not be able to free this ENI because we want at least 5 IPs, which requires at least three ENIs
+	eni = ds.RemoveUnusedENIFromStore(1, 5)
+	assert.Equal(t, "", eni)
+	// Should be able to free an ENI because both warmIPTarget and minimumWarmIPTarget are both effectively 4
+	removedEni := ds.RemoveUnusedENIFromStore(2, 4)
+	assert.Contains(t, []string{"eni-2", "eni-3"}, removedEni)
+
+	// Should not be able to free an ENI because minimumWarmIPTarget requires at least two ENIs and no warm IP target
+	eni = ds.RemoveUnusedENIFromStore(noWarmIPTarget, 3)
+	assert.Equal(t, "", eni)
+	// Should be able to free an ENI because one ENI can provide a minimum count of 2 IPs
+	secondRemovedEni := ds.RemoveUnusedENIFromStore(noWarmIPTarget, 2)
+	assert.Contains(t, []string{"eni-2", "eni-3"}, secondRemovedEni)
+
+	assert.NotEqual(t, removedEni, secondRemovedEni, "The two removed ENIs should not be the same ENI.")
+}

ipamd/ipamd.go

@@ -68,6 +68,17 @@ const (
 	envWarmIPTarget = "WARM_IP_TARGET"
 	noWarmIPTarget  = 0
 
+	// This environment variable is used to specify the desired minimum number of total IPs.
+	// When it is not set, ipamd defaults to 0.
+	// For example, for a m4.4xlarge node,
+	//     If WARM_IP_TARGET is set to 1 and MINIMUM_IP_TARGET is set to 12, and there are 9 pods running on the node,
+	//     ipamd will make the "warm pool" have 12 IP addresses with 9 being assigned to pods and 3 free IPs.
+	//
+	//     If "MINIMUM_IP_TARGET is not set, it will default to 0, which causes WARM_IP_TARGET settings to be the
+	//	   only settings considered.
+	envMinimumIPTarget = "MINIMUM_IP_TARGET"
+	noMinimumIPTarget  = 0
+
 	// This environment is used to specify the desired number of free ENIs along with all of its IP addresses
 	// always available in "warm pool".
 	// When it is not set, it is default to 1.
@@ -159,6 +170,7 @@ type IPAMContext struct {
 	maxENI               int
 	warmENITarget        int
 	warmIPTarget         int
+	minimumIPTarget      int
 	primaryIP            map[string]string
 	lastNodeIPPoolAction time.Time
 	lastDecreaseIPPool   time.Time
@@ -238,6 +250,7 @@ func New(k8sapiClient k8sapi.K8SAPIs, eniConfig *eniconfig.ENIConfigController)
 	c.reconcileCooldownCache.cache = make(map[string]time.Time)
 	c.warmENITarget = getWarmENITarget()
 	c.warmIPTarget = getWarmIPTarget()
+	c.minimumIPTarget = getMinimumIPTarget()
 	c.useCustomNetworking = UseCustomNetworkCfg()
 
 	err = c.nodeInit()
@@ -463,7 +476,7 @@ func (c *IPAMContext) tryFreeENI() {
 		return
 	}
 
-	eni := c.dataStore.RemoveUnusedENIFromStore(c.warmIPTarget)
+	eni := c.dataStore.RemoveUnusedENIFromStore(c.warmIPTarget, c.minimumIPTarget)
 	if eni == "" {
 		return
 	}
@@ -1046,10 +1059,27 @@ func getWarmIPTarget() int {
 	return noWarmIPTarget
 }
 
-// ipTargetState determines the number of IPs `short` or `over` our WARM_IP_TARGET
+func getMinimumIPTarget() int {
+	inputStr, found := os.LookupEnv(envMinimumIPTarget)
+
+	if !found {
+		return noMinimumIPTarget
+	}
+
+	if input, err := strconv.Atoi(inputStr); err == nil {
+		if input >= 0 {
+			log.Debugf("Using MINIMUM_IP_TARGET %v", input)
+			return input
+		}
+	}
+	return noMinimumIPTarget
+}
+
+// ipTargetState determines the number of IPs `short` or `over` our WARM_IP_TARGET,
+// accounting for the MINIMUM_IP_TARGET
 func (c *IPAMContext) ipTargetState() (short int, over int, enabled bool) {
-	if c.warmIPTarget == noWarmIPTarget {
-		// there is no WARM_IP_TARGET defined, fallback to use all IP addresses on ENI
+	if c.warmIPTarget == noWarmIPTarget && c.minimumIPTarget == noMinimumIPTarget {
+		// there is no WARM_IP_TARGET defined and no MINIMUM_IP_TARGET, fallback to use all IP addresses on ENI
 		return 0, 0, false
 	}
 
@@ -1059,9 +1089,15 @@ func (c *IPAMContext) ipTargetState() (short int, over int, enabled bool) {
 	// short is greater than 0 when we have fewer available IPs than the warm IP target
 	short = max(c.warmIPTarget-available, 0)
 
+	// short is greater than the warm IP target alone when we have fewer total IPs than the minimum target
+	short = max(short, c.minimumIPTarget-total)
+
 	// over is the number of available IPs we have beyond the warm IP target
 	over = max(available-c.warmIPTarget, 0)
 
+	// over is less than the warm IP target alone if it would imply reducing total IPs below the minimum target
+	over = max(min(over, total-c.minimumIPTarget), 0)
+
 	log.Tracef("Current warm IP stats: target: %d, total: %d, assigned: %d, available: %d, short: %d, over %d", c.warmIPTarget, total, assigned, available, short, over)
 	return short, over, true
 }