diff --git a/pkg/awsutils/awsutils.go b/pkg/awsutils/awsutils.go index f2f20facd5..574286e023 100644 --- a/pkg/awsutils/awsutils.go +++ b/pkg/awsutils/awsutils.go @@ -161,10 +161,10 @@ type APIs interface { // GetPrimaryENImac returns the mac address of the primary ENI GetPrimaryENImac() string - //Setunmanaged ENI - SetUnmanagedENIs(eniID []string) error + // SetUnmanagedENIs sets the list of unmanaged ENI IDs + SetUnmanagedENIs(eniIDs []string) - //isUnmanagedENI + // IsUnmanagedENI checks if an ENI is unmanaged IsUnmanagedENI(eniID string) bool // WaitForENIAndIPsAttached waits until the ENI has been attached and the secondary IPs have been added @@ -439,9 +439,6 @@ func (cache *EC2InstanceMetadataCache) refreshSGIDs(mac string) error { cache.securityGroups.Set(sgIDs) if !cache.useCustomNetworking && (addedSGsCount != 0 || deletedSGsCount != 0) { - var sgIDsPtrs []*string - sgIDsPtrs = aws.StringSlice(sgIDs) - allENIs, err := cache.GetAttachedENIs() if err != nil { return errors.Wrap(err, "DescribeAllENIs: failed to get local ENI metadata") @@ -449,7 +446,7 @@ func (cache *EC2InstanceMetadataCache) refreshSGIDs(mac string) error { var eniIDs []string for _, eni := range allENIs { - eniIDs = append(eniIDs, string(eni.ENIID)) + eniIDs = append(eniIDs, eni.ENIID) } newENIs := StringSet{} @@ -457,7 +454,8 @@ func (cache *EC2InstanceMetadataCache) refreshSGIDs(mac string) error { filteredENIs := newENIs.Difference(&cache.unmanagedENIs) - //This will update SG for managed ENIs created by EKS. + sgIDsPtrs := aws.StringSlice(sgIDs) + // This will update SG for managed ENIs created by EKS. for _, eniID := range filteredENIs.SortedList() { log.Debugf("Update ENI %s", eniID) @@ -1554,11 +1552,8 @@ func (cache *EC2InstanceMetadataCache) GetPrimaryENImac() string { } //SetUnmanagedENIs Set unmanaged ENI set -func (cache *EC2InstanceMetadataCache) SetUnmanagedENIs(eniID []string) error { - if len(eniID) != 0 { - cache.unmanagedENIs.Set(eniID) - } - return nil +func (cache *EC2InstanceMetadataCache) SetUnmanagedENIs(eniIDs []string) { + cache.unmanagedENIs.Set(eniIDs) } //IsUnmanagedENI returns if the eni is unmanaged diff --git a/pkg/awsutils/awsutils_test.go b/pkg/awsutils/awsutils_test.go index 9e220780a6..5befa68338 100644 --- a/pkg/awsutils/awsutils_test.go +++ b/pkg/awsutils/awsutils_test.go @@ -878,3 +878,15 @@ func TestEC2InstanceMetadataCache_waitForENIAndIPsAttached(t *testing.T) { }) } } + +func TestEC2InstanceMetadataCache_SetUnmanagedENIs(t *testing.T) { + _, mockMetadata, _ := setup(t) + ins := &EC2InstanceMetadataCache{ec2Metadata: mockMetadata} + ins.SetUnmanagedENIs(nil) + assert.False(t, ins.IsUnmanagedENI("eni-1")) + ins.SetUnmanagedENIs([]string{"eni-1", "eni-2"}) + assert.True(t, ins.IsUnmanagedENI("eni-1")) + assert.False(t, ins.IsUnmanagedENI("eni-99")) + ins.SetUnmanagedENIs(nil) + assert.False(t, ins.IsUnmanagedENI("eni-1")) +} diff --git a/pkg/awsutils/mocks/awsutils_mocks.go b/pkg/awsutils/mocks/awsutils_mocks.go index 16f1d2caa3..a641bf2b2d 100644 --- a/pkg/awsutils/mocks/awsutils_mocks.go +++ b/pkg/awsutils/mocks/awsutils_mocks.go @@ -268,11 +268,9 @@ func (mr *MockAPIsMockRecorder) IsUnmanagedENI(arg0 interface{}) *gomock.Call { } // SetUnmanagedENIs mocks base method -func (m *MockAPIs) SetUnmanagedENIs(arg0 []string) error { +func (m *MockAPIs) SetUnmanagedENIs(arg0 []string) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SetUnmanagedENIs", arg0) - ret0, _ := ret[0].(error) - return ret0 + m.ctrl.Call(m, "SetUnmanagedENIs", arg0) } // SetUnmanagedENIs indicates an expected call of SetUnmanagedENIs diff --git a/pkg/ipamd/datastore/data_store.go b/pkg/ipamd/datastore/data_store.go index f20535f54a..0d17437f30 100644 --- a/pkg/ipamd/datastore/data_store.go +++ b/pkg/ipamd/datastore/data_store.go @@ -690,7 +690,7 @@ func (ds *DataStore) RemoveUnusedENIFromStore(warmIPTarget int, minimumIPTarget return removableENI } -// RemoveENIFromDataStore removes an ENI from the datastore. It return nil on success or an error. +// RemoveENIFromDataStore removes an ENI from the datastore. It returns nil on success, or an error. func (ds *DataStore) RemoveENIFromDataStore(eniID string, force bool) error { ds.lock.Lock() defer ds.lock.Unlock() @@ -704,9 +704,9 @@ func (ds *DataStore) RemoveENIFromDataStore(eniID string, force bool) error { if !force { return errors.New(ENIInUseError) } - // This scenario can occur if the reconciliation process discovered this eni was detached - // from the EC2 instance outside of the control of ipamd. If this happens, there's nothing - // we can do other than force all pods to be unassigned from the IPs on this eni. + // This scenario can occur if the reconciliation process discovered this ENI was detached + // from the EC2 instance outside of the control of ipamd. If this happens, there's nothing + // we can do other than force all pods to be unassigned from the IPs on this ENI. ds.log.Warnf("Force removing eni %s with %d assigned pods", eniID, eni.AssignedIPv4Addresses()) forceRemovedENIs.Inc() forceRemovedIPs.Add(float64(eni.AssignedIPv4Addresses())) diff --git a/pkg/ipamd/ipamd.go b/pkg/ipamd/ipamd.go index bb8e9ac11d..d164cdb4db 100644 --- a/pkg/ipamd/ipamd.go +++ b/pkg/ipamd/ipamd.go @@ -112,7 +112,6 @@ const ( // disableENIProvisioning is used to specify that ENI doesn't need to be synced during initializing a pod. envDisableENIProvisioning = "DISABLE_NETWORK_RESOURCE_PROVISIONING" - noDisableENIProvisioning = false // Specify where ipam should persist its current IP<->container allocations. envBackingStorePath = "AWS_VPC_K8S_CNI_BACKING_STORE" @@ -738,31 +737,37 @@ func (c *IPAMContext) tryAssignIPs() (increasedPool bool, err error) { // 2) set up linux ENI related networking stack. // 3) add all ENI's secondary IP addresses to datastore func (c *IPAMContext) setupENI(eni string, eniMetadata awsutils.ENIMetadata, trunkENI string) error { + primaryENI := c.awsClient.GetPrimaryENI() // Add the ENI to the datastore - err := c.dataStore.AddENI(eni, eniMetadata.DeviceNumber, eni == c.awsClient.GetPrimaryENI(), eni == trunkENI) + err := c.dataStore.AddENI(eni, eniMetadata.DeviceNumber, eni == primaryENI, eni == trunkENI) if err != nil && err.Error() != datastore.DuplicatedENIError { return errors.Wrapf(err, "failed to add ENI %s to data store", eni) } + // Store the primary IP of the ENI + c.primaryIP[eni] = eniMetadata.PrimaryIPv4Address() // For secondary ENIs, set up the network - if eni != c.awsClient.GetPrimaryENI() { - eniPrimaryIP := eniMetadata.PrimaryIPv4Address() - err = c.networkClient.SetupENINetwork(eniPrimaryIP, eniMetadata.MAC, eniMetadata.DeviceNumber, eniMetadata.SubnetIPv4CIDR) + if eni != primaryENI { + err = c.networkClient.SetupENINetwork(c.primaryIP[eni], eniMetadata.MAC, eniMetadata.DeviceNumber, eniMetadata.SubnetIPv4CIDR) if err != nil { + // Failed to set up the ENI + errRemove := c.dataStore.RemoveENIFromDataStore(eni, true) + if errRemove != nil { + log.Warnf("failed to remove ENI %s: %v", eni, errRemove) + } + delete(c.primaryIP, eni) return errors.Wrapf(err, "failed to set up ENI %s network", eni) } } - c.primaryIP[eni] = c.addENIaddressesToDataStore(eniMetadata.IPv4Addresses, eni) + c.addENIaddressesToDataStore(eniMetadata.IPv4Addresses, eni) return nil } // return primary ip address on the interface -func (c *IPAMContext) addENIaddressesToDataStore(ec2Addrs []*ec2.NetworkInterfacePrivateIpAddress, eni string) string { - var primaryIP string +func (c *IPAMContext) addENIaddressesToDataStore(ec2Addrs []*ec2.NetworkInterfacePrivateIpAddress, eni string) { for _, ec2Addr := range ec2Addrs { if aws.BoolValue(ec2Addr.Primary) { - primaryIP = aws.StringValue(ec2Addr.PrivateIpAddress) continue } err := c.dataStore.AddIPv4AddressToStore(eni, aws.StringValue(ec2Addr.PrivateIpAddress)) @@ -774,7 +779,6 @@ func (c *IPAMContext) addENIaddressesToDataStore(ec2Addrs []*ec2.NetworkInterfac } total, assigned := c.dataStore.GetStats() log.Debugf("IP Address Pool stats: total: %d, assigned: %d", total, assigned) - return primaryIP } // getMaxENI returns the maximum number of ENIs to attach to this instance. This is calculated as the lesser of @@ -907,6 +911,12 @@ func (c *IPAMContext) nodeIPPoolReconcile(interval time.Duration) { ipamdErrInc("reconcileFailedGetENIs") return } + // We must always have at least the primary ENI of the instance + if allENIs == nil { + log.Error("IP pool reconcile: No ENI found at all in metadata, unable to reconcile") + ipamdErrInc("reconcileFailedGetENIs") + return + } attachedENIs := c.filterUnmanagedENIs(allENIs) currentENIs := c.dataStore.GetENIInfos().ENIs trunkENI := c.dataStore.GetTrunkENI() @@ -978,6 +988,7 @@ func (c *IPAMContext) nodeIPPoolReconcile(interval time.Duration) { ipamdErrInc("eniReconcileDel") continue } + delete(c.primaryIP, eni) reconcileCnt.With(prometheus.Labels{"fn": "eniReconcileDel"}).Inc() } log.Debug("Successfully Reconciled ENI/IP pool") @@ -987,9 +998,51 @@ func (c *IPAMContext) nodeIPPoolReconcile(interval time.Duration) { } func (c *IPAMContext) eniIPPoolReconcile(ipPool []string, attachedENI awsutils.ENIMetadata, eni string) { - seenIPs := make(map[string]bool) + attachedENIIPs := attachedENI.IPv4Addresses + needEC2Reconcile := true + // Here we can't trust attachedENI since the IMDS metadata can be stale. We need to check with EC2 API. + // +1 is for the primary IP of the ENI that is not added to the ipPool and not available for pods to use. + if 1+len(ipPool) != len(attachedENIIPs) { + log.Warnf("Instance metadata does not match data store! ipPool: %v, metadata: %v", ipPool, attachedENIIPs) + log.Debugf("We need to check the ENI status by calling the EC2 control plane.") + // Call EC2 to verify IPs on this ENI + ec2Addresses, err := c.awsClient.GetIPv4sFromEC2(eni) + if err != nil { + log.Errorf("Failed to fetch ENI IP addresses! Aborting reconcile of ENI %s", eni) + return + } + attachedENIIPs = ec2Addresses + needEC2Reconcile = false + } + + // Add all known attached IPs to the datastore + seenIPs := c.verifyAndAddIPsToDatastore(eni, attachedENIIPs, needEC2Reconcile) + + // Sweep phase, delete remaining IPs since they should not remain in the datastore + for _, existingIP := range ipPool { + if seenIPs[existingIP] { + continue + } - for _, privateIPv4 := range attachedENI.IPv4Addresses { + log.Debugf("Reconcile and delete IP %s on ENI %s", existingIP, eni) + // Force the delete, since we have verified with EC2 that these secondary IPs are no longer assigned to this ENI + err := c.dataStore.DelIPv4AddressFromStore(eni, existingIP, true /* force */) + if err != nil { + log.Errorf("Failed to reconcile and delete IP %s on ENI %s, %v", existingIP, eni, err) + ipamdErrInc("ipReconcileDel") + // continue instead of bailout due to one ip + continue + } + reconcileCnt.With(prometheus.Labels{"fn": "eniIPPoolReconcileDel"}).Inc() + } +} + +// verifyAndAddIPsToDatastore updates the datastore with the known secondary IPs. IPs who are out of cooldown gets added +// back to the datastore after being verified against EC2. +func (c *IPAMContext) verifyAndAddIPsToDatastore(eni string, attachedENIIPs []*ec2.NetworkInterfacePrivateIpAddress, needEC2Reconcile bool) map[string]bool { + var ec2VerifiedAddresses []*ec2.NetworkInterfacePrivateIpAddress + seenIPs := make(map[string]bool) + for _, privateIPv4 := range attachedENIIPs { strPrivateIPv4 := aws.StringValue(privateIPv4.PrivateIpAddress) if strPrivateIPv4 == c.primaryIP[eni] { log.Debugf("Reconcile and skip primary IP %s on ENI %s", strPrivateIPv4, eni) @@ -1003,66 +1056,55 @@ func (c *IPAMContext) eniIPPoolReconcile(ipPool []string, attachedENI awsutils.E log.Debugf("Reconcile skipping IP %s on ENI %s because it was recently unassigned from the ENI.", strPrivateIPv4, eni) continue } else { - log.Debugf("This IP was recently freed, but is out of cooldown. We need to verify with EC2 control plane.") - // Call EC2 to verify IPs on this ENI - ec2Addresses, err := c.awsClient.GetIPv4sFromEC2(eni) - if err != nil { - log.Error("Failed to fetch ENI IP addresses!") - continue - } else { + if needEC2Reconcile { + // IMDS data might be stale + log.Debugf("This IP was recently freed, but is now out of cooldown. We need to verify with EC2 control plane.") + // Only call EC2 once for this ENI + if ec2VerifiedAddresses == nil { + var err error + // Call EC2 to verify IPs on this ENI + ec2VerifiedAddresses, err = c.awsClient.GetIPv4sFromEC2(eni) + if err != nil { + log.Errorf("Failed to fetch ENI IP addresses from EC2! %v", err) + // Do not delete this IP from the datastore or cooldown until we have confirmed with EC2 + seenIPs[strPrivateIPv4] = true + continue + } + } // Verify that the IP really belongs to this ENI isReallyAttachedToENI := false - for _, ec2Addr := range ec2Addresses { + for _, ec2Addr := range ec2VerifiedAddresses { if strPrivateIPv4 == aws.StringValue(ec2Addr.PrivateIpAddress) { isReallyAttachedToENI = true log.Debugf("Verified that IP %s is attached to ENI %s", strPrivateIPv4, eni) break } } - if isReallyAttachedToENI { - c.reconcileCooldownCache.Remove(strPrivateIPv4) - } else { - log.Warnf("Skipping IP %s on ENI %s because it does not belong to this ENI!.", strPrivateIPv4, eni) + if !isReallyAttachedToENI { + log.Warnf("Skipping IP %s on ENI %s because it does not belong to this ENI!", strPrivateIPv4, eni) continue } } + // The IP can be removed from the cooldown cache + // TODO: Here we could check if the IP is still used by a pod stuck in Terminating state. (Issue #1091) + c.reconcileCooldownCache.Remove(strPrivateIPv4) } } + // Try to add the IP err := c.dataStore.AddIPv4AddressToStore(eni, strPrivateIPv4) - if err != nil && err.Error() == datastore.IPAlreadyInStoreError { - // mark action - seenIPs[strPrivateIPv4] = true - continue - } - - if err != nil { + if err != nil && err.Error() != datastore.IPAlreadyInStoreError { log.Errorf("Failed to reconcile IP %s on ENI %s", strPrivateIPv4, eni) ipamdErrInc("ipReconcileAdd") - // continue instead of bailout due to one IP - continue - } - reconcileCnt.With(prometheus.Labels{"fn": "eniIPPoolReconcileAdd"}).Inc() - } - - // Sweep phase, delete remaining IPs - for _, existingIP := range ipPool { - if seenIPs[existingIP] { + // Continue to check the other IPs instead of bailout due to one wrong IP continue - } - log.Debugf("Reconcile and delete IP %s on ENI %s", existingIP, eni) - // Force the delete, since aws local metadata has told us that this ENI is no longer - // attached, so any IPs assigned from this ENI will no longer work. - err := c.dataStore.DelIPv4AddressFromStore(eni, existingIP, true /* force */) - if err != nil { - log.Errorf("Failed to reconcile and delete IP %s on ENI %s, %v", existingIP, eni, err) - ipamdErrInc("ipReconcileDel") - // continue instead of bailout due to one ip - continue } - reconcileCnt.With(prometheus.Labels{"fn": "eniIPPoolReconcileDel"}).Inc() + // Mark action + seenIPs[strPrivateIPv4] = true + reconcileCnt.With(prometheus.Labels{"fn": "eniIPPoolReconcileAdd"}).Inc() } + return seenIPs } // UseCustomNetworkCfg returns whether Pods needs to use pod specific configuration or not. diff --git a/pkg/ipamd/ipamd_test.go b/pkg/ipamd/ipamd_test.go index cf1d1527d7..7cc91fbb4f 100644 --- a/pkg/ipamd/ipamd_test.go +++ b/pkg/ipamd/ipamd_test.go @@ -14,6 +14,7 @@ package ipamd import ( + "errors" "fmt" "net" "os" @@ -270,9 +271,7 @@ func testIncreaseIPPool(t *testing.T, useENIConfig bool) { m.awsutils.EXPECT().GetPrimaryENI().Return(primaryENIid) m.awsutils.EXPECT().WaitForENIAndIPsAttached(secENIid, 14).Return(eniMetadata[1], nil) m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, secDevice, secSubnet) - m.awsutils.EXPECT().AllocIPAddresses(eni2, 14) - m.awsutils.EXPECT().GetPrimaryENI().Return(primaryENIid) mockContext.increaseIPPool() } @@ -339,7 +338,6 @@ func TestTryAddIPToENI(t *testing.T) { m.awsutils.EXPECT().WaitForENIAndIPsAttached(secENIid, 3).Return(eniMetadata[1], nil) m.awsutils.EXPECT().GetPrimaryENI().Return(primaryENIid) m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, secDevice, secSubnet) - m.awsutils.EXPECT().GetPrimaryENI().Return(primaryENIid) mockContext.increaseIPPool() } @@ -358,11 +356,23 @@ func TestNodeIPPoolReconcile(t *testing.T) { mockContext.dataStore = testDatastore() primary := true - notPrimary := false - testAddr1 := ipaddr01 - testAddr2 := ipaddr02 + primaryENIMetadata := getPrimaryENIMetadata() + testAddr1 := *primaryENIMetadata.IPv4Addresses[0].PrivateIpAddress + // Always the primary ENI + m.awsutils.EXPECT().GetPrimaryENI().AnyTimes().Return(primaryENIid) + m.awsutils.EXPECT().IsUnmanagedENI(primaryENIid).AnyTimes().Return(false) + eniMetadataList := []awsutils.ENIMetadata{primaryENIMetadata} + m.awsutils.EXPECT().GetAttachedENIs().Return(eniMetadataList, nil) + m.awsutils.EXPECT().DescribeAllENIs().Return(eniMetadataList, map[string]awsutils.TagMap{}, "", nil) - eniMetadata := []awsutils.ENIMetadata{ + mockContext.nodeIPPoolReconcile(0) + + curENIs := mockContext.dataStore.GetENIInfos() + assert.Equal(t, 1, len(curENIs.ENIs)) + assert.Equal(t, 2, curENIs.TotalIPs) + + // 1 secondary IP lost in IMDS + oneIPUnassigned := []awsutils.ENIMetadata{ { ENIID: primaryENIid, MAC: primaryMAC, @@ -372,50 +382,42 @@ func TestNodeIPPoolReconcile(t *testing.T) { { PrivateIpAddress: &testAddr1, Primary: &primary, }, - { - PrivateIpAddress: &testAddr2, Primary: ¬Primary, - }, }, }, } - m.awsutils.EXPECT().GetAttachedENIs().Return(eniMetadata, nil) - m.awsutils.EXPECT().GetPrimaryENI().Times(2).Return(primaryENIid) - m.awsutils.EXPECT().DescribeAllENIs().Return(eniMetadata, map[string]awsutils.TagMap{}, "", nil) - m.awsutils.EXPECT().IsUnmanagedENI(primaryENIid).Return(false).AnyTimes() + m.awsutils.EXPECT().GetAttachedENIs().Return(oneIPUnassigned, nil) + m.awsutils.EXPECT().GetIPv4sFromEC2(primaryENIid).Return(oneIPUnassigned[0].IPv4Addresses, nil) mockContext.nodeIPPoolReconcile(0) + curENIs = mockContext.dataStore.GetENIInfos() + assert.Equal(t, 1, len(curENIs.ENIs)) + assert.Equal(t, 0, curENIs.TotalIPs) - curENIs := mockContext.dataStore.GetENIInfos() - assert.Equal(t, len(curENIs.ENIs), 1) - assert.Equal(t, curENIs.TotalIPs, 1) + // New ENI attached + newENIMetadata := getSecondaryENIMetadata() - // remove 1 IP - m.awsutils.EXPECT().GetAttachedENIs().Return([]awsutils.ENIMetadata{ - { - ENIID: primaryENIid, - MAC: primaryMAC, - DeviceNumber: primaryDevice, - SubnetIPv4CIDR: primarySubnet, - IPv4Addresses: []*ec2.NetworkInterfacePrivateIpAddress{ - { - PrivateIpAddress: &testAddr1, Primary: &primary, - }, - }, - }, - }, nil) + twoENIs := append(oneIPUnassigned, newENIMetadata) + + // Two ENIs found + m.awsutils.EXPECT().GetAttachedENIs().Return(twoENIs, nil) + m.awsutils.EXPECT().IsUnmanagedENI(secENIid).Times(2).Return(false) + m.awsutils.EXPECT().DescribeAllENIs().Return(twoENIs, map[string]awsutils.TagMap{}, "", nil) + m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, secDevice, primarySubnet) mockContext.nodeIPPoolReconcile(0) + + // Verify that we now have 2 ENIs, primary ENI with 0 secondary IPs, and secondary ENI with 1 secondary IP curENIs = mockContext.dataStore.GetENIInfos() - assert.Equal(t, len(curENIs.ENIs), 1) - assert.Equal(t, curENIs.TotalIPs, 0) + assert.Equal(t, 2, len(curENIs.ENIs)) + assert.Equal(t, 1, curENIs.TotalIPs) - // remove eni - m.awsutils.EXPECT().GetAttachedENIs().Return(nil, nil) + // Remove the secondary ENI in the IMDS metadata + m.awsutils.EXPECT().GetAttachedENIs().Return(oneIPUnassigned, nil) mockContext.nodeIPPoolReconcile(0) curENIs = mockContext.dataStore.GetENIInfos() - assert.Equal(t, len(curENIs.ENIs), 0) - assert.Equal(t, curENIs.TotalIPs, 0) + assert.Equal(t, 1, len(curENIs.ENIs)) + assert.Equal(t, 0, curENIs.TotalIPs) } func TestGetWarmENITarget(t *testing.T) { @@ -589,7 +591,7 @@ func TestIPAMContext_filterUnmanagedENIs(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { c := &IPAMContext{awsClient: mockAWSUtils} - mockAWSUtils.EXPECT().SetUnmanagedENIs(tt.unmanagedenis).Return(nil).AnyTimes() + mockAWSUtils.EXPECT().SetUnmanagedENIs(tt.unmanagedenis).AnyTimes() c.setUnmanagedENIs(tt.tagMap) mockAWSUtils.EXPECT().IsUnmanagedENI(gomock.Any()).DoAndReturn( @@ -635,3 +637,182 @@ func TestPodENIConfigFlag(t *testing.T) { disabled = enablePodENI() assert.False(t, disabled) } + +func TestNodeIPPoolReconcileBadIMDSData(t *testing.T) { + m := setup(t) + defer m.ctrl.Finish() + + mockContext := &IPAMContext{ + awsClient: m.awsutils, + networkClient: m.network, + primaryIP: make(map[string]string), + terminating: int32(0), + } + + mockContext.dataStore = testDatastore() + + primaryENIMetadata := getPrimaryENIMetadata() + testAddr1 := *primaryENIMetadata.IPv4Addresses[0].PrivateIpAddress + // Add ENI and IPs to datastore + eniID := primaryENIMetadata.ENIID + _ = mockContext.dataStore.AddENI(eniID, primaryENIMetadata.DeviceNumber, true, false) + mockContext.primaryIP[eniID] = testAddr1 + mockContext.addENIaddressesToDataStore(primaryENIMetadata.IPv4Addresses, eniID) + curENIs := mockContext.dataStore.GetENIInfos() + assert.Equal(t, 1, len(curENIs.ENIs)) + assert.Equal(t, 2, curENIs.TotalIPs) + eniMetadataList := []awsutils.ENIMetadata{primaryENIMetadata} + m.awsutils.EXPECT().GetAttachedENIs().Return(eniMetadataList, nil) + m.awsutils.EXPECT().IsUnmanagedENI(eniID).Return(false).AnyTimes() + + // First reconcile, IMDS returns correct IPs so no change needed + mockContext.nodeIPPoolReconcile(0) + + // IMDS returns no secondary IPs, the EC2 call fails + primary := true + m.awsutils.EXPECT().GetAttachedENIs().Return([]awsutils.ENIMetadata{ + { + ENIID: primaryENIid, + MAC: primaryMAC, + DeviceNumber: primaryDevice, + SubnetIPv4CIDR: primarySubnet, + IPv4Addresses: []*ec2.NetworkInterfacePrivateIpAddress{ + { + PrivateIpAddress: &testAddr1, Primary: &primary, + }, + }, + }, + }, nil) + + // eniIPPoolReconcile() calls EC2 to get the actual count, but that call fails + m.awsutils.EXPECT().GetIPv4sFromEC2(primaryENIid).Return(nil, errors.New("ec2 API call failed")) + mockContext.nodeIPPoolReconcile(0) + curENIs = mockContext.dataStore.GetENIInfos() + assert.Equal(t, 1, len(curENIs.ENIs)) + assert.Equal(t, 2, curENIs.TotalIPs) + + // IMDS returns no secondary IPs + m.awsutils.EXPECT().GetAttachedENIs().Return([]awsutils.ENIMetadata{ + { + ENIID: primaryENIid, + MAC: primaryMAC, + DeviceNumber: primaryDevice, + SubnetIPv4CIDR: primarySubnet, + IPv4Addresses: []*ec2.NetworkInterfacePrivateIpAddress{ + { + PrivateIpAddress: &testAddr1, Primary: &primary, + }, + }, + }, + }, nil) + + // eniIPPoolReconcile() calls EC2 to get the actual count that should still be 2 + m.awsutils.EXPECT().GetIPv4sFromEC2(primaryENIid).Return(primaryENIMetadata.IPv4Addresses, nil) + mockContext.nodeIPPoolReconcile(0) + curENIs = mockContext.dataStore.GetENIInfos() + assert.Equal(t, 1, len(curENIs.ENIs)) + assert.Equal(t, 2, curENIs.TotalIPs) + + // If no ENI is found, we abort the reconcile + m.awsutils.EXPECT().GetAttachedENIs().Return(nil, nil) + mockContext.nodeIPPoolReconcile(0) + curENIs = mockContext.dataStore.GetENIInfos() + assert.Equal(t, 1, len(curENIs.ENIs)) + assert.Equal(t, 2, curENIs.TotalIPs) +} + +func getPrimaryENIMetadata() awsutils.ENIMetadata { + primary := true + notPrimary := false + testAddr1 := ipaddr01 + testAddr2 := ipaddr02 + testAddr3 := ipaddr03 + + eniMetadata := awsutils.ENIMetadata{ + ENIID: primaryENIid, + MAC: primaryMAC, + DeviceNumber: primaryDevice, + SubnetIPv4CIDR: primarySubnet, + IPv4Addresses: []*ec2.NetworkInterfacePrivateIpAddress{ + { + PrivateIpAddress: &testAddr1, Primary: &primary, + }, + { + PrivateIpAddress: &testAddr2, Primary: ¬Primary, + }, + { + PrivateIpAddress: &testAddr3, Primary: ¬Primary, + }, + }, + } + return eniMetadata +} + +func getSecondaryENIMetadata() awsutils.ENIMetadata { + primary := true + notPrimary := false + testAddr3 := ipaddr11 + testAddr4 := ipaddr12 + newENIMetadata := awsutils.ENIMetadata{ + ENIID: secENIid, + MAC: secMAC, + DeviceNumber: secDevice, + SubnetIPv4CIDR: primarySubnet, + IPv4Addresses: []*ec2.NetworkInterfacePrivateIpAddress{ + { + PrivateIpAddress: &testAddr3, Primary: &primary, + }, + { + PrivateIpAddress: &testAddr4, Primary: ¬Primary, + }, + }, + } + return newENIMetadata +} + +func TestIPAMContext_setupENI(t *testing.T) { + m := setup(t) + defer m.ctrl.Finish() + + mockContext := &IPAMContext{ + awsClient: m.awsutils, + networkClient: m.network, + primaryIP: make(map[string]string), + terminating: int32(0), + } + //mockContext.primaryIP[] + + mockContext.dataStore = testDatastore() + primary := true + notPrimary := false + testAddr1 := ipaddr01 + testAddr2 := ipaddr02 + primaryENIMetadata := awsutils.ENIMetadata{ + ENIID: primaryENIid, + MAC: primaryMAC, + DeviceNumber: primaryDevice, + SubnetIPv4CIDR: primarySubnet, + IPv4Addresses: []*ec2.NetworkInterfacePrivateIpAddress{ + { + PrivateIpAddress: &testAddr1, Primary: &primary, + }, + { + PrivateIpAddress: &testAddr2, Primary: ¬Primary, + }, + }, + } + m.awsutils.EXPECT().GetPrimaryENI().Return(primaryENIid) + err := mockContext.setupENI(primaryENIMetadata.ENIID, primaryENIMetadata, "") + assert.NoError(t, err) + // Primary ENI added + assert.Equal(t, 1, len(mockContext.primaryIP)) + + newENIMetadata := getSecondaryENIMetadata() + m.awsutils.EXPECT().GetPrimaryENI().Return(primaryENIid) + m.network.EXPECT().SetupENINetwork(gomock.Any(), secMAC, secDevice, primarySubnet).Return(errors.New("not able to set route 0.0.0.0/0 via 10.10.10.1 table 2")) + + err = mockContext.setupENI(newENIMetadata.ENIID, newENIMetadata, "") + assert.Error(t, err) + assert.Equal(t, 1, len(mockContext.primaryIP)) + +}