add warm target integration tests (#1438)

Tests
- WARM_ENI_TARGET
- WARM_IP_TARGET
- MAX_ENI
- MINIMUM_IP_TARGET

test/framework/resources/k8s/utils/container.go

@@ -60,16 +60,16 @@ func AddOrUpdateEnvironmentVariable(containers []v1.Container, containerName str
 // RemoveEnvironmentVariables removes the environment variable from the specified container
 func RemoveEnvironmentVariables(containers []v1.Container, containerName string,
 	envVars map[string]struct{}) error {
+	var updatedEnvVar []v1.EnvVar
 	containerIndex := -1
 	for i, container := range containers {
 		if container.Name != containerName {
 			continue
 		}
 		containerIndex = i
 		for j := 0; j < len(container.Env); j++ {
-			if _, ok := envVars[container.Env[j].Name]; ok {
-				container.Env = append(container.Env[:j], container.Env[j+1:]...)
-				j--
+			if _, ok := envVars[container.Env[j].Name]; !ok {
+				updatedEnvVar = append(updatedEnvVar, container.Env[j])
 			}
 		}
 	}
@@ -79,5 +79,7 @@ func RemoveEnvironmentVariables(containers []v1.Container, containerName string,
 			containerName)
 	}
 
+	containers[containerIndex].Env = updatedEnvVar
+
 	return nil
 }

test/framework/utils/const.go

@@ -17,6 +17,8 @@ import "time"
 
 const (
 	DefaultTestNamespace = "cni-automation"
+	AwsNodeNamespace     = "kube-system"
+	AwsNodeName          = "aws-node"
 )
 
 const (

test/integration-new/README.md

@@ -7,6 +7,7 @@ The integration test requires
 - At least 2 nodes in a node group.
 - Nodes in the nodegroup shouldn't have existing pods.
 - Ginkgo installed on your environment. To install `go get github.com/onsi/ginkgo/ginkgo`
+- Supports instance types having at least 3 ENIs and 16+ Secondary IPv4 Addresses across all ENIs.
 
 ####Testing
 Set the environment variables that will be passed to Ginkgo script. If you want to directly pass the arguments you can skip to next step.

test/integration-new/ipamd/ipamd_suite_test.go

@@ -0,0 +1,63 @@
+// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//     http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package ipamd
+
+import (
+	"testing"
+
+	"github.com/aws/amazon-vpc-cni-k8s/test/framework"
+	k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils"
+	"github.com/aws/amazon-vpc-cni-k8s/test/framework/utils"
+
+	"github.com/aws/aws-sdk-go/service/ec2"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	v1 "k8s.io/api/core/v1"
+)
+
+var err error
+var f *framework.Framework
+var primaryNode v1.Node
+var primaryInstance *ec2.Instance
+
+func TestIPAMD(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "VPC IPAMD Test Suite")
+}
+
+var _ = BeforeSuite(func() {
+	f = framework.New(framework.GlobalOptions)
+
+	By("creating test namespace")
+	f.K8sResourceManagers.NamespaceManager().
+		CreateNamespace(utils.DefaultTestNamespace)
+
+	nodeList, err := f.K8sResourceManagers.NodeManager().GetNodes(f.Options.NgNameLabelKey,
+		f.Options.NgNameLabelVal)
+	Expect(err).ToNot(HaveOccurred())
+	Expect(len(nodeList.Items)).Should(BeNumerically(">", 1))
+
+	// Nominate the first node as the primary node
+	primaryNode = nodeList.Items[0]
+
+	instanceID := k8sUtils.GetInstanceIDFromNode(primaryNode)
+	primaryInstance, err = f.CloudServices.EC2().DescribeInstance(instanceID)
+	Expect(err).ToNot(HaveOccurred())
+})
+
+var _ = AfterSuite(func() {
+	By("deleting test namespace")
+	f.K8sResourceManagers.NamespaceManager().
+		DeleteAndWaitTillNamespaceDeleted(utils.DefaultTestNamespace)
+})

test/integration-new/ipamd/warm_target_test.go

@@ -0,0 +1,190 @@
+// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//     http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package ipamd
+
+import (
+	"strconv"
+	"time"
+
+	k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils"
+	"github.com/aws/amazon-vpc-cni-k8s/test/framework/utils"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+// IMPORTANT: THE NODEGROUP TO RUN THE TEST MUST NOT HAVE ANY POD
+// Ideally we should drain the node, but drain from go client is non trivial
+// IMPORTANT: Only support nodes that can have 16+ Secondary IPV4s across at least 3 ENI
+var _ = Describe("test warm target variables", func() {
+
+	Context("when warm ENI target is used", func() {
+		var warmENITarget int
+		var maxENI int
+
+		JustBeforeEach(func() {
+			k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f,
+				utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName,
+				map[string]string{
+					"WARM_ENI_TARGET": strconv.Itoa(warmENITarget),
+					"MAX_ENI":         strconv.Itoa(maxENI),
+				})
+
+			// Allow for IPAMD to reconcile it's state
+			time.Sleep(utils.PollIntervalLong * 5)
+
+			primaryInstance, err = f.CloudServices.
+				EC2().DescribeInstance(*primaryInstance.InstanceId)
+			Expect(err).ToNot(HaveOccurred())
+
+			Expect(len(primaryInstance.NetworkInterfaces)).
+				Should(Equal(MinIgnoreZero(warmENITarget, maxENI)))
+		})
+
+		JustAfterEach(func() {
+			k8sUtils.RemoveVarFromDaemonSetAndWaitTillUpdated(f,
+				utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName,
+				map[string]struct{}{"WARM_ENI_TARGET": {}, "MAX_ENI": {}})
+		})
+
+		Context("when WARM_ENI_TARGET = 2 and MAX_ENI = 1", func() {
+			BeforeEach(func() {
+				warmENITarget = 2
+				maxENI = 1
+			})
+
+			It("instance should have only 1 ENI", func() {})
+		})
+
+		Context("when WARM_ENI_TARGET = 3", func() {
+			BeforeEach(func() {
+				warmENITarget = 3
+				maxENI = 0
+			})
+
+			It("instance should have only 3 ENIs", func() {})
+		})
+
+		Context("when WARM_ENI_TARGET = 1", func() {
+			BeforeEach(func() {
+				warmENITarget = 1
+				maxENI = 0
+			})
+
+			It("instance should have only 1 ENI", func() {})
+		})
+
+	})
+
+	Context("when warm IP target is set", func() {
+		var warmIPTarget int
+		var minIPTarget int
+
+		JustBeforeEach(func() {
+			var availIPs int
+
+			// Set the WARM IP TARGET
+			k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f,
+				utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName,
+				map[string]string{
+					"WARM_IP_TARGET":    strconv.Itoa(warmIPTarget),
+					"MINIMUM_IP_TARGET": strconv.Itoa(minIPTarget),
+				})
+
+			// Allow for IPAMD to reconcile it's state
+			time.Sleep(utils.PollIntervalLong)
+
+			// Query the EC2 Instance to get the list of available IPs on the instance
+			primaryInstance, err = f.CloudServices.
+				EC2().DescribeInstance(*primaryInstance.InstanceId)
+			Expect(err).ToNot(HaveOccurred())
+
+			// Sum all the IPs on all network interfaces minus the primary IPv4 address per ENI
+			for _, networkInterface := range primaryInstance.NetworkInterfaces {
+				availIPs += len(networkInterface.PrivateIpAddresses) - 1
+			}
+
+			// Validated avail IP equals the warm IP Size
+			Expect(availIPs).Should(Equal(Max(warmIPTarget, minIPTarget)))
+		})
+
+		JustAfterEach(func() {
+			k8sUtils.RemoveVarFromDaemonSetAndWaitTillUpdated(f,
+				utils.AwsNodeName, utils.AwsNodeNamespace, utils.AwsNodeName,
+				map[string]struct{}{"WARM_IP_TARGET": {}, "MINIMUM_IP_TARGET": {}})
+		})
+
+		Context("when WARM_IP_TARGET = 2", func() {
+			BeforeEach(func() {
+				warmIPTarget = 2
+				minIPTarget = 0
+			})
+
+			It("should have 2 secondary IPv4 addresses", func() {})
+		})
+
+		Context("when WARM_IP_TARGET = 16", func() {
+			BeforeEach(func() {
+				warmIPTarget = 16
+				minIPTarget = 0
+			})
+
+			It("should have 16 secondary IPv4 addresses", func() {})
+		})
+
+		Context("when MINIMUM_IP_TARGET = 2", func() {
+			BeforeEach(func() {
+				warmIPTarget = 0
+				minIPTarget = 2
+			})
+
+			It("should have 2 secondary IPv4 addresses", func() {})
+		})
+
+		Context("when MINIMUM_IP_TARGET = 16", func() {
+			BeforeEach(func() {
+				warmIPTarget = 0
+				minIPTarget = 16
+			})
+
+			It("should have 16 secondary IPv4 addresses", func() {})
+		})
+
+		Context("when MINIMUM_IP_TARGET = 6 and WARM_IP_TARGET = 10", func() {
+			BeforeEach(func() {
+				warmIPTarget = 6
+				minIPTarget = 10
+			})
+
+			It("should have 10 secondary IPv4 addresses", func() {})
+		})
+	})
+})
+
+func Max(x, y int) int {
+	if x < y {
+		return y
+	}
+	return x
+}
+
+// MinIgnoreZero returns smaller of two number, if any number is zero returns the other number
+func MinIgnoreZero(x, y int) int {
+	if x == 0 {return y}
+	if y == 0 {return x}
+	if x < y {
+		return x
+	}
+	return y
+}