Added Test cases for EnvVars check on CNI daemonset (#1431)

test/agent/go.mod

@@ -3,7 +3,6 @@ module github.com/aws/amazon-vpc-cni-k8s/test/agent
 go 1.14
 
 require (
-	github.com/go-kit/kit v0.10.0
 	github.com/vishvananda/netlink v1.1.0
 	golang.org/x/sys v0.0.0-20210426230700-d19ff857e887
 )

test/framework/options.go

@@ -59,12 +59,5 @@ func (options *Options) Validate() error {
 	if len(options.AWSVPCID) == 0 {
 		return errors.Errorf("%s must be set!", "aws-vpc-id")
 	}
-	if len(options.NgNameLabelKey) == 0 {
-		return errors.Errorf("%s must be set!", "ng-name-label-key")
-	}
-	if len(options.NgNameLabelVal) == 0 {
-		return errors.Errorf("%s must be set!", "ng-name-label-val")
-	}
-
 	return nil
 }

test/framework/resources/k8s/manifest/deployment.go

@@ -31,6 +31,9 @@ type DeploymentBuilder struct {
 	nodeSelector           map[string]string
 	terminationGracePeriod int
 	nodeName               string
+	hostNetwork            bool
+	volume                 []corev1.Volume
+	volumeMount            []corev1.VolumeMount
 }
 
 func NewBusyBoxDeploymentBuilder() *DeploymentBuilder {
@@ -93,8 +96,19 @@ func (d *DeploymentBuilder) PodLabel(labelKey string, labelValue string) *Deploy
 	return d
 }
 
+func (d *DeploymentBuilder) HostNetwork(hostNetwork bool) *DeploymentBuilder {
+	d.hostNetwork = hostNetwork
+	return d
+}
+
+func (d *DeploymentBuilder) MountVolume(volume []corev1.Volume, volumeMount []corev1.VolumeMount) *DeploymentBuilder {
+	d.volume = volume
+	d.volumeMount = volumeMount
+	return d
+}
+
 func (d *DeploymentBuilder) Build() *v1.Deployment {
-	return &v1.Deployment{
+	deploymentSpec := &v1.Deployment{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      d.name,
 			Namespace: d.namespace,
@@ -110,6 +124,7 @@ func (d *DeploymentBuilder) Build() *v1.Deployment {
 					Labels: d.labels,
 				},
 				Spec: corev1.PodSpec{
+					HostNetwork:                   d.hostNetwork,
 					NodeSelector:                  d.nodeSelector,
 					Containers:                    []corev1.Container{d.container},
 					TerminationGracePeriodSeconds: aws.Int64(int64(d.terminationGracePeriod)),
@@ -118,4 +133,10 @@ func (d *DeploymentBuilder) Build() *v1.Deployment {
 			},
 		},
 	}
+
+	if len(d.volume) > 0 && len(d.volumeMount) > 0 {
+		deploymentSpec.Spec.Template.Spec.Volumes = d.volume
+		deploymentSpec.Spec.Template.Spec.Containers[0].VolumeMounts = d.volumeMount
+	}
+	return deploymentSpec
 }

test/framework/resources/k8s/resources/node.go

@@ -25,6 +25,7 @@ import (
 
 type NodeManager interface {
 	GetNodes(nodeLabelKey string, nodeLabelVal string) (v1.NodeList, error)
+	GetAllNodes() (v1.NodeList, error)
 	UpdateNode(oldNode *v1.Node, newNode *v1.Node) error
 	WaitTillNodesReady(nodeLabelKey string, nodeLabelVal string, asgSize int) error
 }
@@ -38,6 +39,9 @@ func NewDefaultNodeManager(k8sClient client.DelegatingClient) NodeManager {
 }
 
 func (d *defaultNodeManager) GetNodes(nodeLabelKey string, nodeLabelVal string) (v1.NodeList, error) {
+	if nodeLabelVal == "" {
+		return d.GetAllNodes()
+	}
 	ctx := context.Background()
 	nodeList := v1.NodeList{}
 	err := d.k8sClient.List(ctx, &nodeList, client.MatchingLabels{
@@ -46,6 +50,13 @@ func (d *defaultNodeManager) GetNodes(nodeLabelKey string, nodeLabelVal string)
 	return nodeList, err
 }
 
+func (d *defaultNodeManager) GetAllNodes() (v1.NodeList, error) {
+	ctx := context.Background()
+	nodeList := v1.NodeList{}
+	err := d.k8sClient.List(ctx, &nodeList)
+	return nodeList, err
+}
+
 func (d *defaultNodeManager) UpdateNode(oldNode *v1.Node, newNode *v1.Node) error {
 	return d.k8sClient.Patch(context.Background(), newNode, client.MergeFrom(oldNode))
 }

test/integration-new/ipamd/eni_ip_leak_test.go

@@ -0,0 +1,91 @@
+package ipamd
+
+import (
+	"time"
+
+	"github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest"
+	k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils"
+	"github.com/aws/amazon-vpc-cni-k8s/test/framework/utils"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+const (
+	NAMESPACE          = "kube-system"
+	DAEMONSET          = "aws-node"
+	HOST_POD_LABEL_KEY = "network"
+	HOST_POD_LABEL_VAL = "host"
+)
+
+var _ = Describe("ENI/IP Leak Test", func() {
+	Context("ENI/IP Released on Pod Deletion", func() {
+		It("Verify that on Pod Deletion, ENI/IP State is restored", func() {
+			// Set the WARM_ENI_TARGET to 0 to prevent all pods being scheduled on secondary ENI
+			By("Setting WARM_ENI_TARGET to 0")
+			k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, "aws-node", "kube-system",
+				"aws-node", map[string]string{"WARM_IP_TARGET": "3", "WARM_ENI_TARGET": "0"})
+
+			By("Recording the initial count of IP before new deployment")
+			oldIP, oldENI := getCountOfIPandENIOnPrimaryInstance()
+
+			maxPods := getMaxApplicationPodsOnPrimaryInstance()
+			deploymentSpec := manifest.NewBusyBoxDeploymentBuilder().
+				Namespace("default").
+				Name("busybox").
+				NodeName(primaryNode.Name).
+				Replicas(int(maxPods)).
+				Build()
+
+			By("Deploying a max number of Busybox pods")
+			_, err := f.K8sResourceManagers.
+				DeploymentManager().
+				CreateAndWaitTillDeploymentIsReady(deploymentSpec, utils.DefaultDeploymentReadyTimeout*5)
+			Expect(err).ToNot(HaveOccurred())
+
+			By("Deleting the deployment")
+			err = f.K8sResourceManagers.DeploymentManager().DeleteAndWaitTillDeploymentIsDeleted(deploymentSpec)
+			Expect(err).NotTo(HaveOccurred())
+
+			By("Validating that count of ENI/IP is same as before")
+			ip := 0
+			eni := 0
+			for i := 0; i < 3; i++ {
+				// It takes some time to unassign IP addresses
+				time.Sleep(120 * time.Second)
+				ip, eni = getCountOfIPandENIOnPrimaryInstance()
+				if ip == oldIP {
+					break
+				}
+			}
+			Expect(ip).To(Equal(oldIP))
+			Expect(eni).To(Equal(oldENI))
+
+			By("Restoring WARM ENI Target value")
+			k8sUtils.RemoveVarFromDaemonSetAndWaitTillUpdated(f, "aws-node", "kube-system",
+				"aws-node", map[string]struct{}{"WARM_IP_TARGET": {}, "WARM_ENI_TARGET": {}})
+		})
+	})
+})
+
+func getCountOfIPandENIOnPrimaryInstance() (int, int) {
+	eni := len(primaryInstance.NetworkInterfaces)
+	ip := 0
+	for _, ni := range primaryInstance.NetworkInterfaces {
+		ip += len(ni.PrivateIpAddresses)
+	}
+	return ip, eni
+}
+
+func getMaxApplicationPodsOnPrimaryInstance() int64 {
+	instanceType := primaryInstance.InstanceType
+	instaceInfo, err := f.CloudServices.EC2().DescribeInstanceType(*instanceType)
+	Expect(err).NotTo(HaveOccurred())
+
+	currInstance := instaceInfo[0]
+	maxENI := currInstance.NetworkInfo.MaximumNetworkInterfaces
+	maxIPPerENI := currInstance.NetworkInfo.Ipv4AddressesPerInterface
+
+	// If core-dns pods are running on this instance then we need to exclude them as well
+	maxPods := *maxENI*(*maxIPPerENI-1) - int64(numOfNodes)
+	return maxPods
+}