Added Test cases for EnvVars check on CNI daemonset

test/agent/go.mod

@@ -3,7 +3,6 @@ module github.com/aws/amazon-vpc-cni-k8s/test/agent
 go 1.14
 
 require (
-	github.com/go-kit/kit v0.10.0
 	github.com/vishvananda/netlink v1.1.0
 	golang.org/x/sys v0.0.0-20210426230700-d19ff857e887
 )

test/framework/options.go

@@ -59,12 +59,5 @@ func (options *Options) Validate() error {
 	if len(options.AWSVPCID) == 0 {
 		return errors.Errorf("%s must be set!", "aws-vpc-id")
 	}
-	if len(options.NgNameLabelKey) == 0 {
-		return errors.Errorf("%s must be set!", "ng-name-label-key")
-	}
-	if len(options.NgNameLabelVal) == 0 {
-		return errors.Errorf("%s must be set!", "ng-name-label-val")
-	}
-
 	return nil
 }

test/framework/resources/k8s/manifest/deployment.go

@@ -31,6 +31,9 @@ type DeploymentBuilder struct {
 	nodeSelector           map[string]string
 	terminationGracePeriod int
 	nodeName               string
+	hostNetwork            bool
+	volume                 []corev1.Volume
+	volumeMount            []corev1.VolumeMount
 }
 
 func NewBusyBoxDeploymentBuilder() *DeploymentBuilder {
@@ -93,8 +96,19 @@ func (d *DeploymentBuilder) PodLabel(labelKey string, labelValue string) *Deploy
 	return d
 }
 
+func (d *DeploymentBuilder) HostNetwork(hostNetwork bool) *DeploymentBuilder {
+	d.hostNetwork = hostNetwork
+	return d
+}
+
+func (d *DeploymentBuilder) MountVolume(volume []corev1.Volume, volumeMount []corev1.VolumeMount) *DeploymentBuilder {
+	d.volume = volume
+	d.volumeMount = volumeMount
+	return d
+}
+
 func (d *DeploymentBuilder) Build() *v1.Deployment {
-	return &v1.Deployment{
+	deploymentSpec := &v1.Deployment{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      d.name,
 			Namespace: d.namespace,
@@ -110,6 +124,7 @@ func (d *DeploymentBuilder) Build() *v1.Deployment {
 					Labels: d.labels,
 				},
 				Spec: corev1.PodSpec{
+					HostNetwork:                   d.hostNetwork,
 					NodeSelector:                  d.nodeSelector,
 					Containers:                    []corev1.Container{d.container},
 					TerminationGracePeriodSeconds: aws.Int64(int64(d.terminationGracePeriod)),
@@ -118,4 +133,10 @@ func (d *DeploymentBuilder) Build() *v1.Deployment {
 			},
 		},
 	}
+
+	if len(d.volume) > 0 && len(d.volumeMount) > 0 {
+		deploymentSpec.Spec.Template.Spec.Volumes = d.volume
+		deploymentSpec.Spec.Template.Spec.Containers[0].VolumeMounts = d.volumeMount
+	}
+	return deploymentSpec
 }

test/framework/resources/k8s/resources/node.go

@@ -25,6 +25,7 @@ import (
 
 type NodeManager interface {
 	GetNodes(nodeLabelKey string, nodeLabelVal string) (v1.NodeList, error)
+	GetAllNodes() (v1.NodeList, error)
 	UpdateNode(oldNode *v1.Node, newNode *v1.Node) error
 	WaitTillNodesReady(nodeLabelKey string, nodeLabelVal string, asgSize int) error
 }
@@ -46,6 +47,13 @@ func (d *defaultNodeManager) GetNodes(nodeLabelKey string, nodeLabelVal string)
 	return nodeList, err
 }
 
+func (d *defaultNodeManager) GetAllNodes() (v1.NodeList, error) {
+	ctx := context.Background()
+	nodeList := v1.NodeList{}
+	err := d.k8sClient.List(ctx, &nodeList)
+	return nodeList, err
+}
+
 func (d *defaultNodeManager) UpdateNode(oldNode *v1.Node, newNode *v1.Node) error {
 	return d.k8sClient.Patch(context.Background(), newNode, client.MergeFrom(oldNode))
 }

test/go.sum

@@ -530,9 +530,11 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb
 github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
 github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0=
 github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
+github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852 h1:cPXZWzzG0NllBLdjWoD1nDfaqu98YMv+OneaKc8sPOA=
 github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho=
 github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df h1:OviZH7qLw/7ZovXvuNyL3XQl8UFofeikI1NW1Gypu7k=
 github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
+github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae h1:4hwBBUfQCFe3Cym0ZtKyq7L16eZUtYKs+BaHDN6mAns=
 github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=

test/integration-new/env_vars/env_vars_suite_test.go

@@ -0,0 +1,80 @@
+package env_vars
+
+import (
+	"testing"
+
+	"github.com/aws/amazon-vpc-cni-k8s/test/framework"
+	"github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest"
+	k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	appsV1 "k8s.io/api/apps/v1"
+	v1 "k8s.io/api/core/v1"
+)
+
+const (
+	NAMESPACE          = "kube-system"
+	DAEMONSET          = "aws-node"
+	HOST_POD_LABEL_KEY = "network"
+	HOST_POD_LABEL_VAL = "host"
+	VOLUME_NAME        = "ipamd-logs"
+	VOLUME_MOUNT_PATH  = "/var/log/aws-routed-eni/"
+)
+
+var (
+	primaryNode               v1.Node
+	primaryInstanceId         string
+	ds                        *appsV1.DaemonSet
+	f                         *framework.Framework
+	hostNetworkDeploymentSpec *appsV1.Deployment
+	hostNetworkDeployment     *appsV1.Deployment
+	err                       error
+	hostNetworkPod            v1.Pod
+	primaryNodePublicIP       string
+)
+
+func TestCni(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Cni Suite")
+}
+
+var _ = BeforeSuite(func() {
+	f = framework.New(framework.GlobalOptions)
+	ds, err = f.K8sResourceManagers.DaemonSetManager().GetDaemonSet(NAMESPACE, DAEMONSET)
+	Expect(err).NotTo(HaveOccurred())
+
+	nodes, err := f.K8sResourceManagers.NodeManager().GetAllNodes()
+	Expect(err).NotTo(HaveOccurred())
+	Expect(len(nodes.Items)).To(BeNumerically(">", 0))
+
+	primaryNode = nodes.Items[0]
+	primaryInstanceId = k8sUtils.GetInstanceIDFromNode(primaryNode)
+	instance, err := f.CloudServices.EC2().DescribeInstance(primaryInstanceId)
+	Expect(err).NotTo(HaveOccurred())
+
+	primaryNodePublicIP = *instance.PublicIpAddress
+
+	hostNetworkDeploymentSpec = manifest.NewBusyBoxDeploymentBuilder().
+		Namespace("default").
+		Name("host-network").
+		Replicas(1).
+		HostNetwork(true).
+		PodLabel(HOST_POD_LABEL_KEY, HOST_POD_LABEL_VAL).
+		NodeName(primaryNode.Name).
+		Build()
+
+	hostNetworkDeployment, err = f.K8sResourceManagers.
+		DeploymentManager().
+		CreateAndWaitTillDeploymentIsReady(hostNetworkDeploymentSpec)
+	Expect(err).NotTo(HaveOccurred())
+
+	pods, err := f.K8sResourceManagers.PodManager().GetPodsWithLabelSelector(HOST_POD_LABEL_KEY, HOST_POD_LABEL_VAL)
+	Expect(err).NotTo(HaveOccurred())
+
+	hostNetworkPod = pods.Items[0]
+})
+
+var _ = AfterSuite(func() {
+	err = f.K8sResourceManagers.DeploymentManager().DeleteAndWaitTillDeploymentIsDeleted(hostNetworkDeploymentSpec)
+	Expect(err).NotTo(HaveOccurred())
+})

test/integration-new/env_vars/env_vars_test.go

@@ -0,0 +1,180 @@
+package env_vars
+
+import (
+	"regexp"
+
+	"github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest"
+	k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils"
+	v1 "k8s.io/api/core/v1"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+)
+
+const (
+	AWS_VPC_ENI_MTU            = "AWS_VPC_ENI_MTU"
+	AWS_VPC_K8S_CNI_LOG_FILE   = "AWS_VPC_K8S_CNI_LOG_FILE"
+	AWS_VPC_K8S_CNI_VETHPREFIX = "AWS_VPC_K8S_CNI_VETHPREFIX"
+	POD_VOL_LABEL_KEY          = "MountVolume"
+	POD_VOL_LABEL_VAL          = "true"
+)
+
+var _ = Describe("cni env test", func() {
+
+	Context("CNI Environment Variables", func() {
+		It("Verifying that secondary ENI is created", func() {
+			nodes, err := f.K8sResourceManagers.NodeManager().GetAllNodes()
+			Expect(err).NotTo(HaveOccurred())
+
+			for _, node := range nodes.Items {
+				instanceId := k8sUtils.GetInstanceIDFromNode(node)
+				instance, err := f.CloudServices.EC2().DescribeInstance(instanceId)
+				Expect(err).NotTo(HaveOccurred())
+
+				len := len(instance.NetworkInterfaces)
+				Expect(len).To(BeNumerically(">=", 2))
+			}
+		})
+
+		FIt("Changing AWS_VPC_ENI_MTU and AWS_VPC_K8S_CNI_VETHPREFIX", func() {
+			currMTUVal := getEnvValueForKey(AWS_VPC_ENI_MTU)
+			Expect(currMTUVal).NotTo(Equal(""))
+
+			currVETHPrefix := getEnvValueForKey(AWS_VPC_K8S_CNI_VETHPREFIX)
+			Expect(currVETHPrefix).NotTo(Equal(""))
+
+			k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, DAEMONSET, NAMESPACE, DAEMONSET, map[string]string{
+				AWS_VPC_ENI_MTU:            "1300",
+				AWS_VPC_K8S_CNI_VETHPREFIX: "veth",
+			})
+
+			By("Deploying a BusyBox deployment")
+			deploymentSpec := manifest.NewBusyBoxDeploymentBuilder().
+				Namespace("default").
+				Name("busybox").
+				Replicas(1).
+				NodeName(primaryNode.Name).
+				Build()
+
+			_, err := f.K8sResourceManagers.
+				DeploymentManager().
+				CreateAndWaitTillDeploymentIsReady(deploymentSpec)
+			Expect(err).ToNot(HaveOccurred())
+
+			stdout, _, err := f.K8sResourceManagers.PodManager().PodExec("default", hostNetworkPod.Name, []string{"ifconfig"})
+			Expect(err).NotTo(HaveOccurred())
+
+			re := regexp.MustCompile(`\n`)
+			input := re.ReplaceAllString(stdout, "")
+
+			re = regexp.MustCompile(`eth.*lo`)
+			eth := re.FindStringSubmatch(input)[0]
+
+			re = regexp.MustCompile(`MTU:[0-9]*`)
+			mtus := re.FindAllStringSubmatch(eth, -1)
+
+			By("Validating new MTU value")
+			// Validate MTU
+			for _, m := range mtus {
+				Expect(m[0]).To(Equal("MTU:1300"))
+			}
+
+			By("Validating new VETH Prefix")
+			// Validate VETH Prefix
+			// Adding the new MTU value to below regex ensures that we are checking the recently created
+			// veth and not any older entries
+			re = regexp.MustCompile(`veth.*MTU:1300`)
+			veth := re.FindAllString(input, -1)
+
+			Expect(len(veth)).NotTo(Equal(0))
+
+			By("Deleting BusyBox Deployment")
+			err = f.K8sResourceManagers.DeploymentManager().DeleteAndWaitTillDeploymentIsDeleted(deploymentSpec)
+			Expect(err).NotTo(HaveOccurred())
+
+			By("Restoring old value on daemonset")
+			restoreOldValues(map[string]string{
+				AWS_VPC_ENI_MTU:            currMTUVal,
+				AWS_VPC_K8S_CNI_VETHPREFIX: currVETHPrefix,
+			})
+		})
+
+		It("Changing AWS_VPC_K8S_CNI_LOG_FILE", func() {
+			By("Deploying a host network deployment with Volume mount")
+			curlContainer := manifest.NewBusyBoxContainerBuilder().Image("curlimages/curl:7.76.1").Name("curler").Build()
+
+			volume := []v1.Volume{
+				{
+					Name: VOLUME_NAME,
+					VolumeSource: v1.VolumeSource{
+						HostPath: &v1.HostPathVolumeSource{
+							Path: VOLUME_MOUNT_PATH,
+						},
+					},
+				},
+			}
+
+			volumeMount := []v1.VolumeMount{
+				{
+					Name:      VOLUME_NAME,
+					MountPath: VOLUME_NAME,
+				},
+			}
+
+			deploymentSpecWithVol := manifest.NewDefaultDeploymentBuilder().
+				Namespace("default").
+				Name("host-network").
+				Replicas(1).
+				HostNetwork(true).
+				Container(curlContainer).
+				PodLabel(POD_VOL_LABEL_KEY, POD_VOL_LABEL_VAL).
+				MountVolume(volume, volumeMount).
+				NodeName(primaryNode.Name).
+				Build()
+
+			_, err := f.K8sResourceManagers.
+				DeploymentManager().
+				CreateAndWaitTillDeploymentIsReady(deploymentSpecWithVol)
+			Expect(err).NotTo(HaveOccurred())
+
+			pods, err := f.K8sResourceManagers.PodManager().GetPodsWithLabelSelector(POD_VOL_LABEL_KEY, POD_VOL_LABEL_VAL)
+			Expect(err).NotTo(HaveOccurred())
+
+			podWithVol := pods.Items[0]
+			currLogFilepath := getEnvValueForKey(AWS_VPC_K8S_CNI_LOG_FILE)
+			Expect(currLogFilepath).NotTo(Equal(""))
+
+			newLogFile := "ipamd_test.log"
+			k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, DAEMONSET, NAMESPACE, DAEMONSET, map[string]string{
+				AWS_VPC_K8S_CNI_LOG_FILE: "/host/var/log/aws-routed-eni/" + newLogFile,
+			})
+
+			stdout, _, err := f.K8sResourceManagers.PodManager().PodExec("default", podWithVol.Name, []string{"tail", "-n", "5", "ipamd-logs/ipamd_test.log"})
+			Expect(err).NotTo(HaveOccurred())
+			Expect(stdout).NotTo(Equal(""))
+
+			By("Restoring old value on daemonset")
+			restoreOldValues(map[string]string{
+				AWS_VPC_K8S_CNI_LOG_FILE: currLogFilepath,
+			})
+
+			By("Deleing deployment with Volume Mount")
+			err = f.K8sResourceManagers.DeploymentManager().DeleteAndWaitTillDeploymentIsDeleted(hostNetworkDeploymentSpec)
+			Expect(err).NotTo(HaveOccurred())
+		})
+	})
+})
+
+func getEnvValueForKey(key string) string {
+	envVar := ds.Spec.Template.Spec.Containers[0].Env
+	for _, env := range envVar {
+		if env.Name == key {
+			return env.Value
+		}
+	}
+	return ""
+}
+
+func restoreOldValues(oldVals map[string]string) {
+	k8sUtils.AddEnvVarToDaemonSetAndWaitTillUpdated(f, DAEMONSET, NAMESPACE, DAEMONSET, oldVals)
+}