Refactored code

Addon upgrade/downgrade test similar to aws#1795

Added tests for addon upgrade/downgrade

test/framework/options.go

@@ -34,6 +34,8 @@ type Options struct {
 	NgNameLabelKey string
 	NgNameLabelVal string
 	EKSEndpoint    string
+	InitialAddon   string
+	TargetAddon    string
 }
 
 func (options *Options) BindFlags() {
@@ -44,6 +46,8 @@ func (options *Options) BindFlags() {
 	flag.StringVar(&options.NgNameLabelKey, "ng-name-label-key", "eks.amazonaws.com/nodegroup", "label key used to identify nodegroup name")
 	flag.StringVar(&options.NgNameLabelVal, "ng-name-label-val", "", "label value with the nodegroup name")
 	flag.StringVar(&options.EKSEndpoint, "eks-endpoint", "", "optional eks api server endpoint")
+	flag.StringVar(&options.InitialAddon, "initial-addon-version", "", "Initial CNI addon version before upgrade applied")
+	flag.StringVar(&options.TargetAddon, "target-addon-version", "", "Target CNI addon version after upgrade applied")
 }
 
 func (options *Options) Validate() error {

test/framework/resources/aws/services/eks.go

@@ -22,14 +22,31 @@ import (
 
 type EKS interface {
 	DescribeCluster(clusterName string) (*eks.DescribeClusterOutput, error)
-	CreateAddon(addon string, clusterName string) (*eks.CreateAddonOutput, error)
-	DeleteAddon(addon string, clusterName string) (*eks.DeleteAddonOutput, error)
+	CreateAddon(addonInput *AddonInput) (*eks.CreateAddonOutput, error)
+	DescribeAddonVersions(AddonInput *AddonInput) (*eks.DescribeAddonVersionsOutput, error)
+	DescribeAddon(addonInput *AddonInput) (*eks.DescribeAddonOutput, error)
+	DeleteAddon(AddOnInput *AddonInput) (*eks.DeleteAddonOutput, error)
 }
 
 type defaultEKS struct {
 	eksiface.EKSAPI
 }
 
+// Internal Addon Input struct
+// subset of eks.AddonInput
+// used by ginkgo tests
+type AddonInput struct {
+	AddonName    string
+	ClusterName  string
+	AddonVersion string
+	K8sVersion   string
+}
+
+type VersionWithBuild struct {
+	version uint
+	build   uint
+}
+
 func NewEKS(session *session.Session, endpoint string) EKS {
 	return &defaultEKS{
 		EKSAPI: eks.New(session, &aws.Config{
@@ -39,22 +56,42 @@ func NewEKS(session *session.Session, endpoint string) EKS {
 	}
 }
 
-func (d defaultEKS) CreateAddon(addon string, clusterName string) (*eks.CreateAddonOutput, error) {
+func (d defaultEKS) CreateAddon(addonInput *AddonInput) (*eks.CreateAddonOutput, error) {
 	createAddonInput := &eks.CreateAddonInput{
-		AddonName:   aws.String(addon),
-		ClusterName: aws.String(clusterName),
+		AddonName:   aws.String(addonInput.AddonName),
+		ClusterName: aws.String(addonInput.ClusterName),
+	}
+	if addonInput.AddonVersion != "" {
+		createAddonInput.SetAddonVersion(addonInput.AddonVersion)
+		createAddonInput.SetResolveConflicts("OVERWRITE")
 	}
 	return d.EKSAPI.CreateAddon(createAddonInput)
 }
 
-func (d defaultEKS) DeleteAddon(addon string, clusterName string) (*eks.DeleteAddonOutput, error) {
+func (d defaultEKS) DeleteAddon(addonInput *AddonInput) (*eks.DeleteAddonOutput, error) {
 	deleteAddonInput := &eks.DeleteAddonInput{
-		AddonName:   aws.String(addon),
-		ClusterName: aws.String(clusterName),
+		AddonName:   aws.String(addonInput.AddonName),
+		ClusterName: aws.String(addonInput.ClusterName),
 	}
 	return d.EKSAPI.DeleteAddon(deleteAddonInput)
 }
 
+func (d defaultEKS) DescribeAddonVersions(addonInput *AddonInput) (*eks.DescribeAddonVersionsOutput, error) {
+	describeAddonVersionsInput := &eks.DescribeAddonVersionsInput{
+		AddonName:         aws.String(addonInput.AddonName),
+		KubernetesVersion: aws.String(addonInput.K8sVersion),
+	}
+	return d.EKSAPI.DescribeAddonVersions(describeAddonVersionsInput)
+}
+
+func (d defaultEKS) DescribeAddon(addonInput *AddonInput) (*eks.DescribeAddonOutput, error) {
+	describeAddonInput := &eks.DescribeAddonInput{
+		AddonName:   aws.String(addonInput.AddonName),
+		ClusterName: aws.String(addonInput.ClusterName),
+	}
+	return d.EKSAPI.DescribeAddon(describeAddonInput)
+}
+
 func (d defaultEKS) DescribeCluster(clusterName string) (*eks.DescribeClusterOutput, error) {
 	describeClusterInput := &eks.DescribeClusterInput{
 		Name: aws.String(clusterName),

test/framework/resources/k8s/utils/addon.go

@@ -0,0 +1,52 @@
+package utils
+
+import (
+	"context"
+
+	"github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/aws/services"
+	"github.com/aws/amazon-vpc-cni-k8s/test/framework/utils"
+	"k8s.io/apimachinery/pkg/util/wait"
+)
+
+func WaitTillAddonIsDeleted(eks services.EKS, addonName string, clusterName string) error {
+	ctx := context.Background()
+	return wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) {
+		_, err := eks.DescribeAddon(&services.AddonInput{
+			AddonName:   addonName,
+			ClusterName: clusterName,
+		})
+		if err != nil {
+			return false, err
+		}
+		return false, nil
+	}, ctx.Done())
+}
+
+func WaitTillAddonIsActive(eks services.EKS, addonName string, clusterName string) error {
+	ctx := context.Background()
+	return wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) {
+		describeAddonOutput, err := eks.DescribeAddon(&services.AddonInput{
+			AddonName:   addonName,
+			ClusterName: clusterName,
+		})
+		status := *describeAddonOutput.Addon.Status
+		if status == "ACTIVE" {
+			return true, nil
+		}
+		if err != nil {
+			return false, err
+		}
+		return false, nil
+	}, ctx.Done())
+}
+
+func GetCurrentAddonVersion(eks services.EKS, addonName string, clusterName string) string {
+	describeAddonOutput, err := eks.DescribeAddon(&services.AddonInput{
+		AddonName:   addonName,
+		ClusterName: clusterName,
+	})
+	if err == nil {
+		return *describeAddonOutput.Addon.AddonVersion
+	}
+	return ""
+}

test/integration-new/cni/host_networking_test.go

@@ -14,32 +14,21 @@
 package cni
 
 import (
-	"fmt"
 	"strconv"
 	"time"
 
-	"github.com/aws/amazon-vpc-cni-k8s/test/agent/pkg/input"
 	v1 "k8s.io/api/core/v1"
 
 	"github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest"
 	k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils"
 	"github.com/aws/amazon-vpc-cni-k8s/test/framework/utils"
+	"github.com/aws/amazon-vpc-cni-k8s/test/integration-new/common"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
 // TODO: Instead of passing the list of pods to the test helper, have the test helper get the pod on node
-
-type TestType int
-
-const (
-	NetworkingTearDownSucceeds TestType = iota
-	NetworkingTearDownFails
-	NetworkingSetupSucceeds
-	NetworkingSetupFails
-)
-
 const (
 	AWS_VPC_ENI_MTU            = "AWS_VPC_ENI_MTU"
 	AWS_VPC_K8S_CNI_VETHPREFIX = "AWS_VPC_K8S_CNI_VETHPREFIX"
@@ -76,8 +65,7 @@ var _ = Describe("test host networking", func() {
 			Expect(err).ToNot(HaveOccurred())
 
 			By("getting the list of pods using IP from primary and secondary ENI")
-			interfaceTypeToPodList :=
-				GetPodsOnPrimaryAndSecondaryInterface(primaryNode, podLabelKey, podLabelVal)
+			interfaceTypeToPodList := common.GetPodsOnPrimaryAndSecondaryInterface(primaryNode, podLabelKey, podLabelVal, f)
 
 			// Primary ENI and Secondary ENI IPs are handled differently when setting up
 			// the host networking rule hence this check
@@ -87,11 +75,11 @@ var _ = Describe("test host networking", func() {
 				Should(BeNumerically(">", 0))
 
 			By("generating the pod networking validation input to be passed to tester")
-			input, err := GetPodNetworkingValidationInput(interfaceTypeToPodList).Serialize()
+			input, err := common.GetPodNetworkingValidationInput(interfaceTypeToPodList, vpcCIDRs).Serialize()
 			Expect(err).NotTo(HaveOccurred())
 
 			By("validating host networking setup is setup correctly")
-			ValidateHostNetworking(NetworkingSetupSucceeds, input)
+			common.ValidateHostNetworking(common.NetworkingSetupSucceeds, input, primaryNode.Name, f)
 
 			By("deleting the deployment to test teardown")
 			err = f.K8sResourceManagers.DeploymentManager().
@@ -102,7 +90,7 @@ var _ = Describe("test host networking", func() {
 			time.Sleep(time.Second * 60)
 
 			By("validating host networking is teared down correctly")
-			ValidateHostNetworking(NetworkingTearDownSucceeds, input)
+			common.ValidateHostNetworking(common.NetworkingTearDownSucceeds, input, primaryNode.Name, f)
 		})
 
 		It("Validate Host Networking setup after changing MTU and Veth Prefix", func() {
@@ -125,18 +113,18 @@ var _ = Describe("test host networking", func() {
 
 			By("getting the list of pods using IP from primary and secondary ENI")
 			interfaceTypeToPodList :=
-				GetPodsOnPrimaryAndSecondaryInterface(primaryNode, podLabelKey, podLabelVal)
+				common.GetPodsOnPrimaryAndSecondaryInterface(primaryNode, podLabelKey, podLabelVal, f)
 
 			By("generating the pod networking validation input to be passed to tester")
-			podNetworkingValidationInput := GetPodNetworkingValidationInput(interfaceTypeToPodList)
+			podNetworkingValidationInput := common.GetPodNetworkingValidationInput(interfaceTypeToPodList, vpcCIDRs)
 			podNetworkingValidationInput.VethPrefix = NEW_VETH_PREFIX
 			podNetworkingValidationInput.ValidateMTU = true
 			podNetworkingValidationInput.MTU = NEW_MTU_VAL
 			input, err := podNetworkingValidationInput.Serialize()
 			Expect(err).NotTo(HaveOccurred())
 
 			By("validating host networking setup is setup correctly with MTU check as well")
-			ValidateHostNetworking(NetworkingSetupSucceeds, input)
+			common.ValidateHostNetworking(common.NetworkingSetupSucceeds, input, primaryNode.Name, f)
 
 			By("deleting the deployment to test teardown")
 			err = f.K8sResourceManagers.DeploymentManager().
@@ -147,7 +135,7 @@ var _ = Describe("test host networking", func() {
 			time.Sleep(time.Second * 60)
 
 			By("validating host networking is teared down correctly")
-			ValidateHostNetworking(NetworkingTearDownSucceeds, input)
+			common.ValidateHostNetworking(common.NetworkingTearDownSucceeds, input, primaryNode.Name, f)
 		})
 	})
 
@@ -165,39 +153,39 @@ var _ = Describe("test host networking", func() {
 				CreatAndWaitTillRunning(parkingPod)
 			Expect(err).ToNot(HaveOccurred())
 
-			validInput, err := GetPodNetworkingValidationInput(InterfaceTypeToPodList{
+			validInput, err := common.GetPodNetworkingValidationInput(common.InterfaceTypeToPodList{
 				PodsOnPrimaryENI: []v1.Pod{*parkingPod},
-			}).Serialize()
+			}, vpcCIDRs).Serialize()
 			Expect(err).NotTo(HaveOccurred())
 
 			By("first validating the tester work on valid input")
-			ValidateHostNetworking(NetworkingSetupSucceeds, validInput)
+			common.ValidateHostNetworking(common.NetworkingSetupSucceeds, validInput, primaryNode.Name, f)
 
 			By("validating tester fails when invalid IP is passed")
 			invalidPod := parkingPod.DeepCopy()
 			invalidPod.Status.PodIP = "1.1.1.1"
 
-			invalidInput, err := GetPodNetworkingValidationInput(InterfaceTypeToPodList{
+			invalidInput, err := common.GetPodNetworkingValidationInput(common.InterfaceTypeToPodList{
 				PodsOnPrimaryENI: []v1.Pod{*invalidPod},
-			}).Serialize()
+			}, vpcCIDRs).Serialize()
 			Expect(err).NotTo(HaveOccurred())
 
-			ValidateHostNetworking(NetworkingSetupFails, invalidInput)
+			common.ValidateHostNetworking(common.NetworkingSetupFails, invalidInput, primaryNode.Name, f)
 
 			By("validating the tester fails when invalid namespace is passed")
 			invalidPod = parkingPod.DeepCopy()
 			// veth pair name is generated using namespace+name so the test should fail
 			invalidPod.Namespace = "different"
 
-			invalidInput, err = GetPodNetworkingValidationInput(InterfaceTypeToPodList{
+			invalidInput, err = common.GetPodNetworkingValidationInput(common.InterfaceTypeToPodList{
 				PodsOnPrimaryENI: []v1.Pod{*invalidPod},
-			}).Serialize()
+			}, vpcCIDRs).Serialize()
 			Expect(err).NotTo(HaveOccurred())
 
-			ValidateHostNetworking(NetworkingSetupFails, invalidInput)
+			common.ValidateHostNetworking(common.NetworkingSetupFails, invalidInput, primaryNode.Name, f)
 
 			By("validating the tester fails when tear down check is run on running pod")
-			ValidateHostNetworking(NetworkingTearDownFails, validInput)
+			common.ValidateHostNetworking(common.NetworkingTearDownFails, validInput, primaryNode.Name, f)
 
 			By("deleting the parking pod")
 			err = f.K8sResourceManagers.PodManager().
@@ -206,85 +194,3 @@ var _ = Describe("test host networking", func() {
 		})
 	})
 })
-
-// Validate host networking for the list of pods supplied
-func ValidateHostNetworking(testType TestType, podValidationInputString string) {
-	testerArgs := []string{fmt.Sprintf("-pod-networking-validation-input=%s",
-		podValidationInputString)}
-
-	var shouldTestPodError bool
-	if NetworkingSetupSucceeds == testType {
-		testerArgs = append(testerArgs, "-test-setup=true")
-	} else if NetworkingSetupFails == testType {
-		testerArgs = append(testerArgs, "-test-setup=true")
-		shouldTestPodError = true
-	} else if NetworkingTearDownSucceeds == testType {
-		testerArgs = append(testerArgs, "-test-cleanup=true")
-	} else if NetworkingTearDownFails == testType {
-		testerArgs = append(testerArgs, "-test-cleanup=true")
-		shouldTestPodError = true
-	}
-
-	testContainer := manifest.NewTestHelperContainer().
-		Command([]string{"./networking"}).
-		Args(testerArgs).
-		Build()
-
-	testPod := manifest.NewDefaultPodBuilder().
-		Container(testContainer).
-		NodeName(primaryNode.Name).
-		HostNetwork(true).
-		Build()
-
-	By("creating pod to test host networking setup")
-	testPod, err := f.K8sResourceManagers.PodManager().
-		CreateAndWaitTillPodCompleted(testPod)
-	logs, errLogs := f.K8sResourceManagers.PodManager().
-		PodLogs(testPod.Namespace, testPod.Name)
-	Expect(errLogs).ToNot(HaveOccurred())
-
-	fmt.Fprintln(GinkgoWriter, logs)
-
-	if shouldTestPodError {
-		Expect(err).To(HaveOccurred())
-	} else {
-		Expect(err).ToNot(HaveOccurred())
-	}
-
-	By("deleting the host networking setup pod")
-	err = f.K8sResourceManagers.PodManager().
-		DeleteAndWaitTillPodDeleted(testPod)
-	Expect(err).ToNot(HaveOccurred())
-}
-
-// GetPodNetworkingValidationInput returns input string containing the list of pods for which
-// the host networking has to be tested
-func GetPodNetworkingValidationInput(interfaceTypeToPodList InterfaceTypeToPodList) input.PodNetworkingValidationInput {
-
-	ip := input.PodNetworkingValidationInput{
-		VPCCidrRange: vpcCIDRs,
-		VethPrefix:   "eni",
-		PodList:      []input.Pod{},
-		ValidateMTU:  false,
-	}
-
-	for _, primaryENIPod := range interfaceTypeToPodList.PodsOnPrimaryENI {
-		ip.PodList = append(ip.PodList, input.Pod{
-			PodName:              primaryENIPod.Name,
-			PodNamespace:         primaryENIPod.Namespace,
-			PodIPv4Address:       primaryENIPod.Status.PodIP,
-			IsIPFromSecondaryENI: false,
-		})
-	}
-
-	for _, secondaryENIPod := range interfaceTypeToPodList.PodsOnSecondaryENI {
-		ip.PodList = append(ip.PodList, input.Pod{
-			PodName:              secondaryENIPod.Name,
-			PodNamespace:         secondaryENIPod.Namespace,
-			PodIPv4Address:       secondaryENIPod.Status.PodIP,
-			IsIPFromSecondaryENI: true,
-		})
-	}
-
-	return ip
-}