Refactored code and Added cni addon upgrade/downgrade regression test

scripts/run-cni-release-tests.sh

@@ -16,7 +16,7 @@ set -e
 
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 TEST_DIR="$SCRIPT_DIR/../test"
-INTEGRATION_TEST_DIR="$TEST_DIR/integration-new"
+INTEGRATION_TEST_DIR="$TEST_DIR/integration"
 CALICO_TEST_DIR="$TEST_DIR/e2e/calico"
 
 source "$SCRIPT_DIR"/lib/cluster.sh

scripts/run-multus-tests.sh

@@ -5,7 +5,7 @@
 set -e
 
 SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-INTEGRATION_TEST_DIR="$SCRIPT_DIR/../test/integration-new"
+INTEGRATION_TEST_DIR="$SCRIPT_DIR/../test/integration"
 
 source "$SCRIPT_DIR"/lib/common.sh
 source "$SCRIPT_DIR"/lib/cluster.sh

test/framework/options.go

@@ -37,6 +37,8 @@ type Options struct {
 	CalicoVersion    string
 	ContainerRuntime string
 	InstanceType     string
+	InitialAddon     string
+	TargetAddon      string
 }
 
 func (options *Options) BindFlags() {
@@ -47,6 +49,8 @@ func (options *Options) BindFlags() {
 	flag.StringVar(&options.NgNameLabelKey, "ng-name-label-key", "eks.amazonaws.com/nodegroup", "label key used to identify nodegroup name")
 	flag.StringVar(&options.NgNameLabelVal, "ng-name-label-val", "", "label value with the nodegroup name")
 	flag.StringVar(&options.EKSEndpoint, "eks-endpoint", "", "optional eks api server endpoint")
+	flag.StringVar(&options.InitialAddon, "initial-addon-version", "", "Initial CNI addon version before upgrade applied")
+	flag.StringVar(&options.TargetAddon, "target-addon-version", "", "Target CNI addon version after upgrade applied")
 	flag.StringVar(&options.CalicoVersion, "calico-version", "3.22.0", "calico version to be tested")
 	flag.StringVar(&options.ContainerRuntime, "container-runtime", "", "Optionally can specify it as 'containerd' for the test nodes")
 	flag.StringVar(&options.InstanceType, "instance-type", "amd64", "Optionally specify instance type as arm64 for the test nodes")

test/framework/resources/aws/services/eks.go

@@ -22,14 +22,27 @@ import (
 
 type EKS interface {
 	DescribeCluster(clusterName string) (*eks.DescribeClusterOutput, error)
-	CreateAddon(addon string, clusterName string) (*eks.CreateAddonOutput, error)
-	DeleteAddon(addon string, clusterName string) (*eks.DeleteAddonOutput, error)
+	CreateAddon(addonInput *AddonInput) (*eks.CreateAddonOutput, error)
+	DescribeAddonVersions(AddonInput *AddonInput) (*eks.DescribeAddonVersionsOutput, error)
+	DescribeAddon(addonInput *AddonInput) (*eks.DescribeAddonOutput, error)
+	DeleteAddon(AddOnInput *AddonInput) (*eks.DeleteAddonOutput, error)
+	GetLatestVersion(addonInput *AddonInput) (string, error)
 }
 
 type defaultEKS struct {
 	eksiface.EKSAPI
 }
 
+// Internal Addon Input struct
+// subset of eks.AddonInput
+// used by ginkgo tests
+type AddonInput struct {
+	AddonName    string
+	ClusterName  string
+	AddonVersion string
+	K8sVersion   string
+}
+
 func NewEKS(session *session.Session, endpoint string) EKS {
 	return &defaultEKS{
 		EKSAPI: eks.New(session, &aws.Config{
@@ -39,26 +52,54 @@ func NewEKS(session *session.Session, endpoint string) EKS {
 	}
 }
 
-func (d defaultEKS) CreateAddon(addon string, clusterName string) (*eks.CreateAddonOutput, error) {
+func (d defaultEKS) CreateAddon(addonInput *AddonInput) (*eks.CreateAddonOutput, error) {
 	createAddonInput := &eks.CreateAddonInput{
-		AddonName:   aws.String(addon),
-		ClusterName: aws.String(clusterName),
+		AddonName:   aws.String(addonInput.AddonName),
+		ClusterName: aws.String(addonInput.ClusterName),
+	}
+	if addonInput.AddonVersion != "" {
+		createAddonInput.SetAddonVersion(addonInput.AddonVersion)
+		createAddonInput.SetResolveConflicts("OVERWRITE")
 	}
 	return d.EKSAPI.CreateAddon(createAddonInput)
 }
 
-func (d defaultEKS) DeleteAddon(addon string, clusterName string) (*eks.DeleteAddonOutput, error) {
+func (d defaultEKS) DeleteAddon(addonInput *AddonInput) (*eks.DeleteAddonOutput, error) {
 	deleteAddonInput := &eks.DeleteAddonInput{
-		AddonName:   aws.String(addon),
-		ClusterName: aws.String(clusterName),
+		AddonName:   aws.String(addonInput.AddonName),
+		ClusterName: aws.String(addonInput.ClusterName),
 	}
 	return d.EKSAPI.DeleteAddon(deleteAddonInput)
 }
 
+func (d defaultEKS) DescribeAddonVersions(addonInput *AddonInput) (*eks.DescribeAddonVersionsOutput, error) {
+	describeAddonVersionsInput := &eks.DescribeAddonVersionsInput{
+		AddonName:         aws.String(addonInput.AddonName),
+		KubernetesVersion: aws.String(addonInput.K8sVersion),
+	}
+	return d.EKSAPI.DescribeAddonVersions(describeAddonVersionsInput)
+}
+
+func (d defaultEKS) DescribeAddon(addonInput *AddonInput) (*eks.DescribeAddonOutput, error) {
+	describeAddonInput := &eks.DescribeAddonInput{
+		AddonName:   aws.String(addonInput.AddonName),
+		ClusterName: aws.String(addonInput.ClusterName),
+	}
+	return d.EKSAPI.DescribeAddon(describeAddonInput)
+}
+
 func (d defaultEKS) DescribeCluster(clusterName string) (*eks.DescribeClusterOutput, error) {
 	describeClusterInput := &eks.DescribeClusterInput{
 		Name: aws.String(clusterName),
 	}
 
 	return d.EKSAPI.DescribeCluster(describeClusterInput)
 }
+
+func (d defaultEKS) GetLatestVersion(addonInput *AddonInput) (string, error) {
+	addonOutput, err := d.DescribeAddonVersions(addonInput)
+	if err != nil {
+		return "", err
+	}
+	return *addonOutput.Addons[0].AddonVersions[0].AddonVersion, nil
+}

test/framework/resources/k8s/utils/addon.go

@@ -0,0 +1,46 @@
+package utils
+
+import (
+	"context"
+
+	"github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/aws/services"
+	"github.com/aws/amazon-vpc-cni-k8s/test/framework/utils"
+	"github.com/pkg/errors"
+	"k8s.io/apimachinery/pkg/util/wait"
+)
+
+func WaitTillAddonIsDeleted(eks services.EKS, addonName string, clusterName string) error {
+	ctx := context.Background()
+	return wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) {
+		_, err := eks.DescribeAddon(&services.AddonInput{
+			AddonName:   addonName,
+			ClusterName: clusterName,
+		})
+		if err != nil {
+			return false, err
+		}
+		return false, nil
+	}, ctx.Done())
+}
+
+func WaitTillAddonIsActive(eks services.EKS, addonName string, clusterName string) error {
+	ctx := context.Background()
+	return wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) {
+		describeAddonOutput, err := eks.DescribeAddon(&services.AddonInput{
+			AddonName:   addonName,
+			ClusterName: clusterName,
+		})
+		if err != nil {
+			return false, err
+		}
+
+		status := *describeAddonOutput.Addon.Status
+		if status == "CREATE_FAILED" || status == "DEGRADED" {
+			return false, errors.Errorf("Create Addon Failed, addon status: %s", status)
+		}
+		if status == "ACTIVE" {
+			return true, nil
+		}
+		return false, nil
+	}, ctx.Done())
+}

test/framework/utils/const.go

@@ -30,5 +30,5 @@ const (
 	PollIntervalMedium = time.Second * 5
 	PollIntervalLong   = time.Second * 20
 
-	DefaultDeploymentReadyTimeout = time.Second * 120
+	DefaultDeploymentReadyTimeout = time.Second * 300
 )

test/integration/README.md

@@ -95,7 +95,7 @@ all tests ran successfully in 0 minutes and 27 seconds
 ```
 
 ### Running release tests with scripts/run-cni-release-tests.sh
-`run-cni-release-tests.sh` will run cni, ipamd, and cni-metrics-helper (integration tests)[https://github.com/aws/amazon-vpc-cni-k8s/tree/master/test/integration-new] and (Calico tests)[https://github.com/aws/amazon-vpc-cni-k8s/tree/master/test/e2e/calico]. The script _does not_ create a test cluster, instead it will run the test on cluster specified via variables required in the script. The tests are run on the vpc-cni version installed on the cluster(it does not upgrade/install any specific vpc-cni version). See script `update-cni-images.sh` to update the test cluster with required cni version before running the tests.
+`run-cni-release-tests.sh` will run cni, ipamd, and cni-metrics-helper (integration tests)[https://github.com/aws/amazon-vpc-cni-k8s/tree/master/test/integration] and (Calico tests)[https://github.com/aws/amazon-vpc-cni-k8s/tree/master/test/e2e/calico]. The script _does not_ create a test cluster, instead it will run the test on cluster specified via variables required in the script. The tests are run on the vpc-cni version installed on the cluster(it does not upgrade/install any specific vpc-cni version). See script `update-cni-images.sh` to update the test cluster with required cni version before running the tests.
 
 ## Development of New Integration Tests
 
@@ -159,7 +159,7 @@ import (
 	k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils"
 	"github.com/aws/amazon-vpc-cni-k8s/test/framework/utils"
         //ginkgo and the assertion library: gomega are imported below
-	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
         //v1 imported for Node libraries
 	v1 "k8s.io/api/core/v1"

test/integration/addon-tests/cni_addon_suite_test.go

@@ -0,0 +1,84 @@
+package addon_tests
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/aws/amazon-vpc-cni-k8s/test/framework"
+	"github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/aws/services"
+	"github.com/aws/amazon-vpc-cni-k8s/test/framework/utils"
+	v1 "k8s.io/api/core/v1"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+const InstanceTypeNodeLabelKey = "beta.kubernetes.io/instance-type"
+
+var f *framework.Framework
+var maxIPPerInterface int
+var primaryNode v1.Node
+var secondaryNode v1.Node
+var vpcCIDRs []string
+var clusterVersion string
+var latestAddonVersion string
+
+func TestCNIPodNetworking(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "CNI Pod Networking Suite")
+}
+
+var _ = BeforeSuite(func() {
+	f = framework.New(framework.GlobalOptions)
+
+	By("creating test namespace")
+	f.K8sResourceManagers.NamespaceManager().
+		CreateNamespace(utils.DefaultTestNamespace)
+
+	By(fmt.Sprintf("getting the node with the node label key %s and value %s",
+		f.Options.NgNameLabelKey, f.Options.NgNameLabelVal))
+	nodes, err := f.K8sResourceManagers.NodeManager().GetNodes(f.Options.NgNameLabelKey, f.Options.NgNameLabelVal)
+	Expect(err).ToNot(HaveOccurred())
+
+	By("verifying that atleast 1 nodes is present for the test")
+	Expect(len(nodes.Items)).Should(BeNumerically(">", 0))
+
+	// Set the primary and secondary node for testing
+	primaryNode = nodes.Items[0]
+	secondaryNode = nodes.Items[1]
+
+	By("getting the instance type from node label " + InstanceTypeNodeLabelKey)
+	instanceType := primaryNode.Labels[InstanceTypeNodeLabelKey]
+
+	By("getting the network interface details from ec2")
+	instanceOutput, err := f.CloudServices.EC2().DescribeInstanceType(instanceType)
+	Expect(err).ToNot(HaveOccurred())
+
+	maxIPPerInterface = int(*instanceOutput[0].NetworkInfo.Ipv4AddressesPerInterface)
+
+	By("describing the VPC to get the VPC CIDRs")
+	describeVPCOutput, err := f.CloudServices.EC2().DescribeVPC(f.Options.AWSVPCID)
+	Expect(err).ToNot(HaveOccurred())
+
+	for _, cidrBlockAssociationSet := range describeVPCOutput.Vpcs[0].CidrBlockAssociationSet {
+		vpcCIDRs = append(vpcCIDRs, *cidrBlockAssociationSet.CidrBlock)
+	}
+
+	By("getting current cluster version")
+	clusterOutput, err := f.CloudServices.EKS().DescribeCluster(f.Options.ClusterName)
+	Expect(err).NotTo(HaveOccurred())
+	clusterVersion = *clusterOutput.Cluster.Version
+
+	By("getting latest vpc-cni addon version")
+	latestAddonVersion, err = f.CloudServices.EKS().GetLatestVersion(&services.AddonInput{
+		AddonName:  "vpc-cni",
+		K8sVersion: clusterVersion,
+	})
+	Expect(err).NotTo(HaveOccurred())
+})
+
+var _ = AfterSuite(func() {
+	By("deleting test namespace")
+	f.K8sResourceManagers.NamespaceManager().
+		DeleteAndWaitTillNamespaceDeleted(utils.DefaultTestNamespace)
+})