Remove self-managed node group from custom-networking suite (#2590)

* remove self-managed node group from custom-networking suite

test/framework/resources/aws/utils/nodegroup.go

@@ -297,8 +297,8 @@ func GetClusterVPCConfig(f *framework.Framework) (*ClusterVPCConfig, error) {
 	return clusterConfig, nil
 }
 
-func TerminateInstances(f *framework.Framework, ngLabelKey string, ngLabelVal string) error {
-	nodeList, err := f.K8sResourceManagers.NodeManager().GetNodes(ngLabelKey, ngLabelVal)
+func TerminateInstances(f *framework.Framework) error {
+	nodeList, err := f.K8sResourceManagers.NodeManager().GetNodes(f.Options.NgNameLabelKey, f.Options.NgNameLabelVal)
 	if err != nil {
 		return fmt.Errorf("failed to get list of nodes created: %v", err)
 	}
@@ -314,6 +314,6 @@ func TerminateInstances(f *framework.Framework, ngLabelKey string, ngLabelVal st
 	}
 
 	// Wait for instances to be replaced
-	time.Sleep(time.Second * 450)
+	time.Sleep(time.Minute * 8)
 	return nil
 }

test/integration/custom-networking/custom_networking_suite_test.go

@@ -18,7 +18,6 @@ import (
 	"fmt"
 	"net"
 	"testing"
-	"time"
 
 	"github.com/aws/amazon-vpc-cni-k8s/pkg/apis/crd/v1alpha1"
 	"github.com/aws/amazon-vpc-cni-k8s/test/framework"
@@ -48,18 +47,13 @@ var (
 	cidrRangeString        string
 	cidrRange              *net.IPNet
 	cidrBlockAssociationID string
-	// Key Pair is required for creating a self managed node group
-	keyPairName = "custom-networking-key"
 	// Security Group that will be used in ENIConfig
 	customNetworkingSGID         string
 	customNetworkingSGOpenPort   = 8080
 	customNetworkingSubnetIDList []string
 	// List of ENIConfig per Availability Zone
 	eniConfigList        []*v1alpha1.ENIConfig
 	eniConfigBuilderList []*manifest.ENIConfigBuilder
-	// Properties of the self managed node group created using CFN template
-	nodeGroupProperties awsUtils.NodeGroupProperties
-	err                 error
 )
 
 // Parse test specific variable from flag
@@ -70,6 +64,7 @@ func init() {
 var _ = BeforeSuite(func() {
 	f = framework.New(framework.GlobalOptions)
 
+	var err error
 	_, cidrRange, err = net.ParseCIDR(cidrRangeString)
 	Expect(err).ToNot(HaveOccurred())
 
@@ -80,10 +75,6 @@ var _ = BeforeSuite(func() {
 	clusterVPCConfig, err = awsUtils.GetClusterVPCConfig(f)
 	Expect(err).ToNot(HaveOccurred())
 
-	By("creating ec2 key-pair for the new node group")
-	_, err := f.CloudServices.EC2().CreateKey(keyPairName)
-	Expect(err).ToNot(HaveOccurred())
-
 	By("creating security group to be used by custom networking")
 	createSecurityGroupOutput, err := f.CloudServices.EC2().
 		CreateSecurityGroup("custom-networking-test", "custom networking", f.Options.AWSVPCID)
@@ -142,25 +133,8 @@ var _ = BeforeSuite(func() {
 			"WARM_ENI_TARGET":                    "0",
 		})
 
-	nodeGroupProperties = awsUtils.NodeGroupProperties{
-		NgLabelKey:                "node-type",
-		NgLabelVal:                "custom-networking-node",
-		AsgSize:                   2,
-		NodeGroupName:             "custom-networking-node",
-		IsCustomNetworkingEnabled: true,
-		Subnet:                    clusterVPCConfig.PublicSubnetList,
-		InstanceType:              "c5.xlarge",
-		KeyPairName:               keyPairName,
-		ContainerRuntime:          f.Options.ContainerRuntime,
-	}
-
-	if f.Options.InstanceType == "arm64" {
-		nodeGroupProperties.InstanceType = "m6g.large"
-		nodeGroupProperties.NodeImageId = "ami-087fca294139386b6"
-	}
-
-	By("creating a new self managed node group")
-	err = awsUtils.CreateAndWaitTillSelfManagedNGReady(f, nodeGroupProperties)
+	By("terminating instances")
+	err = awsUtils.TerminateInstances(f)
 	Expect(err).ToNot(HaveOccurred())
 })
 
@@ -169,16 +143,22 @@ var _ = AfterSuite(func() {
 	f.K8sResourceManagers.NamespaceManager().
 		DeleteAndWaitTillNamespaceDeleted(utils.DefaultTestNamespace)
 
-	By("waiting for some time to allow CNI to delete ENI for IP being cooled down")
-	time.Sleep(time.Second * 60)
-
 	var errs prometheus.MultiError
-	By("deleting the self managed node group")
-	// we just accumulate errors instead of immediately failing so we can attempt to clean up everything
-	errs.Append(awsUtils.DeleteAndWaitTillSelfManagedNGStackDeleted(f, nodeGroupProperties))
+	for _, eniConfig := range eniConfigList {
+		By("deleting ENIConfig")
+		errs.Append(f.K8sResourceManagers.CustomResourceManager().DeleteResource(eniConfig))
+	}
 
-	By("deleting the key pair")
-	errs.Append(f.CloudServices.EC2().DeleteKey(keyPairName))
+	By("disabling custom networking on aws-node DaemonSet")
+	k8sUtils.RemoveVarFromDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName,
+		utils.AwsNodeNamespace, utils.AwsNodeName, map[string]struct{}{
+			"AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG": {},
+			"ENI_CONFIG_LABEL_DEF":               {},
+			"WARM_ENI_TARGET":                    {},
+		})
+
+	By("terminating instances")
+	errs.Append(awsUtils.TerminateInstances(f))
 
 	By("deleting security group")
 	errs.Append(f.CloudServices.EC2().DeleteSecurityGroup(customNetworkingSGID))
@@ -191,17 +171,5 @@ var _ = AfterSuite(func() {
 	By("disassociating the CIDR range to the VPC")
 	errs.Append(f.CloudServices.EC2().DisAssociateVPCCIDRBlock(cidrBlockAssociationID))
 
-	By("disabling custom networking on aws-node DaemonSet")
-	k8sUtils.RemoveVarFromDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName,
-		utils.AwsNodeNamespace, utils.AwsNodeName, map[string]struct{}{
-			"AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG": {},
-			"ENI_CONFIG_LABEL_DEF":               {},
-			"WARM_ENI_TARGET":                    {},
-		})
-
-	for _, eniConfig := range eniConfigList {
-		By("deleting ENIConfig")
-		errs.Append(f.K8sResourceManagers.CustomResourceManager().DeleteResource(eniConfig))
-	}
 	Expect(errs.MaybeUnwrap()).ToNot(HaveOccurred())
 })

test/integration/custom-networking/custom_networking_test.go

@@ -17,6 +17,7 @@ import (
 	"fmt"
 	"net"
 	"strconv"
+	"time"
 
 	awsUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/aws/utils"
 	"github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest"
@@ -51,17 +52,21 @@ var _ = Describe("Custom Networking Test", func() {
 				Args([]string{"-k", "-l", strconv.Itoa(port)}).
 				Build()
 
-			deployment = manifest.NewBusyBoxDeploymentBuilder(f.Options.TestImageRegistry).
+			deploymentBuilder := manifest.NewBusyBoxDeploymentBuilder(f.Options.TestImageRegistry).
 				Container(container).
 				Replicas(replicaCount).
-				NodeSelector(nodeGroupProperties.NgLabelKey, nodeGroupProperties.NgLabelVal).
 				PodLabel(podLabelKey, podLabelVal).
 				Build()
 
+			var err error
 			deployment, err = f.K8sResourceManagers.DeploymentManager().
-				CreateAndWaitTillDeploymentIsReady(deployment, utils.DefaultDeploymentReadyTimeout)
+				CreateAndWaitTillDeploymentIsReady(deploymentBuilder, utils.DefaultDeploymentReadyTimeout)
 			Expect(err).ToNot(HaveOccurred())
 
+			// Wait for deployment to settle, as if any pods restart, their pod IP will change between
+			// the GET and the validation.
+			time.Sleep(5 * time.Second)
+
 			podList, err = f.K8sResourceManagers.PodManager().
 				GetPodsWithLabelSelector(podLabelKey, podLabelVal)
 			Expect(err).ToNot(HaveOccurred())
@@ -100,14 +105,14 @@ var _ = Describe("Custom Networking Test", func() {
 		})
 
 		JustAfterEach(func() {
-			err = f.K8sResourceManagers.DeploymentManager().DeleteAndWaitTillDeploymentIsDeleted(deployment)
+			err := f.K8sResourceManagers.DeploymentManager().DeleteAndWaitTillDeploymentIsDeleted(deployment)
 			Expect(err).ToNot(HaveOccurred())
 		})
 
 		Context("when connecting to reachable port", func() {
 			BeforeEach(func() {
 				port = customNetworkingSGOpenPort
-				replicaCount = 16
+				replicaCount = 10
 				shouldConnect = true
 			})
 			It("should connect", func() {})
@@ -127,29 +132,28 @@ var _ = Describe("Custom Networking Test", func() {
 		JustBeforeEach(func() {
 			By("deleting ENIConfig for all availability zones")
 			for _, eniConfig := range eniConfigList {
-				err = f.K8sResourceManagers.CustomResourceManager().DeleteResource(eniConfig)
+				err := f.K8sResourceManagers.CustomResourceManager().DeleteResource(eniConfig)
 				Expect(err).ToNot(HaveOccurred())
 			}
 		})
 
 		JustAfterEach(func() {
 			By("re-creating ENIConfig for all availability zones")
 			for _, eniConfig := range eniConfigList {
-				err = f.K8sResourceManagers.CustomResourceManager().CreateResource(eniConfig)
+				err := f.K8sResourceManagers.CustomResourceManager().CreateResource(eniConfig)
 				Expect(err).ToNot(HaveOccurred())
 			}
 		})
 
 		It("deployment should not become ready", func() {
 			By("terminating instances")
-			err := awsUtils.TerminateInstances(f, nodeGroupProperties.NgLabelKey, nodeGroupProperties.NgLabelVal)
+			err := awsUtils.TerminateInstances(f)
 			Expect(err).ToNot(HaveOccurred())
 
 			// Nodes should be stuck in NotReady state since no ENIs could be attached and no pod
 			// IP addresses are available.
 			deployment := manifest.NewBusyBoxDeploymentBuilder(f.Options.TestImageRegistry).
 				Replicas(2).
-				NodeSelector(nodeGroupProperties.NgLabelKey, nodeGroupProperties.NgLabelVal).
 				Build()
 
 			By("verifying deployment should not succeed")
@@ -168,7 +172,7 @@ var _ = Describe("Custom Networking Test", func() {
 		JustBeforeEach(func() {
 			By("deleting ENIConfig for each availability zone")
 			for _, eniConfig := range eniConfigList {
-				err = f.K8sResourceManagers.CustomResourceManager().DeleteResource(eniConfig)
+				err := f.K8sResourceManagers.CustomResourceManager().DeleteResource(eniConfig)
 				Expect(err).ToNot(HaveOccurred())
 			}
 			By("re-creating ENIConfigs with no security group")
@@ -185,12 +189,11 @@ var _ = Describe("Custom Networking Test", func() {
 
 		It("deployment should become ready", func() {
 			By("terminating instances")
-			err := awsUtils.TerminateInstances(f, nodeGroupProperties.NgLabelKey, nodeGroupProperties.NgLabelVal)
+			err := awsUtils.TerminateInstances(f)
 			Expect(err).ToNot(HaveOccurred())
 
 			deployment := manifest.NewBusyBoxDeploymentBuilder(f.Options.TestImageRegistry).
 				Replicas(2).
-				NodeSelector(nodeGroupProperties.NgLabelKey, nodeGroupProperties.NgLabelVal).
 				Build()
 
 			By("verifying deployment succeeds")

test/integration/pod-eni/security_group_per_pod_suite_test.go

@@ -105,7 +105,7 @@ var _ = BeforeSuite(func() {
 		})
 
 	By("terminating instances")
-	err = awsUtils.TerminateInstances(f, f.Options.NgNameLabelKey, f.Options.NgNameLabelVal)
+	err = awsUtils.TerminateInstances(f)
 	Expect(err).ToNot(HaveOccurred())
 
 	By("getting target node")
@@ -122,7 +122,7 @@ var _ = AfterSuite(func() {
 		})
 
 	By("terminating instances")
-	err := awsUtils.TerminateInstances(f, f.Options.NgNameLabelKey, f.Options.NgNameLabelVal)
+	err := awsUtils.TerminateInstances(f)
 	Expect(err).ToNot(HaveOccurred())
 
 	By("deleting the security group")