diff --git a/test/framework/resources/aws/utils/nodegroup.go b/test/framework/resources/aws/utils/nodegroup.go index 59554450fa..6d395d8ca4 100644 --- a/test/framework/resources/aws/utils/nodegroup.go +++ b/test/framework/resources/aws/utils/nodegroup.go @@ -297,8 +297,8 @@ func GetClusterVPCConfig(f *framework.Framework) (*ClusterVPCConfig, error) { return clusterConfig, nil } -func TerminateInstances(f *framework.Framework, ngLabelKey string, ngLabelVal string) error { - nodeList, err := f.K8sResourceManagers.NodeManager().GetNodes(ngLabelKey, ngLabelVal) +func TerminateInstances(f *framework.Framework) error { + nodeList, err := f.K8sResourceManagers.NodeManager().GetNodes(f.Options.NgNameLabelKey, f.Options.NgNameLabelVal) if err != nil { return fmt.Errorf("failed to get list of nodes created: %v", err) } @@ -314,6 +314,6 @@ func TerminateInstances(f *framework.Framework, ngLabelKey string, ngLabelVal st } // Wait for instances to be replaced - time.Sleep(time.Second * 450) + time.Sleep(time.Minute * 8) return nil } diff --git a/test/integration/custom-networking/custom_networking_suite_test.go b/test/integration/custom-networking/custom_networking_suite_test.go index 52c4e95ce0..0ff08007de 100644 --- a/test/integration/custom-networking/custom_networking_suite_test.go +++ b/test/integration/custom-networking/custom_networking_suite_test.go @@ -18,7 +18,6 @@ import ( "fmt" "net" "testing" - "time" "github.com/aws/amazon-vpc-cni-k8s/pkg/apis/crd/v1alpha1" "github.com/aws/amazon-vpc-cni-k8s/test/framework" @@ -48,8 +47,6 @@ var ( cidrRangeString string cidrRange *net.IPNet cidrBlockAssociationID string - // Key Pair is required for creating a self managed node group - keyPairName = "custom-networking-key" // Security Group that will be used in ENIConfig customNetworkingSGID string customNetworkingSGOpenPort = 8080 @@ -57,9 +54,6 @@ var ( // List of ENIConfig per Availability Zone eniConfigList []*v1alpha1.ENIConfig eniConfigBuilderList []*manifest.ENIConfigBuilder - // Properties of the self managed node group created using CFN template - nodeGroupProperties awsUtils.NodeGroupProperties - err error ) // Parse test specific variable from flag @@ -70,6 +64,7 @@ func init() { var _ = BeforeSuite(func() { f = framework.New(framework.GlobalOptions) + var err error _, cidrRange, err = net.ParseCIDR(cidrRangeString) Expect(err).ToNot(HaveOccurred()) @@ -80,10 +75,6 @@ var _ = BeforeSuite(func() { clusterVPCConfig, err = awsUtils.GetClusterVPCConfig(f) Expect(err).ToNot(HaveOccurred()) - By("creating ec2 key-pair for the new node group") - _, err := f.CloudServices.EC2().CreateKey(keyPairName) - Expect(err).ToNot(HaveOccurred()) - By("creating security group to be used by custom networking") createSecurityGroupOutput, err := f.CloudServices.EC2(). CreateSecurityGroup("custom-networking-test", "custom networking", f.Options.AWSVPCID) @@ -142,25 +133,8 @@ var _ = BeforeSuite(func() { "WARM_ENI_TARGET": "0", }) - nodeGroupProperties = awsUtils.NodeGroupProperties{ - NgLabelKey: "node-type", - NgLabelVal: "custom-networking-node", - AsgSize: 2, - NodeGroupName: "custom-networking-node", - IsCustomNetworkingEnabled: true, - Subnet: clusterVPCConfig.PublicSubnetList, - InstanceType: "c5.xlarge", - KeyPairName: keyPairName, - ContainerRuntime: f.Options.ContainerRuntime, - } - - if f.Options.InstanceType == "arm64" { - nodeGroupProperties.InstanceType = "m6g.large" - nodeGroupProperties.NodeImageId = "ami-087fca294139386b6" - } - - By("creating a new self managed node group") - err = awsUtils.CreateAndWaitTillSelfManagedNGReady(f, nodeGroupProperties) + By("terminating instances") + err = awsUtils.TerminateInstances(f) Expect(err).ToNot(HaveOccurred()) }) @@ -169,16 +143,22 @@ var _ = AfterSuite(func() { f.K8sResourceManagers.NamespaceManager(). DeleteAndWaitTillNamespaceDeleted(utils.DefaultTestNamespace) - By("waiting for some time to allow CNI to delete ENI for IP being cooled down") - time.Sleep(time.Second * 60) - var errs prometheus.MultiError - By("deleting the self managed node group") - // we just accumulate errors instead of immediately failing so we can attempt to clean up everything - errs.Append(awsUtils.DeleteAndWaitTillSelfManagedNGStackDeleted(f, nodeGroupProperties)) + for _, eniConfig := range eniConfigList { + By("deleting ENIConfig") + errs.Append(f.K8sResourceManagers.CustomResourceManager().DeleteResource(eniConfig)) + } - By("deleting the key pair") - errs.Append(f.CloudServices.EC2().DeleteKey(keyPairName)) + By("disabling custom networking on aws-node DaemonSet") + k8sUtils.RemoveVarFromDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, + utils.AwsNodeNamespace, utils.AwsNodeName, map[string]struct{}{ + "AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG": {}, + "ENI_CONFIG_LABEL_DEF": {}, + "WARM_ENI_TARGET": {}, + }) + + By("terminating instances") + errs.Append(awsUtils.TerminateInstances(f)) By("deleting security group") errs.Append(f.CloudServices.EC2().DeleteSecurityGroup(customNetworkingSGID)) @@ -191,17 +171,5 @@ var _ = AfterSuite(func() { By("disassociating the CIDR range to the VPC") errs.Append(f.CloudServices.EC2().DisAssociateVPCCIDRBlock(cidrBlockAssociationID)) - By("disabling custom networking on aws-node DaemonSet") - k8sUtils.RemoveVarFromDaemonSetAndWaitTillUpdated(f, utils.AwsNodeName, - utils.AwsNodeNamespace, utils.AwsNodeName, map[string]struct{}{ - "AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG": {}, - "ENI_CONFIG_LABEL_DEF": {}, - "WARM_ENI_TARGET": {}, - }) - - for _, eniConfig := range eniConfigList { - By("deleting ENIConfig") - errs.Append(f.K8sResourceManagers.CustomResourceManager().DeleteResource(eniConfig)) - } Expect(errs.MaybeUnwrap()).ToNot(HaveOccurred()) }) diff --git a/test/integration/custom-networking/custom_networking_test.go b/test/integration/custom-networking/custom_networking_test.go index 4b00e40d2a..ac8e91ec66 100644 --- a/test/integration/custom-networking/custom_networking_test.go +++ b/test/integration/custom-networking/custom_networking_test.go @@ -17,6 +17,7 @@ import ( "fmt" "net" "strconv" + "time" awsUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/aws/utils" "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest" @@ -51,17 +52,21 @@ var _ = Describe("Custom Networking Test", func() { Args([]string{"-k", "-l", strconv.Itoa(port)}). Build() - deployment = manifest.NewBusyBoxDeploymentBuilder(f.Options.TestImageRegistry). + deploymentBuilder := manifest.NewBusyBoxDeploymentBuilder(f.Options.TestImageRegistry). Container(container). Replicas(replicaCount). - NodeSelector(nodeGroupProperties.NgLabelKey, nodeGroupProperties.NgLabelVal). PodLabel(podLabelKey, podLabelVal). Build() + var err error deployment, err = f.K8sResourceManagers.DeploymentManager(). - CreateAndWaitTillDeploymentIsReady(deployment, utils.DefaultDeploymentReadyTimeout) + CreateAndWaitTillDeploymentIsReady(deploymentBuilder, utils.DefaultDeploymentReadyTimeout) Expect(err).ToNot(HaveOccurred()) + // Wait for deployment to settle, as if any pods restart, their pod IP will change between + // the GET and the validation. + time.Sleep(5) + podList, err = f.K8sResourceManagers.PodManager(). GetPodsWithLabelSelector(podLabelKey, podLabelVal) Expect(err).ToNot(HaveOccurred()) @@ -100,14 +105,14 @@ var _ = Describe("Custom Networking Test", func() { }) JustAfterEach(func() { - err = f.K8sResourceManagers.DeploymentManager().DeleteAndWaitTillDeploymentIsDeleted(deployment) + err := f.K8sResourceManagers.DeploymentManager().DeleteAndWaitTillDeploymentIsDeleted(deployment) Expect(err).ToNot(HaveOccurred()) }) Context("when connecting to reachable port", func() { BeforeEach(func() { port = customNetworkingSGOpenPort - replicaCount = 16 + replicaCount = 10 shouldConnect = true }) It("should connect", func() {}) @@ -127,7 +132,7 @@ var _ = Describe("Custom Networking Test", func() { JustBeforeEach(func() { By("deleting ENIConfig for all availability zones") for _, eniConfig := range eniConfigList { - err = f.K8sResourceManagers.CustomResourceManager().DeleteResource(eniConfig) + err := f.K8sResourceManagers.CustomResourceManager().DeleteResource(eniConfig) Expect(err).ToNot(HaveOccurred()) } }) @@ -135,21 +140,20 @@ var _ = Describe("Custom Networking Test", func() { JustAfterEach(func() { By("re-creating ENIConfig for all availability zones") for _, eniConfig := range eniConfigList { - err = f.K8sResourceManagers.CustomResourceManager().CreateResource(eniConfig) + err := f.K8sResourceManagers.CustomResourceManager().CreateResource(eniConfig) Expect(err).ToNot(HaveOccurred()) } }) It("deployment should not become ready", func() { By("terminating instances") - err := awsUtils.TerminateInstances(f, nodeGroupProperties.NgLabelKey, nodeGroupProperties.NgLabelVal) + err := awsUtils.TerminateInstances(f) Expect(err).ToNot(HaveOccurred()) // Nodes should be stuck in NotReady state since no ENIs could be attached and no pod // IP addresses are available. deployment := manifest.NewBusyBoxDeploymentBuilder(f.Options.TestImageRegistry). Replicas(2). - NodeSelector(nodeGroupProperties.NgLabelKey, nodeGroupProperties.NgLabelVal). Build() By("verifying deployment should not succeed") @@ -168,7 +172,7 @@ var _ = Describe("Custom Networking Test", func() { JustBeforeEach(func() { By("deleting ENIConfig for each availability zone") for _, eniConfig := range eniConfigList { - err = f.K8sResourceManagers.CustomResourceManager().DeleteResource(eniConfig) + err := f.K8sResourceManagers.CustomResourceManager().DeleteResource(eniConfig) Expect(err).ToNot(HaveOccurred()) } By("re-creating ENIConfigs with no security group") @@ -185,12 +189,11 @@ var _ = Describe("Custom Networking Test", func() { It("deployment should become ready", func() { By("terminating instances") - err := awsUtils.TerminateInstances(f, nodeGroupProperties.NgLabelKey, nodeGroupProperties.NgLabelVal) + err := awsUtils.TerminateInstances(f) Expect(err).ToNot(HaveOccurred()) deployment := manifest.NewBusyBoxDeploymentBuilder(f.Options.TestImageRegistry). Replicas(2). - NodeSelector(nodeGroupProperties.NgLabelKey, nodeGroupProperties.NgLabelVal). Build() By("verifying deployment succeeds") diff --git a/test/integration/pod-eni/security_group_per_pod_suite_test.go b/test/integration/pod-eni/security_group_per_pod_suite_test.go index 9ae1966b00..82a396aa24 100644 --- a/test/integration/pod-eni/security_group_per_pod_suite_test.go +++ b/test/integration/pod-eni/security_group_per_pod_suite_test.go @@ -105,7 +105,7 @@ var _ = BeforeSuite(func() { }) By("terminating instances") - err = awsUtils.TerminateInstances(f, f.Options.NgNameLabelKey, f.Options.NgNameLabelVal) + err = awsUtils.TerminateInstances(f) Expect(err).ToNot(HaveOccurred()) By("getting target node") @@ -122,7 +122,7 @@ var _ = AfterSuite(func() { }) By("terminating instances") - err := awsUtils.TerminateInstances(f, f.Options.NgNameLabelKey, f.Options.NgNameLabelVal) + err := awsUtils.TerminateInstances(f) Expect(err).ToNot(HaveOccurred()) By("deleting the security group")