From 88e0d300e9cca99901502678525e9e0d0bdc8aad Mon Sep 17 00:00:00 2001 From: Carlos Salas Date: Thu, 18 Apr 2024 13:41:52 +0200 Subject: [PATCH 1/2] test: cover cluster re-import for v1 clusters Signed-off-by: Carlos Salas --- test/e2e/specs/import_gitops.go | 104 +++++++++++++----- .../import-gitops/import_gitops_test.go | 2 +- 2 files changed, 75 insertions(+), 31 deletions(-) diff --git a/test/e2e/specs/import_gitops.go b/test/e2e/specs/import_gitops.go index 54d8fa1d..05e2541f 100644 --- a/test/e2e/specs/import_gitops.go +++ b/test/e2e/specs/import_gitops.go @@ -76,6 +76,9 @@ type CreateUsingGitOpsSpecInput struct { SkipDeletionTest bool LabelNamespace bool + + // TestClusterReimport defines whether to test un-importing and re-importing the cluster after initial test. + TestClusterReimport bool } // CreateUsingGitOpsSpec implements a spec that will create a cluster via Fleet and test that it @@ -96,6 +99,43 @@ func CreateUsingGitOpsSpec(ctx context.Context, inputGetter func() CreateUsingGi deleteClusterWait []interface{} ) + validateRancherCluster := func() { + By("Waiting for the rancher cluster record to appear") + rancherCluster = &provisioningv1.Cluster{ObjectMeta: metav1.ObjectMeta{ + Namespace: namespace.Name, + Name: turtlesnaming.Name(capiCluster.Name).ToRancherName(), + }} + Eventually(komega.Get(rancherCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(Succeed()) + + By("Waiting for the rancher cluster to have a deployed agent") + Eventually(komega.Object(rancherCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(HaveField("Status.AgentDeployed", BeTrue())) + + By("Waiting for the rancher cluster to be ready") + Eventually(komega.Object(rancherCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(HaveField("Status.Ready", BeTrue())) + + By("Waiting for the CAPI cluster to be connectable using Rancher kubeconfig") + turtlesframework.RancherGetClusterKubeconfig(ctx, turtlesframework.RancherGetClusterKubeconfigInput{ + Getter: input.BootstrapClusterProxy.GetClient(), + SecretName: fmt.Sprintf("%s-capi-kubeconfig", capiCluster.Name), + Namespace: capiCluster.Namespace, + RancherServerURL: input.RancherServerURL, + WriteToTempFile: true, + }, rancherKubeconfig) + + turtlesframework.RunCommand(ctx, turtlesframework.RunCommandInput{ + Command: "kubectl", + Args: []string{ + "--kubeconfig", + rancherKubeconfig.TempFilePath, + "get", + "nodes", + "--insecure-skip-tls-verify", + }, + }, rancherConnectRes) + Expect(rancherConnectRes.Error).NotTo(HaveOccurred(), "Failed getting nodes with Rancher Kubeconfig") + Expect(rancherConnectRes.ExitCode).To(Equal(0), "Getting nodes return non-zero exit code") + } + BeforeEach(func() { Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName) input = inputGetter() @@ -241,40 +281,44 @@ func CreateUsingGitOpsSpec(ctx context.Context, inputGetter func() CreateUsingGi WriteToTempFile: true, }, originalKubeconfig) - By("Waiting for the rancher cluster record to appear") - rancherCluster = &provisioningv1.Cluster{ObjectMeta: metav1.ObjectMeta{ - Namespace: namespace.Name, - Name: turtlesnaming.Name(capiCluster.Name).ToRancherName(), - }} - Eventually(komega.Get(rancherCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(Succeed()) + By("Running checks on Rancher cluster") + validateRancherCluster() - By("Waiting for the rancher cluster to have a deployed agent") - Eventually(komega.Object(rancherCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(HaveField("Status.AgentDeployed", BeTrue())) + if input.TestClusterReimport { + By("Deleting Rancher cluster record to simulate unimporting the cluster") + err := input.BootstrapClusterProxy.GetClient().Delete(ctx, rancherCluster) + Expect(err).NotTo(HaveOccurred(), "Failed to delete rancher cluster") - By("Waiting for the rancher cluster to be ready") - Eventually(komega.Object(rancherCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(HaveField("Status.Ready", BeTrue())) + By("CAPI cluster should have the 'imported' annotation") + Eventually(func() bool { + Eventually(komega.Get(capiCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(Succeed()) + annotations := capiCluster.GetAnnotations() - By("Waiting for the CAPI cluster to be connectable using Rancher kubeconfig") - turtlesframework.RancherGetClusterKubeconfig(ctx, turtlesframework.RancherGetClusterKubeconfigInput{ - Getter: input.BootstrapClusterProxy.GetClient(), - SecretName: fmt.Sprintf("%s-capi-kubeconfig", capiCluster.Name), - Namespace: capiCluster.Namespace, - RancherServerURL: input.RancherServerURL, - WriteToTempFile: true, - }, rancherKubeconfig) + return annotations["imported"] == "true" + }, capiClusterCreateWait...).Should(BeTrue(), "Failed to detect 'imported' annotation on CAPI cluster") - turtlesframework.RunCommand(ctx, turtlesframework.RunCommandInput{ - Command: "kubectl", - Args: []string{ - "--kubeconfig", - rancherKubeconfig.TempFilePath, - "get", - "nodes", - "--insecure-skip-tls-verify", - }, - }, rancherConnectRes) - Expect(rancherConnectRes.Error).NotTo(HaveOccurred(), "Failed getting nodes with Rancher Kubeconfig") - Expect(rancherConnectRes.ExitCode).To(Equal(0), "Getting nodes return non-zero exit code") + By("Waiting for the Rancher cluster record to be removed") + Eventually(komega.Get(rancherCluster), deleteClusterWait...).Should(MatchError(ContainSubstring("not found")), "Rancher cluster should be unimported (deleted)") + + By("Removing 'imported' annotation from CAPI cluster") + Eventually(komega.Get(capiCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(Succeed()) + annotations := capiCluster.GetAnnotations() + delete(annotations, "imported") + capiCluster.SetAnnotations(annotations) + err = input.BootstrapClusterProxy.GetClient().Update(ctx, capiCluster) + Expect(err).NotTo(HaveOccurred(), "Failed to remove 'imported' annotation from CAPI cluster") + + By("Validating annotation is removed from CAPI cluster") + Eventually(func() bool { + Eventually(komega.Get(capiCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(Succeed()) + annotations := capiCluster.GetAnnotations() + + return annotations["imported"] != "true" + }, capiClusterCreateWait...).Should(BeTrue(), "CAPI cluster still contains the 'imported' annotation") + + By("Rancher should be available after removing 'imported' annotation") + validateRancherCluster() + } }) AfterEach(func() { diff --git a/test/e2e/suites/import-gitops/import_gitops_test.go b/test/e2e/suites/import-gitops/import_gitops_test.go index adff4043..a93d6023 100644 --- a/test/e2e/suites/import-gitops/import_gitops_test.go +++ b/test/e2e/suites/import-gitops/import_gitops_test.go @@ -33,7 +33,6 @@ import ( ) var _ = Describe("[Docker] [Kubeadm] Create and delete CAPI cluster functionality should work with namespace auto-import", Label(e2e.ShortTestLabel), func() { - BeforeEach(func() { SetClient(setupClusterResult.BootstrapClusterProxy.GetClient()) SetContext(ctx) @@ -55,6 +54,7 @@ var _ = Describe("[Docker] [Kubeadm] Create and delete CAPI cluster functionalit SkipCleanup: false, SkipDeletionTest: false, LabelNamespace: true, + TestClusterReimport: true, RancherServerURL: hostName, CAPIClusterCreateWaitName: "wait-rancher", DeleteClusterWaitName: "wait-controllers", From baf7661180a3f418b91b858d2ad23bb0b19f1a00 Mon Sep 17 00:00:00 2001 From: Carlos Salas Date: Thu, 18 Apr 2024 13:43:33 +0200 Subject: [PATCH 2/2] test: cover cluster re-import for v3 clusters Signed-off-by: Carlos Salas --- test/e2e/specs/import_gitops_mgmtv3.go | 135 ++++++++++++------ .../suites/managementv3/managementv3_test.go | 1 + 2 files changed, 91 insertions(+), 45 deletions(-) diff --git a/test/e2e/specs/import_gitops_mgmtv3.go b/test/e2e/specs/import_gitops_mgmtv3.go index e8360bc5..823abca7 100644 --- a/test/e2e/specs/import_gitops_mgmtv3.go +++ b/test/e2e/specs/import_gitops_mgmtv3.go @@ -77,6 +77,9 @@ type CreateMgmtV3UsingGitOpsSpecInput struct { LabelNamespace bool + // TestClusterReimport defines whether to test un-importing and re-importing the cluster after initial test. + TestClusterReimport bool + // management.cattle.io specifc CapiClusterOwnerLabel string CapiClusterOwnerNamespaceLabel string @@ -101,6 +104,58 @@ func CreateMgmtV3UsingGitOpsSpec(ctx context.Context, inputGetter func() CreateM deleteClusterWait []interface{} ) + validateRancherCluster := func() { + By("Waiting for the rancher cluster record to appear") + rancherClusters := &managementv3.ClusterList{} + selectors := []client.ListOption{ + client.MatchingLabels{ + input.CapiClusterOwnerLabel: capiCluster.Name, + input.CapiClusterOwnerNamespaceLabel: capiCluster.Namespace, + input.OwnedLabelName: "", + }, + } + Eventually(func() bool { + Eventually(komega.List(rancherClusters, selectors...)).Should(Succeed()) + return len(rancherClusters.Items) == 1 + }, input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(BeTrue()) + rancherCluster = &rancherClusters.Items[0] + Eventually(komega.Get(rancherCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(Succeed()) + + By("Waiting for the rancher cluster to have a deployed agent") + Eventually(func() bool { + Eventually(komega.Get(rancherCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(Succeed()) + return conditions.IsTrue(rancherCluster, managementv3.ClusterConditionAgentDeployed) + }, input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(BeTrue()) + + By("Waiting for the rancher cluster to be ready") + Eventually(func() bool { + Eventually(komega.Get(rancherCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(Succeed()) + return conditions.IsTrue(rancherCluster, managementv3.ClusterConditionReady) + }, input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(BeTrue()) + + By("Waiting for the CAPI cluster to be connectable using Rancher kubeconfig") + turtlesframework.RancherGetClusterKubeconfig(ctx, turtlesframework.RancherGetClusterKubeconfigInput{ + Getter: input.BootstrapClusterProxy.GetClient(), + SecretName: fmt.Sprintf("%s-kubeconfig", rancherCluster.Name), + Namespace: rancherCluster.Spec.FleetWorkspaceName, + RancherServerURL: input.RancherServerURL, + WriteToTempFile: true, + }, rancherKubeconfig) + + turtlesframework.RunCommand(ctx, turtlesframework.RunCommandInput{ + Command: "kubectl", + Args: []string{ + "--kubeconfig", + rancherKubeconfig.TempFilePath, + "get", + "nodes", + "--insecure-skip-tls-verify", + }, + }, rancherConnectRes) + Expect(rancherConnectRes.Error).NotTo(HaveOccurred(), "Failed getting nodes with Rancher Kubeconfig") + Expect(rancherConnectRes.ExitCode).To(Equal(0), "Getting nodes return non-zero exit code") + } + BeforeEach(func() { Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName) input = inputGetter() @@ -246,55 +301,45 @@ func CreateMgmtV3UsingGitOpsSpec(ctx context.Context, inputGetter func() CreateM WriteToTempFile: true, }, originalKubeconfig) - By("Waiting for the rancher cluster record to appear") - rancherClusters := &managementv3.ClusterList{} - selectors := []client.ListOption{ - client.MatchingLabels{ - input.CapiClusterOwnerLabel: capiCluster.Name, - input.CapiClusterOwnerNamespaceLabel: capiCluster.Namespace, - input.OwnedLabelName: "", - }, - } - Eventually(func() bool { - Eventually(komega.List(rancherClusters, selectors...)).Should(Succeed()) - return len(rancherClusters.Items) == 1 - }, input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(BeTrue()) - rancherCluster = &rancherClusters.Items[0] - Eventually(komega.Get(rancherCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(Succeed()) + By("Running checks on Rancher cluster") + validateRancherCluster() - By("Waiting for the rancher cluster to have a deployed agent") - Eventually(func() bool { - Eventually(komega.Get(rancherCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(Succeed()) - return conditions.IsTrue(rancherCluster, managementv3.ClusterConditionAgentDeployed) - }, input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(BeTrue()) + if input.TestClusterReimport { + By("Deleting Rancher cluster record to simulate unimporting the cluster") + err := input.BootstrapClusterProxy.GetClient().Delete(ctx, rancherCluster) + Expect(err).NotTo(HaveOccurred(), "Failed to delete rancher cluster") - By("Waiting for the rancher cluster to be ready") - Eventually(func() bool { - Eventually(komega.Get(rancherCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(Succeed()) - return conditions.IsTrue(rancherCluster, managementv3.ClusterConditionReady) - }, input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(BeTrue()) + By("CAPI cluster should have the 'imported' annotation") + Eventually(func() bool { + Eventually(komega.Get(capiCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(Succeed()) + annotations := capiCluster.GetAnnotations() - By("Waiting for the CAPI cluster to be connectable using Rancher kubeconfig") - turtlesframework.RancherGetClusterKubeconfig(ctx, turtlesframework.RancherGetClusterKubeconfigInput{ - Getter: input.BootstrapClusterProxy.GetClient(), - SecretName: fmt.Sprintf("%s-kubeconfig", rancherCluster.Name), - Namespace: rancherCluster.Spec.FleetWorkspaceName, - RancherServerURL: input.RancherServerURL, - WriteToTempFile: true, - }, rancherKubeconfig) + return annotations["imported"] == "true" + }, capiClusterCreateWait...).Should(BeTrue(), "Failed to detect 'imported' annotation on CAPI cluster") - turtlesframework.RunCommand(ctx, turtlesframework.RunCommandInput{ - Command: "kubectl", - Args: []string{ - "--kubeconfig", - rancherKubeconfig.TempFilePath, - "get", - "nodes", - "--insecure-skip-tls-verify", - }, - }, rancherConnectRes) - Expect(rancherConnectRes.Error).NotTo(HaveOccurred(), "Failed getting nodes with Rancher Kubeconfig") - Expect(rancherConnectRes.ExitCode).To(Equal(0), "Getting nodes return non-zero exit code") + By("Waiting for the Rancher cluster record to be removed") + Eventually(komega.Get(rancherCluster), deleteClusterWait...).Should(MatchError(ContainSubstring("not found")), "Rancher cluster should be unimported (deleted)") + + By("Removing 'imported' annotation from CAPI cluster") + Eventually(komega.Get(capiCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(Succeed()) + annotations := capiCluster.GetAnnotations() + delete(annotations, "imported") + capiCluster.SetAnnotations(annotations) + err = input.BootstrapClusterProxy.GetClient().Update(ctx, capiCluster) + Expect(err).NotTo(HaveOccurred(), "Failed to remove 'imported' annotation from CAPI cluster") + + By("Validating annotation is removed from CAPI cluster") + Eventually(func() bool { + Eventually(komega.Get(capiCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(Succeed()) + annotations := capiCluster.GetAnnotations() + fmt.Printf("Annotations: %v\n", annotations) + + return annotations["imported"] != "true" + }, capiClusterCreateWait...).Should(BeTrue(), "CAPI cluster still contains the 'imported' annotation") + + By("Rancher should be available after removing 'imported' annotation") + validateRancherCluster() + } }) AfterEach(func() { diff --git a/test/e2e/suites/managementv3/managementv3_test.go b/test/e2e/suites/managementv3/managementv3_test.go index ceb6fb9e..77ce0287 100644 --- a/test/e2e/suites/managementv3/managementv3_test.go +++ b/test/e2e/suites/managementv3/managementv3_test.go @@ -51,6 +51,7 @@ var _ = Describe("[Docker] [Kubeadm] - [management.cattle.io/v3] Create and dele SkipCleanup: false, SkipDeletionTest: false, LabelNamespace: true, + TestClusterReimport: true, RancherServerURL: hostName, CAPIClusterCreateWaitName: "wait-rancher", DeleteClusterWaitName: "wait-controllers",