Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

🌱 cover cluster reimport after running e2e #484

Merged
merged 2 commits into from
Apr 19, 2024
Merged

🌱 cover cluster reimport after running e2e #484

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
88e0d30
test: cover cluster re-import for v1 clusters
salasberryfin Apr 18, 2024
baf7661
test: cover cluster re-import for v3 clusters
salasberryfin Apr 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
104 changes: 74 additions & 30 deletions test/e2e/specs/import_gitops.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,9 @@ type CreateUsingGitOpsSpecInput struct {
SkipDeletionTest bool

LabelNamespace bool

// TestClusterReimport defines whether to test un-importing and re-importing the cluster after initial test.
TestClusterReimport bool
}

// CreateUsingGitOpsSpec implements a spec that will create a cluster via Fleet and test that it
Expand All @@ -96,6 +99,43 @@ func CreateUsingGitOpsSpec(ctx context.Context, inputGetter func() CreateUsingGi
deleteClusterWait []interface{}
)

validateRancherCluster := func() {
By("Waiting for the rancher cluster record to appear")
rancherCluster = &provisioningv1.Cluster{ObjectMeta: metav1.ObjectMeta{
Namespace: namespace.Name,
Name: turtlesnaming.Name(capiCluster.Name).ToRancherName(),
}}
Eventually(komega.Get(rancherCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(Succeed())

By("Waiting for the rancher cluster to have a deployed agent")
Eventually(komega.Object(rancherCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(HaveField("Status.AgentDeployed", BeTrue()))

By("Waiting for the rancher cluster to be ready")
Eventually(komega.Object(rancherCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(HaveField("Status.Ready", BeTrue()))

By("Waiting for the CAPI cluster to be connectable using Rancher kubeconfig")
turtlesframework.RancherGetClusterKubeconfig(ctx, turtlesframework.RancherGetClusterKubeconfigInput{
Getter: input.BootstrapClusterProxy.GetClient(),
SecretName: fmt.Sprintf("%s-capi-kubeconfig", capiCluster.Name),
Namespace: capiCluster.Namespace,
RancherServerURL: input.RancherServerURL,
WriteToTempFile: true,
}, rancherKubeconfig)

turtlesframework.RunCommand(ctx, turtlesframework.RunCommandInput{
Command: "kubectl",
Args: []string{
"--kubeconfig",
rancherKubeconfig.TempFilePath,
"get",
"nodes",
"--insecure-skip-tls-verify",
},
}, rancherConnectRes)
Expect(rancherConnectRes.Error).NotTo(HaveOccurred(), "Failed getting nodes with Rancher Kubeconfig")
Expect(rancherConnectRes.ExitCode).To(Equal(0), "Getting nodes return non-zero exit code")
}

BeforeEach(func() {
Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName)
input = inputGetter()
Expand Down Expand Up @@ -241,40 +281,44 @@ func CreateUsingGitOpsSpec(ctx context.Context, inputGetter func() CreateUsingGi
WriteToTempFile: true,
}, originalKubeconfig)

By("Waiting for the rancher cluster record to appear")
rancherCluster = &provisioningv1.Cluster{ObjectMeta: metav1.ObjectMeta{
Namespace: namespace.Name,
Name: turtlesnaming.Name(capiCluster.Name).ToRancherName(),
}}
Eventually(komega.Get(rancherCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(Succeed())
By("Running checks on Rancher cluster")
validateRancherCluster()

By("Waiting for the rancher cluster to have a deployed agent")
Eventually(komega.Object(rancherCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(HaveField("Status.AgentDeployed", BeTrue()))
if input.TestClusterReimport {
By("Deleting Rancher cluster record to simulate unimporting the cluster")
err := input.BootstrapClusterProxy.GetClient().Delete(ctx, rancherCluster)
Expect(err).NotTo(HaveOccurred(), "Failed to delete rancher cluster")

By("Waiting for the rancher cluster to be ready")
Eventually(komega.Object(rancherCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(HaveField("Status.Ready", BeTrue()))
By("CAPI cluster should have the 'imported' annotation")
Eventually(func() bool {
Eventually(komega.Get(capiCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(Succeed())
annotations := capiCluster.GetAnnotations()

By("Waiting for the CAPI cluster to be connectable using Rancher kubeconfig")
turtlesframework.RancherGetClusterKubeconfig(ctx, turtlesframework.RancherGetClusterKubeconfigInput{
Getter: input.BootstrapClusterProxy.GetClient(),
SecretName: fmt.Sprintf("%s-capi-kubeconfig", capiCluster.Name),
Namespace: capiCluster.Namespace,
RancherServerURL: input.RancherServerURL,
WriteToTempFile: true,
}, rancherKubeconfig)
return annotations["imported"] == "true"
}, capiClusterCreateWait...).Should(BeTrue(), "Failed to detect 'imported' annotation on CAPI cluster")

turtlesframework.RunCommand(ctx, turtlesframework.RunCommandInput{
Command: "kubectl",
Args: []string{
"--kubeconfig",
rancherKubeconfig.TempFilePath,
"get",
"nodes",
"--insecure-skip-tls-verify",
},
}, rancherConnectRes)
Expect(rancherConnectRes.Error).NotTo(HaveOccurred(), "Failed getting nodes with Rancher Kubeconfig")
Expect(rancherConnectRes.ExitCode).To(Equal(0), "Getting nodes return non-zero exit code")
By("Waiting for the Rancher cluster record to be removed")
Eventually(komega.Get(rancherCluster), deleteClusterWait...).Should(MatchError(ContainSubstring("not found")), "Rancher cluster should be unimported (deleted)")

By("Removing 'imported' annotation from CAPI cluster")
Eventually(komega.Get(capiCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(Succeed())
annotations := capiCluster.GetAnnotations()
delete(annotations, "imported")
capiCluster.SetAnnotations(annotations)
err = input.BootstrapClusterProxy.GetClient().Update(ctx, capiCluster)
Expect(err).NotTo(HaveOccurred(), "Failed to remove 'imported' annotation from CAPI cluster")

By("Validating annotation is removed from CAPI cluster")
Eventually(func() bool {
Eventually(komega.Get(capiCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(Succeed())
annotations := capiCluster.GetAnnotations()

return annotations["imported"] != "true"
}, capiClusterCreateWait...).Should(BeTrue(), "CAPI cluster still contains the 'imported' annotation")

By("Rancher should be available after removing 'imported' annotation")
validateRancherCluster()
}
})

AfterEach(func() {
Expand Down
135 changes: 90 additions & 45 deletions test/e2e/specs/import_gitops_mgmtv3.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,9 @@ type CreateMgmtV3UsingGitOpsSpecInput struct {

LabelNamespace bool

// TestClusterReimport defines whether to test un-importing and re-importing the cluster after initial test.
TestClusterReimport bool

// management.cattle.io specifc
CapiClusterOwnerLabel string
CapiClusterOwnerNamespaceLabel string
Expand All @@ -101,6 +104,58 @@ func CreateMgmtV3UsingGitOpsSpec(ctx context.Context, inputGetter func() CreateM
deleteClusterWait []interface{}
)

validateRancherCluster := func() {
By("Waiting for the rancher cluster record to appear")
rancherClusters := &managementv3.ClusterList{}
selectors := []client.ListOption{
client.MatchingLabels{
input.CapiClusterOwnerLabel: capiCluster.Name,
input.CapiClusterOwnerNamespaceLabel: capiCluster.Namespace,
input.OwnedLabelName: "",
},
}
Eventually(func() bool {
Eventually(komega.List(rancherClusters, selectors...)).Should(Succeed())
return len(rancherClusters.Items) == 1
}, input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(BeTrue())
rancherCluster = &rancherClusters.Items[0]
Eventually(komega.Get(rancherCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(Succeed())

By("Waiting for the rancher cluster to have a deployed agent")
Eventually(func() bool {
Eventually(komega.Get(rancherCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(Succeed())
return conditions.IsTrue(rancherCluster, managementv3.ClusterConditionAgentDeployed)
}, input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(BeTrue())

By("Waiting for the rancher cluster to be ready")
Eventually(func() bool {
Eventually(komega.Get(rancherCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(Succeed())
return conditions.IsTrue(rancherCluster, managementv3.ClusterConditionReady)
}, input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(BeTrue())

By("Waiting for the CAPI cluster to be connectable using Rancher kubeconfig")
turtlesframework.RancherGetClusterKubeconfig(ctx, turtlesframework.RancherGetClusterKubeconfigInput{
Getter: input.BootstrapClusterProxy.GetClient(),
SecretName: fmt.Sprintf("%s-kubeconfig", rancherCluster.Name),
Namespace: rancherCluster.Spec.FleetWorkspaceName,
RancherServerURL: input.RancherServerURL,
WriteToTempFile: true,
}, rancherKubeconfig)

turtlesframework.RunCommand(ctx, turtlesframework.RunCommandInput{
Command: "kubectl",
Args: []string{
"--kubeconfig",
rancherKubeconfig.TempFilePath,
"get",
"nodes",
"--insecure-skip-tls-verify",
},
}, rancherConnectRes)
Expect(rancherConnectRes.Error).NotTo(HaveOccurred(), "Failed getting nodes with Rancher Kubeconfig")
Expect(rancherConnectRes.ExitCode).To(Equal(0), "Getting nodes return non-zero exit code")
}

BeforeEach(func() {
Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName)
input = inputGetter()
Expand Down Expand Up @@ -246,55 +301,45 @@ func CreateMgmtV3UsingGitOpsSpec(ctx context.Context, inputGetter func() CreateM
WriteToTempFile: true,
}, originalKubeconfig)

By("Waiting for the rancher cluster record to appear")
rancherClusters := &managementv3.ClusterList{}
selectors := []client.ListOption{
client.MatchingLabels{
input.CapiClusterOwnerLabel: capiCluster.Name,
input.CapiClusterOwnerNamespaceLabel: capiCluster.Namespace,
input.OwnedLabelName: "",
},
}
Eventually(func() bool {
Eventually(komega.List(rancherClusters, selectors...)).Should(Succeed())
return len(rancherClusters.Items) == 1
}, input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(BeTrue())
rancherCluster = &rancherClusters.Items[0]
Eventually(komega.Get(rancherCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(Succeed())
By("Running checks on Rancher cluster")
validateRancherCluster()

By("Waiting for the rancher cluster to have a deployed agent")
Eventually(func() bool {
Eventually(komega.Get(rancherCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(Succeed())
return conditions.IsTrue(rancherCluster, managementv3.ClusterConditionAgentDeployed)
}, input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(BeTrue())
if input.TestClusterReimport {
By("Deleting Rancher cluster record to simulate unimporting the cluster")
err := input.BootstrapClusterProxy.GetClient().Delete(ctx, rancherCluster)
Expect(err).NotTo(HaveOccurred(), "Failed to delete rancher cluster")

By("Waiting for the rancher cluster to be ready")
Eventually(func() bool {
Eventually(komega.Get(rancherCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(Succeed())
return conditions.IsTrue(rancherCluster, managementv3.ClusterConditionReady)
}, input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(BeTrue())
By("CAPI cluster should have the 'imported' annotation")
Eventually(func() bool {
Eventually(komega.Get(capiCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(Succeed())
annotations := capiCluster.GetAnnotations()

By("Waiting for the CAPI cluster to be connectable using Rancher kubeconfig")
turtlesframework.RancherGetClusterKubeconfig(ctx, turtlesframework.RancherGetClusterKubeconfigInput{
Getter: input.BootstrapClusterProxy.GetClient(),
SecretName: fmt.Sprintf("%s-kubeconfig", rancherCluster.Name),
Namespace: rancherCluster.Spec.FleetWorkspaceName,
RancherServerURL: input.RancherServerURL,
WriteToTempFile: true,
}, rancherKubeconfig)
return annotations["imported"] == "true"
}, capiClusterCreateWait...).Should(BeTrue(), "Failed to detect 'imported' annotation on CAPI cluster")

turtlesframework.RunCommand(ctx, turtlesframework.RunCommandInput{
Command: "kubectl",
Args: []string{
"--kubeconfig",
rancherKubeconfig.TempFilePath,
"get",
"nodes",
"--insecure-skip-tls-verify",
},
}, rancherConnectRes)
Expect(rancherConnectRes.Error).NotTo(HaveOccurred(), "Failed getting nodes with Rancher Kubeconfig")
Expect(rancherConnectRes.ExitCode).To(Equal(0), "Getting nodes return non-zero exit code")
By("Waiting for the Rancher cluster record to be removed")
Eventually(komega.Get(rancherCluster), deleteClusterWait...).Should(MatchError(ContainSubstring("not found")), "Rancher cluster should be unimported (deleted)")

By("Removing 'imported' annotation from CAPI cluster")
Eventually(komega.Get(capiCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(Succeed())
annotations := capiCluster.GetAnnotations()
delete(annotations, "imported")
capiCluster.SetAnnotations(annotations)
err = input.BootstrapClusterProxy.GetClient().Update(ctx, capiCluster)
Expect(err).NotTo(HaveOccurred(), "Failed to remove 'imported' annotation from CAPI cluster")

By("Validating annotation is removed from CAPI cluster")
Eventually(func() bool {
Eventually(komega.Get(capiCluster), input.E2EConfig.GetIntervals(input.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(Succeed())
annotations := capiCluster.GetAnnotations()
fmt.Printf("Annotations: %v\n", annotations)

return annotations["imported"] != "true"
}, capiClusterCreateWait...).Should(BeTrue(), "CAPI cluster still contains the 'imported' annotation")

By("Rancher should be available after removing 'imported' annotation")
validateRancherCluster()
}
})

AfterEach(func() {
Expand Down
2 changes: 1 addition & 1 deletion test/e2e/suites/import-gitops/import_gitops_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ import (
)

var _ = Describe("[Docker] [Kubeadm] Create and delete CAPI cluster functionality should work with namespace auto-import", Label(e2e.ShortTestLabel), func() {

BeforeEach(func() {
SetClient(setupClusterResult.BootstrapClusterProxy.GetClient())
SetContext(ctx)
Expand All @@ -55,6 +54,7 @@ var _ = Describe("[Docker] [Kubeadm] Create and delete CAPI cluster functionalit
SkipCleanup: false,
SkipDeletionTest: false,
LabelNamespace: true,
TestClusterReimport: true,
RancherServerURL: hostName,
CAPIClusterCreateWaitName: "wait-rancher",
DeleteClusterWaitName: "wait-controllers",
Expand Down
1 change: 1 addition & 0 deletions test/e2e/suites/managementv3/managementv3_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ var _ = Describe("[Docker] [Kubeadm] - [management.cattle.io/v3] Create and dele
SkipCleanup: false,
SkipDeletionTest: false,
LabelNamespace: true,
TestClusterReimport: true,
RancherServerURL: hostName,
CAPIClusterCreateWaitName: "wait-rancher",
DeleteClusterWaitName: "wait-controllers",
Expand Down
Loading