Merge pull request #335 from Danil-Grigorev/collect-cluster-artifacts

🌱 Implement full cluster artifact gathering for both clusters after each test run

Makefile

@@ -50,6 +50,8 @@ $(TOOLS_BIN_DIR):
 	mkdir -p $@
 
 export PATH := $(abspath $(TOOLS_BIN_DIR)):$(PATH)
+export KREW_ROOT := $(abspath $(TOOLS_BIN_DIR))
+export PATH := $(KREW_ROOT)/bin:$(PATH)
 
 # Set --output-base for conversion-gen if we are not within GOPATH
 ifneq ($(abspath $(ROOT_DIR)),$(shell go env GOPATH)/src/github.com/rancher-sandbox/rancher-turtles)

hack/ensure-kubectl.sh

@@ -61,4 +61,19 @@ EOF
   fi
 }
 
+install_plugins() {
+  (
+    set -x; cd "$(mktemp -d)" &&
+    OS="$(uname | tr '[:upper:]' '[:lower:]')" &&
+    ARCH="$(uname -m | sed -e 's/x86_64/amd64/' -e 's/\(arm\)\(64\)\?.*/\1\2/' -e 's/aarch64$/arm64/')" &&
+    KREW="krew-${OS}_${ARCH}" &&
+    curl -fsSLO "https://github.com/kubernetes-sigs/krew/releases/latest/download/${KREW}.tar.gz" &&
+    tar zxvf "${KREW}.tar.gz" &&
+    ./"${KREW}" install krew
+  )
+  kubectl krew index add crust-gather https://github.com/crust-gather/crust-gather.git || true
+  kubectl krew install crust-gather/crust-gather
+}
+
 verify_kubectl_version
+install_plugins

test/e2e/specs/import_gitops.go

@@ -23,6 +23,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"path"
 	"path/filepath"
 	"strconv"
 
@@ -40,6 +41,7 @@ import (
 	provisioningv1 "github.com/rancher-sandbox/rancher-turtles/internal/rancher/provisioning/v1"
 	"github.com/rancher-sandbox/rancher-turtles/test/e2e"
 	turtlesframework "github.com/rancher-sandbox/rancher-turtles/test/framework"
+	"github.com/rancher-sandbox/rancher-turtles/test/testenv"
 	turtlesnaming "github.com/rancher-sandbox/rancher-turtles/util/naming"
 )
 
@@ -86,7 +88,9 @@ func CreateUsingGitOpsSpec(ctx context.Context, inputGetter func() CreateUsingGi
 		cancelWatches         context.CancelFunc
 		capiCluster           *types.NamespacedName
 		rancherKubeconfig     *turtlesframework.RancherGetClusterKubeconfigResult
+		originalKubeconfig    *turtlesframework.RancherGetClusterKubeconfigResult
 		rancherConnectRes     *turtlesframework.RunCommandResult
+		rancherCluster        *provisioningv1.Cluster
 		capiClusterCreateWait []interface{}
 		deleteClusterWait     []interface{}
 	)
@@ -115,6 +119,7 @@ func CreateUsingGitOpsSpec(ctx context.Context, inputGetter func() CreateUsingGi
 		}
 
 		rancherKubeconfig = new(turtlesframework.RancherGetClusterKubeconfigResult)
+		originalKubeconfig = new(turtlesframework.RancherGetClusterKubeconfigResult)
 		rancherConnectRes = new(turtlesframework.RunCommandResult)
 
 		komega.SetClient(input.BootstrapClusterProxy.GetClient())
@@ -227,8 +232,16 @@ func CreateUsingGitOpsSpec(ctx context.Context, inputGetter func() CreateUsingGi
 			return remoteClient.List(ctx, namespaces)
 		}, capiClusterCreateWait...).Should(Succeed(), "Failed to connect to workload cluster using CAPI kubeconfig")
 
+		By("Storing the original CAPI cluster kubeconfig")
+		turtlesframework.RancherGetOriginalKubeconfig(ctx, turtlesframework.RancherGetClusterKubeconfigInput{
+			Getter:          input.BootstrapClusterProxy.GetClient(),
+			SecretName:      fmt.Sprintf("%s-kubeconfig", capiCluster.Name),
+			Namespace:       capiCluster.Namespace,
+			WriteToTempFile: true,
+		}, originalKubeconfig)
+
 		By("Waiting for the rancher cluster record to appear")
-		rancherCluster := &provisioningv1.Cluster{ObjectMeta: metav1.ObjectMeta{
+		rancherCluster = &provisioningv1.Cluster{ObjectMeta: metav1.ObjectMeta{
 			Namespace: namespace.Name,
 			Name:      turtlesnaming.Name(capiCluster.Name).ToRancherName(),
 		}}
@@ -262,6 +275,14 @@ func CreateUsingGitOpsSpec(ctx context.Context, inputGetter func() CreateUsingGi
 		Expect(rancherConnectRes.Error).NotTo(HaveOccurred(), "Failed getting nodes with Rancher Kubeconfig")
 		Expect(rancherConnectRes.ExitCode).To(Equal(0), "Getting nodes return non-zero exit code")
 
+	})
+
+	AfterEach(func() {
+		err := testenv.CollectArtifacts(ctx, originalKubeconfig.TempFilePath, path.Join(input.ArtifactFolder, input.BootstrapClusterProxy.GetName(), input.ClusterName))
+		if err != nil {
+			fmt.Printf("Failed to collect artifacts for the child cluster: %v\n", err)
+		}
+
 		By("Deleting GitRepo from Rancher")
 		turtlesframework.FleetDeleteGitRepo(ctx, turtlesframework.FleetDeleteGitRepoInput{
 			Name:         repoName,
@@ -272,9 +293,6 @@ func CreateUsingGitOpsSpec(ctx context.Context, inputGetter func() CreateUsingGi
 		By("Waiting for the rancher cluster record to be removed")
 		Eventually(komega.Get(rancherCluster), deleteClusterWait...).Should(MatchError(ContainSubstring("not found")), "Rancher cluster should be deleted")
 
-	})
-
-	AfterEach(func() {
 		e2e.DumpSpecResourcesAndCleanup(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder, namespace, cancelWatches, capiCluster, input.E2EConfig.GetIntervals, input.SkipCleanup)
 	})
 }

test/e2e/suites/update-labels/update_labels_test.go

@@ -22,6 +22,7 @@ package update_labels
 import (
 	"fmt"
 	"os"
+	"path"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -34,13 +35,23 @@ import (
 	provisioningv1 "github.com/rancher-sandbox/rancher-turtles/internal/rancher/provisioning/v1"
 	"github.com/rancher-sandbox/rancher-turtles/test/e2e"
 	turtlesframework "github.com/rancher-sandbox/rancher-turtles/test/framework"
+	"github.com/rancher-sandbox/rancher-turtles/test/testenv"
 )
 
 var _ = Describe("[v2prov] [Azure] Creating a cluster with v2prov should still work with CAPI 1.5.x and label renaming", Label(e2e.FullTestLabel), func() {
 
+	var (
+		rancherKubeconfig *turtlesframework.RancherGetClusterKubeconfigResult
+		clusterName       string
+		rancherCluster    *provisioningv1.Cluster
+	)
+
 	BeforeEach(func() {
 		komega.SetClient(setupClusterResult.BootstrapClusterProxy.GetClient())
 		komega.SetContext(ctx)
+
+		rancherKubeconfig = new(turtlesframework.RancherGetClusterKubeconfigResult)
+		clusterName = "az-cluster1"
 	})
 
 	It("Should create a RKE2 cluster in Azure", func() {
@@ -57,7 +68,6 @@ var _ = Describe("[v2prov] [Azure] Creating a cluster with v2prov should still w
 		credsSecretName := "cc-test99"
 		credsName := "az-ecm"
 		poolName := "az-test-pool"
-		clusterName := "az-cluster1"
 
 		lookupResult := &turtlesframework.RancherLookupUserResult{}
 		turtlesframework.RancherLookupUser(ctx, turtlesframework.RancherLookupUserInput{
@@ -120,7 +130,7 @@ var _ = Describe("[v2prov] [Azure] Creating a cluster with v2prov should still w
 		Expect(setupClusterResult.BootstrapClusterProxy.Apply(ctx, []byte(cluster))).To(Succeed(), "Failed apply Digital Ocean cluster config")
 
 		By("Waiting for the rancher cluster record to appear")
-		rancherCluster := &provisioningv1.Cluster{ObjectMeta: metav1.ObjectMeta{
+		rancherCluster = &provisioningv1.Cluster{ObjectMeta: metav1.ObjectMeta{
 			Namespace: "fleet-default",
 			Name:      clusterName,
 		}}
@@ -133,7 +143,6 @@ var _ = Describe("[v2prov] [Azure] Creating a cluster with v2prov should still w
 		Eventually(komega.Object(rancherCluster), e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(HaveField("Status.Ready", BeTrue()))
 
 		By("Getting kubeconfig from Rancher for new cluster")
-		rancherKubeconfig := &turtlesframework.RancherGetClusterKubeconfigResult{}
 		turtlesframework.RancherGetClusterKubeconfig(ctx, turtlesframework.RancherGetClusterKubeconfigInput{
 			Getter:           setupClusterResult.BootstrapClusterProxy.GetClient(),
 			SecretName:       fmt.Sprintf("%s-kubeconfig", rancherCluster.Name),
@@ -156,6 +165,13 @@ var _ = Describe("[v2prov] [Azure] Creating a cluster with v2prov should still w
 		}, rancherConnectRes)
 		Expect(rancherConnectRes.Error).NotTo(HaveOccurred(), "Failed getting nodes with Rancher Kubeconfig")
 		Expect(rancherConnectRes.ExitCode).To(Equal(0), "Getting nodes return non-zero exit code")
+	})
+
+	AfterEach(func() {
+		err := testenv.CollectArtifacts(ctx, rancherKubeconfig.TempFilePath, path.Join(flagVals.ArtifactFolder, setupClusterResult.BootstrapClusterProxy.GetName(), clusterName))
+		if err != nil {
+			fmt.Printf("Failed to collect artifacts for the child cluster: %v\n", err)
+		}
 
 		By("Deleting cluster from Rancher")
 		err = setupClusterResult.BootstrapClusterProxy.GetClient().Delete(ctx, rancherCluster)

test/e2e/suites/v2prov/v2prov_test.go

@@ -22,6 +22,7 @@ package v2prov
 import (
 	"fmt"
 	"os"
+	"path"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -34,13 +35,23 @@ import (
 	provisioningv1 "github.com/rancher-sandbox/rancher-turtles/internal/rancher/provisioning/v1"
 	"github.com/rancher-sandbox/rancher-turtles/test/e2e"
 	turtlesframework "github.com/rancher-sandbox/rancher-turtles/test/framework"
+	"github.com/rancher-sandbox/rancher-turtles/test/testenv"
 )
 
 var _ = Describe("[v2prov] [Azure] Creating a cluster with v2prov should still work", Label(e2e.FullTestLabel), func() {
 
+	var (
+		rancherKubeconfig *turtlesframework.RancherGetClusterKubeconfigResult
+		clusterName       string
+		rancherCluster    *provisioningv1.Cluster
+	)
+
 	BeforeEach(func() {
 		komega.SetClient(setupClusterResult.BootstrapClusterProxy.GetClient())
 		komega.SetContext(ctx)
+
+		rancherKubeconfig = new(turtlesframework.RancherGetClusterKubeconfigResult)
+		clusterName = "az-cluster1"
 	})
 
 	It("Should create a RKE2 cluster in Azure", func() {
@@ -57,7 +68,6 @@ var _ = Describe("[v2prov] [Azure] Creating a cluster with v2prov should still w
 		credsSecretName := "cc-test99"
 		credsName := "az-ecm"
 		poolName := "az-test-pool"
-		clusterName := "az-cluster1"
 
 		lookupResult := &turtlesframework.RancherLookupUserResult{}
 		turtlesframework.RancherLookupUser(ctx, turtlesframework.RancherLookupUserInput{
@@ -120,7 +130,7 @@ var _ = Describe("[v2prov] [Azure] Creating a cluster with v2prov should still w
 		Expect(setupClusterResult.BootstrapClusterProxy.Apply(ctx, []byte(cluster))).To(Succeed(), "Failed apply Digital Ocean cluster config")
 
 		By("Waiting for the rancher cluster record to appear")
-		rancherCluster := &provisioningv1.Cluster{ObjectMeta: metav1.ObjectMeta{
+		rancherCluster = &provisioningv1.Cluster{ObjectMeta: metav1.ObjectMeta{
 			Namespace: "fleet-default",
 			Name:      clusterName,
 		}}
@@ -133,7 +143,6 @@ var _ = Describe("[v2prov] [Azure] Creating a cluster with v2prov should still w
 		Eventually(komega.Object(rancherCluster), e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-rancher")...).Should(HaveField("Status.Ready", BeTrue()))
 
 		By("Waiting for the CAPI cluster to be connectable using Rancher kubeconfig")
-		rancherKubeconfig := &turtlesframework.RancherGetClusterKubeconfigResult{}
 		turtlesframework.RancherGetClusterKubeconfig(ctx, turtlesframework.RancherGetClusterKubeconfigInput{
 			Getter:           setupClusterResult.BootstrapClusterProxy.GetClient(),
 			SecretName:       fmt.Sprintf("%s-kubeconfig", rancherCluster.Name),
@@ -155,6 +164,13 @@ var _ = Describe("[v2prov] [Azure] Creating a cluster with v2prov should still w
 		}, rancherConnectRes)
 		Expect(rancherConnectRes.Error).NotTo(HaveOccurred(), "Failed getting nodes with Rancher Kubeconfig")
 		Expect(rancherConnectRes.ExitCode).To(Equal(0), "Getting nodes return non-zero exit code")
+	})
+
+	AfterEach(func() {
+		err := testenv.CollectArtifacts(ctx, rancherKubeconfig.TempFilePath, path.Join(flagVals.ArtifactFolder, setupClusterResult.BootstrapClusterProxy.GetName(), clusterName))
+		if err != nil {
+			fmt.Printf("Failed to collect artifacts for the child cluster: %v\n", err)
+		}
 
 		By("Deleting cluster from Rancher")
 		err = setupClusterResult.BootstrapClusterProxy.GetClient().Delete(ctx, rancherCluster)

test/framework/rancher_helpers.go

@@ -20,16 +20,22 @@ import (
 	"context"
 	"net/url"
 	"os"
+	"runtime"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
 	corev1 "k8s.io/api/core/v1"
+	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/tools/clientcmd/api"
 	"sigs.k8s.io/cluster-api/test/framework"
+	"sigs.k8s.io/cluster-api/test/infrastructure/container"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 // RancherGetClusterKubeconfigInput is the input to RancherGetClusterKubeconfig.
@@ -62,7 +68,7 @@ func RancherGetClusterKubeconfig(ctx context.Context, input RancherGetClusterKub
 	secret := &corev1.Secret{}
 
 	err := input.Getter.Get(ctx, types.NamespacedName{Namespace: input.Namespace, Name: input.SecretName}, secret)
-	Expect(err).ShouldNot(HaveOccurred(), "Getting Rancher kubeconfig secret %s", input.SecretName)
+	Expect(err).ShouldNot(HaveOccurred(), "Getting Rancher kubeconfig secret for %s", input.SecretName)
 
 	content, ok := secret.Data["value"]
 	Expect(ok).To(BeTrue(), "Failed to find expected key in kubeconfig secret")
@@ -101,6 +107,101 @@ func RancherGetClusterKubeconfig(ctx context.Context, input RancherGetClusterKub
 	result.TempFilePath = tempFile.Name()
 }
 
+// RancherGetOriginalKubeconfig will get the unmodified Kubeconfig for a cluster from Rancher.
+func RancherGetOriginalKubeconfig(ctx context.Context, input RancherGetClusterKubeconfigInput, result *RancherGetClusterKubeconfigResult) {
+	Expect(ctx).NotTo(BeNil(), "ctx is required for RancherGetOriginalKubeconfig")
+	Expect(input.Getter).ToNot(BeNil(), "Invalid argument. input.Getter can't be nil when calling RancherGetOriginalKubeconfig")
+	Expect(input.SecretName).ToNot(BeEmpty(), "Invalid argument. input.SecretName can't be nil when calling RancherGetOriginalKubeconfig")
+
+	if input.Namespace == "" {
+		input.Namespace = DefaultNamespace
+	}
+
+	By("Getting Rancher kubeconfig secret")
+	secret := &corev1.Secret{}
+
+	err := input.Getter.Get(ctx, types.NamespacedName{Namespace: input.Namespace, Name: input.SecretName}, secret)
+	Expect(err).ShouldNot(HaveOccurred(), "Getting Rancher kubeconfig secret for %s", input.SecretName)
+
+	content, ok := secret.Data["value"]
+	Expect(ok).To(BeTrue(), "Failed to find expected key in kubeconfig secret")
+
+	By("Loading secret data into kubeconfig")
+
+	cfg, err := clientcmd.Load(content)
+	Expect(err).ShouldNot(HaveOccurred(), "Failed to unmarshall data into kubeconfig")
+
+	// if we are on mac and the cluster is a DockerCluster, it is required to fix the control plane address
+	// by using localhost:load-balancer-host-port instead of the address used in the docker network.
+	if runtime.GOOS == "darwin" && input.isDockerCluster(ctx) {
+		fixConfig(ctx, input.SecretName, cfg)
+	}
+
+	content, err = clientcmd.Write(*cfg)
+	Expect(err).NotTo(HaveOccurred(), "Failed to save original kubeconfig")
+
+	result.KubeconfigData = content
+
+	if !input.WriteToTempFile {
+		return
+	}
+
+	tempFile, err := os.CreateTemp("", "kubeconfig-original")
+	Expect(err).NotTo(HaveOccurred(), "Failed to create temp file for original kubeconfig")
+
+	Byf("Writing original kubeconfig to temp file %s", tempFile.Name())
+
+	err = clientcmd.WriteToFile(*cfg, tempFile.Name())
+	Expect(err).ShouldNot(HaveOccurred(), "Failed to write kubeconfig to file %s", tempFile.Name())
+
+	result.TempFilePath = tempFile.Name()
+}
+
+func (i *RancherGetClusterKubeconfigInput) isDockerCluster(ctx context.Context) bool {
+	cluster := &clusterv1.Cluster{}
+	key := client.ObjectKey{
+		Name:      i.SecretName,
+		Namespace: i.Namespace,
+	}
+
+	Eventually(func() error {
+		return i.Getter.Get(ctx, key, cluster)
+	}, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to get %s", key)
+
+	return cluster.Spec.InfrastructureRef.Kind == "DockerCluster"
+}
+
+func fixConfig(ctx context.Context, name string, config *api.Config) {
+	containerRuntime, err := container.NewDockerClient()
+	Expect(err).ToNot(HaveOccurred(), "Failed to get Docker runtime client")
+	ctx = container.RuntimeInto(ctx, containerRuntime)
+
+	lbContainerName := name + "-lb"
+
+	// Check if the container exists locally.
+	filters := container.FilterBuilder{}
+	filters.AddKeyValue("name", lbContainerName)
+	containers, err := containerRuntime.ListContainers(ctx, filters)
+	Expect(err).ToNot(HaveOccurred())
+	if len(containers) == 0 {
+		// Return without changing the config if the container does not exist locally.
+		// Note: This is necessary when running the tests with Tilt and a remote Docker
+		// engine as the lb container running on the remote Docker engine is accessible
+		// under its normal address but not via 127.0.0.1.
+		return
+	}
+
+	port, err := containerRuntime.GetHostPort(ctx, lbContainerName, "6443/tcp")
+	Expect(err).ToNot(HaveOccurred(), "Failed to get load balancer port")
+
+	controlPlaneURL := &url.URL{
+		Scheme: "https",
+		Host:   "127.0.0.1:" + port,
+	}
+	currentCluster := config.Contexts[config.CurrentContext].Cluster
+	config.Clusters[currentCluster].Server = controlPlaneURL.String()
+}
+
 type RancherLookupUserInput struct {
 	ClusterProxy framework.ClusterProxy
 	Username     string