🌱 Add more supervisor tests

Makefile

@@ -377,6 +377,8 @@ generate-e2e-templates-main: $(KUSTOMIZE) ## Generate test templates for the mai
 	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/clusterclass" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/clusterclass-quick-start-supervisor.yaml"
 	cp "$(RELEASE_DIR)/main/cluster-template-topology-supervisor.yaml" "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/topology/cluster-template-topology-supervisor.yaml"
 	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/topology" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-topology-supervisor.yaml"
+	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/conformance" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-conformance-supervisor.yaml"
+	"$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/install-on-bootstrap" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-install-on-bootstrap-supervisor.yaml"
 
 .PHONY: generate-e2e-templates-v1.9
 generate-e2e-templates-v1.9: $(KUSTOMIZE)

test/e2e/cluster_upgrade_test.go

@@ -26,7 +26,7 @@ import (
 	"sigs.k8s.io/cluster-api/test/framework/kubernetesversions"
 )
 
-var _ = Describe("When upgrading a workload cluster using ClusterClass and testing K8S conformance [Conformance] [K8s-Upgrade] [ClusterClass]", func() {
+var _ = Describe("When upgrading a workload cluster using ClusterClass and testing K8S conformance [supervisor] [Conformance] [K8s-Upgrade] [ClusterClass]", func() {
 	// Note: This installs a cluster based on KUBERNETES_VERSION_UPGRADE_FROM and then upgrades to
 	// KUBERNETES_VERSION_UPGRADE_TO and runs conformance tests.
 	// Note: We are resolving KUBERNETES_VERSION_UPGRADE_FROM and KUBERNETES_VERSION_UPGRADE_TO and then setting
@@ -58,7 +58,7 @@ var _ = Describe("When upgrading a workload cluster using ClusterClass and testi
 	})
 })
 
-var _ = Describe("When upgrading a workload cluster using ClusterClass [ClusterClass]", func() {
+var _ = Describe("When upgrading a workload cluster using ClusterClass [supervisor] [ClusterClass]", func() {
 	// Note: This installs a cluster based on KUBERNETES_VERSION_UPGRADE_FROM and then upgrades to
 	// KUBERNETES_VERSION_UPGRADE_TO.
 	const specName = "k8s-upgrade" // aligned to CAPI

test/e2e/config/vsphere.yaml

@@ -175,6 +175,8 @@ providers:
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-topology-supervisor.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-supervisor.yaml"
           - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass-quick-start-supervisor.yaml"
+          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-install-on-bootstrap-supervisor.yaml"
+          - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-conformance-supervisor.yaml"
           - sourcePath: "../data/shared/capv/main/metadata.yaml"
       - name: v1.9.0
         # Use manifest from source files
@@ -246,6 +248,9 @@ variables:
   VSPHERE_MACHINE_CLASS_NAME: "test-machine-class"
   VSPHERE_MACHINE_CLASS_CPU: "4"
   VSPHERE_MACHINE_CLASS_MEMORY: "8Gi"
+  VSPHERE_MACHINE_CLASS_NAME_CONFORMANCE: "test-machine-class-conformance"
+  VSPHERE_MACHINE_CLASS_CPU_CONFORMANCE: "8"
+  VSPHERE_MACHINE_CLASS_MEMORY_CONFORMANCE: "8Gi"
   VSPHERE_CONTENT_LIBRARY: "capv"
   VSPHERE_IMAGE_NAME: "ubuntu-2204-kube-v1.29.0"
   VSPHERE_NETWORK: "sddc-cgw-network-6"

test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass/patch-prekubeadmscript.yaml

@@ -4,7 +4,8 @@
     definitions:
     - jsonPatches:
       - op: add
-        path: /spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/0
+        # Note: We are adding prekubeadmscript.sh (install on bootstrap) at the end of the preKubeadmCommands because getting an ip for the machine (dhclient eth0) must always be executed as a first command.
+        path: /spec/template/spec/kubeadmConfigSpec/preKubeadmCommands/-
         value: "/opt/prekubeadmscript.sh"
       - op: add
         path: /spec/template/spec/kubeadmConfigSpec/files/-
@@ -21,7 +22,8 @@
           controlPlane: true
     - jsonPatches:
       - op: add
-        path: /spec/template/spec/preKubeadmCommands/0
+        # Note: We are adding prekubeadmscript.sh (install on bootstrap) at the end of the preKubeadmCommands because getting an ip for the machine (dhclient eth0) must always be executed as a first command.
+        path: /spec/template/spec/preKubeadmCommands/-
         value: "/opt/prekubeadmscript.sh"
       - op: add
         path: /spec/template/spec/files/-

test/e2e/data/infrastructure-vsphere-supervisor/main/conformance/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ../base
+patchesStrategicMerge:
+  - worker-node-size.yaml

test/e2e/data/infrastructure-vsphere-supervisor/main/conformance/worker-node-size.yaml

@@ -0,0 +1,10 @@
+---
+apiVersion: vmware.infrastructure.cluster.x-k8s.io/v1beta1
+kind: VSphereMachineTemplate
+metadata:
+  name: ${CLUSTER_NAME}-worker
+  namespace: ${NAMESPACE}
+spec:
+  template:
+    spec:
+      className: ${VSPHERE_MACHINE_CLASS_NAME_CONFORMANCE}

test/e2e/data/infrastructure-vsphere-supervisor/main/install-on-bootstrap/inject-install-on-bootstrap.yaml

@@ -0,0 +1,109 @@
+# Note: Prior art: https://github.com/kubernetes-sigs/cluster-api/blob/21a350718f5e84a580bc677213bc25faec3291e7/test/framework/kubernetesversions/data/debian_injection_script.envsubst.sh.tpl
+- op: add
+  path: /spec/topology/variables/-
+  value:
+    name: preKubeadmScript
+    # Note: This file has to be escaped as it is run through envsubst by clusterctl
+    value: |
+      #!/bin/bash
+
+      set -o nounset
+      set -o pipefail
+      set -o errexit
+
+      function retry {
+        attempt=0
+        max_attempts=$${1}
+        interval=$${2}
+        shift; shift
+        until [[ $${attempt} -ge "$${max_attempts}" ]] ; do
+          attempt=$((attempt+1))
+          set +e
+          eval "$*" && return || echo "failed $${attempt} times: $*"
+          set -e
+          sleep "$${interval}"
+        done
+        echo "error: reached max attempts at retry($*)"
+        return 1
+      }
+
+      [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO=""
+
+      # This test installs release packages or binaries that are a result of the CI and release builds.
+      # It runs '... --version' commands to verify that the binaries are correctly installed
+      # and finally uninstalls the packages.
+      # For the release packages it tests all versions in the support skew.
+      LINE_SEPARATOR="*************************************************"
+      echo "$${LINE_SEPARATOR}"
+
+      ## Variables (replaced by JSON patching)
+      KUBERNETES_VERSION=$${KUBERNETES_VERSION}
+      ##
+
+      # Note: We assume if kubectl has the right version, everything else has as well
+      if [[ $(kubectl version --client=true -o json | jq '.clientVersion.gitVersion' -r) = "$${KUBERNETES_VERSION}" ]]; then
+        echo "Detected Kubernetes $${KUBERNETES_VERSION} via kubectl version, nothing to do"
+        exit 0
+      fi
+
+      if [[ "$${KUBERNETES_VERSION}" != "" ]]; then
+        CI_DIR=/tmp/k8s-ci
+        mkdir -p "$${CI_DIR}"
+        declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm")
+        # Let's just also download the control plane images for worker nodes. It's easier then optimizing it.
+        declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler")
+        CONTAINER_EXT="tar"
+        echo "* testing CI version $${KUBERNETES_VERSION}"
+        # Check for semver
+        if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+          CI_URL="https://dl.k8s.io/release/$${KUBERNETES_VERSION}/bin/linux/amd64"
+          VERSION_WITHOUT_PREFIX="$${KUBERNETES_VERSION#v}"
+          export DEBIAN_FRONTEND=noninteractive
+          # sometimes the network is not immediately available, so we have to retry the apt-get update
+          retry 10 5 "apt-get update"
+          apt-get install -y apt-transport-https ca-certificates curl gpg
+          mkdir -p -m 755 /etc/apt/keyrings
+          curl -fsSL https://pkgs.k8s.io/core:/stable:/$${KUBERNETES_VERSION:0:5}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
+          echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/$${KUBERNETES_VERSION:0:5}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list
+          apt-get update
+          # replace . with \.
+          VERSION_REGEX="$${VERSION_WITHOUT_PREFIX//./\\.}"
+          PACKAGE_VERSION="$(apt-cache madison kubelet | grep "$${VERSION_REGEX}-" | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')"
+          for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
+            echo "* installing package: $${CI_PACKAGE} $${PACKAGE_VERSION}"
+            apt-mark unhold "$${CI_PACKAGE}"
+            apt-get install --allow-downgrades -y "$${CI_PACKAGE}=$${PACKAGE_VERSION}"
+            apt-mark hold "$${CI_PACKAGE}"
+          done
+          systemctl restart kubelet
+        else
+          CI_URL="https://dl.k8s.io/ci/$${KUBERNETES_VERSION}/bin/linux/amd64"
+          for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do
+            # Browser: https://console.cloud.google.com/storage/browser/k8s-release-dev?project=k8s-release-dev
+            # e.g.: https://storage.googleapis.com/k8s-release-dev/ci/v1.21.0-beta.1.378+cf3374e43491c5/bin/linux/amd64/kubectl
+            echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}"
+            wget "$${CI_URL}/$${CI_PACKAGE}" -O "$${CI_DIR}/$${CI_PACKAGE}"
+            chmod +x "$${CI_DIR}/$${CI_PACKAGE}"
+            mv "$${CI_DIR}/$${CI_PACKAGE}" "/usr/bin/$${CI_PACKAGE}"
+          done
+          systemctl restart kubelet
+        fi
+        IMAGE_REGISTRY_PREFIX=registry.k8s.io
+        # Kubernetes builds from 1.20 through 1.24 are tagged with k8s.gcr.io
+        if [[ "$${KUBERNETES_VERSION}" =~ ^v1\.(1[0-9]|2[0-4])[\.[0-9]+ ]]; then
+          IMAGE_REGISTRY_PREFIX=k8s.gcr.io
+        fi
+        for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do
+          echo "* downloading package: $${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}"
+          wget "$${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" -O "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}"
+          $${SUDO} ctr -n k8s.io images import "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" || echo "* ignoring expected 'ctr images import' result"
+          $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}"
+          $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "gcr.io/k8s-staging-ci-images/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}"
+        done
+      fi
+      echo "* checking binary versions"
+      echo "ctr version: " "$(ctr version)"
+      echo "kubeadm version: " "$(kubeadm version -o=short)"
+      echo "kubectl version: " "$(kubectl version --client=true)"
+      echo "kubelet version: " "$(kubelet --version)"
+      echo "$${LINE_SEPARATOR}"

test/e2e/data/infrastructure-vsphere-supervisor/main/install-on-bootstrap/kustomization.yaml

@@ -0,0 +1,8 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ../topology
+patches:
+  - target:
+      kind: Cluster
+    path: ./inject-install-on-bootstrap.yaml

test/e2e/e2e_setup_test.go

@@ -287,6 +287,11 @@ func setupNamespaceWithVMOperatorDependenciesVCenter(managementClusterProxy fram
 					Cpus:   mustParseInt64(e2eConfig.GetVariable("VSPHERE_MACHINE_CLASS_CPU")),
 					Memory: resource.MustParse(e2eConfig.GetVariable("VSPHERE_MACHINE_CLASS_MEMORY")),
 				},
+				{
+					Name:   e2eConfig.GetVariable("VSPHERE_MACHINE_CLASS_NAME_CONFORMANCE"),
+					Cpus:   mustParseInt64(e2eConfig.GetVariable("VSPHERE_MACHINE_CLASS_CPU_CONFORMANCE")),
+					Memory: resource.MustParse(e2eConfig.GetVariable("VSPHERE_MACHINE_CLASS_MEMORY_CONFORMANCE")),
+				},
 			},
 		},
 	}

test/e2e/k8s_conformance_test.go

@@ -25,7 +25,7 @@ import (
 	"sigs.k8s.io/cluster-api/test/framework/kubernetesversions"
 )
 
-var _ = Describe("When testing K8S conformance [Conformance] [K8s-Install]", func() {
+var _ = Describe("When testing K8S conformance [supervisor] [Conformance] [K8s-Install]", func() {
 	// Note: This installs a cluster based on KUBERNETES_VERSION and runs conformance tests.
 	const specName = "k8s-conformance" // copied from CAPI
 	Setup(specName, func(testSpecificSettingsGetter func() testSettings) {
@@ -43,7 +43,7 @@ var _ = Describe("When testing K8S conformance [Conformance] [K8s-Install]", fun
 	})
 })
 
-var _ = Describe("When testing K8S conformance with K8S latest ci [Conformance] [K8s-Install-ci-latest]", func() {
+var _ = Describe("When testing K8S conformance with K8S latest ci [supervisor] [Conformance] [K8s-Install-ci-latest]", func() {
 	// Note: This installs a cluster based on KUBERNETES_VERSION_LATEST_CI and runs conformance tests.
 	// Note: We are resolving KUBERNETES_VERSION_LATEST_CI and then setting the resolved version as
 	// KUBERNETES_VERSION env var. This only works without side effects on other tests because we are

test/e2e/quick_start_test.go

@@ -40,7 +40,7 @@ var _ = Describe("Cluster Creation using Cluster API quick-start test [vcsim] [s
 	})
 })
 
-var _ = Describe("ClusterClass Creation using Cluster API quick-start test [PR-Blocking] [ClusterClass] [vcsim] [supervisor]", func() {
+var _ = Describe("ClusterClass Creation using Cluster API quick-start test [vcsim] [supervisor] [PR-Blocking] [ClusterClass]", func() {
 	const specName = "quick-start-cluster-class" // prefix (quick-start) copied from CAPI
 	Setup(specName, func(testSpecificSettingsGetter func() testSettings) {
 		capi_e2e.QuickStartSpec(ctx, func() capi_e2e.QuickStartSpecInput {