From 9098a21c532b7a67e19748926fc58944201bb4a5 Mon Sep 17 00:00:00 2001 From: fabriziopandini Date: Thu, 4 Apr 2024 16:37:05 +0200 Subject: [PATCH] Add more supervisor tests --- Makefile | 2 + test/e2e/cluster_upgrade_test.go | 4 +- test/e2e/config/vsphere.yaml | 5 + .../main/conformance/kustomization.yaml | 6 + .../main/conformance/worker-node-size.yaml | 10 ++ .../inject-install-on-bootstrap.yaml | 109 ++++++++++++++++++ .../install-on-bootstrap/kustomization.yaml | 8 ++ test/e2e/e2e_setup_test.go | 5 + test/e2e/k8s_conformance_test.go | 4 +- test/e2e/quick_start_test.go | 2 +- 10 files changed, 150 insertions(+), 5 deletions(-) create mode 100644 test/e2e/data/infrastructure-vsphere-supervisor/main/conformance/kustomization.yaml create mode 100644 test/e2e/data/infrastructure-vsphere-supervisor/main/conformance/worker-node-size.yaml create mode 100644 test/e2e/data/infrastructure-vsphere-supervisor/main/install-on-bootstrap/inject-install-on-bootstrap.yaml create mode 100644 test/e2e/data/infrastructure-vsphere-supervisor/main/install-on-bootstrap/kustomization.yaml diff --git a/Makefile b/Makefile index 6f9185175a..ca839b4aed 100644 --- a/Makefile +++ b/Makefile @@ -377,6 +377,8 @@ generate-e2e-templates-main: $(KUSTOMIZE) ## Generate test templates for the mai "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/clusterclass" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/clusterclass-quick-start-supervisor.yaml" cp "$(RELEASE_DIR)/main/cluster-template-topology-supervisor.yaml" "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/topology/cluster-template-topology-supervisor.yaml" "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/topology" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-topology-supervisor.yaml" + "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/conformance" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-conformance-supervisor.yaml" + "$(KUSTOMIZE)" --load-restrictor LoadRestrictionsNone build "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/install-on-bootstrap" > "$(E2E_SUPERVISOR_TEMPLATE_DIR)/main/cluster-template-install-on-bootstrap-supervisor.yaml" .PHONY: generate-e2e-templates-v1.9 generate-e2e-templates-v1.9: $(KUSTOMIZE) diff --git a/test/e2e/cluster_upgrade_test.go b/test/e2e/cluster_upgrade_test.go index 223aac3769..506a7c6ad6 100644 --- a/test/e2e/cluster_upgrade_test.go +++ b/test/e2e/cluster_upgrade_test.go @@ -26,7 +26,7 @@ import ( "sigs.k8s.io/cluster-api/test/framework/kubernetesversions" ) -var _ = Describe("When upgrading a workload cluster using ClusterClass and testing K8S conformance [Conformance] [K8s-Upgrade] [ClusterClass]", func() { +var _ = Describe("When upgrading a workload cluster using ClusterClass and testing K8S conformance [supervisor] [Conformance] [K8s-Upgrade] [ClusterClass]", func() { // Note: This installs a cluster based on KUBERNETES_VERSION_UPGRADE_FROM and then upgrades to // KUBERNETES_VERSION_UPGRADE_TO and runs conformance tests. // Note: We are resolving KUBERNETES_VERSION_UPGRADE_FROM and KUBERNETES_VERSION_UPGRADE_TO and then setting @@ -58,7 +58,7 @@ var _ = Describe("When upgrading a workload cluster using ClusterClass and testi }) }) -var _ = Describe("When upgrading a workload cluster using ClusterClass [ClusterClass]", func() { +var _ = Describe("When upgrading a workload cluster using ClusterClass [supervisor] [ClusterClass]", func() { // Note: This installs a cluster based on KUBERNETES_VERSION_UPGRADE_FROM and then upgrades to // KUBERNETES_VERSION_UPGRADE_TO. const specName = "k8s-upgrade" // aligned to CAPI diff --git a/test/e2e/config/vsphere.yaml b/test/e2e/config/vsphere.yaml index 23ce28a712..eac23f3741 100644 --- a/test/e2e/config/vsphere.yaml +++ b/test/e2e/config/vsphere.yaml @@ -175,6 +175,8 @@ providers: - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-topology-supervisor.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-supervisor.yaml" - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/clusterclass-quick-start-supervisor.yaml" + - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-install-on-bootstrap-supervisor.yaml" + - sourcePath: "../../../test/e2e/data/infrastructure-vsphere-supervisor/main/cluster-template-conformance-supervisor.yaml" - sourcePath: "../data/shared/capv/main/metadata.yaml" - name: v1.9.0 # Use manifest from source files @@ -246,6 +248,9 @@ variables: VSPHERE_MACHINE_CLASS_NAME: "test-machine-class" VSPHERE_MACHINE_CLASS_CPU: "4" VSPHERE_MACHINE_CLASS_MEMORY: "8Gi" + VSPHERE_MACHINE_CLASS_NAME_CONFORMANCE: "test-machine-class-conformance" + VSPHERE_MACHINE_CLASS_CPU_CONFORMANCE: "8" + VSPHERE_MACHINE_CLASS_MEMORY_CONFORMANCE: "8Gi" VSPHERE_CONTENT_LIBRARY: "capv" VSPHERE_IMAGE_NAME: "ubuntu-2204-kube-v1.29.0" VSPHERE_NETWORK: "sddc-cgw-network-6" diff --git a/test/e2e/data/infrastructure-vsphere-supervisor/main/conformance/kustomization.yaml b/test/e2e/data/infrastructure-vsphere-supervisor/main/conformance/kustomization.yaml new file mode 100644 index 0000000000..b9feefcfb4 --- /dev/null +++ b/test/e2e/data/infrastructure-vsphere-supervisor/main/conformance/kustomization.yaml @@ -0,0 +1,6 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ../base +patchesStrategicMerge: + - worker-node-size.yaml diff --git a/test/e2e/data/infrastructure-vsphere-supervisor/main/conformance/worker-node-size.yaml b/test/e2e/data/infrastructure-vsphere-supervisor/main/conformance/worker-node-size.yaml new file mode 100644 index 0000000000..ffe3a2499b --- /dev/null +++ b/test/e2e/data/infrastructure-vsphere-supervisor/main/conformance/worker-node-size.yaml @@ -0,0 +1,10 @@ +--- +apiVersion: vmware.infrastructure.cluster.x-k8s.io/v1beta1 +kind: VSphereMachineTemplate +metadata: + name: ${CLUSTER_NAME}-worker + namespace: ${NAMESPACE} +spec: + template: + spec: + className: ${VSPHERE_MACHINE_CLASS_NAME_CONFORMANCE} diff --git a/test/e2e/data/infrastructure-vsphere-supervisor/main/install-on-bootstrap/inject-install-on-bootstrap.yaml b/test/e2e/data/infrastructure-vsphere-supervisor/main/install-on-bootstrap/inject-install-on-bootstrap.yaml new file mode 100644 index 0000000000..6abed91df8 --- /dev/null +++ b/test/e2e/data/infrastructure-vsphere-supervisor/main/install-on-bootstrap/inject-install-on-bootstrap.yaml @@ -0,0 +1,109 @@ +# Note: Prior art: https://github.com/kubernetes-sigs/cluster-api/blob/21a350718f5e84a580bc677213bc25faec3291e7/test/framework/kubernetesversions/data/debian_injection_script.envsubst.sh.tpl +- op: add + path: /spec/topology/variables/- + value: + name: preKubeadmScript + # Note: This file has to be escaped as it is run through envsubst by clusterctl + value: | + #!/bin/bash + + set -o nounset + set -o pipefail + set -o errexit + + function retry { + attempt=0 + max_attempts=$${1} + interval=$${2} + shift; shift + until [[ $${attempt} -ge "$${max_attempts}" ]] ; do + attempt=$((attempt+1)) + set +e + eval "$*" && return || echo "failed $${attempt} times: $*" + set -e + sleep "$${interval}" + done + echo "error: reached max attempts at retry($*)" + return 1 + } + + [[ $(id -u) != 0 ]] && SUDO="sudo" || SUDO="" + + # This test installs release packages or binaries that are a result of the CI and release builds. + # It runs '... --version' commands to verify that the binaries are correctly installed + # and finally uninstalls the packages. + # For the release packages it tests all versions in the support skew. + LINE_SEPARATOR="*************************************************" + echo "$${LINE_SEPARATOR}" + + ## Variables (replaced by JSON patching) + KUBERNETES_VERSION=$${KUBERNETES_VERSION} + ## + + # Note: We assume if kubectl has the right version, everything else has as well + if [[ $(kubectl version --client=true -o json | jq '.clientVersion.gitVersion' -r) = "$${KUBERNETES_VERSION}" ]]; then + echo "Detected Kubernetes $${KUBERNETES_VERSION} via kubectl version, nothing to do" + exit 0 + fi + + if [[ "$${KUBERNETES_VERSION}" != "" ]]; then + CI_DIR=/tmp/k8s-ci + mkdir -p "$${CI_DIR}" + declare -a PACKAGES_TO_TEST=("kubectl" "kubelet" "kubeadm") + # Let's just also download the control plane images for worker nodes. It's easier then optimizing it. + declare -a CONTAINERS_TO_TEST=("kube-apiserver" "kube-controller-manager" "kube-proxy" "kube-scheduler") + CONTAINER_EXT="tar" + echo "* testing CI version $${KUBERNETES_VERSION}" + # Check for semver + if [[ "$${KUBERNETES_VERSION}" =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + CI_URL="https://dl.k8s.io/release/$${KUBERNETES_VERSION}/bin/linux/amd64" + VERSION_WITHOUT_PREFIX="$${KUBERNETES_VERSION#v}" + export DEBIAN_FRONTEND=noninteractive + # sometimes the network is not immediately available, so we have to retry the apt-get update + retry 10 5 "apt-get update" + apt-get install -y apt-transport-https ca-certificates curl gpg + mkdir -p -m 755 /etc/apt/keyrings + curl -fsSL https://pkgs.k8s.io/core:/stable:/$${KUBERNETES_VERSION:0:5}/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg + echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://pkgs.k8s.io/core:/stable:/$${KUBERNETES_VERSION:0:5}/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list + apt-get update + # replace . with \. + VERSION_REGEX="$${VERSION_WITHOUT_PREFIX//./\\.}" + PACKAGE_VERSION="$(apt-cache madison kubelet | grep "$${VERSION_REGEX}-" | head -n1 | cut -d '|' -f 2 | tr -d '[:space:]')" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + echo "* installing package: $${CI_PACKAGE} $${PACKAGE_VERSION}" + apt-mark unhold "$${CI_PACKAGE}" + apt-get install --allow-downgrades -y "$${CI_PACKAGE}=$${PACKAGE_VERSION}" + apt-mark hold "$${CI_PACKAGE}" + done + systemctl restart kubelet + else + CI_URL="https://dl.k8s.io/ci/$${KUBERNETES_VERSION}/bin/linux/amd64" + for CI_PACKAGE in "$${PACKAGES_TO_TEST[@]}"; do + # Browser: https://console.cloud.google.com/storage/browser/k8s-release-dev?project=k8s-release-dev + # e.g.: https://storage.googleapis.com/k8s-release-dev/ci/v1.21.0-beta.1.378+cf3374e43491c5/bin/linux/amd64/kubectl + echo "* downloading binary: $${CI_URL}/$${CI_PACKAGE}" + wget "$${CI_URL}/$${CI_PACKAGE}" -O "$${CI_DIR}/$${CI_PACKAGE}" + chmod +x "$${CI_DIR}/$${CI_PACKAGE}" + mv "$${CI_DIR}/$${CI_PACKAGE}" "/usr/bin/$${CI_PACKAGE}" + done + systemctl restart kubelet + fi + IMAGE_REGISTRY_PREFIX=registry.k8s.io + # Kubernetes builds from 1.20 through 1.24 are tagged with k8s.gcr.io + if [[ "$${KUBERNETES_VERSION}" =~ ^v1\.(1[0-9]|2[0-4])[\.[0-9]+ ]]; then + IMAGE_REGISTRY_PREFIX=k8s.gcr.io + fi + for CI_CONTAINER in "$${CONTAINERS_TO_TEST[@]}"; do + echo "* downloading package: $${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + wget "$${CI_URL}/$${CI_CONTAINER}.$${CONTAINER_EXT}" -O "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" + $${SUDO} ctr -n k8s.io images import "$${CI_DIR}/$${CI_CONTAINER}.$${CONTAINER_EXT}" || echo "* ignoring expected 'ctr images import' result" + $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + $${SUDO} ctr -n k8s.io images tag "$${IMAGE_REGISTRY_PREFIX}/$${CI_CONTAINER}-amd64:$${KUBERNETES_VERSION//+/_}" "gcr.io/k8s-staging-ci-images/$${CI_CONTAINER}:$${KUBERNETES_VERSION//+/_}" + done + fi + echo "* checking binary versions" + echo "ctr version: " "$(ctr version)" + echo "kubeadm version: " "$(kubeadm version -o=short)" + echo "kubectl version: " "$(kubectl version --client=true)" + echo "kubelet version: " "$(kubelet --version)" + echo "$${LINE_SEPARATOR}" diff --git a/test/e2e/data/infrastructure-vsphere-supervisor/main/install-on-bootstrap/kustomization.yaml b/test/e2e/data/infrastructure-vsphere-supervisor/main/install-on-bootstrap/kustomization.yaml new file mode 100644 index 0000000000..1a90a2c199 --- /dev/null +++ b/test/e2e/data/infrastructure-vsphere-supervisor/main/install-on-bootstrap/kustomization.yaml @@ -0,0 +1,8 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - ../topology +patches: + - target: + kind: Cluster + path: ./inject-install-on-bootstrap.yaml diff --git a/test/e2e/e2e_setup_test.go b/test/e2e/e2e_setup_test.go index 26792e6390..49d1b3d671 100644 --- a/test/e2e/e2e_setup_test.go +++ b/test/e2e/e2e_setup_test.go @@ -287,6 +287,11 @@ func setupNamespaceWithVMOperatorDependenciesVCenter(managementClusterProxy fram Cpus: mustParseInt64(e2eConfig.GetVariable("VSPHERE_MACHINE_CLASS_CPU")), Memory: resource.MustParse(e2eConfig.GetVariable("VSPHERE_MACHINE_CLASS_MEMORY")), }, + { + Name: e2eConfig.GetVariable("VSPHERE_MACHINE_CLASS_NAME_CONFORMANCE"), + Cpus: mustParseInt64(e2eConfig.GetVariable("VSPHERE_MACHINE_CLASS_CPU_CONFORMANCE")), + Memory: resource.MustParse(e2eConfig.GetVariable("VSPHERE_MACHINE_CLASS_MEMORY_CONFORMANCE")), + }, }, }, } diff --git a/test/e2e/k8s_conformance_test.go b/test/e2e/k8s_conformance_test.go index cd5ab9c0e7..27ee4955b9 100644 --- a/test/e2e/k8s_conformance_test.go +++ b/test/e2e/k8s_conformance_test.go @@ -25,7 +25,7 @@ import ( "sigs.k8s.io/cluster-api/test/framework/kubernetesversions" ) -var _ = Describe("When testing K8S conformance [Conformance] [K8s-Install]", func() { +var _ = Describe("When testing K8S conformance [supervisor] [Conformance] [K8s-Install]", func() { // Note: This installs a cluster based on KUBERNETES_VERSION and runs conformance tests. const specName = "k8s-conformance" // copied from CAPI Setup(specName, func(testSpecificSettingsGetter func() testSettings) { @@ -43,7 +43,7 @@ var _ = Describe("When testing K8S conformance [Conformance] [K8s-Install]", fun }) }) -var _ = Describe("When testing K8S conformance with K8S latest ci [Conformance] [K8s-Install-ci-latest]", func() { +var _ = Describe("When testing K8S conformance with K8S latest ci [supervisor] [Conformance] [K8s-Install-ci-latest]", func() { // Note: This installs a cluster based on KUBERNETES_VERSION_LATEST_CI and runs conformance tests. // Note: We are resolving KUBERNETES_VERSION_LATEST_CI and then setting the resolved version as // KUBERNETES_VERSION env var. This only works without side effects on other tests because we are diff --git a/test/e2e/quick_start_test.go b/test/e2e/quick_start_test.go index 3240a3a42f..c4f0cc96cc 100644 --- a/test/e2e/quick_start_test.go +++ b/test/e2e/quick_start_test.go @@ -40,7 +40,7 @@ var _ = Describe("Cluster Creation using Cluster API quick-start test [vcsim] [s }) }) -var _ = Describe("ClusterClass Creation using Cluster API quick-start test [PR-Blocking] [ClusterClass] [vcsim] [supervisor]", func() { +var _ = Describe("ClusterClass Creation using Cluster API quick-start test [vcsim] [supervisor] [PR-Blocking] [ClusterClass]", func() { const specName = "quick-start-cluster-class" // prefix (quick-start) copied from CAPI Setup(specName, func(testSpecificSettingsGetter func() testSettings) { capi_e2e.QuickStartSpec(ctx, func() capi_e2e.QuickStartSpecInput {