From 3e2240b9133932476c16812f6f5f6e51dfb64a4c Mon Sep 17 00:00:00 2001 From: Sid Shukla Date: Tue, 5 Mar 2024 20:21:09 +0100 Subject: [PATCH] Add support for Kubernetes v1.29 clusters (#382) This required two changes: - Remove hostAliases from kube-vip podspec This is addressed by adding entries directly to the /etc/hosts - Do a super-admin.conf switcheroo for the kube-vip static pod Add pre and post kubeadm commands for handling kubernetes versions v1.29.0+. The prekubeadm command checks if kubeadm init has been run and if it is, it replaces the kubeconfig hostPath in kube-vip static pod from admin.conf to the super-admin.conf. The postkubeadm command checks if kubeadm init has been run and if it is, it changes the hostPath in kube-vip static pod from super-admin.conf back to admin.conf. --- Makefile | 4 +-- templates/base/cluster-with-kcp.yaml | 26 ++++++++++++++----- templates/cluster-template-csi.yaml | 24 ++++++++++++++--- templates/cluster-template.yaml | 24 ++++++++++++++--- .../v1beta1/cluster-template-csi/kcp.yaml | 12 +++++++++ 5 files changed, 74 insertions(+), 16 deletions(-) diff --git a/Makefile b/Makefile index 1d64da7d3d..941cd506ef 100644 --- a/Makefile +++ b/Makefile @@ -217,7 +217,7 @@ help: ## Display this help. ##@ Development .PHONY: manifests -manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. +manifests: $(CONTROLLER_GEN_BIN) ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects. $(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases .PHONY: release-manifests @@ -228,7 +228,7 @@ release-manifests: manifests cluster-templates cp $(REPO_ROOT)/metadata.yaml $(RELEASE_DIR)/metadata.yaml .PHONY: generate -generate: controller-gen conversion-gen ## Generate code containing DeepCopy, DeepCopyInto, DeepCopyObject method implementations and API conversion implementations. +generate: $(CONTROLLER_GEN_BIN) $(CONVERSION_GEN_BIN) ## Generate code containing DeepCopy, DeepCopyInto, DeepCopyObject method implementations and API conversion implementations. $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..." $(CONVERSION_GEN) \ diff --git a/templates/base/cluster-with-kcp.yaml b/templates/base/cluster-with-kcp.yaml index f6ba2dc11b..8de174d131 100644 --- a/templates/base/cluster-with-kcp.yaml +++ b/templates/base/cluster-with-kcp.yaml @@ -40,7 +40,6 @@ spec: apiVersion: infrastructure.cluster.x-k8s.io/v1beta1 kind: NutanixCluster name: "${CLUSTER_NAME}" - --- apiVersion: controlplane.cluster.x-k8s.io/v1beta1 kind: KubeadmControlPlane @@ -125,10 +124,6 @@ spec: name: kubeconfig resources: {} hostNetwork: true - hostAliases: - - hostnames: - - kubernetes - ip: 127.0.0.1 volumes: - name: kubeconfig hostPath: @@ -162,12 +157,31 @@ spec: preKubeadmCommands: - echo "before kubeadm call" > /var/log/prekubeadm.log - hostnamectl set-hostname "{{ ds.meta_data.hostname }}" + - echo "::1 ipv6-localhost ipv6-loopback" >/etc/hosts + - echo "127.0.0.1 localhost" >>/etc/hosts + - echo "127.0.0.1 kubernetes" >>/etc/hosts + - echo "127.0.0.1 {{ ds.meta_data.hostname }}" >> /etc/hosts + - | + KUBERNETES_VERSION_NO_V=${KUBERNETES_VERSION#v} + VERSION_TO_COMPARE=1.29.0 + if [ "$(printf '%s\n' "$KUBERNETES_VERSION_NO_V" "$VERSION_TO_COMPARE" | sort -V | head -n1)" != "$KUBERNETES_VERSION_NO_V" ]; then + if [ -f /run/kubeadm/kubeadm.yaml ]; then + sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' /etc/kubernetes/manifests/kube-vip.yaml; + fi + fi postKubeadmCommands: - echo export KUBECONFIG=/etc/kubernetes/admin.conf >> /root/.bashrc + - | + KUBERNETES_VERSION_NO_V=${KUBERNETES_VERSION#v} + VERSION_TO_COMPARE=1.29.0 + if [ "$(printf '%s\n' "$KUBERNETES_VERSION_NO_V" "$VERSION_TO_COMPARE" | sort -V | head -n1)" != "$KUBERNETES_VERSION_NO_V" ]; then + if [ -f /run/kubeadm/kubeadm.yaml ]; then + sed -i 's#path: /etc/kubernetes/super-admin.conf#path: /etc/kubernetes/admin.conf#' /etc/kubernetes/manifests/kube-vip.yaml; + fi + fi - echo "after kubeadm call" > /var/log/postkubeadm.log useExperimentalRetryJoin: true verbosity: 10 - --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KubeadmConfigTemplate diff --git a/templates/cluster-template-csi.yaml b/templates/cluster-template-csi.yaml index 6163807ccc..abfa5a8a70 100644 --- a/templates/cluster-template-csi.yaml +++ b/templates/cluster-template-csi.yaml @@ -1788,10 +1788,6 @@ spec: name: kubeconfig resources: {} hostNetwork: true - hostAliases: - - hostnames: - - kubernetes - ip: 127.0.0.1 volumes: - name: kubeconfig hostPath: @@ -1814,10 +1810,30 @@ spec: tls-cipher-suites: ${TLS_CIPHER_SUITES=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256} postKubeadmCommands: - echo export KUBECONFIG=/etc/kubernetes/admin.conf >> /root/.bashrc + - | + KUBERNETES_VERSION_NO_V=${KUBERNETES_VERSION#v} + VERSION_TO_COMPARE=1.29.0 + if [ "$(printf '%s\n' "$KUBERNETES_VERSION_NO_V" "$VERSION_TO_COMPARE" | sort -V | head -n1)" != "$KUBERNETES_VERSION_NO_V" ]; then + if [ -f /run/kubeadm/kubeadm.yaml ]; then + sed -i 's#path: /etc/kubernetes/super-admin.conf#path: /etc/kubernetes/admin.conf#' /etc/kubernetes/manifests/kube-vip.yaml; + fi + fi - echo "after kubeadm call" > /var/log/postkubeadm.log preKubeadmCommands: - echo "before kubeadm call" > /var/log/prekubeadm.log - hostnamectl set-hostname "{{ ds.meta_data.hostname }}" + - echo "::1 ipv6-localhost ipv6-loopback" >/etc/hosts + - echo "127.0.0.1 localhost" >>/etc/hosts + - echo "127.0.0.1 kubernetes" >>/etc/hosts + - echo "127.0.0.1 {{ ds.meta_data.hostname }}" >> /etc/hosts + - | + KUBERNETES_VERSION_NO_V=${KUBERNETES_VERSION#v} + VERSION_TO_COMPARE=1.29.0 + if [ "$(printf '%s\n' "$KUBERNETES_VERSION_NO_V" "$VERSION_TO_COMPARE" | sort -V | head -n1)" != "$KUBERNETES_VERSION_NO_V" ]; then + if [ -f /run/kubeadm/kubeadm.yaml ]; then + sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' /etc/kubernetes/manifests/kube-vip.yaml; + fi + fi useExperimentalRetryJoin: true users: - lockPassword: false diff --git a/templates/cluster-template.yaml b/templates/cluster-template.yaml index c83529564d..b6c5351250 100644 --- a/templates/cluster-template.yaml +++ b/templates/cluster-template.yaml @@ -455,10 +455,6 @@ spec: name: kubeconfig resources: {} hostNetwork: true - hostAliases: - - hostnames: - - kubernetes - ip: 127.0.0.1 volumes: - name: kubeconfig hostPath: @@ -481,10 +477,30 @@ spec: tls-cipher-suites: ${TLS_CIPHER_SUITES=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256} postKubeadmCommands: - echo export KUBECONFIG=/etc/kubernetes/admin.conf >> /root/.bashrc + - | + KUBERNETES_VERSION_NO_V=${KUBERNETES_VERSION#v} + VERSION_TO_COMPARE=1.29.0 + if [ "$(printf '%s\n' "$KUBERNETES_VERSION_NO_V" "$VERSION_TO_COMPARE" | sort -V | head -n1)" != "$KUBERNETES_VERSION_NO_V" ]; then + if [ -f /run/kubeadm/kubeadm.yaml ]; then + sed -i 's#path: /etc/kubernetes/super-admin.conf#path: /etc/kubernetes/admin.conf#' /etc/kubernetes/manifests/kube-vip.yaml; + fi + fi - echo "after kubeadm call" > /var/log/postkubeadm.log preKubeadmCommands: - echo "before kubeadm call" > /var/log/prekubeadm.log - hostnamectl set-hostname "{{ ds.meta_data.hostname }}" + - echo "::1 ipv6-localhost ipv6-loopback" >/etc/hosts + - echo "127.0.0.1 localhost" >>/etc/hosts + - echo "127.0.0.1 kubernetes" >>/etc/hosts + - echo "127.0.0.1 {{ ds.meta_data.hostname }}" >> /etc/hosts + - | + KUBERNETES_VERSION_NO_V=${KUBERNETES_VERSION#v} + VERSION_TO_COMPARE=1.29.0 + if [ "$(printf '%s\n' "$KUBERNETES_VERSION_NO_V" "$VERSION_TO_COMPARE" | sort -V | head -n1)" != "$KUBERNETES_VERSION_NO_V" ]; then + if [ -f /run/kubeadm/kubeadm.yaml ]; then + sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' /etc/kubernetes/manifests/kube-vip.yaml; + fi + fi useExperimentalRetryJoin: true users: - lockPassword: false diff --git a/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-csi/kcp.yaml b/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-csi/kcp.yaml index ed3785662f..f436eb90b8 100644 --- a/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-csi/kcp.yaml +++ b/test/e2e/data/infrastructure-nutanix/v1beta1/cluster-template-csi/kcp.yaml @@ -12,3 +12,15 @@ spec: - apt update - apt install -y nfs-common open-iscsi lvm2 xfsprogs - systemctl enable --now iscsid + - echo "::1 ipv6-localhost ipv6-loopback" >/etc/hosts + - echo "127.0.0.1 localhost" >>/etc/hosts + - echo "127.0.0.1 kubernetes" >>/etc/hosts + - echo "127.0.0.1 {{ ds.meta_data.hostname }}" >> /etc/hosts + - | + KUBERNETES_VERSION_NO_V=${KUBERNETES_VERSION#v} + VERSION_TO_COMPARE=1.29.0 + if [ "$(printf '%s\n' "$KUBERNETES_VERSION_NO_V" "$VERSION_TO_COMPARE" | sort -V | head -n1)" != "$KUBERNETES_VERSION_NO_V" ]; then + if [ -f /run/kubeadm/kubeadm.yaml ]; then + sed -i 's#path: /etc/kubernetes/admin.conf#path: /etc/kubernetes/super-admin.conf#' /etc/kubernetes/manifests/kube-vip.yaml; + fi + fi