Support for Kubernetes v1.23 (#431)

* Add Kubernetes 1.23 support to README.md * Upgrade github.com/gardener/cloud-provider-azure ```feature dependency github.com/gardener/cloud-provider-azure $805c4a0edd656951b540790e3b69b17e70beeb75 `k8s.io/legacy-cloud-providers` is now updated to `v0.22.6`. ``` ```feature dependency github.com/gardener/cloud-provider-azure $753148b7a46234b519e996430078dda63db1c46a `k8s.io/legacy-cloud-providers` is now updated to `v0.21.9`. ``` ```feature dependency github.com/gardener/cloud-provider-azure $7e0c69a4fa110858b8774c6338f5d2d52de4fd2c `k8s.io/legacy-cloud-providers` is now updated to `v0.20.15`. ``` * Use upstream `cloud-provider-azure` for 1.23+ * Replace deprecated kubelet flags
gardener · Feb 15, 2022 · e68a978 · e68a978
1 parent 423407f
commit e68a978
Show file tree

Hide file tree

Showing 15 changed files with 369 additions and 148 deletions.
diff --git a/README.md b/README.md
@@ -23,6 +23,7 @@ This extension controller supports the following Kubernetes versions:
 
 | Version         | Support     | Conformance test results |
 | --------------- | ----------- | ------------------------ |
+| Kubernetes 1.23 | 1.23.0+     | N/A |
 | Kubernetes 1.22 | 1.22.0+     | [![Gardener v1.22 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.22%20Azure/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.22%20Azure) |
 | Kubernetes 1.21 | 1.21.0+     | [![Gardener v1.21 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.21%20Azure/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.21%20Azure) |
 | Kubernetes 1.20 | 1.20.0+     | [![Gardener v1.20 Conformance Tests](https://testgrid.k8s.io/q/summary/conformance-gardener/Gardener,%20v1.20%20Azure/tests_status?style=svg)](https://testgrid.k8s.io/conformance-gardener#Gardener,%20v1.20%20Azure) |

diff --git a/charts/images.yaml b/charts/images.yaml
@@ -26,18 +26,27 @@ images:
 - name: cloud-controller-manager
   sourceRepository: github.com/gardener/cloud-provider-azure
   repository: eu.gcr.io/gardener-project/kubernetes/cloud-provider-azure
-  tag: "v1.20.12"
+  tag: "v1.20.15"
   targetVersion: "1.20.x"
 - name: cloud-controller-manager
   sourceRepository: github.com/gardener/cloud-provider-azure
   repository: eu.gcr.io/gardener-project/kubernetes/cloud-provider-azure
-  tag: "v1.21.6"
+  tag: "v1.21.9"
   targetVersion: "1.21.x"
 - name: cloud-controller-manager
   sourceRepository: github.com/gardener/cloud-provider-azure
   repository: eu.gcr.io/gardener-project/kubernetes/cloud-provider-azure
-  tag: "v1.22.3"
-  targetVersion: ">= 1.22"
+  tag: "v1.22.6"
+  targetVersion: "1.22.x"
+- name: cloud-controller-manager
+  sourceRepository: github.com/kubernetes-sigs/cloud-provider-azure
+  repository: mcr.microsoft.com/oss/kubernetes/azure-cloud-controller-manager
+  tag: "v1.23.2"
+  targetVersion: ">= 1.23"
+- name: cloud-node-manager
+  sourceRepository: github.com/kubernetes-sigs/cloud-provider-azure
+  repository: mcr.microsoft.com/oss/kubernetes/azure-cloud-node-manager
+  tag: "v1.23.2"
 - name: machine-controller-manager
   sourceRepository: github.com/gardener/machine-controller-manager
   repository: eu.gcr.io/gardener-project/gardener/machine-controller-manager

diff --git a/...seed-controlplane/charts/cloud-controller-manager/templates/cloud-controller-manager.yaml b/...seed-controlplane/charts/cloud-controller-manager/templates/cloud-controller-manager.yaml
@@ -40,6 +40,8 @@ spec:
         {{- if semverCompare "< 1.17" .Values.kubernetesVersion }}
         - /hyperkube
         - cloud-controller-manager
+        {{- else if semverCompare ">= 1.23" .Values.kubernetesVersion }}
+        - /usr/local/bin/cloud-controller-manager
         {{- else }}
         - /azure-cloud-controller-manager
         {{- end }}
@@ -50,6 +52,10 @@ spec:
         - --cluster-name={{ .Values.clusterName }}
         - --concurrent-service-syncs=1
         - --configure-cloud-routes=true
+        {{- if semverCompare ">= 1.23" .Values.kubernetesVersion }}
+        - --controllers=*,-cloud-node
+        - --route-reconciliation-period=10s
+        {{- end }}
         {{- include "cloud-controller-manager.featureGates" . | trimSuffix "," | indent 8 }}
         {{- if .Values.global.useTokenRequestor }}
         - --kubeconfig=/var/run/secrets/gardener.cloud/shoot/generic-kubeconfig/kubeconfig

diff --git a/...shoot-system-components/charts/cloud-controller-manager/templates/cloud-node-manager.yaml b/...shoot-system-components/charts/cloud-controller-manager/templates/cloud-node-manager.yaml
@@ -0,0 +1,123 @@
+{{- if semverCompare ">= 1.23" .Capabilities.KubeVersion.GitVersion -}}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  labels:
+    k8s-app: cloud-node-manager
+  name: cloud-node-manager
+  namespace: {{ .Release.Namespace }}
+{{- if .Values.global.useProjectedTokenMount }}
+automountServiceAccountToken: false
+{{- end }}
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: cloud-node-manager
+  labels:
+    k8s-app: cloud-node-manager
+rules:
+- apiGroups: [""]
+  resources: ["nodes"]
+  verbs: ["watch","list","get","update", "patch"]
+- apiGroups: [""]
+  resources: ["nodes/status"]
+  verbs: ["patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: cloud-node-manager
+  labels:
+    k8s-app: cloud-node-manager
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: cloud-node-manager
+subjects:
+- kind: ServiceAccount
+  name: cloud-node-manager
+  namespace: {{ .Release.Namespace }}
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: cloud-node-manager
+  namespace: {{ .Release.Namespace }}
+  labels:
+    component: cloud-node-manager
+spec:
+  selector:
+    matchLabels:
+      k8s-app: cloud-node-manager
+  template:
+    metadata:
+      labels:
+        k8s-app: cloud-node-manager
+      annotations:
+        cluster-autoscaler.kubernetes.io/daemonset-pod: "true"
+        {{- if .Values.global.useProjectedTokenMount }}
+        # TODO(rfranzke): Remove in a future release.
+        security.gardener.cloud/trigger: rollout
+        {{- end }}
+    spec:
+      priorityClassName: system-node-critical
+      serviceAccountName: cloud-node-manager
+      hostNetwork: true   # required to fetch correct hostname
+      nodeSelector:
+        kubernetes.io/os: linux
+      tolerations:
+      - key: CriticalAddonsOnly
+        operator: Exists
+      - key: node-role.kubernetes.io/master
+        operator: Equal
+        value: "true"
+        effect: NoSchedule
+      - operator: "Exists"
+        effect: NoExecute
+      - operator: "Exists"
+        effect: NoSchedule
+      containers:
+      - name: cloud-node-manager
+        image: {{ index .Values.images "cloud-node-manager" }}
+        imagePullPolicy: IfNotPresent
+        command:
+        - cloud-node-manager
+        - --node-name=$(NODE_NAME)
+        - --wait-routes=true   # only set to true when --configure-cloud-routes=true in cloud-controller-manager.
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
+        resources:
+          requests:
+            cpu: 50m
+            memory: 50Mi
+          limits:
+            cpu: 200m
+            memory: 200Mi
+
+{{- if .Values.global.vpaEnabled }}
+---
+apiVersion: "autoscaling.k8s.io/v1beta2"
+kind: VerticalPodAutoscaler
+metadata:
+  name: cloud-node-manager
+  namespace: {{ .Release.Namespace }}
+spec:
+  resourcePolicy:
+    containerPolicies:
+    - containerName: '*'
+      minAllowed:
+        cpu: 20m
+        memory: 25Mi
+  targetRef:
+    apiVersion: apps/v1
+    kind: DaemonSet
+    name: cloud-node-manager
+  updatePolicy:
+    updateMode: "Auto"
+{{- end }}
+{{- end }}
diff --git a/...ot-system-components/charts/cloud-controller-manager/templates/rbac-cloud-controller.yaml b/...ot-system-components/charts/cloud-controller-manager/templates/rbac-cloud-controller.yaml
@@ -1,3 +1,112 @@
+{{- if semverCompare ">= 1.23" .Capabilities.KubeVersion.GitVersion }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: system:cloud-controller-manager
+  annotations:
+    rbac.authorization.kubernetes.io/autoupdate: "true"
+  labels:
+    k8s-app: cloud-controller-manager
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - events
+    verbs:
+      - create
+      - patch
+      - update
+  - apiGroups:
+      - ""
+    resources:
+      - nodes
+    verbs:
+      - "*"
+  - apiGroups:
+      - ""
+    resources:
+      - nodes/status
+    verbs:
+      - patch
+  - apiGroups:
+      - ""
+    resources:
+      - services
+    verbs:
+      - list
+      - patch
+      - update
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - services/status
+    verbs:
+      - list
+      - patch
+      - update
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - serviceaccounts
+    verbs:
+      - create
+      - get
+      - list
+      - watch
+      - update
+  - apiGroups:
+      - ""
+    resources:
+      - persistentvolumes
+    verbs:
+      - get
+      - list
+      - update
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - endpoints
+    verbs:
+      - create
+      - get
+      - list
+      - watch
+      - update
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - get
+      - list
+      - watch
+  - apiGroups:
+      - coordination.k8s.io
+    resources:
+      - leases
+    verbs:
+      - get
+      - create
+      - update
+---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: system:cloud-controller-manager
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:cloud-controller-manager
+subjects:
+- kind: User
+  name: system:cloud-controller-manager
+- kind: User
+  name: cloud-controller-manager
+{{- else }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
@@ -23,3 +132,4 @@ subjects:
 - kind: ServiceAccount
   name: azure-cloud-provider
   namespace: kube-system
+{{- end }}
diff --git a/...oot-system-components/charts/cloud-controller-manager/templates/rbac-node-controller.yaml b/...oot-system-components/charts/cloud-controller-manager/templates/rbac-node-controller.yaml
@@ -1,3 +1,4 @@
+{{- if semverCompare "< 1.23" .Capabilities.KubeVersion.GitVersion -}}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
@@ -41,3 +42,4 @@ subjects:
 - kind: ServiceAccount
   name: cloud-node-controller
   namespace: kube-system
+{{- end -}}
diff --git a/charts/internal/shoot-system-components/charts/cloud-controller-manager/values.yaml b/charts/internal/shoot-system-components/charts/cloud-controller-manager/values.yaml
@@ -0,0 +1,5 @@
+images:
+  cloud-node-manager: image-repository:image-tag
+
+global:
+  vpaEnabled: false
diff --git a/charts/internal/shoot-system-components/charts/csi-driver-node/templates/_vpa.tpl b/charts/internal/shoot-system-components/charts/csi-driver-node/templates/_vpa.tpl
@@ -1,5 +1,5 @@
 {{- define "csi-driver-node.vpa" -}}
-{{- if .Values.vpaEnabled -}}
+{{- if .Values.global.vpaEnabled -}}
 apiVersion: "autoscaling.k8s.io/v1beta2"
 kind: VerticalPodAutoscaler
 metadata:

diff --git a/charts/internal/shoot-system-components/charts/csi-driver-node/values.yaml b/charts/internal/shoot-system-components/charts/csi-driver-node/values.yaml
@@ -9,7 +9,9 @@ images:
   csi-liveness-probe: image-repository:image-tag
 
 socketPath: /csi/csi.sock
-vpaEnabled: false
+
+global:
+  vpaEnabled: false
 
 resources:
   driver:

diff --git a/charts/internal/shoot-system-components/values.yaml b/charts/internal/shoot-system-components/values.yaml
@@ -9,3 +9,6 @@ csi-driver-node:
   enabled: false
 remedy-controller-azure:
   enabled: true
+
+global:
+  vpaEnabled: false
diff --git a/pkg/azure/types.go b/pkg/azure/types.go
@@ -32,6 +32,8 @@ const (
 
 	// CloudControllerManagerImageName is the name of the cloud-controller-manager image.
 	CloudControllerManagerImageName = "cloud-controller-manager"
+	// CloudNodeManagerImageName is the name of the cloud-node-manager image.
+	CloudNodeManagerImageName = "cloud-node-manager"
 	// CSIDriverDiskImageName is the name of the csi-driver-disk image.
 	CSIDriverDiskImageName = "csi-driver-disk"
 	// CSIDriverFileImageName is the name of the csi-driver-file image.

diff --git a/pkg/controller/controlplane/valuesprovider.go b/pkg/controller/controlplane/valuesprovider.go
@@ -318,6 +318,9 @@ var (
 			},
 			{
 				Name: azure.CloudControllerManagerName,
+				Images: []string{
+					azure.CloudNodeManagerImageName,
+				},
 				Objects: []*chart.Object{
 					{Type: &rbacv1.ClusterRole{}, Name: "system:controller:cloud-node-controller"},
 					{Type: &rbacv1.ClusterRoleBinding{}, Name: "system:controller:cloud-node-controller"},
@@ -810,18 +813,20 @@ func getControlPlaneShootChartValues(
 		"global": map[string]interface{}{
 			"useTokenRequestor":      useTokenRequestor,
 			"useProjectedTokenMount": useProjectedTokenMount,
+			"vpaEnabled":             gardencorev1beta1helper.ShootWantsVerticalPodAutoscaler(cluster.Shoot),
 		},
 		azure.AllowEgressName:            map[string]interface{}{"enabled": infraStatus.Zoned || azureapihelper.IsVmoRequired(infraStatus)},
 		azure.CloudControllerManagerName: map[string]interface{}{"enabled": true},
 		azure.CSINodeName: map[string]interface{}{
 			"enabled":           !k8sVersionLessThan121,
 			"kubernetesVersion": cluster.Shoot.Spec.Kubernetes.Version,
-			"vpaEnabled":        gardencorev1beta1helper.ShootWantsVerticalPodAutoscaler(cluster.Shoot),
 			"podAnnotations": map[string]interface{}{
 				"checksum/configmap-" + azure.CloudProviderDiskConfigName: cloudProviderDiskConfigChecksum,
 			},
 			"cloudProviderConfig": cloudProviderDiskConfig,
 		},
-		azure.RemedyControllerName: map[string]interface{}{"enabled": !disableRemedyController},
+		azure.RemedyControllerName: map[string]interface{}{
+			"enabled": !disableRemedyController,
+		},
 	}
 }