fix(kubernetes): taint nodes on cluster upgrade

docs/vars.md

@@ -245,7 +245,7 @@ node_labels:
   label2_name: label2_value
 ```
 
-* *node_taints* - Taints applied to nodes via kubelet --register-with-taints parameter.
+* *node_taints* - Taints applied to nodes via `kubectl taint node`.
   For example, taints can be set in the inventory as variables or more widely in group_vars.
   *node_taints* has to be defined as a list of strings in format `key=value:effect`, e.g.:
 

playbooks/cluster.yml

@@ -48,6 +48,7 @@
     - { role: kubespray-defaults }
     - { role: kubernetes/kubeadm, tags: kubeadm}
     - { role: kubernetes/node-label, tags: node-label }
+    - { role: kubernetes/node-taint, tags: node-taint }
     - { role: network_plugin, tags: network }
     - { role: kubernetes-apps/kubelet-csr-approver, tags: kubelet-csr-approver }
 

playbooks/scale.yml

@@ -91,6 +91,7 @@
     - { role: kubespray-defaults }
     - { role: kubernetes/kubeadm, tags: kubeadm }
     - { role: kubernetes/node-label, tags: node-label }
+    - { role: kubernetes/node-taint, tags: node-taint }
     - { role: network_plugin, tags: network }
 
 - name: Apply resolv.conf changes now that cluster DNS is up

playbooks/upgrade_cluster.yml

@@ -55,6 +55,7 @@
     - { role: kubernetes/control-plane, tags: master, upgrade_cluster_setup: true }
     - { role: kubernetes/client, tags: client }
     - { role: kubernetes/node-label, tags: node-label }
+    - { role: kubernetes/node-taint, tags: node-taint }
     - { role: kubernetes-apps/cluster_roles, tags: cluster-roles }
     - { role: kubernetes-apps, tags: csi-driver }
     - { role: upgrade/post-upgrade, tags: post-upgrade }
@@ -87,6 +88,7 @@
     - { role: kubernetes/node, tags: node }
     - { role: kubernetes/kubeadm, tags: kubeadm }
     - { role: kubernetes/node-label, tags: node-label }
+    - { role: kubernetes/node-taint, tags: node-taint }
     - { role: upgrade/post-upgrade, tags: post-upgrade }
 
 - name: Patch Kubernetes for Windows

roles/kubernetes/node-taint/tasks/main.yml

@@ -0,0 +1,35 @@
+---
+- name: Set role and inventory node taint to empty list
+  set_fact:
+    role_node_taints: []
+    inventory_node_taints: []
+
+- name: Node taint for nvidia GPU nodes
+  set_fact:
+    role_node_taints: "{{ role_node_taints + ['nvidia.com/gpu=:NoSchedule'] }}"
+  when:
+    - nvidia_gpu_nodes is defined
+    - nvidia_accelerator_enabled | bool
+    - inventory_hostname in nvidia_gpu_nodes
+
+- name: Populate inventory node taint
+  set_fact:
+    inventory_node_taints: "{{ inventory_node_taints + ['%s' | format(item)] }}"
+  loop: "{{ node_taints | d([]) }}"
+  when:
+    - node_taints is defined
+    - node_taints is not string
+    - node_taints is not mapping
+    - node_taints is iterable
+- debug:  # noqa name[missing]
+    var: role_node_taints
+- debug:  # noqa name[missing]
+    var: inventory_node_taints
+
+- name: Set taint to node
+  command: >-
+      {{ kubectl }} taint node {{ kube_override_hostname | default(inventory_hostname) }} {{ (role_node_taints + inventory_node_taints) | join(' ') }} --overwrite=true
+  delegate_to: "{{ groups['kube_control_plane'][0] }}"
+  changed_when: false
+  when:
+    - (role_node_taints + inventory_node_taints) | length > 0

roles/kubernetes/node/templates/kubelet.env.v1beta1.j2

@@ -15,17 +15,7 @@ KUBELET_HOSTNAME="--hostname-override={{ kube_override_hostname }}"
 --runtime-cgroups={{ kubelet_runtime_cgroups }} \
 {% endset %}
 
-{# Kubelet node taints for gpu #}
-{% if nvidia_gpu_nodes is defined and nvidia_accelerator_enabled|bool %}
-{%   if inventory_hostname in nvidia_gpu_nodes and node_taints is defined %}
-{%       set dummy = node_taints.append('nvidia.com/gpu=:NoSchedule') %}
-{%   elif inventory_hostname in nvidia_gpu_nodes and node_taints is not defined %}
-{%       set node_taints = [] %}
-{%       set dummy = node_taints.append('nvidia.com/gpu=:NoSchedule') %}
-{%   endif %}
-{% endif %}
-
-KUBELET_ARGS="{{ kubelet_args_base }} {% if node_taints|default([]) %}--register-with-taints={{ node_taints | join(',') }} {% endif %} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube_node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}"
+KUBELET_ARGS="{{ kubelet_args_base }} {% if kubelet_custom_flags is string %} {{kubelet_custom_flags}} {% else %}{% for flag in kubelet_custom_flags %} {{flag}} {% endfor %}{% endif %}{% if inventory_hostname in groups['kube_node'] %}{% if kubelet_node_custom_flags is string %} {{kubelet_node_custom_flags}} {% else %}{% for flag in kubelet_node_custom_flags %} {{flag}} {% endfor %}{% endif %}{% endif %}"
 {% if kubelet_flexvolumes_plugins_dir is defined %}
 KUBELET_VOLUME_PLUGIN="--volume-plugin-dir={{ kubelet_flexvolumes_plugins_dir }}"
 {% endif %}