feat(on-premises): node labels and annotations

schemas/public/onpremises-kfd-v1alpha2.json

@@ -298,6 +298,14 @@
           "items": {
             "$ref": "#/$defs/Spec.Kubernetes.Masters.Host"
           }
+        },
+        "labels": {
+          "description": "Optional additional Kubernetes labels that will be added to the control-plane nodes. Follows Kubernetes labels format. **Existing labels with the same key will be overwritten**.",
+          "$ref": "#/$defs/Types.KubeLabels"
+        },
+        "annotations": {
+          "description": "Optional additinal Kubernetes annotations that will be added to the control-plane nodes. Follows Kubernetes annotations format. **Existing annotations with the same key will be overwritten**.",
+          "$ref": "#/$defs/Types.KubeAnnotations"
         }
       },
       "required": [
@@ -353,6 +361,14 @@
           "items": {
             "$ref": "#/$defs/Types.KubeTaints"
           }
+        },
+        "labels": {
+          "description": "Optional additional Kubernetes labels that will be added to the nodes in this node group. Follows Kubernetes labels format. **Existing labels with the same key will be overwritten**.",
+          "$ref": "#/$defs/Types.KubeLabels"
+        },
+        "annotations": {
+          "description": "Optional additinal Kubernetes annotations that will be added to the nodes in this node group. Follows Kubernetes annotations format. **Existing annotations with the same key will be overwritten**.",
+          "$ref": "#/$defs/Types.KubeAnnotations"
         }
       },
       "required": [
@@ -2263,7 +2279,29 @@
       "pattern": "^(http|https)\\:\\/\\/.+$"
     },
     "Types.KubeLabels": {
-      "type": "object",
+      "type": [
+        "object",
+        "null"
+      ],
+      "propertyNames": {
+        "pattern": "^([a-zA-Z0-9][a-zA-Z0-9-.]*[a-zA-Z0-9]/)?([a-zA-Z0-9][-a-zA-Z0-9_.]*)?[a-zA-Z0-9]$",
+        "maxLength": 253
+      },
+      "additionalProperties": {
+        "type": "string",
+        "pattern": "^(([a-zA-Z0-9][-a-zA-Z0-9_.]*)?[a-zA-Z0-9])?$",
+        "maxLength": 63
+      }
+    },
+    "Types.KubeAnnotations": {
+      "type": [
+        "object",
+        "null"
+      ],
+      "propertyNames": {
+        "pattern": "^([a-zA-Z0-9][a-zA-Z0-9-.]*[a-zA-Z0-9]/)?([a-zA-Z0-9][-a-zA-Z0-9_.]*)?[a-zA-Z0-9]$",
+        "maxLength": 253
+      },
       "additionalProperties": {
         "type": "string"
       }

templates/kubernetes/onpremises/create-playbook.yaml.tpl

@@ -122,19 +122,84 @@
   tags:
     - kube-worker
 
-# We set the node's role to the node group's name in furyctl.yaml
-# We set a custom label with the role as part of the kubeadm boostrap of the
-# node via a kubelet flag.
-# The command below sets also the standard label that kubectl uses to show
-# the node role when you do `kubectl get nodes`. This label cannot be set via
-# the kubelet flag for security reasons.
-- name: Label nodes with role
+- name: Gather needed information for updating control-plane and nodes labels and annotations
+  hosts: nodes, master
+  tasks:
+    # TODO: furyctl has already checked for this secret, maybe we can pass it on from furyctl to the template engine
+    # somehow instead of downloading it again.
+    - name: Get previous cluster configuration
+      delegate_to: localhost
+      ansible.builtin.command: "{{ .paths.kubectl }} get secrets -n kube-system furyctl-config -o jsonpath='{.data.config}'"
+      register: previous_state
+      # We ignore the secret not found error because when we init the cluster the secret does not exist yet, so the command fails.
+      # Notice that all conditions must be true.
+      failed_when:
+        - previous_state.rc != 0
+        - '"Error from server (NotFound): secrets \"furyctl-config\" not found" not in previous_state.stderr'
+      # This is common for all the nodes, just run it once.
+      run_once: true
+
+    - name: Deserialize previous cluster configuration into a variable
+      delegate_to: localhost
+      ansible.builtin.set_fact:
+        furyctl_yaml: "{{ print "{{ previous_state.stdout | b64decode | from_yaml }}" }}"
+      when: previous_state.rc == 0 and previous_state.stdout != "null"
+      # This is common for all the nodes, just run it once.
+      run_once: true
+
+- name: Preparing control-plane labels and annotations
+  hosts: master
+  tasks:
+    - name: Format control-plane labels and annotations for usage with kubectl
+      vars:
+        # We calculate the removed labels and annotations so we can pass them as parameters with the appended `-` to delete them from the nodes.
+        # If they were not defined in the previous configuration we default to an empty list for calculating the difference.
+        removed_cp_annotations: "{{ "{{ furyctl_yaml.spec.kubernetes.masters.annotations | default([], true) | difference(kubernetes_node_annotations|default([], true)) }}" }}"
+        removed_cp_labels: "{{ "{{ furyctl_yaml.spec.kubernetes.masters.labels | default([], true) | difference(kubernetes_node_labels|default([], true)) }}" }}"
+      ansible.builtin.set_fact:
+        # We apply all the labels defined in the new configuration and delete the removed ones. We don't care if the rest are new or existed before, we just overwrite.
+        node_labels: "{{ "{% for l in kubernetes_node_labels|default([], true) %}{{l}}={{kubernetes_node_labels[l]}} {% endfor %}{% for rl in removed_cp_labels %}{{rl}}- {% endfor %}" }}"
+        node_annotations: "{{ "{% for a in kubernetes_node_annotations|default([], true) %} {{a}}={{kubernetes_node_annotations[a]|quote}} {% endfor %} {% for ra in removed_cp_annotations %}{{ra}}- {% endfor %}" }}"
+      # We run this once because labels are common for all the control plane hosts.
+      run_once: true
+
+- name: Preparing nodes labels and annotations
   hosts: nodes
   tasks:
-    - name: Get node's name and role
-      set_fact:
-        node_name: "{{ print "{{ kubernetes_hostname }}" }}"
-        node_role:  "{{ print "{{ kubernetes_role }}" }}"
-    - name: Label node
+    - name: Format nodes labels and annotations for usage with kubectl
+      vars:
+        # For the nodes we can't access directly the labels and annotations properties like we do for the masters
+        # because nodes is a list of node groups.
+        # We need to identify which element of the `nodes` list is the right one for this node.
+        node_group_details: "{{ "{{ furyctl_yaml.spec.kubernetes.nodes | selectattr('name', '==', kubernetes_role) | first }}" }}"
+        # We calculate the removed labels and annotations accessing the element of the `nodes` property we got in the previous line.
+        removed_node_labels: "{{ "{{ node_group_details.labels|default([], true) | difference(kubernetes_node_labels|default([], true)) }}" }}"
+        removed_node_annotations: "{{ "{{ node_group_details.annotations|default([], true) | difference(kubernetes_node_annotations|default([], true)) }}" }}"
+      ansible.builtin.set_fact:
+        node_labels: "{{ "{% for l in kubernetes_node_labels|default([], true) %}{{l}}={{kubernetes_node_labels[l]}} {% endfor %}{% for rl in removed_node_labels %}{{rl}}- {% endfor %}" }}"
+        node_annotations: "{{ "{% for a in kubernetes_node_annotations|default([], true) %} {{a}}={{kubernetes_node_annotations[a]|quote}} {% endfor %} {% for ra in removed_node_annotations %}{{ra}}- {% endfor %}" }}"
+
+- name: Update control-plane and nodes labels and annotations
+  hosts: nodes, master
+  tasks:
+
+    # We set here the label that determines the node role because the kubelet can't do it for security reasons.
+    # We set by default the role to be the name of the node group in the furyctl.yaml file.
+    # We do this only for the regular nodes. We don't need to do this for the control plane because kubeadm configures
+    # the kubelet to do it automatically.
+    - name: Set nodes role based on the node group's name
+      delegate_to: localhost
+      ansible.builtin.command: "{{ .paths.kubectl }} {{ "label node {{ kubernetes_hostname }} node-role.kubernetes.io/{{ kubernetes_role }}= --kubeconfig={{ kubernetes_kubeconfig_path }}admin.conf" }}"
+      when: kubernetes_role is defined
+
+    # Update the control plane and nodes labels with what we calculated before only if needed.
+    - name: Update node labels
+      delegate_to: localhost
+      ansible.builtin.command: "{{ .paths.kubectl }} {{ print "label node {{ kubernetes_hostname }} {{ node_labels }} --overwrite --kubeconfig={{ kubernetes_kubeconfig_path }}admin.conf" }}"
+      when: node_labels is defined and node_labels|trim != ''
+
+    # Update the control plane and nodes annotations with what we calculated before only if needed.
+    - name: Update node annotations
       delegate_to: localhost
-      shell: "{{ .paths.kubectl }} {{ print "label node {{ node_name }} node-role.kubernetes.io/{{ node_role }}= --kubeconfig={{ kubernetes_kubeconfig_path }}admin.conf" }}"
+      ansible.builtin.command: "{{ .paths.kubectl }} {{ print "annotate node {{ kubernetes_hostname }} {{ node_annotations }} --overwrite --kubeconfig={{ kubernetes_kubeconfig_path }}admin.conf" }}"
+      when: node_annotations is defined and node_annotations|trim != ''

templates/kubernetes/onpremises/hosts.yaml.tpl

@@ -28,6 +28,14 @@ all:
           ansible_host: "{{ $h.ip }}"
           kubernetes_apiserver_advertise_address: "{{ $h.ip }}"
           kubernetes_hostname: "{{ $h.name }}.{{ $dnsZone }}"
+          {{- if index $.spec.kubernetes.masters "labels" }}
+          kubernetes_node_labels:
+            {{ $.spec.kubernetes.masters.labels | toYaml | indent 12 | trim }}
+          {{- end }}
+          {{- if index $.spec.kubernetes.masters "annotations" }}
+          kubernetes_node_annotations:
+            {{ $.spec.kubernetes.masters.annotations | toYaml | indent 12 | trim }}
+          {{- end }}
         {{- end }}
       vars:
         dns_zone: "{{ $dnsZone }}"
@@ -102,6 +110,14 @@ all:
             kubernetes_taints:
               {{ $n.taints | toYaml | indent 14 | trim }}
             {{- end }}
+            {{- if index $n "labels" }}
+            kubernetes_node_labels:
+              {{ $n.labels | toYaml | indent 14 | trim }}
+            {{- end -}}
+            {{- if index $n "annotations" }}
+            kubernetes_node_annotations:
+              {{ $n.annotations | toYaml | indent 14 | trim }}
+            {{- end -}}
       {{- end }}
     ungrouped: {}
   vars: