Remove access to cluster from anonymous users (#11016)

* feat: add user facing variable with default

* feat: remove rolebinding to anonymous users after init and upgrade

* feat: use file discovery for secondary control plane nodes

* feat: use file discovery for nodes

* fix: do not fail if rolebinding does not exist

* docs: add warning about kube_api_anonymous_auth

* style: improve readability of delegate_to parameter

* refactor: rename discovery kubeconfig file

* test: enable new variable in hardening and upgrade test cases

* docs: add option to config parameters

* test: multiple instances and upgrade

docs/vars.md

@@ -281,6 +281,11 @@ node_taints:
   * `audit_webhook_batch_max_wait`: 1s
 * *kubectl_alias* - Bash alias of kubectl to interact with Kubernetes cluster much easier.
 
+* *remove_anonymous_access* - When set to `true`, removes the `kubeadm:bootstrap-signer-clusterinfo` rolebinding created by kubeadm.
+  By default, kubeadm creates a rolebinding in the `kube-public` namespace which grants permissions to anonymous users. This rolebinding allows kubeadm to discover and validate cluster information during the join phase.
+  In a nutshell, this option removes the rolebinding after the init phase of the first control plane node and then configures kubeadm to use file discovery for the join phase of other nodes.
+  This option does not remove the anonymous authentication feature of the API server.
+
 ### Custom flags for Kube Components
 
 For all kube components, custom flags can be passed in. This allows for edge cases where users need changes to the default deployment that may not be applicable to all deployments.

inventory/sample/group_vars/k8s_cluster/k8s-cluster.yml

@@ -371,3 +371,6 @@ kubeadm_patches:
   enabled: false
   source_dir: "{{ inventory_dir }}/patches"
   dest_dir: "{{ kube_config_dir }}/patches"
+
+# Set to true to remove the role binding to anonymous users created by kubeadm
+remove_anonymous_access: false

roles/kubernetes/control-plane/defaults/main/main.yml

@@ -240,3 +240,6 @@ kubeadm_upgrade_auto_cert_renewal: true
 kube_apiserver_tracing: false
 kube_apiserver_tracing_endpoint: 0.0.0.0:4317
 kube_apiserver_tracing_sampling_rate_per_million: 100
+
+# Enable kubeadm file discovery if anonymous access has been removed
+kubeadm_use_file_discovery: "{{ remove_anonymous_access }}"

roles/kubernetes/control-plane/tasks/kubeadm-secondary.yml

@@ -63,6 +63,26 @@
     - kubeadm_already_run is not defined or not kubeadm_already_run.stat.exists
     - not kube_external_ca_mode
 
+- name: Get kubeconfig for join discovery process
+  command: "{{ kubectl }} -n kube-public get cm cluster-info -o jsonpath='{.data.kubeconfig}'"
+  register: kubeconfig_file_discovery
+  run_once: true
+  delegate_to: "{{ groups['kube_control_plane'] | first }}"
+  when:
+    - kubeadm_use_file_discovery
+    - kubeadm_already_run is not defined or not kubeadm_already_run.stat.exists
+
+- name: Copy discovery kubeconfig
+  copy:
+    dest: "{{ kube_config_dir }}/cluster-info-discovery-kubeconfig.yaml"
+    content: "{{ kubeconfig_file_discovery.stdout }}"
+    owner: "root"
+    mode: 0644
+  when:
+    - inventory_hostname != first_kube_control_plane
+    - kubeadm_use_file_discovery
+    - kubeadm_already_run is not defined or not kubeadm_already_run.stat.exists
+
 - name: Joining control plane node to the cluster.
   command: >-
     {{ bin_dir }}/kubeadm join

roles/kubernetes/control-plane/tasks/kubeadm-setup.yml

@@ -221,12 +221,16 @@
     {{ bin_dir }}/kubeadm --kubeconfig {{ kube_config_dir }}/admin.conf token create {{ kubeadm_token }}
   changed_when: false
   when:
-    - inventory_hostname ==  first_kube_control_plane
+    - inventory_hostname == first_kube_control_plane
     - kubeadm_token is defined
     - kubeadm_refresh_token
   tags:
     - kubeadm_token
 
+- name: Remove binding to anonymous user
+  command: "{{ kubectl }} -n kube-public delete rolebinding kubeadm:bootstrap-signer-clusterinfo --ignore-not-found"
+  when: inventory_hostname == first_kube_control_plane and remove_anonymous_access
+
 - name: Create kubeadm token for joining nodes with 24h expiration (default)
   command: "{{ bin_dir }}/kubeadm --kubeconfig {{ kube_config_dir }}/admin.conf token create"
   changed_when: false

roles/kubernetes/control-plane/tasks/kubeadm-upgrade.yml

@@ -53,6 +53,10 @@
     PATH: "{{ bin_dir }}:{{ ansible_env.PATH }}"
   notify: Master | restart kubelet
 
+- name: Kubeadm | Remove binding to anonymous user
+  command: "{{ kubectl }} -n kube-public delete rolebinding kubeadm:bootstrap-signer-clusterinfo --ignore-not-found"
+  when: remove_anonymous_access
+
 - name: Kubeadm | clean kubectl cache to refresh api types
   file:
     path: "{{ item }}"

roles/kubernetes/control-plane/templates/kubeadm-controlplane.v1beta3.yaml.j2

@@ -1,6 +1,10 @@
 apiVersion: kubeadm.k8s.io/v1beta3
 kind: JoinConfiguration
 discovery:
+{% if kubeadm_use_file_discovery %}
+  file:
+    kubeConfigPath: {{ kube_config_dir }}/cluster-info-discovery-kubeconfig.yaml
+{% else %}
   bootstrapToken:
 {% if kubeadm_config_api_fqdn is defined %}
     apiServerEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
@@ -9,6 +13,7 @@ discovery:
 {% endif %}
     token: {{ kubeadm_token }}
     unsafeSkipCAVerification: true
+{% endif %}
   timeout: {{ discovery_timeout }}
   tlsBootstrapToken: {{ kubeadm_token }}
 controlPlane:

roles/kubernetes/kubeadm/defaults/main.yml

@@ -4,6 +4,9 @@
 discovery_timeout: 60s
 kubeadm_join_timeout: 120s
 
+# Enable kubeadm file discovery if anonymous access has been removed
+kubeadm_use_file_discovery: "{{ remove_anonymous_access }}"
+
 # If non-empty, will use this string as identification instead of the actual hostname
 kube_override_hostname: >-
   {%- if cloud_provider is defined and cloud_provider in ['aws'] -%}

roles/kubernetes/kubeadm/tasks/main.yml

@@ -57,6 +57,24 @@
   set_fact:
     kubeadmConfig_api_version: v1beta3
 
+- name: Get kubeconfig for join discovery process
+  command: "{{ kubectl }} -n kube-public get cm cluster-info -o jsonpath='{.data.kubeconfig}'"
+  register: kubeconfig_file_discovery
+  run_once: true
+  delegate_to: "{{ groups['kube_control_plane'] | first }}"
+  when: kubeadm_use_file_discovery
+
+- name: Copy discovery kubeconfig
+  copy:
+    dest: "{{ kube_config_dir }}/cluster-info-discovery-kubeconfig.yaml"
+    content: "{{ kubeconfig_file_discovery.stdout }}"
+    owner: "root"
+    mode: 0644
+  when:
+    - not is_kube_master
+    - not kubelet_conf.stat.exists
+    - kubeadm_use_file_discovery
+
 - name: Create kubeadm client config
   template:
     src: "kubeadm-client.conf.{{ kubeadmConfig_api_version }}.j2"

roles/kubernetes/kubeadm/templates/kubeadm-client.conf.v1beta3.j2

@@ -2,6 +2,10 @@
 apiVersion: kubeadm.k8s.io/v1beta3
 kind: JoinConfiguration
 discovery:
+{% if kubeadm_use_file_discovery %}
+  file:
+    kubeConfigPath: {{ kube_config_dir }}/cluster-info-discovery-kubeconfig.yaml
+{% else %}
   bootstrapToken:
 {% if kubeadm_config_api_fqdn is defined %}
     apiServerEndpoint: {{ kubeadm_config_api_fqdn }}:{{ loadbalancer_apiserver.port | default(kube_apiserver_port) }}
@@ -14,6 +18,7 @@ discovery:
     - sha256:{{ kubeadm_ca_hash.stdout }}
 {% else %}
     unsafeSkipCAVerification: true
+{% endif %}
 {% endif %}
   timeout: {{ discovery_timeout }}
   tlsBootstrapToken: {{ kubeadm_token }}

roles/kubespray-defaults/defaults/main/main.yml

@@ -6,6 +6,8 @@ ansible_ssh_common_args: "{% if 'bastion' in groups['all'] %} -o ProxyCommand='s
 # selinux state
 preinstall_selinux_state: permissive
 
+# Setting this value to false will fail
+# For details, read this comment https://github.com/kubernetes-sigs/kubespray/pull/11016#issuecomment-2004985001
 kube_api_anonymous_auth: true
 
 # Default value, but will be set to true automatically if detected
@@ -50,6 +52,9 @@ kubeadm_join_phases_skip_default: []
 kubeadm_join_phases_skip: >-
  {{ kubeadm_join_phases_skip_default }}
 
+# Set to true to remove the role binding to anonymous users created by kubeadm
+remove_anonymous_access: false
+
 # A string slice of values which specify the addresses to use for NodePorts.
 # Values may be valid IP blocks (e.g. 1.2.3.0/24, 1.2.3.4/32).
 # The default empty string slice ([]) means to use all local addresses.

tests/files/packet_debian11-calico-upgrade.yml

@@ -11,3 +11,6 @@ calico_network_backend: bird
 
 # Needed to bypass deprecation check
 ignore_assert_errors: true
+
+# Remove anonymous access to cluster
+remove_anonymous_access: true

tests/files/packet_ubuntu20-calico-all-in-one-hardening.yml

@@ -104,3 +104,6 @@ kube_cert_group: root
 # kube-system namespace is exempted by default
 kube_pod_security_use_default: true
 kube_pod_security_default_enforce: restricted
+
+# Remove anonymous access to cluster
+remove_anonymous_access: true

tests/files/packet_ubuntu20-calico-etcd-kubeadm.yml

@@ -9,3 +9,6 @@ etcd_deployment_type: kubeadm
 # Currently ipvs not available on KVM: https://packages.ubuntu.com/search?suite=focal&arch=amd64&mode=exactfilename&searchon=contents&keywords=ip_vs_sh.ko
 kube_proxy_mode: iptables
 enable_nodelocaldns: False
+
+# Remove anonymous access to cluster
+remove_anonymous_access: true