From a3e569f5c4701806c9d0ddbcfa611ee240f1b3a6 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Wed, 18 Dec 2024 21:58:13 +0100 Subject: [PATCH 1/4] kubernetes/preinstall: switch coredns_server to vars/ --- .../preinstall/tasks/0020-set_facts.yml | 15 --------------- roles/kubernetes/preinstall/vars/main.yml | 7 +++++++ 2 files changed, 7 insertions(+), 15 deletions(-) diff --git a/roles/kubernetes/preinstall/tasks/0020-set_facts.yml b/roles/kubernetes/preinstall/tasks/0020-set_facts.yml index 6109479c1a4..3a0f8d8a3fb 100644 --- a/roles/kubernetes/preinstall/tasks/0020-set_facts.yml +++ b/roles/kubernetes/preinstall/tasks/0020-set_facts.yml @@ -162,21 +162,6 @@ supersede_domain: supersede domain-name "{{ dns_domain }}"; -- name: Pick coredns cluster IP or default resolver - set_fact: - coredns_server: |- - {%- if dns_mode == 'coredns' and not dns_early | bool -%} - {{ [skydns_server] }} - {%- elif dns_mode == 'coredns_dual' and not dns_early | bool -%} - {{ [skydns_server] + [skydns_server_secondary] }} - {%- elif dns_mode == 'manual' and not dns_early | bool -%} - {{ (manual_dns_server.split(',') | list) }} - {%- elif dns_mode == 'none' and not dns_early | bool -%} - [] - {%- elif dns_early | bool -%} - {{ upstream_dns_servers | default([]) }} - {%- endif -%} - # This task should only run after cluster/nodelocal DNS is up, otherwise all DNS lookups will timeout - name: Generate nameservers for resolvconf, including cluster DNS set_fact: diff --git a/roles/kubernetes/preinstall/vars/main.yml b/roles/kubernetes/preinstall/vars/main.yml index 699a87de569..12458b5579f 100644 --- a/roles/kubernetes/preinstall/vars/main.yml +++ b/roles/kubernetes/preinstall/vars/main.yml @@ -70,3 +70,10 @@ pkgs: tar: [] unzip: [] xfsprogs: [] + +coredns_server_by_mode: + coredns: "{{ [skydns_server] }}" + coredns_dual: "{{ [skydns_server, skydns_server_secondary] }}" + manual: "{{ manual_dns_server.split(',') }}" + none: [] +coredns_server: "{{ upstream_dns_server if dns_early else coredns_server_by_mode[dns_mode] }}" From 1127a6217672c036dfdd842928345868982fefaa Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Thu, 19 Dec 2024 11:37:40 +0100 Subject: [PATCH 2/4] kubernetes/preinstall: dns setting cleanup(dhclient, resolvconf) We use a lot of facts where variables are enough, and format too early, which prevent reusing the variables in different contexts. - Moves set_fact variables to the vars directory, remove unnecessary intermediate variables, and render them at usage sites to only do logic on native Ansible/Jinja lists. - Use defaults/ rather than default filters for several variables. --- .../docker/tasks/set_facts_dns.yml | 4 +- roles/kubernetes-apps/ansible/tasks/main.yml | 2 +- .../ansible/templates/coredns-config.yml.j2 | 2 +- roles/kubernetes/preinstall/defaults/main.yml | 3 ++ .../preinstall/tasks/0020-set_facts.yml | 39 +------------------ .../preinstall/tasks/0060-resolvconf.yml | 7 ++-- .../tasks/0063-networkmanager-dns.yml | 7 ++-- .../preinstall/tasks/0100-dhclient-hooks.yml | 5 ++- .../preinstall/templates/resolved.conf.j2 | 10 ++--- roles/kubernetes/preinstall/vars/main.yml | 15 ++++++- .../kubespray-defaults/defaults/main/main.yml | 5 +++ 11 files changed, 42 insertions(+), 57 deletions(-) diff --git a/roles/container-engine/docker/tasks/set_facts_dns.yml b/roles/container-engine/docker/tasks/set_facts_dns.yml index d0ccd745a70..42a2c64bc89 100644 --- a/roles/container-engine/docker/tasks/set_facts_dns.yml +++ b/roles/container-engine/docker/tasks/set_facts_dns.yml @@ -10,12 +10,12 @@ - name: Add upstream dns servers set_fact: - docker_dns_servers: "{{ docker_dns_servers + upstream_dns_servers | default([]) }}" + docker_dns_servers: "{{ docker_dns_servers + upstream_dns_servers }}" when: dns_mode in ['coredns', 'coredns_dual'] - name: Add global searchdomains set_fact: - docker_dns_search_domains: "{{ docker_dns_search_domains + searchdomains | default([]) }}" + docker_dns_search_domains: "{{ docker_dns_search_domains + searchdomains }}" - name: Check system nameservers shell: set -o pipefail && grep "^nameserver" /etc/resolv.conf | sed -r 's/^nameserver\s*([^#\s]+)\s*(#.*)?/\1/' diff --git a/roles/kubernetes-apps/ansible/tasks/main.yml b/roles/kubernetes-apps/ansible/tasks/main.yml index 5622f3b89f5..3f998f58d00 100644 --- a/roles/kubernetes-apps/ansible/tasks/main.yml +++ b/roles/kubernetes-apps/ansible/tasks/main.yml @@ -66,7 +66,7 @@ {{ primaryClusterIP }} {%- endif -%} upstreamForwardTarget: >- - {%- if upstream_dns_servers is defined and upstream_dns_servers | length > 0 -%} + {%- if upstream_dns_servers | length > 0 -%} {{ upstream_dns_servers | join(' ') }} {%- else -%} /etc/resolv.conf diff --git a/roles/kubernetes-apps/ansible/templates/coredns-config.yml.j2 b/roles/kubernetes-apps/ansible/templates/coredns-config.yml.j2 index 841dc6501d0..7671c0c95d6 100644 --- a/roles/kubernetes-apps/ansible/templates/coredns-config.yml.j2 +++ b/roles/kubernetes-apps/ansible/templates/coredns-config.yml.j2 @@ -68,7 +68,7 @@ data: {% endif %} } prometheus :9153 - forward . {{ upstream_dns_servers | join(' ') if upstream_dns_servers is defined and upstream_dns_servers | length > 0 else '/etc/resolv.conf' }} { + forward . {{ upstream_dns_servers | join(' ') if upstream_dns_servers | length > 0 else '/etc/resolv.conf' }} { prefer_udp max_concurrent 1000 {% if dns_upstream_forward_extra_opts is defined %} diff --git a/roles/kubernetes/preinstall/defaults/main.yml b/roles/kubernetes/preinstall/defaults/main.yml index ec0309d30c0..74b962c3e53 100644 --- a/roles/kubernetes/preinstall/defaults/main.yml +++ b/roles/kubernetes/preinstall/defaults/main.yml @@ -2,6 +2,9 @@ # Set to true to allow pre-checks to fail and continue deployment ignore_assert_errors: false +nameservers: [] +cloud_resolver: [] +disable_host_nameservers: false epel_enabled: false # Kubespray sets this to true after clusterDNS is running to apply changes to the host resolv.conf dns_late: false diff --git a/roles/kubernetes/preinstall/tasks/0020-set_facts.yml b/roles/kubernetes/preinstall/tasks/0020-set_facts.yml index 3a0f8d8a3fb..c8303e61d25 100644 --- a/roles/kubernetes/preinstall/tasks/0020-set_facts.yml +++ b/roles/kubernetes/preinstall/tasks/0020-set_facts.yml @@ -77,7 +77,7 @@ - name: Set default dns if remove_default_searchdomains is false set_fact: default_searchdomains: ["default.svc.{{ dns_domain }}", "svc.{{ dns_domain }}"] - when: not remove_default_searchdomains | default() | bool or (remove_default_searchdomains | default() | bool and searchdomains | default([]) | length==0) + when: not remove_default_searchdomains | default() | bool or (remove_default_searchdomains | default() | bool and searchdomains | length == 0) - name: Set dns facts set_fact: @@ -151,43 +151,6 @@ dhclienthookfile: /etc/dhcp/dhclient-exit-hooks.d/zdnsupdate when: ansible_os_family == "Debian" -- name: Generate search domains to resolvconf - set_fact: - searchentries: - search {{ (default_searchdomains | default([]) + searchdomains | default([])) | join(' ') }} - domainentry: - domain {{ dns_domain }} - supersede_search: - supersede domain-search "{{ (default_searchdomains | default([]) + searchdomains | default([])) | join('", "') }}"; - supersede_domain: - supersede domain-name "{{ dns_domain }}"; - -# This task should only run after cluster/nodelocal DNS is up, otherwise all DNS lookups will timeout -- name: Generate nameservers for resolvconf, including cluster DNS - set_fact: - nameserverentries: |- - {{ (([nodelocaldns_ip] if enable_nodelocaldns else []) + (coredns_server | d([]) if not enable_nodelocaldns else []) + nameservers | d([]) + cloud_resolver | d([]) + (configured_nameservers | d([]) if not disable_host_nameservers | d() | bool else [])) | unique | join(',') }} - dhclient_supersede_nameserver_entries_list: |- - {{ (([nodelocaldns_ip] if enable_nodelocaldns else []) + (coredns_server | d([]) if not enable_nodelocaldns else []) + nameservers | d([]) + cloud_resolver | d([]) + (configured_nameservers | d([]) if not disable_host_nameservers | d() | bool else [])) | unique }} - when: not dns_early or dns_late - -# This task should run instead of the above task when cluster/nodelocal DNS hasn't -# been deployed yet (like scale.yml/cluster.yml) or when it's down (reset.yml) -- name: Generate nameservers for resolvconf, not including cluster DNS - set_fact: - nameserverentries: |- - {{ (nameservers | d([]) + cloud_resolver | d([]) + configured_nameservers | d([])) | unique | join(',') }} - dhclient_supersede_nameserver_entries_list: |- - {{ (nameservers | d([]) + cloud_resolver | d([])) | unique }} - when: dns_early and not dns_late - -- name: Generate supersede_nameserver from dhclient_supersede_nameserver_entries_list - set_fact: - supersede_nameserver: |- - {%- if dhclient_supersede_nameserver_entries_list | length > 0 -%} - supersede domain-name-servers {{ dhclient_supersede_nameserver_entries_list | join(', ') }}; - {%- endif -%} - - name: Set etcd vars if using kubeadm mode set_fact: etcd_cert_dir: "{{ kube_cert_dir }}" diff --git a/roles/kubernetes/preinstall/tasks/0060-resolvconf.yml b/roles/kubernetes/preinstall/tasks/0060-resolvconf.yml index 9aad0dba8e3..b7e9a1ec903 100644 --- a/roles/kubernetes/preinstall/tasks/0060-resolvconf.yml +++ b/roles/kubernetes/preinstall/tasks/0060-resolvconf.yml @@ -7,10 +7,9 @@ blockinfile: path: "{{ resolvconffile }}" block: |- - {% for item in [domainentry] + [searchentries] -%} - {{ item }} - {% endfor %} - {% for item in nameserverentries.split(',') %} + domain {{ dns_domain }} + search {{ (default_searchdomains + searchdomains) | join(' ') }} + {% for item in nameserverentries %} nameserver {{ item }} {% endfor %} options ndots:{{ ndots }} timeout:{{ dns_timeout | default('2') }} attempts:{{ dns_attempts | default('2') }} diff --git a/roles/kubernetes/preinstall/tasks/0063-networkmanager-dns.yml b/roles/kubernetes/preinstall/tasks/0063-networkmanager-dns.yml index 6dfa7242643..d223a9468d2 100644 --- a/roles/kubernetes/preinstall/tasks/0063-networkmanager-dns.yml +++ b/roles/kubernetes/preinstall/tasks/0063-networkmanager-dns.yml @@ -4,11 +4,12 @@ path: /etc/NetworkManager/conf.d/dns.conf section: global-dns-domain-* option: servers - value: "{{ nameserverentries }}" + value: "{{ nameserverentries | join(',') }}" mode: '0600' backup: true when: - - nameserverentries != "127.0.0.53" or systemd_resolved_enabled.rc != 0 + - ('127.0.0.53' not in nameserverentries + or systemd_resolved_enabled.rc != 0) notify: Preinstall | update resolvconf for networkmanager - name: Set default dns if remove_default_searchdomains is false @@ -21,7 +22,7 @@ path: /etc/NetworkManager/conf.d/dns.conf section: global-dns option: searches - value: "{{ (default_searchdomains | default([]) + searchdomains | default([])) | join(',') }}" + value: "{{ (default_searchdomains | default([]) + searchdomains) | join(',') }}" mode: '0600' backup: true notify: Preinstall | update resolvconf for networkmanager diff --git a/roles/kubernetes/preinstall/tasks/0100-dhclient-hooks.yml b/roles/kubernetes/preinstall/tasks/0100-dhclient-hooks.yml index 9745ab261bc..24ae1bbc9dd 100644 --- a/roles/kubernetes/preinstall/tasks/0100-dhclient-hooks.yml +++ b/roles/kubernetes/preinstall/tasks/0100-dhclient-hooks.yml @@ -1,9 +1,10 @@ --- - name: Configure dhclient to supersede search/domain/nameservers blockinfile: + # 1 is the 2nd item of a tuple in items() block: |- - {% for item in [supersede_domain, supersede_search, supersede_nameserver] | reject('equalto', '') -%} - {{ item }} + {% for key, val in dhclient_supersede.items() | rejectattr(1, '==', []) -%} + supersede {{ key }} {{ val | join(',') }}; {% endfor %} path: "{{ dhclientconffile }}" create: true diff --git a/roles/kubernetes/preinstall/templates/resolved.conf.j2 b/roles/kubernetes/preinstall/templates/resolved.conf.j2 index edafbf9b9d3..85f02468da0 100644 --- a/roles/kubernetes/preinstall/templates/resolved.conf.j2 +++ b/roles/kubernetes/preinstall/templates/resolved.conf.j2 @@ -1,12 +1,12 @@ [Resolve] {% if not dns_early and dns_late %} -DNS={{ ([nodelocaldns_ip] if enable_nodelocaldns else coredns_server )| list | join(' ') }} +DNS={{ ([nodelocaldns_ip] if enable_nodelocaldns else coredns_server) | list | join(' ') }} {% endif %} -FallbackDNS={{ ( upstream_dns_servers|d([]) + nameservers|d([]) + cloud_resolver|d([])) | unique | join(' ') }} -{% if remove_default_searchdomains and searchdomains|default([])|length != 0 %} -Domains={{ searchdomains|default([]) | join(' ') }} +FallbackDNS={{ ( upstream_dns_servers + nameservers + cloud_resolver) | unique | join(' ') }} +{% if remove_default_searchdomains and searchdomains | length != 0 %} +Domains={{ searchdomains | join(' ') }} {% else %} -Domains={{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains|default([])) | join(' ') }} +Domains={{ ([ 'default.svc.' + dns_domain, 'svc.' + dns_domain ] + searchdomains) | join(' ') }} {% endif %} DNSSEC=no Cache=no-negative diff --git a/roles/kubernetes/preinstall/vars/main.yml b/roles/kubernetes/preinstall/vars/main.yml index 12458b5579f..4052cb26062 100644 --- a/roles/kubernetes/preinstall/vars/main.yml +++ b/roles/kubernetes/preinstall/vars/main.yml @@ -76,4 +76,17 @@ coredns_server_by_mode: coredns_dual: "{{ [skydns_server, skydns_server_secondary] }}" manual: "{{ manual_dns_server.split(',') }}" none: [] -coredns_server: "{{ upstream_dns_server if dns_early else coredns_server_by_mode[dns_mode] }}" +coredns_server: "{{ upstream_dns_servers if dns_early else coredns_server_by_mode[dns_mode] }}" + +_nameserverentries: + late: + - "{{ nodelocaldns_ip if enable_nodelocaldns else coredns_server }}" + early: + - "{{ nameservers }}" + - "{{ cloud_resolver }}" + - "{{ configured_nameservers if not disable_host_nameservers else [] }}" +nameserverentries: "{{ ((_nameserverentries['late'] if not dns_early else []) + _nameserverentries['early']) | flatten | unique }}" +dhclient_supersede: + domain-name-servers: "{{ ([nameservers, cloud_resolver] | flatten | unique) if dns_early else nameserverentries }}" + domain-name: "{{ [dns_domain] }}" + domain-search: "{{ default_searchdomains + searchdomains }}" diff --git a/roles/kubespray-defaults/defaults/main/main.yml b/roles/kubespray-defaults/defaults/main/main.yml index 32a78545a3d..93e957ccf04 100644 --- a/roles/kubespray-defaults/defaults/main/main.yml +++ b/roles/kubespray-defaults/defaults/main/main.yml @@ -114,6 +114,10 @@ dns_mode: coredns # Enable dns autoscaler enable_dns_autoscaler: true +# DNS servers added after the cluster DNS +# These will also be used as upstream by Coredns for out-cluster queries +upstream_dns_servers: [] + # Enable nodelocal dns cache enable_nodelocaldns: true enable_nodelocaldns_secondary: false @@ -137,6 +141,7 @@ dns_domain: "{{ cluster_name }}" docker_dns_search_domains: - 'default.svc.{{ dns_domain }}' - 'svc.{{ dns_domain }}' +searchdomains: [] kube_dns_servers: coredns: ["{{ skydns_server }}"] From 55e095c1c775842e566483874bfed82fc0b4e73c Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Thu, 19 Dec 2024 14:21:22 +0100 Subject: [PATCH 3/4] kubernetes/preinstall: dns vars cleanup - Move validation from facts to verify-settings - Move set_fact to vars/ --- .../preinstall/tasks/0020-set_facts.yml | 28 ++++--------------- .../preinstall/tasks/0040-verify-settings.yml | 9 ++++++ roles/kubernetes/preinstall/vars/main.yml | 2 ++ 3 files changed, 16 insertions(+), 23 deletions(-) diff --git a/roles/kubernetes/preinstall/tasks/0020-set_facts.yml b/roles/kubernetes/preinstall/tasks/0020-set_facts.yml index c8303e61d25..9b1f0faeb05 100644 --- a/roles/kubernetes/preinstall/tasks/0020-set_facts.yml +++ b/roles/kubernetes/preinstall/tasks/0020-set_facts.yml @@ -33,30 +33,12 @@ failed_when: false register: resolvconf_stat + # Used in vars/ - name: Fetch resolvconf - when: resolvconf_stat.stat.exists is defined and resolvconf_stat.stat.exists - block: - - - name: Get content of /etc/resolv.conf - slurp: - src: /etc/resolv.conf - register: resolvconf_slurp - - - name: Get currently configured nameservers - set_fact: - configured_nameservers: "{{ resolvconf_slurp.content | b64decode | regex_findall('^nameserver\\s*(\\S*)', multiline=True) | ansible.utils.ipaddr }}" - when: resolvconf_slurp.content is defined - -- name: Stop if /etc/resolv.conf not configured nameservers - assert: - that: configured_nameservers | length>0 - fail_msg: "nameserver should not empty in /etc/resolv.conf" - when: - - not ignore_assert_errors - - configured_nameservers is defined - - not (upstream_dns_servers is defined and upstream_dns_servers | length > 0) - - not (disable_host_nameservers | default(false)) - - dns_mode in ['coredns', 'coredns_dual'] + when: resolvconf_stat.stat.exists + slurp: + src: /etc/resolv.conf + register: resolvconf_slurp - name: NetworkManager | Check if host has NetworkManager # noqa command-instead-of-module - Should we use service_facts for this? diff --git a/roles/kubernetes/preinstall/tasks/0040-verify-settings.yml b/roles/kubernetes/preinstall/tasks/0040-verify-settings.yml index e911c76ccbd..23a05bf6ec7 100644 --- a/roles/kubernetes/preinstall/tasks/0040-verify-settings.yml +++ b/roles/kubernetes/preinstall/tasks/0040-verify-settings.yml @@ -220,6 +220,15 @@ when: dns_mode is defined run_once: true +- name: Stop if /etc/resolv.conf has no configured nameservers + assert: + that: configured_nameservers | length>0 + fail_msg: "nameserver should not empty in /etc/resolv.conf" + when: + - upstream_dns_servers | length == 0 + - not disable_host_nameservers + - dns_mode in ['coredns', 'coredns_dual'] + - name: Stop if unknown kube proxy mode assert: that: kube_proxy_mode in ['iptables', 'ipvs'] diff --git a/roles/kubernetes/preinstall/vars/main.yml b/roles/kubernetes/preinstall/vars/main.yml index 4052cb26062..872d2ed1eb9 100644 --- a/roles/kubernetes/preinstall/vars/main.yml +++ b/roles/kubernetes/preinstall/vars/main.yml @@ -90,3 +90,5 @@ dhclient_supersede: domain-name-servers: "{{ ([nameservers, cloud_resolver] | flatten | unique) if dns_early else nameserverentries }}" domain-name: "{{ [dns_domain] }}" domain-search: "{{ default_searchdomains + searchdomains }}" +configured_nameservers: "{{ (resolvconf_slurp.content | b64decode | regex_findall('^nameserver\\s*(\\S*)', multiline=True) | ansible.utils.ipaddr) + if resolvconf_stat.stat.exists else [] }}" From 684f52eaf443810277eb9249a1ba32234020f673 Mon Sep 17 00:00:00 2001 From: Max Gautier Date: Thu, 19 Dec 2024 15:24:39 +0100 Subject: [PATCH 4/4] kubernetes/preinstall: remove unused variable --- roles/kubernetes/preinstall/tasks/0020-set_facts.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/roles/kubernetes/preinstall/tasks/0020-set_facts.yml b/roles/kubernetes/preinstall/tasks/0020-set_facts.yml index 9b1f0faeb05..9213f6bce3e 100644 --- a/roles/kubernetes/preinstall/tasks/0020-set_facts.yml +++ b/roles/kubernetes/preinstall/tasks/0020-set_facts.yml @@ -34,7 +34,7 @@ register: resolvconf_stat # Used in vars/ -- name: Fetch resolvconf +- name: Fetch resolv.conf when: resolvconf_stat.stat.exists slurp: src: /etc/resolv.conf @@ -65,10 +65,6 @@ set_fact: resolvconf: >- {%- if resolvconf.rc == 0 and resolvconfd_path.stat.isdir is defined and resolvconfd_path.stat.isdir -%}true{%- else -%}false{%- endif -%} - bogus_domains: |- - {% for d in default_searchdomains | default([]) + searchdomains | default([]) -%} - {{ dns_domain }}.{{ d }}./{{ d }}.{{ d }}./com.{{ d }}./ - {%- endfor %} - name: Check if kubelet is configured stat: