Merge branch 'master' of https://github.com/kubernetes-sigs/kubespray

* 'master' of https://github.com/kubernetes-sigs/kubespray:
  Upgrade etcd to 3.4.3 (kubernetes-sigs#5998)
  add audit webhook support (kubernetes-sigs#6317)

README.md

@@ -116,7 +116,7 @@ Note: Upstart/SysV init based OS types are not supported.
 
 - Core
   - [kubernetes](https://github.com/kubernetes/kubernetes) v1.18.5
-  - [etcd](https://github.com/coreos/etcd) v3.3.12
+  - [etcd](https://github.com/coreos/etcd) v3.4.3
   - [docker](https://www.docker.com/) v19.03 (see note)
   - [containerd](https://containerd.io/) v1.2.13
   - [cri-o](http://cri-o.io/) v1.17 (experimental: see [CRI-O Note](docs/cri-o.md). Only on fedora, ubuntu and centos based OS)

roles/download/defaults/main.yml

@@ -51,7 +51,7 @@ image_arch: "{{host_architecture | default('amd64')}}"
 # Versions
 kube_version: v1.18.5
 kubeadm_version: "{{ kube_version }}"
-etcd_version: v3.3.12
+etcd_version: v3.4.3
 
 # gcr and kubernetes image repo define
 gcr_image_repo: "gcr.io"
@@ -376,8 +376,8 @@ etcd_binary_checksums:
   # Etcd does not have arm32 builds at the moment, having some dummy value is
   # required to avoid "no attribute" error
   arm: 0
-  arm64: 170b848ac1a071fe7d495d404a868a2c0090750b2944f8a260ef1c6125b2b4f4
-  amd64: dc5d82df095dae0a2970e4d870b6929590689dd707ae3d33e7b86da0f7f211b6
+  arm64: 01bd849ad99693600bd59db8d0e66ac64aac1e3801900665c31bd393972e3554
+  amd64: 6c642b723a86941b99753dff6c00b26d3b033209b15ee33325dc8e7f4cd68f07
 cni_binary_checksums:
   arm: 28e61b5847265135dc1ca397bf94322ecce4acab5c79cc7d360ca3f6a655bdb7
   arm64: 43fbf750c5eccb10accffeeb092693c32b236fb25d919cf058c91a677822c999

roles/etcd/handlers/backup.yml

@@ -49,7 +49,7 @@
       snapshot save {{ etcd_backup_directory }}/snapshot.db
   environment:
     ETCDCTL_API: 3
-    ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
+    ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses.split(',') | first }}"
     ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
     ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
     ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"

roles/etcd/tasks/configure.yml

@@ -1,6 +1,6 @@
 ---
 - name: Configure | Check if etcd cluster is healthy
-  shell: "{{ bin_dir }}/etcdctl cluster-health | grep -q 'cluster is healthy'"
+  shell: "{{ bin_dir }}/etcdctl endpoint --cluster status && {{ bin_dir }}/etcdctl endpoint --cluster health  2>&1 | grep -q -v 'Error: unhealthy cluster'"
   register: etcd_cluster_is_healthy
   failed_when: false
   changed_when: false
@@ -10,14 +10,14 @@
   tags:
     - facts
   environment:
-    ETCDCTL_API: 2
+    ETCDCTL_API: 3
+    ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
+    ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
     ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
-    ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
-    ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
-    ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"
 
 - name: Configure | Check if etcd-events cluster is healthy
-  shell: "{{ bin_dir }}/etcdctl cluster-health | grep -q 'cluster is healthy'"
+  shell: "{{ bin_dir }}/etcdctl endpoint --cluster status && {{ bin_dir }}/etcdctl endpoint --cluster health  2>&1 | grep -q -v 'Error: unhealthy cluster'"
   register: etcd_events_cluster_is_healthy
   failed_when: false
   changed_when: false
@@ -27,11 +27,11 @@
   tags:
     - facts
   environment:
-    ETCDCTL_API: 2
+    ETCDCTL_API: 3
+    ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
+    ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
     ETCDCTL_ENDPOINTS: "{{ etcd_events_access_addresses }}"
-    ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
-    ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
-    ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"
 
 - include_tasks: refresh_config.yml
   when: is_etcd_master
@@ -74,12 +74,11 @@
   when: is_etcd_master and etcd_events_cluster_setup
 
 - name: Configure | Wait for etcd cluster to be healthy
-  shell: "{{ bin_dir }}/etcdctl --no-sync cluster-health | grep -q 'cluster is healthy'"
+  shell: "{{ bin_dir }}/etcdctl endpoint --cluster status && {{ bin_dir }}/etcdctl endpoint --cluster health 2>&1 | grep -q -v 'Error: unhealthy cluster'"
   register: etcd_cluster_is_healthy
   until: etcd_cluster_is_healthy.rc == 0
   retries: "{{ etcd_retries }}"
   delay: "{{ retry_stagger | random + 3 }}"
-  ignore_errors: false
   changed_when: false
   check_mode: no
   run_once: yes
@@ -89,19 +88,18 @@
   tags:
     - facts
   environment:
-    ETCDCTL_API: 2
+    ETCDCTL_API: 3
+    ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
+    ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
     ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
-    ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
-    ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
-    ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"
 
 - name: Configure | Wait for etcd-events cluster to be healthy
-  shell: "{{ bin_dir }}/etcdctl --no-sync cluster-health | grep -q 'cluster is healthy'"
+  shell: "{{ bin_dir }}/etcdctl endpoint --cluster status && {{ bin_dir }}/etcdctl endpoint --cluster health 2>&1 | grep -q -v 'Error: unhealthy cluster'"
   register: etcd_events_cluster_is_healthy
   until: etcd_events_cluster_is_healthy.rc == 0
   retries: "{{ etcd_retries }}"
   delay: "{{ retry_stagger | random + 3 }}"
-  ignore_errors: false
   changed_when: false
   check_mode: no
   run_once: yes
@@ -111,14 +109,14 @@
   tags:
     - facts
   environment:
-    ETCDCTL_API: 2
+    ETCDCTL_API: 3
+    ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
+    ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
     ETCDCTL_ENDPOINTS: "{{ etcd_events_access_addresses }}"
-    ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
-    ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
-    ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"
 
 - name: Configure | Check if member is in etcd cluster
-  shell: "{{ bin_dir }}/etcdctl --no-sync member list | grep -q {{ etcd_access_address }}"
+  shell: "{{ bin_dir }}/etcdctl member list | grep -q {{ etcd_access_address }}"
   register: etcd_member_in_cluster
   ignore_errors: true
   changed_when: false
@@ -127,14 +125,14 @@
   tags:
     - facts
   environment:
-    ETCDCTL_API: 2
+    ETCDCTL_API: 3
+    ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
+    ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
     ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
-    ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
-    ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
-    ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"
 
 - name: Configure | Check if member is in etcd-events cluster
-  shell: "{{ bin_dir }}/etcdctl --no-sync member list | grep -q {{ etcd_access_address }}"
+  shell: "{{ bin_dir }}/etcdctl member list | grep -q {{ etcd_access_address }}"
   register: etcd_events_member_in_cluster
   ignore_errors: true
   changed_when: false
@@ -143,11 +141,11 @@
   tags:
     - facts
   environment:
-    ETCDCTL_API: 2
+    ETCDCTL_API: 3
+    ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
+    ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
     ETCDCTL_ENDPOINTS: "{{ etcd_events_access_addresses }}"
-    ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
-    ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
-    ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"
 
 - name: Configure | Join member(s) to etcd cluster one at a time
   include_tasks: join_etcd_member.yml

roles/etcd/tasks/join_etcd-events_member.yml

@@ -1,15 +1,16 @@
 ---
 - name: Join Member | Add member to etcd-events cluster
-  shell: "{{ bin_dir }}/etcdctl member add {{ etcd_member_name }} {{ etcd_events_peer_url }}"
+  shell: "{{ bin_dir }}/etcdctl member add {{ etcd_member_name }} --peer-urls={{ etcd_events_peer_url }}"
   register: member_add_result
   until: member_add_result.rc == 0
   retries: "{{ etcd_retries }}"
   delay: "{{ retry_stagger | random + 3 }}"
   environment:
-    ETCDCTL_API: 2
+    ETCDCTL_API: 3
+    ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
+    ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
     ETCDCTL_ENDPOINTS: "{{ etcd_events_access_addresses }}"
-    ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
-    ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
 
 - include_tasks: refresh_config.yml
   vars:
@@ -24,17 +25,18 @@
       {%- endfor -%}
 
 - name: Join Member | Ensure member is in etcd-events cluster
-  shell: "{{ bin_dir }}/etcdctl --no-sync member list | grep -q {{ etcd_events_access_address }}"
+  shell: "{{ bin_dir }}/etcdctl member list | grep -q {{ etcd_events_access_address }}"
   register: etcd_events_member_in_cluster
   changed_when: false
   check_mode: no
   tags:
     - facts
   environment:
-    ETCDCTL_API: 2
+    ETCDCTL_API: 3
+    ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
+    ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
     ETCDCTL_ENDPOINTS: "{{ etcd_events_access_addresses }}"
-    ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
-    ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
 
 - name: Configure | Ensure etcd-events is running
   service:

roles/etcd/tasks/join_etcd_member.yml

@@ -1,16 +1,16 @@
 ---
 - name: Join Member | Add member to etcd cluster
-  shell: "{{ bin_dir }}/etcdctl member add {{ etcd_member_name }} {{ etcd_peer_url }}"
+  shell: "{{ bin_dir }}/etcdctl member add {{ etcd_member_name }} --peer-urls={{ etcd_peer_url }}"
   register: member_add_result
   until: member_add_result.rc == 0
   retries: "{{ etcd_retries }}"
   delay: "{{ retry_stagger | random + 3 }}"
   environment:
-    ETCDCTL_API: 2
+    ETCDCTL_API: 3
+    ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
+    ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
     ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
-    ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
-    ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
-    ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"
 
 - include_tasks: refresh_config.yml
   vars:
@@ -25,18 +25,18 @@
       {%- endfor -%}
 
 - name: Join Member | Ensure member is in etcd cluster
-  shell: "{{ bin_dir }}/etcdctl --no-sync member list | grep -q {{ etcd_access_address }}"
+  shell: "{{ bin_dir }}/etcdctl member list | grep -q {{ etcd_access_address }}"
   register: etcd_member_in_cluster
   changed_when: false
   check_mode: no
   tags:
     - facts
   environment:
-    ETCDCTL_API: 2
+    ETCDCTL_API: 3
+    ETCDCTL_CERT: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
+    ETCDCTL_KEY: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
+    ETCDCTL_CACERT: "{{ etcd_cert_dir }}/ca.pem"
     ETCDCTL_ENDPOINTS: "{{ etcd_access_addresses }}"
-    ETCDCTL_CERT_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem"
-    ETCDCTL_KEY_FILE: "{{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem"
-    ETCDCTL_CA_FILE: "{{ etcd_cert_dir }}/ca.pem"
 
 - name: Configure | Ensure etcd is running
   service:

roles/etcd/templates/etcd-events.env.j2

@@ -1,7 +1,7 @@
 ETCD_DATA_DIR={{ etcd_events_data_dir }}
 ETCD_ADVERTISE_CLIENT_URLS={{ etcd_events_client_url }}
 ETCD_INITIAL_ADVERTISE_PEER_URLS={{ etcd_events_peer_url }}
-ETCD_INITIAL_CLUSTER_STATE={% if etcd_events_cluster_is_healthy.rc != 0 | bool %}new{% else %}existing{% endif %}
+ETCD_INITIAL_CLUSTER_STATE={% if etcd_events_cluster_is_healthy.rc == 0 | bool %}existing{% else %}new{% endif %}
 
 ETCD_METRICS={{ etcd_metrics }}
 ETCD_LISTEN_CLIENT_URLS=https://{{ etcd_address }}:2381,https://127.0.0.1:2381

roles/etcd/templates/etcd.env.j2

@@ -2,7 +2,7 @@
 ETCD_DATA_DIR={{ etcd_data_dir }}
 ETCD_ADVERTISE_CLIENT_URLS={{ etcd_client_url }}
 ETCD_INITIAL_ADVERTISE_PEER_URLS={{ etcd_peer_url }}
-ETCD_INITIAL_CLUSTER_STATE={% if etcd_cluster_is_healthy.rc != 0 | bool %}new{% else %}existing{% endif %}
+ETCD_INITIAL_CLUSTER_STATE={% if etcd_cluster_is_healthy.rc == 0 | bool %}existing{% else %}new{% endif %}
 
 ETCD_METRICS={{ etcd_metrics }}
 {% if etcd_metrics_port is defined %}
@@ -26,6 +26,8 @@ ETCD_QUOTA_BACKEND_BYTES={{ etcd_quota_backend_bytes }}
 {% if etcd_log_package_levels is defined %}
 ETCD_LOG_PACKAGE_LEVELS={{ etcd_log_package_levels }}
 {% endif %}
+# Flannel need etcd v2 API
+ETCD_ENABLE_V2=true
 
 # TLS settings
 ETCD_TRUSTED_CA_FILE={{ etcd_cert_dir }}/ca.pem
@@ -48,6 +50,6 @@ ETCD_UNSUPPORTED_ARCH={{host_architecture}}
 
 # CLI settings
 ETCDCTL_ENDPOINTS=https://127.0.0.1:2379
-ETCDCTL_CA_FILE={{ etcd_cert_dir }}/ca.pem
-ETCDCTL_KEY_FILE={{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem
-ETCDCTL_CERT_FILE={{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem
+ETCDCTL_CACERT={{ etcd_cert_dir }}/ca.pem
+ETCDCTL_KEY={{ etcd_cert_dir }}/admin-{{ inventory_hostname }}-key.pem
+ETCDCTL_CERT={{ etcd_cert_dir }}/admin-{{ inventory_hostname }}.pem

roles/kubernetes/master/defaults/main/main.yml

@@ -76,6 +76,16 @@ audit_policy_name: audit-policy
 audit_policy_hostpath: "{{ audit_policy_file | dirname }}"
 audit_policy_mountpath: "{{ audit_policy_hostpath }}"
 
+# audit webhook support
+kubernetes_audit_webhook: false
+
+# path to audit webhook config file
+audit_webhook_config_file: "{{ kube_config_dir }}/audit-policy/apiserver-audit-webhook-config.yaml"
+audit_webhook_server_url: "https://audit.app"
+audit_webhook_mode: batch
+audit_webhook_batch_max_size: 100
+audit_webhook_batch_max_wait: 1s
+
 # Limits for kube components
 kube_controller_memory_limit: 512M
 kube_controller_cpu_limit: 250m

roles/kubernetes/master/tasks/kubeadm-setup.yml

@@ -80,13 +80,19 @@
   file:
     path: "{{ audit_policy_file | dirname }}"
     state: directory
-  when: kubernetes_audit|default(false)
+  when: kubernetes_audit|default(false) or kubernetes_audit_webhook|default(false)
 
 - name: Write api audit policy yaml
   template:
     src: apiserver-audit-policy.yaml.j2
     dest: "{{ audit_policy_file }}"
-  when: kubernetes_audit|default(false)
+  when: kubernetes_audit|default(false) or kubernetes_audit_webhook|default(false)
+
+- name: Write api audit webhook config yaml
+  template:
+    src: apiserver-audit-webhook-config.yaml.j2
+    dest: "{{ audit_webhook_config_file }}"
+  when: kubernetes_audit_webhook|default(false)
 
 # Nginx LB(default), If kubeadm_config_api_fqdn is defined, use other LB by kubeadm controlPlaneEndpoint.
 - name: set kubeadm_config_api_fqdn define

roles/kubernetes/master/templates/apiserver-webhook-config.yaml.j2

@@ -0,0 +1,14 @@
+apiVersion: v1
+kind: Config
+clusters:
+- cluster:
+    server: {{ audit_webhook_server_url }}
+  name: auditsink
+contexts:
+- context:
+    cluster: auditsink
+    user: ""
+  name: default-context
+current-context: default-context
+preferences: {}
+users: []

roles/kubernetes/master/templates/kubeadm-config.v1beta2.yaml.j2

@@ -169,6 +169,13 @@ apiServer:
     audit-log-maxsize: "{{ audit_log_maxsize }}"
     audit-policy-file: {{ audit_policy_file }}
 {% endif %}
+{% if kubernetes_audit_webhook %}
+    audit-webhook-config-file: {{ audit_webhook_config_file }}
+    audit-policy-file: {{ audit_policy_file }}
+    audit-webhook-mode: {{ audit_webhook_mode }}
+    audit-webhook-batch-max-size: "{{ audit_webhook_batch_max_size }}"
+    audit-webhook-batch-max-wait: "{{ audit_webhook_batch_max_wait }}"
+{% endif %}
 {% for key in kube_kubeadm_apiserver_extra_args %}
     {{ key }}: "{{ kube_kubeadm_apiserver_extra_args[key] }}"
 {% endfor %}
@@ -211,7 +218,7 @@ apiServer:
     hostPath: {{ kube_config_dir }}/webhook-token-auth-config.yaml
     mountPath: {{ kube_config_dir }}/webhook-token-auth-config.yaml
 {% endif %}
-{% if kubernetes_audit %}
+{% if kubernetes_audit or kubernetes_audit_webhook %}
   - name: {{ audit_policy_name }}
     hostPath: {{ audit_policy_hostpath }}
     mountPath: {{ audit_policy_mountpath }}