Merge develop into master for 0.5.1 release (#863)

CHANGELOG-0.5.md

@@ -1,13 +1,21 @@
 # Changelog 0.5
 
-## [0.5.0] 2020-01-17
+## [0.5.1] 2020-01-23
 
-### Added
+### Hotfixed
 
-- [#820](https://github.com/epiphany-platform/epiphany/pull/820) - Firewall: OS level firewall setup (firewalld)
+- [#849](https://github.com/epiphany-platform/epiphany/issues/849) - Firewall: Do not install firewalld package on Ubuntu
+- [#842](https://github.com/epiphany-platform/epiphany/issues/842) - Firewall: Do not require kubernetes_master and kubernetes_node components
+- Filebeat (Ubuntu): [Installing auditd sometimes fails in post-inst](https://bugs.launchpad.net/ubuntu/+source/auditd/+bug/1848330)
+- Filebeat (Ubuntu): Restarting auditd service sometimes fails with error: "Job for auditd.service failed because a timeout was exceeded"
+- Repository (RHEL/CentOS): Add second try for skopeo to avoid random error on Azure: "pinging docker registry returned: Get https://k8s.gcr.io/v2/: net/http: TLS handshake timeout"
+- [#860](https://github.com/epiphany-platform/epiphany/issues/860) - Prometheus: K8s packages and their dependencies are installed on prometheus host
+
+## [0.5.0] 2020-01-17
 
 ### Added
 
+- [#820](https://github.com/epiphany-platform/epiphany/pull/820) - Firewall: OS level firewall setup (firewalld)
 - [#381](https://github.com/epiphany-platform/epiphany/issues/381) - Add AWS EC2 Root Volume encryption
 - [#782](https://github.com/epiphany-platform/epiphany/issues/781) - All disks encryption documentation - AWS
 - [#782](https://github.com/epiphany-platform/epiphany/issues/782) - All disks encryption documentation - Azure

CHANGELOG.md

@@ -11,6 +11,7 @@ Reference for actual cluster component versions can be found [here](docs/home/CO
 
 ### 0.5.x
 
+- [CHANGELOG-0.5.1](./CHANGELOG-0.5.md#051-2020-01-23)
 - [CHANGELOG-0.5.0](./CHANGELOG-0.5.md#050-2020-01-17)
 
 ## Older releases

core/src/epicli/cli/version.txt.py

@@ -1 +1 @@
-0.5.0
+0.5.1

core/src/epicli/data/common/ansible/playbooks/roles/filebeat/tasks/Debian.yml

@@ -10,4 +10,8 @@
     name:
       - auditd
     update_cache: yes
-    state: present
+    state: present
+  register: result
+  retries: 3 # Installing auditd sometimes fails in post-inst: https://bugs.launchpad.net/ubuntu/+source/auditd/+bug/1848330
+  delay: 1
+  until: result is succeeded

core/src/epicli/data/common/ansible/playbooks/roles/filebeat/tasks/configure-auditd.yml

@@ -12,9 +12,13 @@
     - name: Restart auditd service
       shell: >-
         service auditd restart
-      args:  
-        warn: false  
-      when: modify_audit_epi_rules.changed  
+      args:
+        warn: false
+      register: result
+      retries: 3 # to avoid error "job for auditd.service failed because a timeout was exceeded"
+      delay: 1
+      until: result is succeeded
+      when: modify_audit_epi_rules.changed
 
     - name: Enable and start auditd service
       service:

core/src/epicli/data/common/ansible/playbooks/roles/firewall/tasks/Debian/install-firewall.yml

@@ -16,6 +16,30 @@
     purge: yes
 
 - name: Install firewalld package
-  package:
-    name: firewalld
-    state: present
+  block:
+    - name: Install firewalld package
+      package:
+        name: firewalld
+        state: present
+      register: install_firewalld
+
+    - name: Stop and mask service for consistency with RHEL
+      block:
+        # On Ubuntu firewalld service starts automatically after installing package
+        # Stop to avoid unexpected blocking and for consistency with RHEL
+        - name: Stop firewalld service
+          systemd:
+            name: firewalld
+            state: stopped
+            enabled: no
+
+        # On RHEL firewalld service is initially masked, so mask it for consistency
+        # This task is separated by purpose to mask service AFTER it was stopped
+        - name: Mask firewalld service
+          systemd:
+            name: firewalld
+            masked: yes
+      when:
+        - install_firewalld.changed
+  when:
+    - specification.Debian.install_firewalld

core/src/epicli/data/common/ansible/playbooks/roles/firewall/tasks/configure-firewall.yml

@@ -145,12 +145,43 @@
       when: default_zone_name.stdout != specification.managed_zone_name
   when:
     - specification.apply_configuration
-    - inventory_hostname in groups['kubernetes_master'] or inventory_hostname in groups['kubernetes_node']
+    - (groups['kubernetes_master'] is defined and inventory_hostname in groups['kubernetes_master'])
+      or (groups['kubernetes_node'] is defined and inventory_hostname in groups['kubernetes_node'])
 
-- name: Stop and disable firewalld service
-  systemd:
-    name: firewalld
-    state: stopped
-    enabled: no
-    masked: yes # to make sure it is not started by accessing firewalld D-Bus interface or if other services require firewalld
-  when: not specification.firewall_service_enabled
+- name: Get service facts
+  service_facts:
+
+- name: Print firewalld.service state
+  debug:
+    var: ansible_facts.services['firewalld.service']
+  when:
+    - ansible_facts.services['firewalld.service'] is defined
+
+- name: Stop, disable and mask firewalld service
+  block:
+    - name: Stop and disable firewalld service
+      systemd:
+        name: firewalld
+        state: stopped
+        enabled: no
+
+    # Mask service to make sure it is not started by accessing firewalld D-Bus interface or if other services require firewalld
+    # This task is separated by purpose to mask service AFTER it was stopped
+    - name: Mask firewalld service
+      systemd:
+        name: firewalld
+        masked: yes
+  when:
+    - not specification.firewall_service_enabled
+    - ansible_facts.services['firewalld.service'] is defined
+
+- name: Get firewall state
+  shell: firewall-cmd --state 2>&1
+  register: firewall_state
+  changed_when: false
+  failed_when: firewall_state.rc not in [0, 127, 252] # 252 means not running
+
+- name: Print firewall state
+  debug:
+    var: firewall_state.stdout
+  when: firewall_state.rc in [0, 252]

core/src/epicli/data/common/ansible/playbooks/roles/prometheus/tasks/configure-k8s-apps-monitoring.yml

@@ -7,10 +7,11 @@
   set_fact:
     api_server_address: "https://{{ master_hostname }}:6443"
 
-- name: Set Kubernetes credentials 
+- name: Set Kubernetes credentials
   import_role:
     name: kubernetes_master
     tasks_from: copy-kubeconfig
+  delegate_to: "{{ master_hostname }}"
 
 - name: Deploy rolebinding file to server for prometheus
   copy:
@@ -37,7 +38,8 @@
   set_fact:
     bearer_token: "{{ kube_token.stdout }}"
 
-- name: Remove Kubernetes credentials 
+- name: Remove Kubernetes credentials
   import_role:
     name: kubernetes_master
-    tasks_from: remove-kubeconfig
+    tasks_from: remove-kubeconfig
+  delegate_to: "{{ master_hostname }}"

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/download-requirements.sh

@@ -99,7 +99,9 @@ download_image() {
 		local tmp_file_path=$(mktemp)
 		local skopeo_cmd="$SKOPEO_BIN --insecure-policy copy docker://$image_name docker-archive:$tmp_file_path:$repository:$tag"
 		echol "Downloading image: $image"
+		# try twice to avoid random error on Azure: "pinging docker registry returned: Get https://k8s.gcr.io/v2/: net/http: TLS handshake timeout"
 		{ $skopeo_cmd && chmod 644 $tmp_file_path && mv $tmp_file_path $dest_path; } ||
+		{ echol "Second try:" && $skopeo_cmd && chmod 644 $tmp_file_path && mv $tmp_file_path $dest_path; } ||
 			exit_with_error "skopeo failed, command was: $skopeo_cmd && chmod 644 $tmp_file_path && mv $tmp_file_path $dest_path"
 	fi
 }

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/download-requirements.sh

@@ -99,7 +99,9 @@ download_image() {
 		local tmp_file_path=$(mktemp)
 		local skopeo_cmd="$SKOPEO_BIN --insecure-policy copy docker://$image_name docker-archive:$tmp_file_path:$repository:$tag"
 		echol "Downloading image: $image"
+		# try twice to avoid random error on Azure: "pinging docker registry returned: Get https://k8s.gcr.io/v2/: net/http: TLS handshake timeout"
 		{ $skopeo_cmd && chmod 644 $tmp_file_path && mv $tmp_file_path $dest_path; } ||
+		{ echol "Second try:" && $skopeo_cmd && chmod 644 $tmp_file_path && mv $tmp_file_path $dest_path; } ||
 			exit_with_error "skopeo failed, command was: $skopeo_cmd && chmod 644 $tmp_file_path && mv $tmp_file_path $dest_path"
 	fi
 }

core/src/epicli/data/common/ansible/playbooks/roles/repository/tasks/firewall/clean-up-rule.yml

@@ -4,20 +4,28 @@
 
 # Note: Current zone when run 'teardown' may be different than was while 'setup'
 
-- name: Include vars from firewall role
-  include_vars:
-    file: roles/firewall/vars/main.yml
-    name: firewall_role_config
-
-- name: Get name of Epiphany managed zone from configuration
-  set_fact:
-    managed_zone: "{{ firewall_role_config.specification.managed_zone_name }}"
-
 - name: Check if {{ firewall_rule_added_flag_file }} file exists
   stat:
     path: "{{ firewall_rule_added_flag_file }}"
   register: firewall_rule_added_flag_file_stat
 
+- name: Get firewall settings from Epiphany configuration
+  block:
+    - name: Include vars from firewall role
+      include_vars:
+        file: roles/firewall/vars/main.yml
+        name: firewall_role_config
+      failed_when: false # to not fail if firewall role is disabled
+
+    - name: Get firewall settings from Epiphany configuration
+      set_fact:
+        managed_zone: "{{ firewall_role_config.specification.managed_zone_name }}"
+        apply_firewall_configuration: "{{ firewall_role_config.specification.apply_configuration }}"
+      when:
+        - firewall_role_config.specification is defined
+  when:
+    - "'firewall' in group_names"
+
 - name: Remove temporary firewall rule for epirepo added by repository role
   block:
     - name: Get name of modified zone from file
@@ -26,21 +34,24 @@
 
     - name: Restore configuration of {{ modified_zone | default('modified') }} zone from backup
       copy:
-        remote_src: yes
         src: "{{ zone_config_backup_dest_dir }}/{{ modified_zone }}.xml.bak"
-        dest:  /etc/firewalld/zones/{{ modified_zone }}.xml
+        dest: /etc/firewalld/zones/{{ modified_zone }}.xml
+        remote_src: yes
         mode: preserve
       register: restore_modified_zone_from_backup
-      when: modified_zone != managed_zone
-         or not firewall_role_config.specification.apply_configuration
+      when:
+        - apply_firewall_configuration is undefined
+          or not apply_firewall_configuration
+          or managed_zone is undefined
+          or managed_zone != modified_zone
 
     - name: Clean up temporary firewall files
       file:
         path: "{{ item }}"
         state: absent
       loop: [ firewall_rule_added_flag_file, modified_zone_name_file, "{{ zone_config_backup_dest_dir }}/{{ modified_zone }}.xml.bak" ]
-
-  when: firewall_rule_added_flag_file_stat.stat.exists
+  when:
+    - firewall_rule_added_flag_file_stat.stat.exists
 
 - name: Remove temporary firewall rule for epirepo added by firewall role
   blockinfile:
@@ -51,7 +62,8 @@
     state: absent
   register: remove_http_service_from_managed_zone
   when:
-    - firewall_role_config.specification.apply_configuration
+    - apply_firewall_configuration is defined
+    - apply_firewall_configuration
 
 - name: Reload firewalld to apply permanent configuration to runtime
   command: firewall-cmd --reload

core/src/epicli/data/common/defaults/configuration/firewall.yml

@@ -2,6 +2,8 @@ kind: configuration/firewall
 title: OS level firewall
 name: default
 specification:
+  Debian:                         # On RHEL on Azure firewalld is already in VM image (pre-installed)
+    install_firewalld: false      # false to avoid random issue "No route to host" even when firewalld service is disabled
   firewall_service_enabled: false # for all inventory hosts
   apply_configuration: false      # if false only service state is managed
   managed_zone_name: epiphany