Hotfix: removing hard dependency on Helm (#1360)

* repository,helm: code cleanup

* helm_charts: restricting execution to repository hosts only

* download: cleanup and refactor

- adding ability to list "files" from a running repository
- adding name parser to pick and download latest version of a "file"

* helm: cleanup and refactor

- improving the role ordering inside the feature-mapping
- removing helm version from configs (newest one from epirepo is taken instead)
- adding ability to deploy a Helm chart with a default values.yaml file
- adding simple versioning to the Helm binaries, improving idempotency
- restricting execution to main repository host (detected in preflight)

core/src/epicli/data/common/ansible/playbooks/helm.yml

@@ -1,8 +1,7 @@
 ---
-
-- hosts: kubernetes_master[0]
+- hosts: helm
   become: true
   become_method: sudo
   become_flags: --login
   roles:
-    - helm 
+    - helm

core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/download_file.yml

@@ -1,12 +1,11 @@
 ---
-
 - name: "Download file {{ file_name }}"
   get_url:
     url: "{{ repository_url }}/files/{{ file_name }}"
     dest: "{{ download_directory }}"
     validate_certs: "{{ validate_certs }}"
   register: get_url_result
   until: "'OK' in get_url_result.msg or 'file already exists' in get_url_result.msg"
-  retries: "3"
-  delay: "2"
+  retries: 3
+  delay: 2
   become: false

core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/download_image.yml

@@ -1,12 +1,11 @@
 ---
-
-- name: Download image {{ file_name }}
+- name: "Download image {{ file_name }}"
   get_url:
     url: "{{ repository_url }}/images/{{ file_name }}"
     dest: "{{ download_directory }}"
     validate_certs: "{{ validate_certs }}"
   register: get_url_result
   until: "'OK' in get_url_result.msg or 'file already exists' in get_url_result.msg"
-  retries: "3"
-  delay: "2"
-  become: false
+  retries: 3
+  delay: 2
+  become: false

core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/download_newest_file.yml

@@ -0,0 +1,18 @@
+---
+# Invoke with (example):
+#- set_fact:
+#    archive_name_regexp: '^helm-v([0-9.]+)-linux-amd64.tar.gz$'
+#    archive_version_format: '{:0>4s}.{:0>4s}.{:0>4s}'  # OPTIONAL
+
+- name: Get "files" list from the repository
+  import_tasks: list_files.yml
+
+- name: Pick the newest archive file
+  import_tasks: pick_newest.yml
+  vars:
+    archive_names: "{{ list_files_result }}"
+
+- name: Download the newest archive file
+  import_tasks: download_file.yml
+  vars:
+    file_name: "{{ newest_archive_name }}"

core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/list_files.yml

@@ -0,0 +1,25 @@
+---
+- name: Get file listing
+  uri:
+    method: GET
+    url: "{{ repository_url }}/files/?F=0"  # F=0 formats the listing as a simple list (not FancyIndexed)
+    body_format: raw
+    return_content: true
+    validate_certs: "{{ validate_certs | default(false, true) | bool }}"  # handling "undefined", "null", "empty" and "boolean" values all at once
+  register: uri_list_files
+  until: uri_list_files is success
+  retries: 3
+  delay: 2
+  become: false
+
+# TODO: make it work with yaml or json (instead of html, sic!).
+- name: Parse html response and return file listing
+  set_fact:
+    list_files_result: >-
+      {{ lines | select('match', regexp)
+               | reject('match', '.*Parent Directory.*')
+               | map('regex_replace', regexp, '\1')
+               | list }}
+  vars:
+    lines: "{{ uri_list_files.content.splitlines() }}"
+    regexp: '.*<li><a href="([^"]+)".*'

core/src/epicli/data/common/ansible/playbooks/roles/download/tasks/pick_newest.yml

@@ -0,0 +1,50 @@
+---
+# Invoke with (example):
+#- set_fact:
+#    archive_names:
+#      - "haproxy_exporter-0.10.0.linux-amd64.tar.gz"
+#      - "helm-v2.2.0-linux-amd64.tar.gz"
+#      - "helm-v10.2.3-linux-amd64.tar.gz"
+#      - "elasticsearch-curator_5.5.4_amd64.deb"
+#      - "helm-v3.2.0-linux-amd64.tar.gz"
+#      - "prometheus-2.10.0.linux-amd64.tar.gz"
+#    archive_name_regexp: '^helm-v([0-9.]+)-linux-amd64.tar.gz$'
+#    archive_version_format: '{:0>4s}.{:0>4s}.{:0>4s}'  # OPTIONAL
+
+- name: "Find and return the newest archive's name and version (using pattern: {{ archive_name_regexp }})"
+
+  set_fact:
+    newest_archive_name: >-
+      {{ _newest_archive_name }}
+
+    newest_archive_version: >-
+      {{ _newest_archive_name | regex_replace(archive_name_regexp, '\1') }}
+
+  vars:
+    # Provide a default value for the version format (assuming 3 segments, left-padded up to 4 zeros).
+    _archive_version_format: >-
+      {{ archive_version_format | default( '{:0>4s}.{:0>4s}.{:0>4s}' ) }}
+
+    # Drop all archive names not matching the pattern.
+    _archive_names_filtered: >-
+      {{ archive_names | select('match', archive_name_regexp) | list }}
+
+    # Extract all archive version strings.
+    _archive_versions: >-
+      {{ _archive_names_filtered | map('regex_replace', archive_name_regexp, '\1') | list }}
+
+    # Zero-pad each segment of the version string (returns a list of strings).
+    _archive_versions_padded: >-
+      {%- set output = [] -%}
+      {%- for version in _archive_versions -%}
+        {{- output.append( _archive_version_format.format(*version.split('.')) ) -}}
+      {%- endfor -%}
+      {{- output -}}
+
+    # Create a version (padded) to archive name mapping.
+    _version_to_name_map: >-
+      {{ dict(_archive_versions_padded | zip(_archive_names_filtered)) }}
+
+    # Use map to get the newest archive's name.
+    _newest_archive_name: >-
+      {{ _version_to_name_map[_version_to_name_map.keys() | max] }}

core/src/epicli/data/common/ansible/playbooks/roles/helm/tasks/gather-facts.yml

@@ -0,0 +1,8 @@
+---
+- name: Produce final list of hosts to deploy Helm chart sources on
+  set_fact:
+    target_repository_hostnames: >-
+      {{ groups.helm | difference(available_masters) | union([resolved_repository_hostname]) | unique }}
+  vars:
+    available_masters: >-
+      {{ groups.kubernetes_master | default([]) }}

core/src/epicli/data/common/ansible/playbooks/roles/helm/tasks/install-bin.yml

@@ -1,17 +1,40 @@
 ---
+- name: Download the newest Helm binary package from the epirepo
+  import_role:
+    name: download
+    tasks_from: download_newest_file
+  vars:
+    archive_name_regexp: '^helm-v([0-9.]+)-linux-amd64.tar.gz$'
+    validate_certs: false
+
+- name: Create versioned destination directory for Helm binary
+  file:
+    dest: "/usr/local/bin/helm-{{ newest_archive_version }}/"
+    state: directory
+
 - name: Unpack Helm binary
-  unarchive: 
-    src: "{{ specification.apache_epirepo_path }}/files/helm-{{ specification.version }}-linux-amd64.tar.gz"
-    dest: "/usr/local/bin"
+  unarchive:
+    src: "{{ download_directory }}/{{ newest_archive_name }}"
+    dest: "/usr/local/bin/helm-{{ newest_archive_version }}/"
+    creates: "/usr/local/bin/helm-{{ newest_archive_version }}/helm"
     extra_opts:
       - --strip=1
       - --wildcards
       - '*/helm'
-    remote_src: yes
+    remote_src: true
+  register: unarchive_result
 
-- name: Create symlink for Helm binary
-  file:
-    src: "/usr/local/bin/helm"
-    dest: /usr/local/sbin/helm
-    state: link
-  when: ansible_os_family == "RedHat" 
+- when: unarchive_result is changed
+  block:
+    - name: Create symlink to enable the newest Helm binary
+      file:
+        src: "/usr/local/bin/helm-{{ newest_archive_version }}/helm"
+        dest: "/usr/local/bin/helm"
+        state: link
+
+    - name: Create "sbin" symlink for Helm binary (RedHat fix)
+      file:
+        src: "/usr/local/bin/helm"
+        dest: "/usr/local/sbin/helm"
+        state: link
+      when: ansible_os_family == "RedHat"

core/src/epicli/data/common/ansible/playbooks/roles/helm/tasks/install-chart.yml

@@ -1,24 +1,49 @@
 ---
-- name: Copy Helm charts custom configuration to file
-  copy:
-    content: "{{ helm_chart_values }}"
-    dest: "/tmp/{{ helm_chart_name }}_values.yaml"
-  when: helm_chart_values != ""
+- name: Prepare configuration and upgrade/install Helm chart
+  vars:
+    # Handling "undefined", "null", "empty" and "boolean" values all at once.
+    disable_helm_chart_bool: "{{ disable_helm_chart | default(false, true) | bool }}"
 
-- name: Install Helm chart
-  shell: helm upgrade --install -f /tmp/{{ helm_chart_name }}_values.yaml {{ helm_chart_name }} {{ helm_chart_repo_name }}/{{ helm_chart_name }}
-  when: 
-    - disable_helm_chart != "" 
-    - not disable_helm_chart
-    - helm_chart_values != ""
+    # Handling "undefined", "null" and "empty" values all at once.
+    helm_chart_values_bool: "{{ helm_chart_values | default(false) | ternary(true, false) }}"
 
-- name: Delete Helm chart
-  shell: helm delete {{ helm_chart_name }} --purge
-  when: 
-    - disable_helm_chart != ""
-    - disable_helm_chart
+  always:
+    - name: Clean up temporary Helm chart values file
+      file:
+        state: absent
+        path: /tmp/{{ helm_chart_name }}_values.yaml
 
-- name: Clean up temporary Helm chart values file
-  file:
-    state: absent
-    path: /tmp/{{ helm_chart_name }}_values.yaml"
+  block:
+    # IF `disable_helm_chart`
+    - when: disable_helm_chart_bool
+      block:
+        - name: Delete Helm chart
+          shell: |
+            helm delete --purge {{ helm_chart_name }}
+
+    # ELSE
+    - when: not disable_helm_chart_bool
+      block:
+        # IF `helm_chart_values`
+        - when: helm_chart_values_bool
+          block:
+            - name: Copy Helm chart's custom configuration to file
+              copy:
+                content: "{{ helm_chart_values }}"
+                dest: /tmp/{{ helm_chart_name }}_values.yaml
+
+            - name: Install Helm chart (with custom values.yaml)
+              shell: |
+                helm upgrade --install \
+                  -f /tmp/{{ helm_chart_name }}_values.yaml \
+                  {{ helm_chart_name }} \
+                  {{ helm_chart_repo_name }}/{{ helm_chart_name }}
+
+        # ELSE
+        - when: not helm_chart_values_bool
+          block:
+            - name: Install Helm chart (with default values.yaml)
+              shell: |
+                helm upgrade --install \
+                  {{ helm_chart_name }} \
+                  {{ helm_chart_repo_name }}/{{ helm_chart_name }}

core/src/epicli/data/common/ansible/playbooks/roles/helm/tasks/main.yml

@@ -1,41 +1,48 @@
 ---
-- name: Check if Helm chart repo already exists
-  shell: helm repo list | grep {{ helm_chart_repo_name }}
-  args:
-    executable: /bin/bash
-  register: helm_repo_list
-  failed_when: (helm_repo_list.rc != 0 and not 'no repositories' in helm_repo_list.stderr) 
-               or helm_repo_list.rc > 1
+- import_tasks: gather-facts.yml
 
-- name: Check if files exist in {{ specification.apache_epirepo_path }}/helm-charts/system
-  find:
-    paths: "{{ specification.apache_epirepo_path }}/helm-charts/system"
-  register: helm_charts_files_number
-
-- name: "Add Helm chart repo and install charts"
+- name: Deploy the "system" Helm charts
+  when: inventory_hostname == target_repository_hostnames[0]  # execute on the main repository host only
   block:
-    - name: Get Helm charts list from repo before update 
-      shell: helm search repo {{ helm_chart_repo_name }} --output json | jq -r '.[].name'
-      register: helm_charts_list_before_update
+    - name: Check if Helm chart repo already exists
+      shell: |
+        helm repo list | grep {{ helm_chart_repo_name }}
+      register: helm_repo_list
+      failed_when: (helm_repo_list.rc != 0 and not 'no repositories' in helm_repo_list.stderr)
+                   or helm_repo_list.rc > 1
+
+    - name: "Check if files exist in {{ specification.apache_epirepo_path }}/helm-charts/system"
+      find:
+        paths: "{{ specification.apache_epirepo_path }}/helm-charts/system"
+      register: helm_charts_files_number
 
-    - name: "Add {{ helm_chart_repo_name }} Helm repository from url {{ repository_url }}"
-      shell: helm repo add {{ helm_chart_repo_name }} {{ repository_url }}/helm-charts/system
-      when: 
-        - helm_chart_repo_name not in helm_repo_list.stdout 
+    - name: Add Helm chart repo and install charts
+      when: helm_charts_files_number.matched > 1
+      block:
+        - name: Get Helm charts list from repo before update
+          shell: |
+            helm search repo {{ helm_chart_repo_name }} --output json | jq -r '.[].name'
+          register: helm_charts_list_before_update
 
-    - name: Update Helm repo
-      shell: helm repo update
+        - name: "Add {{ helm_chart_repo_name }} Helm repository from url {{ repository_url }}"
+          shell: |
+            helm repo add {{ helm_chart_repo_name }} {{ repository_url }}/helm-charts/system
+          when:
+            - helm_chart_repo_name not in helm_repo_list.stdout
 
-    - name: Get Helm charts list from repo after update 
-      shell: helm search repo {{ helm_chart_repo_name }} --output json | jq -r '.[].name'
-      register: helm_charts_list_after_update
+        - name: Update Helm repo
+          shell: |
+            helm repo update
 
-    - name: Delete Helm charts not present in repository
-      shell: helm delete {{ chart_subname }} --purge
-      vars:
-        chart_subname: "{{ item.split('/')[1] }}"
-      loop: "{{ helm_charts_list_before_update.stdout_lines }}"
-      when: item not in helm_charts_list_after_update.stdout_lines
-  when: 
-    - helm_charts_files_number.matched > 1
+        - name: Get Helm charts list from repo after update
+          shell: |
+            helm search repo {{ helm_chart_repo_name }} --output json | jq -r '.[].name'
+          register: helm_charts_list_after_update
 
+        - name: Delete Helm charts not present in repository
+          shell: |
+            helm delete {{ chart_subname }} --purge
+          vars:
+            chart_subname: "{{ item.split('/')[1] }}"
+          loop: "{{ helm_charts_list_before_update.stdout_lines }}"
+          when: item not in helm_charts_list_after_update.stdout_lines

core/src/epicli/data/common/ansible/playbooks/roles/helm_charts/tasks/main.yml

@@ -1,12 +1,20 @@
 ---
-- name: Create Helm charts directory
-  file:
-    path: "{{ specification.apache_epirepo_path }}/helm-charts"
-    state: directory
-    mode: u=rwx,go=rx
+- name: Borrow the fact initializer from the "repository" role
+  import_role:
+    name: repository
+    tasks_from: gather-facts
 
-- name: Copy Helm charts to repository server
-  copy:
-    src: "./"
-    dest: "{{ specification.apache_epirepo_path }}/helm-charts"
-    mode: u=rwx,go=rx 
+- name: Copy Helm chart sources (to repository hosts only)
+  when: inventory_hostname == target_repository_hostnames[0]  # execute on the main repository host only
+  block:
+    - name: Create Helm charts directory
+      file:
+        path: "{{ specification.apache_epirepo_path }}/helm-charts"
+        state: directory
+        mode: u=rwx,go=rx
+
+    - name: Copy Helm charts to repository server
+      copy:
+        src: "./"
+        dest: "{{ specification.apache_epirepo_path }}/helm-charts"
+        mode: u=rwx,go=rx