Feature/kafka upgrade

CHANGELOG-0.5.md

@@ -7,3 +7,4 @@
 - [#763](https://github.com/epiphany-platform/epiphany/pull/763) - Elasticsearch Curator: Flexible configuration of cron jobs
 - [#763](https://github.com/epiphany-platform/epiphany/pull/763) - Elasticsearch Curator: Upgrade to v5.8.1
 - [#766](https://github.com/epiphany-platform/epiphany/issues/766) - Elasticsearch: Upgrade to v6.8.5
+- [#752](https://github.com/epiphany-platform/epiphany/pull/752) - Kafka: Upgrade to v2.3.1

core/src/epicli/data/common/ansible/playbooks/roles/kafka/defaults/main.yml

@@ -0,0 +1,5 @@
+specification:
+  kafka_var:
+    version: 2.3.1
+    scala:
+      version: 2.12

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt

@@ -116,7 +116,7 @@ kubelet-1.13.12
 [files]
 https://github.com/prometheus/haproxy_exporter/releases/download/v0.10.0/haproxy_exporter-0.10.0.linux-amd64.tar.gz
 https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.12.0/jmx_prometheus_javaagent-0.12.0.jar
-https://archive.apache.org/dist/kafka/2.0.0/kafka_2.12-2.0.0.tgz
+https://archive.apache.org/dist/kafka/2.3.1/kafka_2.12-2.3.1.tgz
 https://github.com/danielqsj/kafka_exporter/releases/download/v1.2.0/kafka_exporter-1.2.0.linux-amd64.tar.gz
 https://github.com/prometheus/node_exporter/releases/download/v0.16.0/node_exporter-0.16.0.linux-amd64.tar.gz
 https://github.com/prometheus/prometheus/releases/download/v2.10.0/prometheus-2.10.0.linux-amd64.tar.gz

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.txt

@@ -113,7 +113,7 @@ kubelet-1.13.12
 [files]
 https://github.com/prometheus/haproxy_exporter/releases/download/v0.10.0/haproxy_exporter-0.10.0.linux-amd64.tar.gz
 https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.12.0/jmx_prometheus_javaagent-0.12.0.jar
-https://archive.apache.org/dist/kafka/2.0.0/kafka_2.12-2.0.0.tgz
+https://archive.apache.org/dist/kafka/2.3.1/kafka_2.12-2.3.1.tgz
 https://github.com/danielqsj/kafka_exporter/releases/download/v1.2.0/kafka_exporter-1.2.0.linux-amd64.tar.gz
 https://github.com/prometheus/node_exporter/releases/download/v0.16.0/node_exporter-0.16.0.linux-amd64.tar.gz
 https://github.com/prometheus/prometheus/releases/download/v2.10.0/prometheus-2.10.0.linux-amd64.tar.gz

core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.txt

@@ -123,7 +123,7 @@ kubelet 1.13.12
 
 [files]
 https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.12.0/jmx_prometheus_javaagent-0.12.0.jar
-https://archive.apache.org/dist/kafka/2.0.0/kafka_2.12-2.0.0.tgz
+https://archive.apache.org/dist/kafka/2.3.1/kafka_2.12-2.3.1.tgz
 https://archive.apache.org/dist/zookeeper/zookeeper-3.4.12/zookeeper-3.4.12.tar.gz
 https://github.com/danielqsj/kafka_exporter/releases/download/v1.2.0/kafka_exporter-1.2.0.linux-amd64.tar.gz
 https://github.com/prometheus/alertmanager/releases/download/v0.17.0/alertmanager-0.17.0.linux-amd64.tar.gz

core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kafka.yml

@@ -0,0 +1,52 @@
+---
+- name: Include defaults from kafka role
+  include_vars:
+    file: roles/kafka/defaults/main.yml
+
+- name: Include pre-flight checks
+  include_tasks: kafka/preflight-check.yml
+
+- name: Get installed Kafka version
+  shell: >-
+    set -o pipefail &&
+    /opt/kafka/bin/kafka-server-start.sh --version | grep Commit | grep -oP '^\d+\.\d+\.\d+'
+  register: result
+  args:
+    executable: /bin/bash
+
+- name: Set installed Kafka version as fact
+  set_fact:
+    before_upgrade_kafka_version: "{{ result.stdout }}"
+
+- name: Check for upgrade flag file
+  stat:
+    path: "{{ lock_file }}"
+  register: lock_file_status
+
+- name: Include upgrade tasks
+  block:
+    - name: Create upgrade flag file
+      file:
+        path: "{{ lock_file }}"
+        state: touch
+
+    - name: Stop Kafka service
+      service:
+        name: kafka
+        state: stopped
+
+    - name: Include update Kafka properties tasks
+      include_tasks: kafka/update-properties.yml
+
+    - name: Include Kafka upgrade tasks
+      include_tasks: kafka/install-upgrade.yml
+
+    - name: Include set Kafka version tasks
+      include_tasks: kafka/set-updated-version.yml
+  when:
+    - lock_file_status.stat.exists or before_upgrade_kafka_version is version( specification.kafka_var.version, '<' )
+
+- name: Remove Kafka upgrade flag file
+  file:
+    path: "{{ lock_file }}"
+    state: absent

core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kafka/install-upgrade.yml

@@ -0,0 +1,57 @@
+---
+- name: Set Kafka installation file name as fact
+  set_fact:
+    kafka_installation_file_name: "kafka_{{ specification.kafka_var.scala.version }}-{{ specification.kafka_var.version }}.tgz"
+
+- name: Download Kafka binaries
+  include_role:
+    name: download
+    tasks_from: download_file
+  vars:
+    file_name: "{{ kafka_installation_file_name }}"
+
+- name: Uncompress Kafka installation file
+  unarchive:
+    remote_src: yes
+    src: "{{ download_directory }}/{{ kafka_installation_file_name }}"
+    dest: /opt
+
+- name: Change ownership on kafka directory
+  file:
+    path: /opt/kafka_{{ specification.kafka_var.scala.version }}-{{ specification.kafka_var.version }}
+    state: directory
+    owner: kafka
+    group: kafka
+
+- name: Copy configuration from previous version
+  copy:
+    remote_src: yes
+    src: /opt/kafka/config/
+    dest: /opt/kafka_{{ specification.kafka_var.scala.version }}-{{ specification.kafka_var.version }}/config
+    mode: preserve
+
+- name: Link /opt/kafka to recently installed version
+  file:
+    dest: /opt/kafka
+    state: link
+    src: /opt/kafka_{{ specification.kafka_var.scala.version }}-{{ specification.kafka_var.version }}
+    force: yes
+
+- name: Remove previous version binaries
+  file:
+    path: /opt/kafka_{{ specification.kafka_var.scala.version }}-{{ before_upgrade_kafka_version }}
+    state: absent
+
+- name: Get log.dirs property
+  shell: >-
+    set -o pipefail &&
+    grep log.dirs /opt/kafka/config/server.properties | awk -F'=' '{print $2}'
+  register: log_dirs
+  changed_when: False
+  args:
+    executable: /bin/bash
+
+- name: Remove lost+found directory from log.dirs
+  file:
+    path: "{{ log_dirs.stdout }}/lost+found"
+    state: absent

core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kafka/preflight-check.yml

@@ -0,0 +1,11 @@
+---
+  - name: Check if Kafka is installed in default location
+    stat:
+      path: /opt/kafka/bin/kafka-server-start.sh
+    register: kafka_exec_file
+
+  - name: Assert Kafka location
+    assert:
+      that:
+        - kafka_exec_file.stat.exists
+      fail_msg: Kafka not found in /opt/kafka (Epiphany default) - check your configuration

core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kafka/set-updated-version.yml

@@ -0,0 +1,16 @@
+---
+- name: Check if server.properties file exists
+  stat:
+    path: /opt/kafka/config/server.properties
+
+- name: Modify inter.broker.protocol.version property
+  lineinfile:
+    path: /opt/kafka/config/server.properties
+    regexp: "^inter.broker.protocol.version"
+    line: "inter.broker.protocol.version={{ specification.kafka_var.version }}"
+
+- name: Start kafka service
+  systemd:
+    name: kafka
+    state: started
+    daemon-reload: yes

core/src/epicli/data/common/ansible/playbooks/roles/upgrade/tasks/kafka/update-properties.yml

@@ -0,0 +1,34 @@
+---
+- name: Check if current_kafka_version property is defined
+  shell: >-
+    set -o pipefail && 
+    grep "^CURRENT_KAFKA_VERSION" /opt/kafka/config/server.properties
+  register: current_kafka_version_property
+  failed_when:
+    - result.rc == 2
+  changed_when: False
+  args: 
+    executable: /bin/bash
+
+- name: Add current_kafka_version property
+  lineinfile:
+    path: /opt/kafka/config/server.properties
+    line: "CURRENT_KAFKA_VERSION={{ before_upgrade_kafka_version }}"
+  when: current_kafka_version_property.stdout == ""
+
+- name: Check if inter.broker.protocol.version property is defined
+  shell: >-
+    set -o pipefail &&
+    grep "^inter.broker.protocol.version" /opt/kafka/config/server.properties
+  register: inter_broker_protocol_version_property
+  failed_when:
+    - result.rc == 2
+  changed_when: False
+  args:
+    executable: /bin/bash
+
+- name: Add inter.broker.protocol.version property
+  lineinfile:
+    path: /opt/kafka/config/server.properties
+    line: "inter.broker.protocol.version={{ before_upgrade_kafka_version }}"
+  when: inter_broker_protocol_version_property.stdout == ""

core/src/epicli/data/common/ansible/playbooks/upgrade.yml

@@ -68,4 +68,14 @@
   tasks:
     - import_role:
         name: upgrade
-        tasks_from: elasticsearch-curator
+        tasks_from: elasticsearch-curator
+
+- hosts: kafka
+  serial: 1
+  become: true
+  become_method: sudo
+  tasks:
+    - import_role:
+        name: upgrade
+        tasks_from: kafka
+      vars: {lock_file: /var/tmp/kafka-upgrade-in-progress.flag}

core/src/epicli/data/common/defaults/configuration/kafka.yml

@@ -5,10 +5,10 @@ specification:
 
   # Have to use odd name to support valid yaml due to ansible getting confused over playbook names and variables etc.
   kafka_var:
-    version: 2.0.0
+    version: 2.3.1
     scala:
       version: 2.12
-    file_name: "kafka_2.12-2.0.0.tgz"
+    file_name: "kafka_2.12-2.3.1.tgz"
     enable: True
     admin: kafka
     admin_pwd: epiphany

docs/home/COMPONENTS.md

@@ -8,7 +8,7 @@ Note that versions are default versions and can be changed in certain cases thro
 | Calico                | -       | https://github.com/projectcalico/calico               | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) |
 | Flannel               | -       | https://github.com/coreos/flannel/                    | [Apache License](https://www.apache.org/licenses/LICENSE-1.0)     |
 | Canal                 | -       | https://github.com/projectcalico/calico               | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) |
-| Kafka                 | 2.0.0   | https://github.com/apache/kafka                       | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) |
+| Kafka                 | 2.3.1   | https://github.com/apache/kafka                       | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) |
 | Zookeeper             | 3.4.12  | https://github.com/apache/zookeeper                   | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) |
 | RabbitMQ              | 3.7.10  | https://github.com/rabbitmq/rabbitmq-server           | [Mozilla Public License](https://www.mozilla.org/en-US/MPL/)      |
 | Docker-ce             | 18.09.6 | https://github.com/docker/docker-ce/                  | [Apache License](https://www.apache.org/licenses/LICENSE-1.0)     |