From d5f7f542318a8a64c0c65b6bb3b384b6a686d1b9 Mon Sep 17 00:00:00 2001
From: Thomas Widhalm <thomas.widhalm@netways.de>
Date: Mon, 16 Oct 2023 18:37:26 +0200
Subject: [PATCH 1/5] Remove redundant restart (#279)

Restarting Elasticsearch takes quite a while and may lead to connection
issues as well as sync issues. So keeping restarts to a minimum is
important. These changes will make sure that, even when the `Restart
Elasticsearch` handler is notified, it will only restart if
Elasticsearch was running before. If there's a fresh start (after
reconfiguration) we don't need to restart again.

Same goes for Logstash and Kibana. Some restarts of these tools happen
fairly fast. But others (like after fresh installs or updates) will
trigger internal jobs that should not be intercepted by another restart.

Beats restart very fast and as far as I know there's not a big downside
to restarting them right after the first start so I didn't include them
in the change.

Additionally, this PR will make sure some tasks in `verify.yml` of the
full stack are only run when the service to be checked is actually
running on this node. This helps with spreading services over nodes to
save ressources.

Since GitHub hosted runners are quite low on ressources we can't run
every service on every node in a cluster setup anymore. So this PR will
make sure that only Elasticsearch runs everywhere and the others are
spread out.

Caches get cleared after every role in during a Molecule test. This
helps with saving ressources, too.

Elasticsearch still won't sync all shards due to full volumes, the
watermarks for Elasticseach are set to extremely high volumes so that
the cluster can at least get into sync.

fixes #278
fixes #141
fixes #194
---
 molecule/elasticsearch_default/converge.yml   |   1 +
 molecule/elasticstack_default/converge.yml    |   3 +-
 molecule/elasticstack_default/molecule.yml    |   2 -
 molecule/elasticstack_default/verify.yml      |  78 ++++++-------
 roles/beats/tasks/main.yml                    |   7 ++
 roles/elasticsearch/defaults/main.yml         |   6 +
 roles/elasticsearch/handlers/main.yml         |   5 +-
 .../tasks/elasticsearch-security.yml          | 104 +++++++++++++++---
 roles/elasticsearch/tasks/main.yml            |   8 +-
 roles/kibana/defaults/main.yml                |   3 +
 roles/kibana/handlers/main.yml                |   2 +
 roles/kibana/tasks/main.yml                   |   8 ++
 roles/logstash/defaults/main.yml              |   5 +
 roles/logstash/handlers/main.yml              |   4 +-
 roles/logstash/tasks/main.yml                 |   8 ++
 15 files changed, 183 insertions(+), 61 deletions(-)

diff --git a/molecule/elasticsearch_default/converge.yml b/molecule/elasticsearch_default/converge.yml
index 8cba6694..1e836b59 100644
--- a/molecule/elasticsearch_default/converge.yml
+++ b/molecule/elasticsearch_default/converge.yml
@@ -12,6 +12,7 @@
     elasticsearch_disable_systemcallfilterchecks: true
     elasticstack_release: "{{ lookup('env', 'ELASTIC_RELEASE') | int}}"
     elasticsearch_heap: "1"
+    elasticstack_no_log: false
   tasks:
     - name: Include Elastics repos role
       ansible.builtin.include_role:
diff --git a/molecule/elasticstack_default/converge.yml b/molecule/elasticstack_default/converge.yml
index 61ec7344..009b0fdd 100644
--- a/molecule/elasticstack_default/converge.yml
+++ b/molecule/elasticstack_default/converge.yml
@@ -12,8 +12,9 @@
   vars:
     elasticsearch_jna_workaround: true
     elasticsearch_disable_systemcallfilterchecks: true
+    elasticsearch_monitoring_enabled: false
     elasticstack_release: "{{ lookup('env', 'ELASTIC_RELEASE') | int}}"
-    elasticsearch_heap: "1"
+    elasticsearch_heap: "2"
     elasticstack_full_stack: true
     elasticstack_no_log: false
     logstash_pipeline_unsafe_shutdown: true
diff --git a/molecule/elasticstack_default/molecule.yml b/molecule/elasticstack_default/molecule.yml
index d658c84d..513db812 100644
--- a/molecule/elasticstack_default/molecule.yml
+++ b/molecule/elasticstack_default/molecule.yml
@@ -10,7 +10,6 @@ platforms:
     groups:
       - beats
       - logstash
-      - kibana
       - elasticsearch
     image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
@@ -22,7 +21,6 @@ platforms:
   - name: "elasticstack${ELASTIC_RELEASE}-cluster2-${MOLECULE_DISTRO}"
     groups:
       - beats
-      - logstash
       - kibana
       - elasticsearch
     image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest"
diff --git a/molecule/elasticstack_default/verify.yml b/molecule/elasticstack_default/verify.yml
index abceef3f..3a2e8430 100644
--- a/molecule/elasticstack_default/verify.yml
+++ b/molecule/elasticstack_default/verify.yml
@@ -59,42 +59,46 @@
         msg: "Elasticsearch received {{ logstash_count.stdout }} events so far"
       when: "'elasticsearch' in group_names"
 
-    - name: fetch kibana.yml
-      ansible.builtin.command: cat /etc/kibana/kibana.yml
-      register: kibanayml
-
-    - name: Show kibana.yml
-      ansible.builtin.debug:
-        var: kibanayml.stdout_lines
-
-    - name: Check for Kibana port
-      ansible.builtin.wait_for:
-        port: 5601
-        timeout: 120
-
-    - name: Connect to Kibana
-      ansible.builtin.command:
-        curl
-        -s
-        -u elastic:{{ elastic_pass.stdout }}
-        http://{{ ansible_hostname }}:5601/api/status
-      register: curl_out
-      failed_when:
-        - "'green' not in curl_out.stdout"
-        - "'Elasticsearch is available' not in curl_out.stdout"
-
-      # The following might be nicer but doesn't work
-      #- name: Connect to Kibana
-      #  ansible.builtin.uri:
-      #    url: http://ansible-role-kibana_full_stack:5601/api/status
-      #    user: elastic
-      #    password: "{{ elastic_password.stdout }}"
-      #    return_content: yes
-      #  register: kibana_status
-      #  #failed_when: "'"title": "Green"' not in kibana_status.content"
-      #  failed_when: "'Green' not in kibana_status.content"
-
-    - name: Health check
+    - name: Run Kibana checks
+      when: "'kibana' in group_names"
+      block:
+
+        - name: Fetch kibana.yml
+          ansible.builtin.command: cat /etc/kibana/kibana.yml
+          register: kibanayml
+
+        - name: Show kibana.yml
+          ansible.builtin.debug:
+            var: kibanayml.stdout_lines
+
+        - name: Check for Kibana port
+          ansible.builtin.wait_for:
+            port: 5601
+            timeout: 120
+
+        - name: Connect to Kibana
+          ansible.builtin.command:
+            curl
+            -s
+            -u elastic:{{ elastic_pass.stdout }}
+            http://{{ ansible_hostname }}:5601/api/status
+          register: curl_out
+          failed_when:
+            - "'green' not in curl_out.stdout"
+            - "'Elasticsearch is available' not in curl_out.stdout"
+
+          # The following might be nicer but doesn't work
+          #- name: Connect to Kibana
+          #  ansible.builtin.uri:
+          #    url: http://ansible-role-kibana_full_stack:5601/api/status
+          #    user: elastic
+          #    password: "{{ elastic_password.stdout }}"
+          #    return_content: yes
+          #  register: kibana_status
+          #  #failed_when: "'"title": "Green"' not in kibana_status.content"
+          #  failed_when: "'Green' not in kibana_status.content"
+
+    - name: Elasticsearch health check
       ansible.builtin.uri:
         url: https://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/health
         method: GET
@@ -110,7 +114,7 @@
       delay: 10
       when: groups['elasticsearch'] | length > 1
 
-    - name: Node check
+    - name: Elasticsearch Node check
       ansible.builtin.uri:
         url: https://localhost:{{ elasticstack_elasticsearch_http_port }}/_cat/nodes
         method: GET
diff --git a/roles/beats/tasks/main.yml b/roles/beats/tasks/main.yml
index f242c0a8..9d521bb1 100644
--- a/roles/beats/tasks/main.yml
+++ b/roles/beats/tasks/main.yml
@@ -82,3 +82,10 @@
 - name: Import Metricbeat tasks
   ansible.builtin.import_tasks: metricbeat.yml
   when: beats_metricbeat | bool
+
+# Free up some space to let elsticsearch allocate replica in GitHub Action
+- name: Remove cache
+  ansible.builtin.command: >
+    rm -rf /var/cache/*
+  changed_when: false
+  when: ansible_virtualization_type == "container" or ansible_virtualization_type == "docker"
diff --git a/roles/elasticsearch/defaults/main.yml b/roles/elasticsearch/defaults/main.yml
index bcadfb3d..3f531606 100644
--- a/roles/elasticsearch/defaults/main.yml
+++ b/roles/elasticsearch/defaults/main.yml
@@ -48,6 +48,12 @@ elasticsearch_cert_expiration_buffer: 30
 elasticstack_ca_will_expire_soon: false
 elasticsearch_cert_will_expire_soon: false
 
+# only used internally
+elasticsearch_freshstart:
+  changed: false
+elasticsearch_freshstart_security:
+  changed: false
+
 # "global" variables for all roles
 
 elasticstack_release: 8
diff --git a/roles/elasticsearch/handlers/main.yml b/roles/elasticsearch/handlers/main.yml
index b39f884a..ff3b5ab5 100644
--- a/roles/elasticsearch/handlers/main.yml
+++ b/roles/elasticsearch/handlers/main.yml
@@ -5,7 +5,10 @@
     name: elasticsearch
     state: restarted
     daemon_reload: yes
-  when: elasticsearch_enable | bool
+  when:
+    - elasticsearch_enable | bool
+    - not elasticsearch_freshstart.changed | bool
+    - not elasticsearch_freshstart_security.changed | bool
 
 - name: Restart kibana if available for elasticsearch certificates
   ansible.builtin.include_tasks: handlers/restart_kibana.yml
diff --git a/roles/elasticsearch/tasks/elasticsearch-security.yml b/roles/elasticsearch/tasks/elasticsearch-security.yml
index 0b1a95ac..48bcb2aa 100644
--- a/roles/elasticsearch/tasks/elasticsearch-security.yml
+++ b/roles/elasticsearch/tasks/elasticsearch-security.yml
@@ -352,25 +352,31 @@
     name: elasticsearch
     state: started
     enabled: yes
+  register: elasticsearch_freshstart_security
 
 - name: Wait for all instances to start
   ansible.builtin.include_tasks: wait_for_instance.yml
   loop: "{{ groups['elasticsearch'] }}"
 
-- name: Force all notified handlers to run at this point, not waiting for normal sync points
-  ansible.builtin.meta: flush_handlers
-  tags:
-    - certificates
-    - renew_ca
-    - renew_es_cert
-
-- name: Wait for all instances to start
-  ansible.builtin.include_tasks: wait_for_instance.yml
-  loop: "{{ groups['elasticsearch'] }}"
-  tags:
-    - certificates
-    - renew_ca
-    - renew_es_cert
+- name: Restart if Elasticsearch was already running
+  when:
+    - not elasticsearch_freshstart.changed | bool
+    - not elasticsearch_freshstart_security.changed | bool
+  block:
+    - name: Force all notified handlers to run at this point, not waiting for normal sync points
+      ansible.builtin.meta: flush_handlers
+      tags:
+        - certificates
+        - renew_ca
+        - renew_es_cert
+
+    - name: Wait for all instances to start
+      ansible.builtin.include_tasks: wait_for_instance.yml
+      loop: "{{ groups['elasticsearch'] }}"
+      tags:
+        - certificates
+        - renew_ca
+        - renew_es_cert
 
 - name: Check for passwords being set
   ansible.builtin.stat:
@@ -383,6 +389,25 @@
     elasticsearch_http_protocol: "https"
   when: elasticsearch_http_security
 
+- name: Check for API with bootstrap password
+  ansible.builtin.uri:
+    url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}"
+    user: elastic
+    password: "{{ elasticsearch_bootstrap_pw }}"
+    validate_certs: false
+  register: elasticsearch_api_status_bootstrap
+  changed_when: false
+  no_log: "{{ elasticstack_no_log }}"
+  when:
+    - not elasticsearch_passwords_file.stat.exists | bool
+    - groups['elasticsearch'] | length > 1
+  until: elasticsearch_api_status_bootstrap.json.cluster_name is defined
+  retries: 5
+  delay: 10
+
+# We need this check twice. One to wait for the API to be actually available. And a second time to
+# check the actual return code. Should not cause a huge delay.
+
 - name: Check for cluster status with bootstrap password
   ansible.builtin.uri:
     url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/health?pretty"
@@ -410,6 +435,57 @@
   delegate_to: "{{ elasticstack_ca }}"
   when: elasticsearch_passwords_file.stat.exists | bool
 
+- name: Check for API availability with elastic password
+  ansible.builtin.uri:
+    url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}"
+    user: elastic
+    password: "{{ elasticstack_password.stdout }}"
+    validate_certs: false
+  register: elasticsearch_api_status
+  changed_when: false
+  no_log: "{{ elasticstack_no_log }}"
+  when:
+    - elasticsearch_passwords_file.stat.exists | bool
+    - groups['elasticsearch'] | length > 1
+  until: elasticsearch_api_status.json.cluster_name is defined
+  retries: 20
+  delay: 10
+
+- name: Work around low ressources on CI/CD nodes
+  when: ansible_virtualization_type == "container" or ansible_virtualization_type == "docker"
+  block:
+    # Free up some space to let elsticsearch allocate replica in GitHub Action
+    - name: Remove cache
+      ansible.builtin.command: >
+        rm -rf /var/cache/*
+      changed_when: false
+
+    - name: Set persistent watermarks to very high values in Docker # noqa: risky-shell-pipe
+      ansible.builtin.shell: >
+        if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi;
+        curl
+        -k
+        -X PUT
+        "{{ elasticsearch_http_protocol }}://elastic:{{ elasticstack_password.stdout }}@localhost:9200/_cluster/settings"
+        -H 'Content-Type: application/json' -d
+        '
+        {
+          "persistent": {
+            "cluster.routing.allocation.disk.watermark.low": "97%",
+            "cluster.routing.allocation.disk.watermark.high": "98%",
+            "cluster.routing.allocation.disk.watermark.flood_stage": "99%",
+            "cluster.routing.allocation.disk.watermark.flood_stage.frozen": "99%"
+          }
+        }
+        '
+      changed_when: false
+      no_log: "{{ elasticstack_no_log }}"
+      when:
+        - elasticstack_password.stdout is defined
+
+# We need this check twice. One to wait for the API to be actually available. And a second time to
+# check the actual return code. Should not cause a huge delay.
+
 - name: Check for cluster status with elastic password
   ansible.builtin.uri:
     url: "{{ elasticsearch_http_protocol }}://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/health?pretty"
diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml
index 4b813117..0632f870 100644
--- a/roles/elasticsearch/tasks/main.yml
+++ b/roles/elasticsearch/tasks/main.yml
@@ -178,9 +178,8 @@
   when: ansible_virtualization_type == "container" or ansible_virtualization_type == "docker"
 
 # Free up some space to let elsticsearch allocate replica in GitHub Action
-- name: Remove cache # noqa: risky-shell-pipe
-  ansible.builtin.shell: >
-    if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi;
+- name: Remove cache
+  ansible.builtin.command: >
     rm -rf /var/cache/*
   changed_when: false
   when: ansible_virtualization_type == "container" or ansible_virtualization_type == "docker"
@@ -200,6 +199,7 @@
     name: elasticsearch
     state: started
     enabled: yes
+  register: elasticsearch_freshstart
 
 - name: Handle cluster setup without security
   when: not elasticsearch_security | bool
@@ -237,8 +237,6 @@
     group: root
     mode: 0644
     backup: "{{ elasticsearch_config_backup }}"
-  notify:
-    - Restart Elasticsearch
   when: elasticsearch_manage_yaml | bool
 
 - name: Show Info about heap
diff --git a/roles/kibana/defaults/main.yml b/roles/kibana/defaults/main.yml
index cc21f125..7aa06f7b 100644
--- a/roles/kibana/defaults/main.yml
+++ b/roles/kibana/defaults/main.yml
@@ -18,6 +18,9 @@ kibana_cert_will_expire_soon: false
 kibana_sniff_on_start: false
 kibana_sniff_on_connection_fault: false
 
+kibana_freshstart:
+  changed: false
+
 # "global" variables for all roles
 elasticstack_release: 8
 elasticstack_full_stack: true
diff --git a/roles/kibana/handlers/main.yml b/roles/kibana/handlers/main.yml
index 81ffa146..532d014a 100644
--- a/roles/kibana/handlers/main.yml
+++ b/roles/kibana/handlers/main.yml
@@ -4,3 +4,5 @@
   ansible.builtin.service:
     name: kibana
     state: restarted
+  when:
+    - not kibana_freshstart.changed | bool
diff --git a/roles/kibana/tasks/main.yml b/roles/kibana/tasks/main.yml
index b5b3ebb9..74e91344 100644
--- a/roles/kibana/tasks/main.yml
+++ b/roles/kibana/tasks/main.yml
@@ -82,6 +82,7 @@
     state: started
     enabled: yes
   when: kibana_enable | bool
+  register: kibana_freshstart
 
 # the following is useful when running tests or extra tasks that need to
 # have Kibana running. Escape it on Rocky8, because it gets time out with Elastic 8
@@ -90,3 +91,10 @@
   ansible.builtin.wait_for:
     host: localhost
     port: 5601
+
+# Free up some space to let elsticsearch allocate replica in GitHub Action
+- name: Remove cache
+  ansible.builtin.command: >
+    rm -rf /var/cache/*
+  changed_when: false
+  when: ansible_virtualization_type == "container" or ansible_virtualization_type == "docker"
diff --git a/roles/logstash/defaults/main.yml b/roles/logstash/defaults/main.yml
index 8ff97804..1941e792 100644
--- a/roles/logstash/defaults/main.yml
+++ b/roles/logstash/defaults/main.yml
@@ -73,6 +73,11 @@ logstash_pipeline_identifier: true
 logstash_pipeline_identifier_field_name: "[netways][pipeline]"
 logstash_pipeline_identifier_defaults: false
 
+# Only for internal use
+
+logstash_freshstart:
+  changed: false
+
 elasticstack_ca_dir: /opt/es-ca
 elasticstack_initial_passwords: /usr/share/elasticsearch/initial_passwords
 elasticstack_ca_pass: PleaseChangeMe
diff --git a/roles/logstash/handlers/main.yml b/roles/logstash/handlers/main.yml
index eb55a868..08b3b71b 100644
--- a/roles/logstash/handlers/main.yml
+++ b/roles/logstash/handlers/main.yml
@@ -4,7 +4,9 @@
   ansible.builtin.service:
     name: logstash
     state: restarted
-  when: logstash_enable | bool
+  when:
+    - logstash_enable | bool
+    - not logstash_freshstart.changed | bool
 
 - name: Restart Logstash noauto
   ansible.builtin.service:
diff --git a/roles/logstash/tasks/main.yml b/roles/logstash/tasks/main.yml
index 1dcee30b..a1d1b3de 100644
--- a/roles/logstash/tasks/main.yml
+++ b/roles/logstash/tasks/main.yml
@@ -230,3 +230,11 @@
     state: started
     enabled: yes
   when: logstash_enable | bool
+  register: logstash_freshstart
+
+# Free up some space to let elsticsearch allocate replica in GitHub Action
+- name: Remove cache
+  ansible.builtin.command: >
+    rm -rf /var/cache/*
+  changed_when: false
+  when: ansible_virtualization_type == "container" or ansible_virtualization_type == "docker"

From 976d73925fbf92c9518e69ccaa865149efec8169 Mon Sep 17 00:00:00 2001
From: Thomas Widhalm <thomas.widhalm@netways.de>
Date: Tue, 17 Oct 2023 10:00:04 +0200
Subject: [PATCH 2/5] Set Debian 11 as new default distro for molecule (#277)

We had CentOS 7 long enough so I thought it might be nice to go with
Debian instead. This PR will not change much in the behaviour because
usually we overwrite the default. So this is just in case and to stay up
to date.

fixes #245
---
 molecule/beats_default/molecule.yml                   | 4 ++--
 molecule/beats_peculiar/molecule.yml                  | 4 ++--
 molecule/elasticsearch_cluster-oss/molecule.yml       | 4 ++--
 molecule/elasticsearch_default/molecule.yml           | 4 ++--
 molecule/elasticsearch_no-security/molecule.yml       | 4 ++--
 molecule/elasticsearch_roles_calculation/molecule.yml | 6 +++---
 molecule/elasticstack_default/molecule.yml            | 4 ++--
 molecule/kibana_default/molecule.yml                  | 2 +-
 molecule/logstash_full_stack-oss/molecule.yml         | 2 +-
 molecule/logstash_pipelines/molecule.yml              | 2 +-
 molecule/logstash_specific_version/molecule.yml       | 2 +-
 molecule/repos_default/molecule.yml                   | 2 +-
 molecule/repos_oss/molecule.yml                       | 2 +-
 13 files changed, 21 insertions(+), 21 deletions(-)

diff --git a/molecule/beats_default/molecule.yml b/molecule/beats_default/molecule.yml
index 9e57f3c0..51fbb100 100644
--- a/molecule/beats_default/molecule.yml
+++ b/molecule/beats_default/molecule.yml
@@ -4,8 +4,8 @@ dependency:
 driver:
   name: docker
 platforms:
-  - name: beats_default_${MOLECULE_DISTRO:-centos7}
-    image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest"
+  - name: beats_default_${MOLECULE_DISTRO:-debian11}
+    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:rw
diff --git a/molecule/beats_peculiar/molecule.yml b/molecule/beats_peculiar/molecule.yml
index 4eb05f8f..bbbd5849 100644
--- a/molecule/beats_peculiar/molecule.yml
+++ b/molecule/beats_peculiar/molecule.yml
@@ -4,8 +4,8 @@ dependency:
 driver:
   name: docker
 platforms:
-  - name: beats_peculiar_${MOLECULE_DISTRO:-centos7}
-    image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest"
+  - name: beats_peculiar_${MOLECULE_DISTRO:-debian11}
+    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:rw
diff --git a/molecule/elasticsearch_cluster-oss/molecule.yml b/molecule/elasticsearch_cluster-oss/molecule.yml
index 50f31e05..7b78fb80 100644
--- a/molecule/elasticsearch_cluster-oss/molecule.yml
+++ b/molecule/elasticsearch_cluster-oss/molecule.yml
@@ -9,7 +9,7 @@ platforms:
   - name: elasticsearch-cluster1
     groups:
       - elasticsearch
-    image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest"
+    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:rw
@@ -19,7 +19,7 @@ platforms:
   - name: elasticsearch-cluster2
     groups:
       - elasticsearch
-    image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest"
+    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:rw
diff --git a/molecule/elasticsearch_default/molecule.yml b/molecule/elasticsearch_default/molecule.yml
index 25c5022b..7c2c71b7 100644
--- a/molecule/elasticsearch_default/molecule.yml
+++ b/molecule/elasticsearch_default/molecule.yml
@@ -9,7 +9,7 @@ platforms:
   - name: elasticsearch_default1
     groups:
       - elasticsearch
-    image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest"
+    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:rw
@@ -19,7 +19,7 @@ platforms:
   - name: elasticsearch_default2
     groups:
       - elasticsearch
-    image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest"
+    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:rw
diff --git a/molecule/elasticsearch_no-security/molecule.yml b/molecule/elasticsearch_no-security/molecule.yml
index 9855d8e2..8b08e708 100644
--- a/molecule/elasticsearch_no-security/molecule.yml
+++ b/molecule/elasticsearch_no-security/molecule.yml
@@ -9,7 +9,7 @@ platforms:
   - name: elasticsearch-nosecurity1
     groups:
       - elasticsearch
-    image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest"
+    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:rw
@@ -19,7 +19,7 @@ platforms:
   - name: elasticsearch-nosecurity2
     groups:
       - elasticsearch
-    image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest"
+    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:rw
diff --git a/molecule/elasticsearch_roles_calculation/molecule.yml b/molecule/elasticsearch_roles_calculation/molecule.yml
index 6e7d35a3..6d9a59c7 100644
--- a/molecule/elasticsearch_roles_calculation/molecule.yml
+++ b/molecule/elasticsearch_roles_calculation/molecule.yml
@@ -9,7 +9,7 @@ platforms:
   - name: elasticsearch-cluster1
     groups:
       - elasticsearch
-    image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest"
+    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:rw
@@ -19,7 +19,7 @@ platforms:
   - name: elasticsearch-cluster2
     groups:
       - elasticsearch
-    image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest"
+    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:rw
@@ -29,7 +29,7 @@ platforms:
   - name: elasticsearch-cluster3
     groups:
       - elasticsearch
-    image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest"
+    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:rw
diff --git a/molecule/elasticstack_default/molecule.yml b/molecule/elasticstack_default/molecule.yml
index 513db812..122ee248 100644
--- a/molecule/elasticstack_default/molecule.yml
+++ b/molecule/elasticstack_default/molecule.yml
@@ -11,7 +11,7 @@ platforms:
       - beats
       - logstash
       - elasticsearch
-    image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest"
+    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:rw
@@ -23,7 +23,7 @@ platforms:
       - beats
       - kibana
       - elasticsearch
-    image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest"
+    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:rw
diff --git a/molecule/kibana_default/molecule.yml b/molecule/kibana_default/molecule.yml
index d279313b..0ae83520 100644
--- a/molecule/kibana_default/molecule.yml
+++ b/molecule/kibana_default/molecule.yml
@@ -5,7 +5,7 @@ driver:
   name: docker
 platforms:
   - name: kibana_default
-    image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest"
+    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:rw
diff --git a/molecule/logstash_full_stack-oss/molecule.yml b/molecule/logstash_full_stack-oss/molecule.yml
index ebd40da8..199fff10 100644
--- a/molecule/logstash_full_stack-oss/molecule.yml
+++ b/molecule/logstash_full_stack-oss/molecule.yml
@@ -11,7 +11,7 @@ platforms:
       - elasticsearch
       - logstash
       - filebeat
-    image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest"
+    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:rw
diff --git a/molecule/logstash_pipelines/molecule.yml b/molecule/logstash_pipelines/molecule.yml
index e2627f02..51d872e7 100644
--- a/molecule/logstash_pipelines/molecule.yml
+++ b/molecule/logstash_pipelines/molecule.yml
@@ -7,7 +7,7 @@ driver:
   name: docker
 platforms:
   - name: ansible-role-logstash_pipelines
-    image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest"
+    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:rw
diff --git a/molecule/logstash_specific_version/molecule.yml b/molecule/logstash_specific_version/molecule.yml
index 5b4cf3c0..de33f462 100644
--- a/molecule/logstash_specific_version/molecule.yml
+++ b/molecule/logstash_specific_version/molecule.yml
@@ -7,7 +7,7 @@ driver:
   name: docker
 platforms:
   - name: elasticstack_version
-    image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest"
+    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:rw
diff --git a/molecule/repos_default/molecule.yml b/molecule/repos_default/molecule.yml
index 3c857a61..a111002c 100644
--- a/molecule/repos_default/molecule.yml
+++ b/molecule/repos_default/molecule.yml
@@ -7,7 +7,7 @@ driver:
   name: docker
 platforms:
   - name: elastic-repos-default
-    image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest"
+    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:rw
diff --git a/molecule/repos_oss/molecule.yml b/molecule/repos_oss/molecule.yml
index 9527212b..e1e7a330 100644
--- a/molecule/repos_oss/molecule.yml
+++ b/molecule/repos_oss/molecule.yml
@@ -7,7 +7,7 @@ driver:
   name: docker
 platforms:
   - name: elastic-repos-default-oss
-    image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest"
+    image: "geerlingguy/docker-${MOLECULE_DISTRO:-debian11}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
     volumes:
       - /sys/fs/cgroup:/sys/fs/cgroup:rw

From d429b152431d521e15cd6f7f54b6df924b5c2f9b Mon Sep 17 00:00:00 2001
From: Thomas Widhalm <thomas.widhalm@netways.de>
Date: Tue, 17 Oct 2023 10:49:21 +0200
Subject: [PATCH 3/5] Refresh apt cache before installing packages (#179)

Run an apt refresh at the start of each role. We can use grace time so
we don't neet to refresh it every time and safe some time.

fixes #167
---
 roles/beats/tasks/main.yml         | 7 +++++++
 roles/elasticsearch/tasks/main.yml | 8 ++++++++
 roles/kibana/tasks/main.yml        | 7 +++++++
 roles/logstash/tasks/main.yml      | 7 +++++++
 4 files changed, 29 insertions(+)

diff --git a/roles/beats/tasks/main.yml b/roles/beats/tasks/main.yml
index 9d521bb1..cce08b9e 100644
--- a/roles/beats/tasks/main.yml
+++ b/roles/beats/tasks/main.yml
@@ -6,6 +6,13 @@
     - '{{ ansible_os_family }}_{{ ansible_distribution_major_version }}.yml'
     - '{{ ansible_os_family }}.yml'
 
+- name: Update apt cache.
+  ansible.builtin.apt:
+    update_cache: yes
+    cache_valid_time: 600
+  changed_when: false
+  when: ansible_os_family == 'Debian'
+
 - name: Prepare for whole stack roles if used
   when:
     - elasticstack_full_stack | bool
diff --git a/roles/elasticsearch/tasks/main.yml b/roles/elasticsearch/tasks/main.yml
index 0632f870..fdc11ea6 100644
--- a/roles/elasticsearch/tasks/main.yml
+++ b/roles/elasticsearch/tasks/main.yml
@@ -1,4 +1,12 @@
 ---
+
+- name: Update apt cache.
+  ansible.builtin.apt:
+    update_cache: yes
+    cache_valid_time: 600
+  changed_when: false
+  when: ansible_os_family == 'Debian'
+
 - name: Check-set-parameters
   ansible.builtin.include_tasks: elasticsearch-parameters.yml
 
diff --git a/roles/kibana/tasks/main.yml b/roles/kibana/tasks/main.yml
index 74e91344..87638d4d 100644
--- a/roles/kibana/tasks/main.yml
+++ b/roles/kibana/tasks/main.yml
@@ -1,5 +1,12 @@
 ---
 
+- name: Update apt cache.
+  ansible.builtin.apt:
+    update_cache: yes
+    cache_valid_time: 600
+  changed_when: false
+  when: ansible_os_family == 'Debian'
+
 - name: Include OS specific vars
   ansible.builtin.include_vars: '{{ item }}'
   with_first_found:
diff --git a/roles/logstash/tasks/main.yml b/roles/logstash/tasks/main.yml
index a1d1b3de..488926b6 100644
--- a/roles/logstash/tasks/main.yml
+++ b/roles/logstash/tasks/main.yml
@@ -1,5 +1,12 @@
 ---
 
+- name: Update apt cache.
+  ansible.builtin.apt:
+    update_cache: yes
+    cache_valid_time: 600
+  changed_when: false
+  when: ansible_os_family == 'Debian'
+
 - name: Include OS specific vars
   ansible.builtin.include_vars: '{{ item }}'
   with_first_found:

From 60115d101e3522ab13dcbb4c5704f1e1aa19952e Mon Sep 17 00:00:00 2001
From: Thomas Widhalm <thomas.widhalm@netways.de>
Date: Tue, 17 Oct 2023 11:51:34 +0200
Subject: [PATCH 4/5] Remove all max-parallel from GitHub workflows (#284)

This will need #279 to be merged before tests can pass.

fixes #283
---
 .github/workflows/test_full_stack.yml         | 1 -
 .github/workflows/test_plugins.yml            | 4 ----
 .github/workflows/test_role_beats.yml         | 1 -
 .github/workflows/test_role_elasticsearch.yml | 1 -
 .github/workflows/test_role_kibana.yml        | 1 -
 .github/workflows/test_role_logstash.yml      | 1 -
 .github/workflows/test_role_repos.yml         | 1 -
 .github/workflows/test_roles_pr.yml           | 1 -
 8 files changed, 11 deletions(-)

diff --git a/.github/workflows/test_full_stack.yml b/.github/workflows/test_full_stack.yml
index 3dba4e20..4568fdd9 100644
--- a/.github/workflows/test_full_stack.yml
+++ b/.github/workflows/test_full_stack.yml
@@ -31,7 +31,6 @@ jobs:
 
     strategy:
       fail-fast: false
-      max-parallel: 2
       matrix:
         distro:
           - rockylinux8
diff --git a/.github/workflows/test_plugins.yml b/.github/workflows/test_plugins.yml
index 3bd80ab0..8632a0d7 100644
--- a/.github/workflows/test_plugins.yml
+++ b/.github/workflows/test_plugins.yml
@@ -66,7 +66,6 @@ jobs:
 
     strategy:
       fail-fast: false
-      max-parallel: 1
 
     steps:
       - name: Check out code
@@ -111,7 +110,6 @@ jobs:
 
     strategy:
       fail-fast: false
-      max-parallel: 1
       matrix:
         python_version: [ 3.5.10, 3.6.15, 3.7.13, 3.8.16, 3.10.10 ]
 
@@ -151,7 +149,6 @@ jobs:
 
     strategy:
       fail-fast: false
-      max-parallel: 1
       matrix:
         ansible_core_version: [ 2.11.12, 2.12.10, 2.13.8, 2.14.4 ]
 
@@ -191,7 +188,6 @@ jobs:
 
     strategy:
       fail-fast: false
-      max-parallel: 1
       matrix:
         python_cryptography_version: [ 2.5, 3.0, 3.1, 3.2, 3.3, 3.4, 35.0.0, 36.0.0, 38.0.0, 40.0.1]
 
diff --git a/.github/workflows/test_role_beats.yml b/.github/workflows/test_role_beats.yml
index 45544ba6..3f38fed9 100644
--- a/.github/workflows/test_role_beats.yml
+++ b/.github/workflows/test_role_beats.yml
@@ -47,7 +47,6 @@ jobs:
 
     strategy:
       fail-fast: false
-      max-parallel: 4
       matrix:
         distro: [ubuntu2204]
         scenario: [beats_default, beats_peculiar]
diff --git a/.github/workflows/test_role_elasticsearch.yml b/.github/workflows/test_role_elasticsearch.yml
index b0da7d9b..47c8008b 100644
--- a/.github/workflows/test_role_elasticsearch.yml
+++ b/.github/workflows/test_role_elasticsearch.yml
@@ -47,7 +47,6 @@ jobs:
 
     strategy:
       fail-fast: false
-      max-parallel: 4
       matrix:
         distro: [ubuntu2204]
         scenario:
diff --git a/.github/workflows/test_role_kibana.yml b/.github/workflows/test_role_kibana.yml
index 914464eb..f11bc1ff 100644
--- a/.github/workflows/test_role_kibana.yml
+++ b/.github/workflows/test_role_kibana.yml
@@ -48,7 +48,6 @@ jobs:
 
     strategy:
       fail-fast: false
-      max-parallel: 4
       matrix:
         distro: [ubuntu2204]
         scenario: [kibana_default]
diff --git a/.github/workflows/test_role_logstash.yml b/.github/workflows/test_role_logstash.yml
index 14198928..cb959a4c 100644
--- a/.github/workflows/test_role_logstash.yml
+++ b/.github/workflows/test_role_logstash.yml
@@ -48,7 +48,6 @@ jobs:
 
     strategy:
       fail-fast: false
-      max-parallel: 4
       matrix:
         distro: [ubuntu2204]
         scenario:
diff --git a/.github/workflows/test_role_repos.yml b/.github/workflows/test_role_repos.yml
index 7963ce6b..c9d83220 100644
--- a/.github/workflows/test_role_repos.yml
+++ b/.github/workflows/test_role_repos.yml
@@ -46,7 +46,6 @@ jobs:
 
     strategy:
       fail-fast: false
-      max-parallel: 4
       matrix:
 
         distro: [centos7, debian10, debian11, rockylinux8, rockylinux9, ubuntu2004, ubuntu2204]
diff --git a/.github/workflows/test_roles_pr.yml b/.github/workflows/test_roles_pr.yml
index 525f1691..2f18d0a2 100644
--- a/.github/workflows/test_roles_pr.yml
+++ b/.github/workflows/test_roles_pr.yml
@@ -48,7 +48,6 @@ jobs:
 
     strategy:
       fail-fast: false
-      max-parallel: 2
       matrix:
         distro:
           - rockylinux8

From 4077a85b60bce2c22388ab5d2386141ad1082f7b Mon Sep 17 00:00:00 2001
From: Thomas Widhalm <thomas.widhalm@netways.de>
Date: Wed, 29 Nov 2023 18:01:46 +0100
Subject: [PATCH 5/5] Typo in path of slow log (#293)

Just had the wrong path to the logfiles.
---
 roles/beats/templates/filebeat.yml.j2 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/roles/beats/templates/filebeat.yml.j2 b/roles/beats/templates/filebeat.yml.j2
index 5e788d45..43456cde 100644
--- a/roles/beats/templates/filebeat.yml.j2
+++ b/roles/beats/templates/filebeat.yml.j2
@@ -43,7 +43,7 @@ filebeat.inputs:
 - type: log
   enabled: true
   paths:
-    - /var/lib/mysql/*-slow.log
+    - /var/log/mysql/*-slow.log
   multiline.pattern: '^\#[[:space:]]Time'
   multiline.negate: true
   multiline.match: after