Merge branch 'main' into feature/aptcache-167

.github/workflows/test_full_stack.yml

@@ -49,7 +49,7 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Python 3.8
         uses: actions/setup-python@v4

.github/workflows/test_linting.yml

@@ -41,7 +41,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out the codebase.
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Python 3.
         uses: actions/setup-python@v4

.github/workflows/test_plugins.yml

@@ -40,7 +40,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out the codebase.
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Python 3.
         uses: actions/setup-python@v4
@@ -70,7 +70,7 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Python 3.9.14
         uses: actions/setup-python@v4
@@ -117,7 +117,7 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Python ${{ matrix.python_version }}
         uses: actions/setup-python@v4
@@ -157,7 +157,7 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Python 3.9.14
         uses: actions/setup-python@v4
@@ -197,7 +197,7 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Python 3.9.14
         uses: actions/setup-python@v4

.github/workflows/test_role_beats.yml

@@ -55,7 +55,7 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Python 3.8
         uses: actions/setup-python@v4

.github/workflows/test_role_elasticsearch.yml

@@ -61,7 +61,7 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Python 3.8
         uses: actions/setup-python@v4

.github/workflows/test_role_kibana.yml

@@ -56,7 +56,7 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Python 3.8
         uses: actions/setup-python@v4

.github/workflows/test_role_logstash.yml

@@ -61,7 +61,7 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Python 3.8
         uses: actions/setup-python@v4

.github/workflows/test_role_repos.yml

@@ -55,7 +55,7 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Python 3.8
         uses: actions/setup-python@v4

.github/workflows/test_roles_pr.yml

@@ -61,7 +61,7 @@ jobs:
 
     steps:
       - name: Check out code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set up Python 3.8
         uses: actions/setup-python@v4

docs/role-kibana.md

@@ -13,6 +13,7 @@ Role Variables
 * *kibana_tls*: Whether to offer `https` for clients or not (default: `false`)
 * *kibana_tls_cert*: Path to the certificate Kibana should show to its clients (default: `/etc/kibana/certs/cert.pem`)
 * *kibana_tls_key*: Path to the key Kibana should use when communicating with clients (default: `/etc/kibana/certs/key.pem`)
+* *kibana_extra_config*: You can add arbitraty configuration options with this option. Just start it with `|-` and indent the following lines. So you can add as many lines and options to `kibana.yml` as you like. (default: none)
 
 * *kibana_security*: Activate TLS and authentication when connecting to Elasticsearch. **Note**: Only works when `elasticstack_full_stack` is enabled. (default: `true`)
 

molecule/elasticsearch_default/converge.yml

@@ -12,6 +12,7 @@
     elasticsearch_disable_systemcallfilterchecks: true
     elasticstack_release: "{{ lookup('env', 'ELASTIC_RELEASE') | int}}"
     elasticsearch_heap: "1"
+    elasticstack_no_log: false
   tasks:
     - name: Include Elastics repos role
       ansible.builtin.include_role:

molecule/elasticstack_default/converge.yml

@@ -12,8 +12,9 @@
   vars:
     elasticsearch_jna_workaround: true
     elasticsearch_disable_systemcallfilterchecks: true
+    elasticsearch_monitoring_enabled: false
     elasticstack_release: "{{ lookup('env', 'ELASTIC_RELEASE') | int}}"
-    elasticsearch_heap: "1"
+    elasticsearch_heap: "2"
     elasticstack_full_stack: true
     elasticstack_no_log: false
     logstash_pipeline_unsafe_shutdown: true
@@ -23,6 +24,8 @@
       - system
     beats_fields:
       - "testbed: molecule"
+    kibana_extra_config: |-
+      ops.interval: 5000
   tasks:
     - name: Enable Elastic installation on RHEL 9
       ansible.builtin.set_fact:
@@ -49,6 +52,11 @@
     - name: Install rsyslog
       ansible.builtin.package:
         name: rsyslog
+    - name: Remove cache # noqa: risky-shell-pipe
+      ansible.builtin.shell: >
+        if test -n "$(ps -p $$ | grep bash)"; then set -o pipefail; fi;
+        rm -rf /var/cache/*
+      changed_when: false
     - name: Configure rsyslog
       ansible.builtin.lineinfile:
         line: "*.* @@localhost:514"

molecule/elasticstack_default/molecule.yml

@@ -10,7 +10,6 @@ platforms:
     groups:
       - beats
       - logstash
-      - kibana
       - elasticsearch
     image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest"
     command: ${MOLECULE_DOCKER_COMMAND:-""}
@@ -22,7 +21,6 @@ platforms:
   - name: "elasticstack${ELASTIC_RELEASE}-cluster2-${MOLECULE_DISTRO}"
     groups:
       - beats
-      - logstash
       - kibana
       - elasticsearch
     image: "geerlingguy/docker-${MOLECULE_DISTRO:-centos7}-ansible:latest"

molecule/elasticstack_default/verify.yml

@@ -59,42 +59,46 @@
         msg: "Elasticsearch received {{ logstash_count.stdout }} events so far"
       when: "'elasticsearch' in group_names"
 
-    - name: fetch kibana.yml
-      ansible.builtin.command: cat /etc/kibana/kibana.yml
-      register: kibanayml
-
-    - name: Show kibana.yml
-      ansible.builtin.debug:
-        var: kibanayml.stdout_lines
-
-    - name: Check for Kibana port
-      ansible.builtin.wait_for:
-        port: 5601
-        timeout: 120
-
-    - name: Connect to Kibana
-      ansible.builtin.command:
-        curl
-        -s
-        -u elastic:{{ elastic_pass.stdout }}
-        http://{{ ansible_hostname }}:5601/api/status
-      register: curl_out
-      failed_when:
-        - "'green' not in curl_out.stdout"
-        - "'Elasticsearch is available' not in curl_out.stdout"
-
-      # The following might be nicer but doesn't work
-      #- name: Connect to Kibana
-      #  ansible.builtin.uri:
-      #    url: http://ansible-role-kibana_full_stack:5601/api/status
-      #    user: elastic
-      #    password: "{{ elastic_password.stdout }}"
-      #    return_content: yes
-      #  register: kibana_status
-      #  #failed_when: "'"title": "Green"' not in kibana_status.content"
-      #  failed_when: "'Green' not in kibana_status.content"
-
-    - name: Health check
+    - name: Run Kibana checks
+      when: "'kibana' in group_names"
+      block:
+
+        - name: Fetch kibana.yml
+          ansible.builtin.command: cat /etc/kibana/kibana.yml
+          register: kibanayml
+
+        - name: Show kibana.yml
+          ansible.builtin.debug:
+            var: kibanayml.stdout_lines
+
+        - name: Check for Kibana port
+          ansible.builtin.wait_for:
+            port: 5601
+            timeout: 120
+
+        - name: Connect to Kibana
+          ansible.builtin.command:
+            curl
+            -s
+            -u elastic:{{ elastic_pass.stdout }}
+            http://{{ ansible_hostname }}:5601/api/status
+          register: curl_out
+          failed_when:
+            - "'green' not in curl_out.stdout"
+            - "'Elasticsearch is available' not in curl_out.stdout"
+
+          # The following might be nicer but doesn't work
+          #- name: Connect to Kibana
+          #  ansible.builtin.uri:
+          #    url: http://ansible-role-kibana_full_stack:5601/api/status
+          #    user: elastic
+          #    password: "{{ elastic_password.stdout }}"
+          #    return_content: yes
+          #  register: kibana_status
+          #  #failed_when: "'"title": "Green"' not in kibana_status.content"
+          #  failed_when: "'Green' not in kibana_status.content"
+
+    - name: Elasticsearch health check
       ansible.builtin.uri:
         url: https://localhost:{{ elasticstack_elasticsearch_http_port }}/_cluster/health
         method: GET
@@ -110,7 +114,7 @@
       delay: 10
       when: groups['elasticsearch'] | length > 1
 
-    - name: Node check
+    - name: Elasticsearch Node check
       ansible.builtin.uri:
         url: https://localhost:{{ elasticstack_elasticsearch_http_port }}/_cat/nodes
         method: GET

roles/beats/tasks/main.yml

@@ -89,3 +89,10 @@
 - name: Import Metricbeat tasks
   ansible.builtin.import_tasks: metricbeat.yml
   when: beats_metricbeat | bool
+
+# Free up some space to let elsticsearch allocate replica in GitHub Action
+- name: Remove cache
+  ansible.builtin.command: >
+    rm -rf /var/cache/*
+  changed_when: false
+  when: ansible_virtualization_type == "container" or ansible_virtualization_type == "docker"

roles/elasticsearch/defaults/main.yml

@@ -48,6 +48,12 @@ elasticsearch_cert_expiration_buffer: 30
 elasticstack_ca_will_expire_soon: false
 elasticsearch_cert_will_expire_soon: false
 
+# only used internally
+elasticsearch_freshstart:
+  changed: false
+elasticsearch_freshstart_security:
+  changed: false
+
 # "global" variables for all roles
 
 elasticstack_release: 8

roles/elasticsearch/handlers/main.yml

@@ -5,7 +5,10 @@
     name: elasticsearch
     state: restarted
     daemon_reload: yes
-  when: elasticsearch_enable | bool
+  when:
+    - elasticsearch_enable | bool
+    - not elasticsearch_freshstart.changed | bool
+    - not elasticsearch_freshstart_security.changed | bool
 
 - name: Restart kibana if available for elasticsearch certificates
   ansible.builtin.include_tasks: handlers/restart_kibana.yml