Merge pull request #2042 from ProgrammeVitam/story_13392_disable_file…

…beat_health_checks Story #13392: Drop Consul and Prometheus accesslogs from filebeat processor.
ProgrammeVitam · Sep 25, 2024 · a817695 · a817695
2 parents 26d1306 + 1b78293
commit a817695
Show file tree

Hide file tree

Showing 7 changed files with 72 additions and 78 deletions.
diff --git a/deployment/roles/filebeat/defaults/main.yml b/deployment/roles/filebeat/defaults/main.yml
@@ -13,17 +13,6 @@ filebeat_log:
   filename: filebeat.log
   keepfiles: 30
 
-items_to_exclude:
-  - hosts
-  - browse
-  - vitamui
-  - ui_search
-  - zone_admin
-  - zone_cas
-  - zone_data
-  - zone_ui
-  - zone_app
-
 filebeat:
   system:
     enable_log: false
@@ -42,3 +31,7 @@ filebeat:
       enable_ingress: false
     mongodb:
       enable_log: true
+  services:
+    vitamui:
+      api_gateway:
+        enable_management: false
diff --git a/deployment/roles/filebeat/tasks/add_inputs.yml b/deployment/roles/filebeat/tasks/add_inputs.yml
@@ -8,18 +8,15 @@
   notify: "filebeat - restart service"
 
 - set_fact:
-    services_on_vm: "{{ group_names | regex_replace('hosts_', '') | regex_replace('hosts_vitamui_', '') | regex_replace('vitamui_', '') }}"
-
-- set_fact:
-    services_on_vm: "{{ services_on_vm | difference(items_to_exclude) }}"
+    services_on_vm: "{{ group_names | regex_replace('hosts_', '') | regex_replace('vitamui_', '') }}"
 
 - block:
 
   - set_fact:
-      vitamui_service_list: "{{ vitamui.keys() | list }}"
+      vitamui_keys: "{{ vitamui.keys() | list }}"
 
   - set_fact:
-      vitamui_service_list: "{{ services_on_vm | intersect(vitamui_service_list) | regex_replace('_', '-') }}"
+      vitamui_service_list: "{{ services_on_vm | intersect(vitamui_keys) }}"
 
   - name: Add VitamUI filebeat inputs
     template:
@@ -35,4 +32,4 @@
       mode: 0644
     notify: "filebeat - restart service"
 
-  when: inventory_hostname in groups['hosts_vitamui']
+  when: inventory_hostname in groups['hosts_vitamui']
diff --git a/deployment/roles/filebeat/templates/inputs/cots.yml.j2 b/deployment/roles/filebeat/templates/inputs/cots.yml.j2
@@ -2,11 +2,10 @@
 #
 # Consul
 #
-# Disabled by default because of a bug with spring.cloud.consul when calling consul (too many error logs).
 {% if filebeat.cots.consul.enable_log | bool %}
 - type: filestream
   id: {{ vitam_site_name }}-consul
-  enabled: {{ filebeat.cots.consul.enable_log | bool | lower }}
+  enabled: true
   paths:
     - "{{ vitam_defaults.folder.root_path }}/log/consul/consul*.log"
   fields_under_root: true

diff --git a/deployment/roles/filebeat/templates/inputs/vitamui_services.yml.j2 b/deployment/roles/filebeat/templates/inputs/vitamui_services.yml.j2
@@ -1,31 +1,19 @@
 #jinja2: lstrip_blocks: True
 {% for vitamui_service in vitamui_service_list %}
+{% set vitamui_component = vitamui[vitamui_service].vitamui_component %}
 #
-# VITAMUI SERVICE : "{{ vitamui_service }}"
+# VITAMUI SERVICE : "vitamui-{{ vitamui_component }}"
 #
-  {% if filebeat.services[vitamui_service].enable_access | default(true) | bool %}
+  {% if filebeat.services.vitamui[vitamui_service].enable_access | default(true) | bool %}
 - type: filestream
-  id: {{ vitam_site_name }}-{{ vitamui_service }}-access
-  enabled: {{ {'vitamui_iam_internal': filebeat.services.vitamui_iam_internal.enable_access,
-    'vitamui_ingest_internal': filebeat.services.vitamui_ingest_internal.enable_access,
-    'vitamui_referential_internal': filebeat.services.vitamui_referential_internal.enable_access,
-    'vitamui_archive_search_internal': filebeat.services.vitamui_archive_search_internal.enable_access,
-    'vitamui_security_internal': filebeat.services.vitamui_security_internal.enable_access,
-    'vitamui_collect_internal': filebeat.services.vitamui_collect_internal.enable_access,
-    'cas_server': filebeat.services.cas_server.enable_access,
-    'vitamui_iam_external': filebeat.services.vitamui_iam_external.enable_access,
-    'vitamui_ingest_external': filebeat.services.vitamui_ingest_external.enable_access,
-    'vitamui_referential_external': filebeat.services.vitamui_referential_external.enable_access,
-    'vitamui_archive_search_external': filebeat.services.vitamui_archive_search_external.enable_access,
-    'vitamui_collect_external': filebeat.services.vitamui_collect_external.enable_access,
-    'vitamui_pastis_external': filebeat.services.vitamui_pastis_external.enable_access,
-    'vitamui_api_gateway': filebeat.services.vitamui_api_gateway.enable_access} [vitamui_service] | default(true) | bool | lower }}
+  id: {{ vitam_site_name }}-{{ vitamui_component }}-access
+  enabled: true
   paths:
-    - "{{ vitamui_defaults.folder.root_path }}/log/{{ vitamui_service }}/accesslog-{{ vitamui_service }}.*.log"
+    - "{{ vitamui_defaults.folder.root_path }}/log/{{ vitamui_component }}/accesslog-{{ vitamui_component }}.*.log"
   fields_under_root: true
   fields:
     kind: access
-    vitam_component: vitamui-{{ vitamui_service }}
+    vitam_component: vitamui-{{ vitamui_component }}
     source: vitamui
     type: application
   processors:
@@ -34,31 +22,28 @@
         target_prefix: ""
     - drop_fields:
         fields: ["date_time"]
-  {% endif %}
+    {% if vitamui_service == 'api_gateway' and filebeat.disable_health_checks | default(true) | bool %}
+    # Drop health checks
+    - drop_event:
+        when:
+          and:
+            - equals:
+                http_status_code: 200
+            - equals:
+                request_path: "/actuator/health"
+    {% endif %}
 
-  {% if filebeat.services[vitamui_service].enable_management | default(true) | bool %}
+  {% endif %}
+  {% if filebeat.services.vitamui[vitamui_service].enable_management | default(true) | bool %}
 - type: filestream
-  id: {{ vitam_site_name }}-{{ vitamui_service }}-management
-  enabled: {{ {'vitamui_iam_internal': filebeat.services.vitamui_iam_internal.enable_management,
-    'vitamui_ingest_internal': filebeat.services.vitamui_ingest_internal.enable_management,
-    'vitamui_referential_internal': filebeat.services.vitamui_referential_internal.enable_management,
-    'vitamui_archive_search_internal': filebeat.services.vitamui_archive_search_internal.enable_management,
-    'vitamui_security_internal': filebeat.services.vitamui_security_internal.enable_management,
-    'vitamui_collect_internal': filebeat.services.vitamui_collect_internal.enable_management,
-    'cas_server': filebeat.services.cas_server.enable_management,
-    'vitamui_iam_external': filebeat.services.vitamui_iam_external.enable_management,
-    'vitamui_ingest_external': filebeat.services.vitamui_ingest_external.enable_management,
-    'vitamui_referential_external': filebeat.services.vitamui_referential_external.enable_management,
-    'vitamui_archive_search_external': filebeat.services.vitamui_archive_search_external.enable_management,
-    'vitamui_collect_external': filebeat.services.vitamui_collect_external.enable_management,
-    'vitamui_pastis_external': filebeat.services.vitamui_pastis_external.enable_management,
-    'vitamui_api_gateway': filebeat.services.vitamui_api_gateway.enable_management} [vitamui_service] | default(true) | bool | lower }}
+  id: {{ vitam_site_name }}-{{ vitamui_component }}-management
+  enabled: true
   paths:
-    - "{{ vitamui_defaults.folder.root_path }}/log/{{ vitamui_service }}/management_accesslog-{{ vitamui_service }}.*.log"
+    - "{{ vitamui_defaults.folder.root_path }}/log/{{ vitamui_component }}/management_accesslog-{{ vitamui_component }}.*.log"
   fields_under_root: true
   fields:
     kind: management
-    vitam_component: vitamui-{{ vitamui_service }}
+    vitam_component: vitamui-{{ vitamui_component }}
     source: vitamui
     type: application
   processors:
@@ -67,31 +52,28 @@
         target_prefix: ""
     - drop_fields:
         fields: ["date_time"]
-  {% endif %}
+    {% if filebeat.disable_health_checks | default(true) | bool %}
+    # Drop health checks
+    - drop_event:
+        when:
+          and:
+            - equals:
+                http_status_code: 200
+            - equals:
+                request_path: "/actuator/health"
+    {% endif %}
 
-  {% if filebeat.services[vitamui_service].enable_log | default(true) | bool %}
+  {% endif %}
+  {% if filebeat.services.vitamui[vitamui_service].enable_log | default(true) | bool %}
 - type: filestream
-  id: {{ vitam_site_name }}-{{ vitamui_service }}-log
-  enabled: {{ {'vitamui_iam_internal': filebeat.services.vitamui_iam_internal.enable_log,
-    'vitamui_ingest_internal': filebeat.services.vitamui_ingest_internal.enable_log,
-    'vitamui_referential_internal': filebeat.services.vitamui_referential_internal.enable_log,
-    'vitamui_archive_search_internal': filebeat.services.vitamui_archive_search_internal.enable_log,
-    'vitamui_security_internal': filebeat.services.vitamui_security_internal.enable_log,
-    'vitamui_collect_internal': filebeat.services.vitamui_collect_internal.enable_log,
-    'cas_server': filebeat.services.cas_server.enable_log,
-    'vitamui_iam_external': filebeat.services.vitamui_iam_external.enable_log,
-    'vitamui_ingest_external': filebeat.services.vitamui_ingest_external.enable_log,
-    'vitamui_referential_external': filebeat.services.vitamui_referential_external.enable_log,
-    'vitamui_archive_search_external': filebeat.services.vitamui_archive_search_external.enable_log,
-    'vitamui_collect_external': filebeat.services.vitamui_collect_external.enable_log,
-    'vitamui_pastis_external': filebeat.services.vitamui_pastis_external.enable_log,
-    'vitamui_api_gateway': filebeat.services.vitamui_api_gateway.enable_log} [vitamui_service] | default(true) | bool | lower }}
+  id: {{ vitam_site_name }}-{{ vitamui_component }}-log
+  enabled: true
   paths:
-    - "{{ vitamui_defaults.folder.root_path }}/log/{{ vitamui_service }}/{{ vitamui_service }}.*.log"
+    - "{{ vitamui_defaults.folder.root_path }}/log/{{ vitamui_component }}/{{ vitamui_component }}.*.log"
   fields_under_root: true
   fields:
     kind: log
-    vitam_component: vitamui-{{ vitamui_service }}
+    vitam_component: vitamui-{{ vitamui_component }}
     source: vitamui
     type: application
   processors:
@@ -107,5 +89,6 @@
         pattern: '^\d{4}-\d{2}-\d{2}\s+\d{2}:\d{2}:\d{2},\d{3}'
         negate: true
         match: after
+
   {% endif %}
 {% endfor %}
diff --git a/deployment/roles/filebeat/templates/modules/apache.yml.j2 b/deployment/roles/filebeat/templates/modules/apache.yml.j2
@@ -1,3 +1,4 @@
+#jinja2: lstrip_blocks: True
 # Module: apache
 # Docs: https://www.elastic.co/guide/en/beats/filebeat/7.17/filebeat-module-apache.html
 
@@ -22,6 +23,16 @@
             target_prefix: ""
         - drop_fields:
             fields: ["date_time"]
+{% if filebeat.disable_health_checks | default(true) | bool %}
+        # Drop Blackbox health checks
+        - drop_event:
+            when:
+              and:
+                - equals:
+                    http_status_code: 200
+                - regexp:
+                    user_agent: "^Blackbox.*"
+{% endif %}
 
   # Error logs
   error:

diff --git a/deployment/roles/filebeat/templates/modules/nginx.yml.j2 b/deployment/roles/filebeat/templates/modules/nginx.yml.j2
@@ -1,3 +1,4 @@
+#jinja2: lstrip_blocks: True
 # Module: nginx
 # Docs: https://www.elastic.co/guide/en/beats/filebeat/7.17/filebeat-module-nginx.html
 
@@ -22,6 +23,16 @@
             target_prefix: ""
         - drop_fields:
             fields: ["date_time"]
+{% if filebeat.disable_health_checks | default(true) | bool %}
+        # Drop Blackbox health checks
+        - drop_event:
+            when:
+              and:
+                - equals:
+                    http_status_code: 200
+                - regexp:
+                    user_agent: "^Blackbox.*"
+{% endif %}
 
   # Error logs
   error:

diff --git a/deployment/roles/vitamui/templates/logback.xml.j2 b/deployment/roles/vitamui/templates/logback.xml.j2
@@ -39,7 +39,7 @@
     <syslogHost>localhost</syslogHost>
     <facility>local0</facility>
     <port>514</port>
-    <suffixPattern>vitam-vitamui-{{ vitamui_struct.vitamui_component }}: %d{ISO8601} [[%thread]] [%X{X-Request-Id}] %-5level %logger - %replace(%caller{1..2}){'Caller\+1     at |\n',''} : %msg %rootException%n</suffixPattern>
+    <suffixPattern>vitamui-{{ vitamui_struct.vitamui_component }}: %d{ISO8601} [[%thread]] [%X{X-Request-Id}] %-5level %logger - %replace(%caller{1..2}){'Caller\+1     at |\n',''} : %msg %rootException%n</suffixPattern>
   </appender>
 
   <root level="{{ vitamui_struct.log.root_log_level | default(log.root_log_level) }}">