chore: Ensure we can tollerate to opensearch issues (#108)

Signed-off-by: Sergiy Kulanov <sergiy_kulanov@epam.com>
epam · Jul 11, 2024 · 63640c9 · 63640c9
1 parent 5eb1141
commit 63640c9
Show file tree

Hide file tree

Showing 3 changed files with 24 additions and 2 deletions.
diff --git a/README.md b/README.md
@@ -149,6 +149,7 @@ make update-readme
 | defectdojo             | 1.6.127   | 2.34.1       | False             | False    |
 | dependency-track       | 0.9.2     | v4.11.3      | False             | False    |
 | edp                    | 3.9.0     | 3.9.0        | False             | False    |
+| eks                    | 1.0.0     | v1.0.0       | False             | False    |
 | extensions-oidc        | 1.21.0    | 1.21.0       | False             | False    |
 | external-secrets       | 0.9.9     | 1.0          | False             | False    |
 | fluent-bit             | 0.46.11   | 3.0.7        | False             | False    |

diff --git a/add-ons/fluent-bit/README.md b/add-ons/fluent-bit/README.md
@@ -94,8 +94,9 @@ if (startTimeMillis > 0 && completionTimeMillis > 0) {
 | eso.secretName | string | `"/edp/eks/addons/fluent-bit"` | Value name in AWS ParameterStore, AWS SecretsManager or other Secret Store. |
 | eso.secretStoreName | string | `"aws-parameterstore"` | Defines Secret Store name. |
 | eso.type | string | `"aws"` | Defines provider type. One of `aws` or `generic`. |
-| fluent-bit.config.inputs | string | `"[INPUT]\n    Name kubernetes_events\n    # add the tag \"k8s_events\" to all events coming from this input\n    tag k8s_events\n    # ask k8s API for updates every 30 seconds\n    interval_sec 15\n    # fetch at most 250 items per requests (pagination)\n    kube_request_limit 2500\n\n# Use as the synk for the CloudEvents from Tekton Pipelines\n[INPUT]\n    name http\n    listen 0.0.0.0\n    port 8888\n"` |  |
-| fluent-bit.config.outputs | string | `"[OUTPUT]\n    Name            es\n    Match           k8s_events\n    Host            opensearch-cluster-master\n    Port            9200\n    HTTP_User       ${ES_SUPERUSER_USER}\n    HTTP_Passwd     ${ES_SUPERUSER_PASSWORD}\n    Logstash_Format On\n    Logstash_Prefix logstash-events\n    Time_Key        @timestamp\n    Replace_Dots    On\n    Retry_Limit     False\n    Trace_Error     Off\n    Suppress_Type_Name On\n    tls             On\n    tls.verify      Off\n\n[OUTPUT]\n    Name            es\n    Match           http.*\n    Host            opensearch-cluster-master\n    Port            9200\n    HTTP_User       ${ES_SUPERUSER_USER}\n    HTTP_Passwd     ${ES_SUPERUSER_PASSWORD}\n    Logstash_Format On\n    Logstash_Prefix logstash-cloudevents\n    Time_Key        @timestamp\n    Replace_Dots    On\n    Retry_Limit     False\n    Trace_Error     Off\n    Suppress_Type_Name On\n    tls             On\n    tls.verify      Off\n"` |  |
+| fluent-bit.config.inputs | string | `"[INPUT]\n    Name kubernetes_events\n    # add the tag \"k8s_events\" to all events coming from this input\n    tag k8s_events\n    # ask k8s API for updates every 30 seconds\n    interval_sec 15\n    # fetch at most 250 items per requests (pagination)\n    kube_request_limit 2500\n    Storage.type filesystem\n\n# Use as the synk for the CloudEvents from Tekton Pipelines\n[INPUT]\n    name http\n    listen 0.0.0.0\n    port 8888\n    Storage.type filesystem\n"` |  |
+| fluent-bit.config.outputs | string | `"[OUTPUT]\n    Name            es\n    Match           k8s_events\n    Host            opensearch-cluster-master\n    Port            9200\n    HTTP_User       ${ES_SUPERUSER_USER}\n    HTTP_Passwd     ${ES_SUPERUSER_PASSWORD}\n    Logstash_Format On\n    Logstash_Prefix logstash-events\n    Time_Key        @timestamp\n    Replace_Dots    On\n    Retry_Limit     False\n    Trace_Error     Off\n    Suppress_Type_Name On\n    tls             On\n    tls.verify      Off\n    Storage.total_limit_size 1G\n\n[OUTPUT]\n    Name            es\n    Match           http.*\n    Host            opensearch-cluster-master\n    Port            9200\n    HTTP_User       ${ES_SUPERUSER_USER}\n    HTTP_Passwd     ${ES_SUPERUSER_PASSWORD}\n    Logstash_Format On\n    Logstash_Prefix logstash-cloudevents\n    Time_Key        @timestamp\n    Replace_Dots    On\n    Retry_Limit     False\n    Trace_Error     Off\n    Suppress_Type_Name On\n    tls             On\n    tls.verify      Off\n    Storage.total_limit_size 1G\n"` |  |
+| fluent-bit.config.service | string | `"[SERVICE]\n    Daemon Off\n    Flush {{ .Values.flush }}\n    Log_Level {{ .Values.logLevel }}\n    Parsers_File /fluent-bit/etc/parsers.conf\n    Parsers_File /fluent-bit/etc/conf/custom_parsers.conf\n    HTTP_Server On\n    HTTP_Listen 0.0.0.0\n    HTTP_Port {{ .Values.metricsPort }}\n    Health_Check On\n    Storage.path /var/log/flb-storage/\n    Storage.sync normal\n    Storage.checksum off\n    Storage.backlog.mem_limit 5M\n"` |  |
 | fluent-bit.env[0].name | string | `"ES_SUPERUSER_USER"` |  |
 | fluent-bit.env[0].valueFrom.secretKeyRef.key | string | `"username"` |  |
 | fluent-bit.env[0].valueFrom.secretKeyRef.name | string | `"fluentbit-creds"` |  |

diff --git a/add-ons/fluent-bit/values.yaml b/add-ons/fluent-bit/values.yaml
@@ -79,6 +79,22 @@ fluent-bit:
       memory: 128Mi
 
   config:
+    service: |
+      [SERVICE]
+          Daemon Off
+          Flush {{ .Values.flush }}
+          Log_Level {{ .Values.logLevel }}
+          Parsers_File /fluent-bit/etc/parsers.conf
+          Parsers_File /fluent-bit/etc/conf/custom_parsers.conf
+          HTTP_Server On
+          HTTP_Listen 0.0.0.0
+          HTTP_Port {{ .Values.metricsPort }}
+          Health_Check On
+          Storage.path /var/log/flb-storage/
+          Storage.sync normal
+          Storage.checksum off
+          Storage.backlog.mem_limit 5M
+
     inputs: |
       [INPUT]
           Name kubernetes_events
@@ -88,12 +104,14 @@ fluent-bit:
           interval_sec 15
           # fetch at most 250 items per requests (pagination)
           kube_request_limit 2500
+          Storage.type filesystem
 
       # Use as the synk for the CloudEvents from Tekton Pipelines
       [INPUT]
           name http
           listen 0.0.0.0
           port 8888
+          Storage.type filesystem
 
     outputs: |
       [OUTPUT]
@@ -112,6 +130,7 @@ fluent-bit:
           Suppress_Type_Name On
           tls             On
           tls.verify      Off
+          Storage.total_limit_size 1G
 
       [OUTPUT]
           Name            es
@@ -129,6 +148,7 @@ fluent-bit:
           Suppress_Type_Name On
           tls             On
           tls.verify      Off
+          Storage.total_limit_size 1G
 
 # Configure components of the External Secrets Operator (ESO).
 eso: