Merge pull request #7235 from sapcc/opensearch-logs

Opensearch-logs bump OpenSearch to 2.17.1, fix datastream

system/opensearch-logs/Chart.lock

@@ -1,19 +1,19 @@
 dependencies:
 - name: opensearch
   repository: https://opensearch-project.github.io/helm-charts
-  version: 2.22.0
+  version: 2.26.0
 - name: opensearch
   repository: https://opensearch-project.github.io/helm-charts
-  version: 2.22.0
+  version: 2.26.0
 - name: opensearch
   repository: https://opensearch-project.github.io/helm-charts
-  version: 2.22.0
+  version: 2.26.0
 - name: opensearch
   repository: https://opensearch-project.github.io/helm-charts
-  version: 2.22.0
+  version: 2.26.0
 - name: opensearch-dashboards
   repository: https://opensearch-project.github.io/helm-charts
-  version: 2.20.0
+  version: 2.24.0
 - name: owner-info
   repository: oci://keppel.eu-de-1.cloud.sap/ccloud-helm
   version: 0.2.0
@@ -26,5 +26,5 @@ dependencies:
 - name: linkerd-support
   repository: oci://keppel.eu-de-1.cloud.sap/ccloud-helm
   version: 0.1.3
-digest: sha256:ae8256667e36fe195fd28f4b2c32f4f22b11007e71cfc0371403dac06127ee9c
-generated: "2024-08-09T08:48:05.704483+02:00"
+digest: sha256:2e921d396d14202ca910e7b7e0107d146ba5d5702f45709f2f05b819fc756960
+generated: "2024-10-14T09:49:12.435706+02:00"

system/opensearch-logs/Chart.yaml

@@ -1,34 +1,34 @@
 apiVersion: v2
 description: A Helm chart for the Opensearch stack
 name: opensearch-logs
-version: 0.0.26
+version: 0.0.29
 home: https://github.com/sapcc/helm-charts/tree/master/system/opensearch-logs
 dependencies:
   - name: opensearch
     alias: opensearch_master
     condition: opensearch_master.enabled
     repository: https://opensearch-project.github.io/helm-charts
-    version: 2.22.0
+    version: 2.26.0
   - name: opensearch
     alias: opensearch_client
     condition: opensearch_client.enabled
     repository: https://opensearch-project.github.io/helm-charts
-    version: 2.22.0
+    version: 2.26.0
   - name: opensearch
     alias: opensearch_data
     condition: opensearch_data.enabled
     repository: https://opensearch-project.github.io/helm-charts
-    version: 2.22.0
+    version: 2.26.0
   - name: opensearch
     alias: opensearch_ml
     condition: opensearch_ml.enabled
     repository: https://opensearch-project.github.io/helm-charts
-    version: 2.22.0
+    version: 2.26.0
   - name: opensearch-dashboards
     alias: opensearch_dashboards
     condition: opensearch_dashboards.enabled
     repository: https://opensearch-project.github.io/helm-charts
-    version: 2.20.0
+    version: 2.24.0
   - name: owner-info
     repository: oci://keppel.eu-de-1.cloud.sap/ccloud-helm
     version: 0.2.0

system/opensearch-logs/aggregations/retention.rules

@@ -39,3 +39,13 @@ groups:
     expr: count(opensearch_index_doc_number{index=~"storage.+", pod="opensearch-logs-master-0", context="primaries"})
   - record:  opensearch_retention_compute
     expr: count(opensearch_index_doc_number{index=~"compute.+", pod="opensearch-logs-master-0", context="primaries"})
+  - record:  opensearch_retention_dsotel
+    expr: count(opensearch_index_doc_number{index=~".ds-otel-datastream.+", pod="opensearch-logs-master-0", context="primaries"})
+  - record:  opensearch_retention_dsfilelog
+    expr: count(opensearch_index_doc_number{index=~".ds-otel-filelog-datastream.+", pod="opensearch-logs-master-0", context="primaries"})
+  - record:  opensearch_retention_dsotellogs
+    expr: count(opensearch_index_doc_number{index=~".ds-otel-otellogs-datastream.+", pod="opensearch-logs-master-0", context="primaries"})
+  - record:  opensearch_retention_dscompute
+    expr: count(opensearch_index_doc_number{index=~".ds-compute-datastream.+", pod="opensearch-logs-master-0", context="primaries"})
+  - record:  opensearch_retention_dsstorage
+    expr: count(opensearch_index_doc_number{index=~".ds-storage-datastream.+", pod="opensearch-logs-master-0", context="primaries"})

system/opensearch-logs/templates/config/_ds-ism.json.tpl

@@ -60,7 +60,7 @@
         "ism_template":
             {
                 "index_patterns": [
-                    "otel-_DS_NAME_-datastream"
+                    "_DS_NAME_-datastream"
                 ],
                 "priority": 1
             }

system/opensearch-logs/templates/config/_ds.json.tpl

@@ -6,7 +6,7 @@
     "settings": {
       "index.number_of_shards": "4",
       "index.number_of_replicas": "1",
-      "index.refresh_interval": "15s"
+      "index.refresh_interval": "60s"
     },
     "aliases": {
       "_DS_NAME_-ds": {}

system/opensearch-logs/templates/config/_install-ds-templates.sh.tpl

@@ -55,13 +55,16 @@ if [ "${DATA_STREAM_ENABLED}" = true ]; then
         # 2. create datastream, if missing. It's done here, because the normal template has no versioning and can be overwritten during each run.
         #    data_stream example: otellogs-datastream
 
-        export EXISTING_DS=$(curl -s -u "${ADMIN_USER}:${ADMIN_PASSWORD}" -XGET "${CLUSTER_HOST}/_data_stream"|jq .data_streams[].name)
-        echo $EXISTING_DS | grep -q "$e-datastream"
+        export EXISTING_DS=`curl -s -u "${ADMIN_USER}:${ADMIN_PASSWORD}" -XGET "${CLUSTER_HOST}/_data_stream"|jq .data_streams[].name|awk ' {gsub("-datastream","") } 1'|tr  -d \"`
+        echo "\nExisting datastreams: $EXISTING_DS\n"
+        echo "$EXISTING_DS" | grep -q "^$e$"
         if [ $? -eq 1 ]; then
-           echo "Creating datastream $e"
+           echo "\nCreating datastream $e\n"
            curl -u "${ADMIN_USER}:${ADMIN_PASSWORD}" -XPUT "${CLUSTER_HOST}/_data_stream/${e}-datastream"
            # Assign ISM policy to newly created datastream
            curl --header 'content-type: application/JSON' --silent -u "${ADMIN_USER}:${ADMIN_PASSWORD}" -XPOST "${CLUSTER_HOST}/_plugins/_ism/add/.ds-${e}-datastream-000001" -d "{ \"policy_id\": \"ds-${e}-ism\" }"
+        else
+           echo "No datastream creation return code was not 1 for datastream $e"
         fi
      else
        # update of existing policy

system/opensearch-logs/templates/metrics-podmonitor-opensearch-master.yaml

@@ -74,7 +74,7 @@ spec:
           regex: '(opensearch_indices_get_.+|opensearch_indices_indexing_delete_.+|opensearch_index_indexing_index_time_seconds|opensearch_index_indexing_is_throttled_bool|opensearch_index_indexing_noop_update_count|opensearch_index_indexing_throttle_time_seconds|opensearch_indices_merges_.+|opensearch_indices_percolate_.+|opensearch_indices_recovery_.+|opensearch_indices_refresh.+|opensearch_indices_search_.+|opensearch_indices_suggest_.+|opensearch_index_search_fetch_count|opensearch_index_search_open_contexts_number|opensearch_index_search_fetch_time_seconds|opensearch_index_search_query.+|opensearch_index_search_scroll.+|opensearch_indices_segments_.+|opensearch_indices_suggest_.+|opensearch_ingest_pipeline_.+|opensearch_indices_querycache_.+|opensearch_indices_requestcache_.+|opensearch_indices_search_.+|opensearch_indices_f.+|opensearch_threadpool_threads_count|opensearch_threadpool_threads_number|opensearch_jvm_threads_peak_number)'
           sourceLabels: [__name__]
         - sourceLabels: [__name__, index]
-          regex: '(opensearch_index_doc_number|opensearch_index_store_size_bytes|opensearch_index_search_fetch_current_number);(logstash-swift|logstash|virtual|scaleout|jump|admin|kubernikus|qade2-logstash|qade3-logstash|alerts|alerts-other|maillog|maillog-archive|deployments|storage|compute|greenhouse)(.+)'
+          regex: '(opensearch_index_doc_number|opensearch_index_store_size_bytes|opensearch_index_search_fetch_current_number);(logstash-swift|logstash|virtual|scaleout|jump|admin|kubernikus|qade2-logstash|qade3-logstash|alerts|alerts-other|maillog|maillog-archive|deployments|storage|compute|greenhouse|.ds-otelstorage-datastream|.ds-otel-filelog-datastream|.ds-compute-datastream|.ds-otel-datastream)(.+)'
           replacement: '$2'
           targetLabel: index_docs
 {{- end }}

system/opensearch-logs/values.yaml

@@ -37,7 +37,7 @@ opensearch_master:
   nodeGroup: "master"
   masterService: "opensearch-logs-master"
   image:
-    tag: 2.16.0
+    tag: 2.17.1
   roles:
     - master
   replicas: 3
@@ -64,12 +64,12 @@ opensearch_master:
   plugins:
     enabled: true
     installList:
-      - https://github.com/Aiven-Open/prometheus-exporter-plugin-for-opensearch/releases/download/2.16.0.0/prometheus-exporter-2.16.0.0.zip
+      - https://github.com/Aiven-Open/prometheus-exporter-plugin-for-opensearch/releases/download/2.17.1.0/prometheus-exporter-2.17.1.0.zip
 
 opensearch_client:
   enabled: false
   image:
-    tag: 2.16.0
+    tag: 2.17.1
   replicas: 3
   nameOverride: "opensearch-logs-client"
   fullnameOverride: "opensearch-logs-client"
@@ -110,12 +110,12 @@ opensearch_client:
   plugins:
     enabled: true
     installList:
-      - https://github.com/Aiven-Open/prometheus-exporter-plugin-for-opensearch/releases/download/2.16.0.0/prometheus-exporter-2.16.0.0.zip
+      - https://github.com/Aiven-Open/prometheus-exporter-plugin-for-opensearch/releases/download/2.17.1.0/prometheus-exporter-2.17.1.0.zip
 
 opensearch_data:
   enabled: false
   image:
-    tag: 2.16.0
+    tag: 2.17.1
   nameOverride: "opensearch-logs-data"
   fullnameOverride: "opensearch-logs-data"
   nodeGroup: "data"
@@ -139,7 +139,7 @@ opensearch_data:
   plugins:
     enabled: true
     installList:
-      - https://github.com/Aiven-Open/prometheus-exporter-plugin-for-opensearch/releases/download/2.16.0.0/prometheus-exporter-2.16.0.0.zip
+      - https://github.com/Aiven-Open/prometheus-exporter-plugin-for-opensearch/releases/download/2.17.1.0/prometheus-exporter-2.17.1.0.zip
   securityConfig:
     enabled: false
     path: "/usr/share/opensearch/config/opensearch-security"
@@ -152,7 +152,7 @@ opensearch_data:
 opensearch_ml:
   enabled: false
   image:
-    tag: 2.16.0
+    tag: 2.17.1
   replicas: 2
   nameOverride: "opensearch-logs-ml"
   fullnameOverride: "opensearch-logs-ml"
@@ -193,12 +193,12 @@ opensearch_ml:
   plugins:
     enabled: true
     installList:
-      - https://github.com/Aiven-Open/prometheus-exporter-plugin-for-opensearch/releases/download/2.16.0.0/prometheus-exporter-2.16.0.0.zip
+      - https://github.com/Aiven-Open/prometheus-exporter-plugin-for-opensearch/releases/download/2.17.1.0/prometheus-exporter-2.17.1.0.zip
 
 opensearch_dashboards:
   enabled: false
   image:
-    tag: "2.16.0"
+    tag: "2.17.1"
   fullnameOverride: opensearch-logs-dashboards
   nameOverride: opensearch-logs-dashboards
   serviceAccount: