fix idaholab#596, anomaly detection default detectors are not being c…

…reated

two issues were present:

1. opensearch_status.sh -w was no longer behaving as previously, as now an empty index is being created that doesn't have any events in it (see idaholab#527 and idaholab#567). It's been adjusted so that now it waits for an index with docs.count > 0.
2. The shared_object_creation.sh script needed to create the dummy detector if the .opendistro-anomaly-detection-state doesn't exist, so this check has been put in place

Dockerfiles/api.Dockerfile

@@ -68,7 +68,7 @@ COPY --from=ghcr.io/mmguero-dev/gostatic --chmod=755 /goStatic /usr/bin/goStatic
 
 RUN    apt-get -q update \
     && apt-get -y -q --no-install-recommends upgrade \
-    && apt-get -y -q --no-install-recommends install curl netcat-openbsd rsync tini \
+    && apt-get -y -q --no-install-recommends install curl jq netcat-openbsd rsync tini \
     && python3 -m pip install --upgrade pip \
     && python3 -m pip install --no-cache /wheels/* \
     && groupadd --gid ${DEFAULT_GID} ${PGROUP} \

Dockerfiles/dashboards.Dockerfile

@@ -36,7 +36,7 @@ ADD https://github.com/lguillaud/osd_transform_vis/releases/download/$OSD_TRANSF
 
 RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/') && \
     yum upgrade -y && \
-    yum install -y curl-minimal psmisc findutils util-linux openssl rsync python3 zip unzip && \
+    yum install -y curl-minimal psmisc findutils util-linux jq openssl rsync python3 zip unzip && \
     yum remove -y vim-* && \
     usermod -a -G tty ${PUSER} && \
     # Malcolm manages authentication and encryption via NGINX reverse proxy

Dockerfiles/dirinit.Dockerfile

@@ -25,7 +25,7 @@ COPY --chmod=755 shared/bin/docker-uid-gid-setup.sh /usr/local/bin/
 
 RUN apk update --no-cache && \
     apk upgrade --no-cache && \
-    apk --no-cache add bash psmisc rsync shadow tini && \
+    apk --no-cache add bash jq psmisc rsync shadow tini && \
     addgroup -g ${DEFAULT_GID} ${PGROUP} ; \
       adduser -D -H -u ${DEFAULT_UID} -h /nonexistant -s /sbin/nologin -G ${PGROUP} -g ${PUSER} ${PUSER} ; \
       addgroup ${PUSER} tty ; \

Dockerfiles/file-upload.Dockerfile

@@ -74,6 +74,7 @@ RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/')
       ca-certificates \
       curl \
       file \
+      jq \
       less \
       nginx-light \
       openssh-server \

Dockerfiles/freq.Dockerfile

@@ -34,6 +34,7 @@ RUN apt-get -q update && \
     apt-get -y -q --no-install-recommends upgrade && \
     apt-get -y --no-install-recommends install \
       curl \
+      jq \
       procps \
       psmisc \
       python3 \

Dockerfiles/htadmin.Dockerfile

@@ -39,6 +39,7 @@ RUN apt-get -q update && \
     apt-get -y -q --allow-downgrades --allow-remove-essential --allow-change-held-packages --no-install-recommends install \
       ca-certificates \
       curl \
+      jq \
       libmcrypt-dev \
       libmcrypt4 \
       make \

Dockerfiles/logstash.Dockerfile

@@ -49,6 +49,7 @@ RUN set -x && \
         curl \
         gettext \
         git \
+        jq \
         patch \
         python3-setuptools \
         python3-pip \

Dockerfiles/opensearch.Dockerfile

@@ -44,7 +44,7 @@ USER root
 # Remove the performance-analyzer plugin - Reduce resources in docker image
 RUN export BINARCH=$(uname -m | sed 's/x86_64/amd64/' | sed 's/aarch64/arm64/') && \
   yum upgrade -y && \
-  yum install -y openssl util-linux procps rsync findutils && \
+  yum install -y openssl util-linux procps jq rsync findutils && \
   yum remove -y vim-* && \
   /usr/share/opensearch/bin/opensearch-plugin remove opensearch-security --purge && \
   /usr/share/opensearch/bin/opensearch-plugin remove opensearch-performance-analyzer --purge && \

Dockerfiles/pcap-capture.Dockerfile

@@ -67,6 +67,7 @@ RUN apt-get -q update && \
     apt-get install --no-install-recommends -y -q \
       bc \
       ethtool \
+      jq \
       libcap2-bin \
       netsniff-ng \
       openssl \

Dockerfiles/pcap-monitor.Dockerfile

@@ -48,6 +48,7 @@ RUN apt-get -q update && \
     apt-get install --no-install-recommends -y -q \
       file \
       inotify-tools \
+      jq \
       libzmq5 \
       procps \
       psmisc \

Dockerfiles/postgresql.Dockerfile

@@ -28,7 +28,7 @@ COPY --from=ghcr.io/mmguero-dev/gostatic --chmod=755 /goStatic /usr/bin/goStatic
 
 RUN apk update --no-cache && \
     apk upgrade --no-cache && \
-    apk add --no-cache bash procps psmisc rsync shadow tini && \
+    apk add --no-cache bash jq procps psmisc rsync shadow tini && \
     apk add --no-cache --virtual .build-deps rsync && \
     rsync -a /usr/local/bin/ /usr/bin/ && \
     rsync -a /usr/local/share/ /usr/share/ && \

Dockerfiles/redis.Dockerfile

@@ -27,7 +27,7 @@ COPY --from=ghcr.io/mmguero-dev/gostatic --chmod=755 /goStatic /usr/bin/goStatic
 
 RUN apk update --no-cache && \
     apk upgrade --no-cache && \
-    apk --no-cache add bash psmisc rsync shadow tini && \
+    apk --no-cache add bash jq psmisc rsync shadow tini && \
     addgroup ${PUSER} tty
 
 WORKDIR /home/${PUSER}

shared/bin/opensearch_status.sh

@@ -102,21 +102,24 @@ if (( $WAIT_FOR_LOG_DATA == 1 )); then
 
   echo "Waiting until $OPENSEARCH_PRIMARY has logs..." >&2
 
-  # wait until at least one network traffic log index exists
+  # wait until at least one network traffic log index exists (get index count where docs.count > 0)
   FOUND_INDEX=
   while true; do
-    if (( $(curl "${CURL_CONFIG_PARAMS[@]}" -fs -H'Content-Type: application/json' -XGET "$OPENSEARCH_URL/_cat/indices/$MALCOLM_NETWORK_INDEX_PATTERN" 2>/dev/null | wc -l) > 0 )); then
-      FOUND_INDEX="$MALCOLM_NETWORK_INDEX_PATTERN"
-    elif [[ "$MALCOLM_NETWORK_INDEX_PATTERN" != "$ARKIME_NETWORK_INDEX_PATTERN" ]] && (( $(curl "${CURL_CONFIG_PARAMS[@]}" -fs -H'Content-Type: application/json' -XGET "$OPENSEARCH_URL/_cat/indices/$ARKIME_NETWORK_INDEX_PATTERN" 2>/dev/null | wc -l) > 0 )); then
-      FOUND_INDEX="$ARKIME_NETWORK_INDEX_PATTERN"
+    # use jq if it's available to parse the machine-readable index list as JSON, fall back to awk if it's not
+    if command -v jq >/dev/null 2>&1; then
+      if (( $(curl "${CURL_CONFIG_PARAMS[@]}" -fs -H'Content-Type: application/json' -XGET "$OPENSEARCH_URL/_cat/indices/$MALCOLM_NETWORK_INDEX_PATTERN?format=json" 2>/dev/null | jq '[.[] | select(.["docs.count"] != "0")] | length' 2>/dev/null) > 0 )); then
+        FOUND_INDEX="$MALCOLM_NETWORK_INDEX_PATTERN"
+      elif [[ "$MALCOLM_NETWORK_INDEX_PATTERN" != "$ARKIME_NETWORK_INDEX_PATTERN" ]] && (( $(curl "${CURL_CONFIG_PARAMS[@]}" -fs -H'Content-Type: application/json' -XGET "$OPENSEARCH_URL/_cat/indices/$ARKIME_NETWORK_INDEX_PATTERN?format=json" 2>/dev/null | jq '[.[] | select(.["docs.count"] != "0")] | length' 2>/dev/null) > 0 )); then
+        FOUND_INDEX="$ARKIME_NETWORK_INDEX_PATTERN"
+      fi
+    else
+      if (( $(curl "${CURL_CONFIG_PARAMS[@]}" -fs -H'Content-Type: application/json' -XGET "$OPENSEARCH_URL/_cat/indices/$MALCOLM_NETWORK_INDEX_PATTERN" 2>/dev/null | awk '$7 != "0"' | wc -l) > 0 )); then
+        FOUND_INDEX="$MALCOLM_NETWORK_INDEX_PATTERN"
+      elif [[ "$MALCOLM_NETWORK_INDEX_PATTERN" != "$ARKIME_NETWORK_INDEX_PATTERN" ]] && (( $(curl "${CURL_CONFIG_PARAMS[@]}" -fs -H'Content-Type: application/json' -XGET "$OPENSEARCH_URL/_cat/indices/$ARKIME_NETWORK_INDEX_PATTERN" 2>/dev/null | awk '$7 != "0"' | wc -l) > 0 )); then
+        FOUND_INDEX="$ARKIME_NETWORK_INDEX_PATTERN"
+      fi
     fi
     [[ -n "$FOUND_INDEX" ]] && break || sleep 5
   done
-  echo "Log indices exist." >&2
-
-  # wait until at least one record with @timestamp exists
-  until curl "${CURL_CONFIG_PARAMS[@]}" -fs -H'Content-Type: application/json' -XPOST "$OPENSEARCH_URL/$FOUND_INDEX/_search" -d'{ "sort": { "@timestamp" : "desc" }, "size" : 1 }' >/dev/null 2>&1 ; do
-    sleep 5
-  done
   echo "Logs exist." >&2
 fi