idaholab#471, allow setting spiDataMaxIndices for Arkime's config.ini

arkime/scripts/docker_entrypoint.sh

@@ -15,6 +15,7 @@ ARKIME_PASSWORD_SECRET=${ARKIME_PASSWORD_SECRET:-"Malcolm"}
 ARKIME_FREESPACEG=${ARKIME_FREESPACEG:-"10%"}
 ARKIME_ROTATE_INDEX=${ARKIME_ROTATE_INDEX:-"daily"}
 ARKIME_QUERY_ALL_INDICES=${ARKIME_QUERY_ALL_INDICES:-"false"}
+ARKIME_SPI_DATA_MAX_INDICES=${ARKIME_SPI_DATA_MAX_INDICES:-7}
 MALCOLM_NETWORK_INDEX_PATTERN=${MALCOLM_NETWORK_INDEX_PATTERN:-}
 ARKIME_DEBUG_LEVEL=${ARKIME_DEBUG_LEVEL:-0}
 CAPTURE_INTERFACE=${PCAP_IFACE:-}
@@ -64,6 +65,7 @@ if [[ ! -f "${ARKIME_CONFIG_FILE}" ]] && [[ -r "${ARKIME_DIR}"/etc/config.orig.i
     sed -i "s/^\(freeSpaceG=\).*/\1"${ARKIME_FREESPACEG}"/" "${ARKIME_CONFIG_FILE}"
     sed -i "s/^\(rotateIndex=\).*/\1"${ARKIME_ROTATE_INDEX}"/" "${ARKIME_CONFIG_FILE}"
     sed -i "s/^\(queryAllIndices=\).*/\1"${ARKIME_QUERY_ALL_INDICES}"/" "${ARKIME_CONFIG_FILE}"
+    sed -i "s/^\(spiDataMaxIndices=\).*/\1"${ARKIME_SPI_DATA_MAX_INDICES}"/" "${ARKIME_SPI_DATA_MAX_INDICES}"
     sed -i "s/^\(queryExtraIndices=\).*/\1"${MALCOLM_NETWORK_INDEX_PATTERN}"/" "${MALCOLM_NETWORK_INDEX_PATTERN}" "${ARKIME_CONFIG_FILE}"
     sed -i "s/^\(debug=\).*/\1"${ARKIME_DEBUG_LEVEL}"/" "${ARKIME_CONFIG_FILE}"
     sed -i "s/^\(viewPort=\).*/\1"${VIEWER_PORT}"/" "${ARKIME_CONFIG_FILE}"

config/arkime.env.example

@@ -6,6 +6,10 @@ ARKIME_FREESPACEG=10%
 # How often to create a new index in OpenSearch/Elasticsearch
 #   https://arkime.com/settings#rotateIndex
 ARKIME_ROTATE_INDEX=daily
+# Specify the max number of indices for querying SPI data, or set to -1 to disable any max.
+#   The Arkime documentation warns "OpenSearch/Elasticsearch MAY blow up if we ... search too many indices."
+#   https://arkime.com/settings#spiDataMaxIndices
+ARKIME_SPI_DATA_MAX_INDICES=7
 # Always query all indices instead of trying to calculate which ones
 #   https://arkime.com/settings#queryAllIndices
 ARKIME_QUERY_ALL_INDICES=false

docs/malcolm-config.md

@@ -13,6 +13,7 @@ Although the configuration script automates many of the following configuration
     - `ARKIME_PASSWORD_SECRET` - the password hash secret for the Arkime viewer cluster (see `passwordSecret` in [Arkime INI Settings](https://arkime.com/settings)) used to secure the connection used when Arkime viewer retrieves a PCAP payload for display in its user interface
     - `ARKIME_ROTATE_INDEX` - how often (based on network traffic timestamp) to [create a new index](https://arkime.com/settings#rotateIndex) in OpenSearch
     - `ARKIME_QUERY_ALL_INDICES` - whether or not Arkime should [query all indices](https://arkime.com/settings#queryAllIndices) instead of trying to calculate which ones pertain to the search time frame (default `false`)
+    - `ARKIME_SPI_DATA_MAX_INDICES` - the maximum number of indices for querying SPI data, or set to `-1` to disable any max. The [Arkime documentation](https://arkime.com/settings#spiDataMaxIndices) warns "OpenSearch/Elasticsearch MAY blow up if we ... search too many indices." (default `7`)
     - `MANAGE_PCAP_FILES` and `ARKIME_FREESPACEG` - these variables deal with PCAP [deletion by Arkime](https://arkime.com/faq#pcap-deletion), see [**Managing disk usage**](#DiskUsage) below
     - `MAXMIND_GEOIP_DB_LICENSE_KEY` - Malcolm uses MaxMind's free GeoLite2 databases for GeoIP lookups. As of December 30, 2019, these databases are [no longer available](https://blog.maxmind.com/2019/12/18/significant-changes-to-accessing-and-using-geolite2-databases/) for download via a public URL. Instead, they must be downloaded using a MaxMind license key (available without charge [from MaxMind](https://www.maxmind.com/en/geolite2/signup)). The license key can be specified here for GeoIP database downloads during build- and run-time.
     - The following variables configure [Arkime's use](index-management.md#ArkimeIndexPolicies) of OpenSearch [Index State Management (ISM)](https://opensearch.org/docs/latest/im-plugin/ism/index/) or Elasticsearch [Index Lifecycle Management (ILM)](https://www.elastic.co/guide/en/elasticsearch/reference/current/index-lifecycle-management.html):