idaholab#585, fall back to alternative Zeek .deb download URL

.github/workflows/hedgehog-iso-build-docker-wrap-push-ghcr.yml

@@ -129,6 +129,7 @@ jobs:
           echo "${{ steps.extract_malcolm_version.outputs.mversion }}" > ./shared/version.txt
           echo "${{ secrets.MAXMIND_GEOIP_DB_LICENSE_KEY }}" > ./shared/maxmind_license.txt
           echo "${{ secrets.MAXMIND_GEOIP_DB_ALTERNATE_DOWNLOAD_URL }}" > ./shared/maxmind_url.txt
+          echo "${{ secrets.ZEEK_DEB_ALTERNATE_DOWNLOAD_URL }}" > ./shared/zeek_url.txt
           echo "GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}" > ./shared/environment.chroot
           echo "VCS_REVSION=${{ steps.extract_commit_sha.outputs.sha }}" > ./shared/environment.chroot
           echo "BUILD_JOBS=2" > ./shared/environment.chroot

.github/workflows/hedgehog-raspi-build-docker-wrap-push-ghcr.yml

@@ -86,6 +86,7 @@ jobs:
           echo "${{ steps.extract_malcolm_version.outputs.mversion }}" > ./shared/version.txt
           echo "${{ secrets.MAXMIND_GEOIP_DB_LICENSE_KEY }}" > ./shared/maxmind_license.txt
           echo "${{ secrets.MAXMIND_GEOIP_DB_ALTERNATE_DOWNLOAD_URL }}" > ./shared/maxmind_url.txt
+          echo "${{ secrets.ZEEK_DEB_ALTERNATE_DOWNLOAD_URL }}" > ./shared/zeek_url.txt
           echo "GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}" > ./shared/environment.chroot
           echo "VCS_REVSION=${{ steps.extract_commit_sha.outputs.sha }}" > ./shared/environment.chroot
           echo "BUILD_JOBS=2" > ./shared/environment.chroot

.github/workflows/zeek-build-and-push-ghcr.yml

@@ -105,6 +105,7 @@ jobs:
             MALCOLM_VERSION=${{ steps.extract_malcolm_version.outputs.mversion }}
             BUILD_DATE=${{ steps.generate_build_timestamp.outputs.btimestamp }}
             VCS_REVISION=${{ steps.extract_commit_sha.outputs.sha }}
+            ZEEK_DEB_ALTERNATE_DOWNLOAD_URL=${{ secrets.ZEEK_DEB_ALTERNATE_DOWNLOAD_URL }}
           push: true
           provenance: false
           platforms: ${{ matrix.platform }}

Dockerfiles/zeek.Dockerfile

@@ -35,6 +35,7 @@ USER root
 # for download and install
 ARG ZEEK_VERSION=7.0.2-0
 ENV ZEEK_VERSION $ZEEK_VERSION
+ARG ZEEK_DEB_ALTERNATE_DOWNLOAD_URL=""
 
 # put Zeek and Spicy in PATH
 ENV ZEEK_DIR "/opt/zeek"

hedgehog-iso/build.sh

@@ -190,7 +190,7 @@ if [ -d "$WORKDIR" ]; then
     "https://github.com/arkime/arkime/releases/download/v${ARKIME_VER}/arkime_${ARKIME_VER}-1.debian12_amd64.deb"
 
   # download Zeek .deb packages
-  bash "$SCRIPT_PATH/shared/bin/zeek-deb-download.sh" -o ./config/packages.chroot/
+  bash "$SCRIPT_PATH/shared/bin/zeek-deb-download.sh" -o ./config/packages.chroot/ -f "$SCRIPT_PATH/shared/zeek_url.txt"
 
   # reclaim some space
   docker system prune --volumes --force

hedgehog-iso/build_via_vagrant.sh

@@ -100,6 +100,7 @@ YML_IMAGE_VERSION="$(grep -P "^\s+image:.*/malcolm/" "$SCRIPT_PATH"/../docker-co
 [[ -n $YML_IMAGE_VERSION ]] && echo "$YML_IMAGE_VERSION" > "$SCRIPT_PATH"/shared/version.txt
 [[ ${#MAXMIND_GEOIP_DB_LICENSE_KEY} -gt 1 ]] && echo "$MAXMIND_GEOIP_DB_LICENSE_KEY" > "$SCRIPT_PATH"/shared/maxmind_license.txt
 [[ ${#MAXMIND_GEOIP_DB_ALTERNATE_DOWNLOAD_URL} -gt 1 ]] && echo "$MAXMIND_GEOIP_DB_ALTERNATE_DOWNLOAD_URL" > "$SCRIPT_PATH"/shared/maxmind_url.txt
+[[ ${#ZEEK_DEB_ALTERNATE_DOWNLOAD_URL} -gt 1 ]] && echo "$ZEEK_DEB_ALTERNATE_DOWNLOAD_URL" > "$SCRIPT_PATH"/shared/zeek_url.txt
 [[ ${#GITHUB_TOKEN} -gt 1 ]] && echo "GITHUB_TOKEN=$GITHUB_TOKEN" >> "$SCRIPT_PATH"/shared/environment.chroot
 echo "VCS_REVSION=$( git rev-parse --short HEAD 2>/dev/null || echo main )" >> "$SCRIPT_PATH"/shared/environment.chroot
 trap cleanup_shared_and_docs EXIT

hedgehog-raspi/build_via_vagrant.sh

@@ -99,6 +99,7 @@ YML_IMAGE_VERSION="$(grep -P "^\s+image:.*/malcolm/" "$SCRIPT_PATH"/../docker-co
 [[ -n $YML_IMAGE_VERSION ]] && echo "$YML_IMAGE_VERSION" > "$SCRIPT_PATH"/shared/version.txt
 [[ ${#MAXMIND_GEOIP_DB_LICENSE_KEY} -gt 1 ]] && echo "$MAXMIND_GEOIP_DB_LICENSE_KEY" > "$SCRIPT_PATH"/shared/maxmind_license.txt
 [[ ${#MAXMIND_GEOIP_DB_ALTERNATE_DOWNLOAD_URL} -gt 1 ]] && echo "$MAXMIND_GEOIP_DB_ALTERNATE_DOWNLOAD_URL" > "$SCRIPT_PATH"/shared/maxmind_url.txt
+[[ ${#ZEEK_DEB_ALTERNATE_DOWNLOAD_URL} -gt 1 ]] && echo "$ZEEK_DEB_ALTERNATE_DOWNLOAD_URL" > "$SCRIPT_PATH"/shared/zeek_url.txt
 [[ ${#GITHUB_TOKEN} -gt 1 ]] && echo "GITHUB_TOKEN=$GITHUB_TOKEN" >> "$SCRIPT_PATH"/shared/environment.chroot
 echo "VCS_REVSION=$( git rev-parse --short HEAD 2>/dev/null || echo main )" >> "$SCRIPT_PATH"/shared/environment.chroot
 trap cleanup_shared_and_docs EXIT

hedgehog-raspi/sensor_install.sh

@@ -197,7 +197,7 @@ build_yara_src() {
 build_zeek() {
     # install zeek from debs from OpenSUSE
     mkdir -p /tmp/zeek-debs
-    /bin/bash /usr/local/bin/zeek-deb-download.sh -o /tmp/zeek-debs
+    /bin/bash /usr/local/bin/zeek-deb-download.sh -o /tmp/zeek-debs -f "$SHARED_DIR/zeek_url.txt"
     dpkg -i /tmp/zeek-debs/*.deb
 }
 

scripts/build.sh

@@ -122,9 +122,9 @@ fi
 # build the image(s)
 DOCKER_COMPOSE_COMMAND="${DOCKER_COMPOSE_BIN[@]} --profile malcolm -f "$CONFIG_FILE""
 if [[ $CONFIRMATION =~ ^[Yy] ]]; then
-  $DOCKER_COMPOSE_COMMAND --progress=plain build --force-rm --no-cache --build-arg TARGETPLATFORM="$TARGET_PLATFORM" --build-arg GITHUB_TOKEN="$GITHUB_API_TOKEN" --build-arg MAXMIND_GEOIP_DB_LICENSE_KEY="$MAXMIND_API_KEY" --build-arg MAXMIND_GEOIP_DB_ALTERNATE_DOWNLOAD_URL="${MAXMIND_GEOIP_DB_ALTERNATE_DOWNLOAD_URL:-}" --build-arg BUILD_DATE="$BUILD_DATE" --build-arg MALCOLM_VERSION="$MALCOLM_VERSION" --build-arg VCS_REVISION="$VCS_REVISION" "$@"
+  $DOCKER_COMPOSE_COMMAND --progress=plain build --force-rm --no-cache --build-arg TARGETPLATFORM="$TARGET_PLATFORM" --build-arg GITHUB_TOKEN="$GITHUB_API_TOKEN" --build-arg MAXMIND_GEOIP_DB_LICENSE_KEY="$MAXMIND_API_KEY" --build-arg MAXMIND_GEOIP_DB_ALTERNATE_DOWNLOAD_URL="${MAXMIND_GEOIP_DB_ALTERNATE_DOWNLOAD_URL:-}" --build-arg ZEEK_DEB_ALTERNATE_DOWNLOAD_URL="${ZEEK_DEB_ALTERNATE_DOWNLOAD_URL:-}" --build-arg BUILD_DATE="$BUILD_DATE" --build-arg MALCOLM_VERSION="$MALCOLM_VERSION" --build-arg VCS_REVISION="$VCS_REVISION" "$@"
 else
-  $DOCKER_COMPOSE_COMMAND --progress=plain build --build-arg TARGETPLATFORM="$TARGET_PLATFORM" --build-arg GITHUB_TOKEN="$GITHUB_API_TOKEN" --build-arg MAXMIND_GEOIP_DB_LICENSE_KEY="$MAXMIND_API_KEY" --build-arg MAXMIND_GEOIP_DB_ALTERNATE_DOWNLOAD_URL="${MAXMIND_GEOIP_DB_ALTERNATE_DOWNLOAD_URL:-}" --build-arg BUILD_DATE="$BUILD_DATE" --build-arg MALCOLM_VERSION="$MALCOLM_VERSION" --build-arg VCS_REVISION="$VCS_REVISION" "$@"
+  $DOCKER_COMPOSE_COMMAND --progress=plain build --build-arg TARGETPLATFORM="$TARGET_PLATFORM" --build-arg GITHUB_TOKEN="$GITHUB_API_TOKEN" --build-arg MAXMIND_GEOIP_DB_LICENSE_KEY="$MAXMIND_API_KEY" --build-arg MAXMIND_GEOIP_DB_ALTERNATE_DOWNLOAD_URL="${MAXMIND_GEOIP_DB_ALTERNATE_DOWNLOAD_URL:-}" --build-arg ZEEK_DEB_ALTERNATE_DOWNLOAD_URL="${ZEEK_DEB_ALTERNATE_DOWNLOAD_URL:-}" --build-arg BUILD_DATE="$BUILD_DATE" --build-arg MALCOLM_VERSION="$MALCOLM_VERSION" --build-arg VCS_REVISION="$VCS_REVISION" "$@"
 fi
 
 # we're going to do some validation that some things got pulled/built correctly

shared/bin/zeek-deb-download.sh

@@ -7,12 +7,18 @@ command -v dpkg >/dev/null 2>&1 && ARCH="$(dpkg --print-architecture)" || ARCH=a
 DISTRO=Debian_12
 OUTPUT_DIR=/tmp
 ZEEK_VERSION=7.0.2-0
+PRESERVE_HIERARCHY=false
+ZEEK_DEB_ALTERNATE_DOWNLOAD_URL=${ZEEK_DEB_ALTERNATE_DOWNLOAD_URL:-}
+ZEEK_DEB_ALTERNATE_DOWNLOAD_URL_FILE=${ZEEK_DEB_ALTERNATE_DOWNLOAD_URL_FILE:-}
 
-while getopts a:d:o:vz: opts; do
+while getopts a:d:f:ho:u:vz: opts; do
    case ${opts} in
       a) ARCH=${OPTARG} ;;
       d) DISTRO=${OPTARG} ;;
+      f) ZEEK_DEB_ALTERNATE_DOWNLOAD_URL_FILE=${OPTARG} ;;
+      h) PRESERVE_HIERARCHY=true ;;
       o) OUTPUT_DIR=${OPTARG} ;;
+      u) ZEEK_DEB_ALTERNATE_DOWNLOAD_URL=${OPTARG} ;;
       v) VERBOSE=1 ;;
       z) ZEEK_VERSION=${OPTARG} ;;
    esac
@@ -23,23 +29,37 @@ if [[ -n $VERBOSE ]]; then
   set -x
 fi
 
-URL_PREFIX="https://downloadcontentcdn.opensuse.org/repositories/security:/zeek/${DISTRO}"
-URLS=(
-  "${URL_PREFIX}/${ARCH}/libbroker-dev_${ZEEK_VERSION}_${ARCH}.deb"
-  "${URL_PREFIX}/${ARCH}/zeek-core-dev_${ZEEK_VERSION}_${ARCH}.deb"
-  "${URL_PREFIX}/${ARCH}/zeek-core_${ZEEK_VERSION}_${ARCH}.deb"
-  "${URL_PREFIX}/${ARCH}/zeek-spicy-dev_${ZEEK_VERSION}_${ARCH}.deb"
-  "${URL_PREFIX}/${ARCH}/zeek_${ZEEK_VERSION}_${ARCH}.deb"
-  "${URL_PREFIX}/${ARCH}/zeekctl_${ZEEK_VERSION}_${ARCH}.deb"
-  "${URL_PREFIX}/all/zeek-client_${ZEEK_VERSION}_all.deb"
-  "${URL_PREFIX}/all/zeek-zkg_${ZEEK_VERSION}_all.deb"
-  "${URL_PREFIX}/all/zeek-btest_${ZEEK_VERSION}_all.deb"
-  "${URL_PREFIX}/all/zeek-btest-data_${ZEEK_VERSION}_all.deb"
+if [[ -z "${ZEEK_DEB_ALTERNATE_DOWNLOAD_URL}" ]] && [[ -f "${ZEEK_DEB_ALTERNATE_DOWNLOAD_URL_FILE}" ]]; then
+  ZEEK_DEB_ALTERNATE_DOWNLOAD_URL="$(head -n 1 "${ZEEK_DEB_ALTERNATE_DOWNLOAD_URL_FILE}")"
+fi
+
+URL_PREFIXES=(
+  "https://downloadcontentcdn.opensuse.org/repositories/security:/zeek"
+)
+[[ -n "$ZEEK_DEB_ALTERNATE_DOWNLOAD_URL" ]] && URL_PREFIXES+=( "$ZEEK_DEB_ALTERNATE_DOWNLOAD_URL" )
+
+URL_SUFFIXES=(
+  "${DISTRO}/${ARCH}/libbroker-dev_${ZEEK_VERSION}_${ARCH}.deb"
+  "${DISTRO}/${ARCH}/zeek-core-dev_${ZEEK_VERSION}_${ARCH}.deb"
+  "${DISTRO}/${ARCH}/zeek-core_${ZEEK_VERSION}_${ARCH}.deb"
+  "${DISTRO}/${ARCH}/zeek-spicy-dev_${ZEEK_VERSION}_${ARCH}.deb"
+  "${DISTRO}/${ARCH}/zeek_${ZEEK_VERSION}_${ARCH}.deb"
+  "${DISTRO}/${ARCH}/zeekctl_${ZEEK_VERSION}_${ARCH}.deb"
+  "${DISTRO}/all/zeek-client_${ZEEK_VERSION}_all.deb"
+  "${DISTRO}/all/zeek-zkg_${ZEEK_VERSION}_all.deb"
+  "${DISTRO}/all/zeek-btest_${ZEEK_VERSION}_all.deb"
+  "${DISTRO}/all/zeek-btest-data_${ZEEK_VERSION}_all.deb"
 )
 
 pushd "$OUTPUT_DIR" >/dev/null 2>&1
-for URL in ${URLS[@]}; do
-  curl -fsSL -O -J "${URL}"
+for URL_SUFFIX in ${URL_SUFFIXES[@]}; do
+  [[ "$PRESERVE_HIERARCHY" == "true" ]] && OUTPUT_DIR_REL="$(dirname "$URL_SUFFIX")" || OUTPUT_DIR_REL=.
+  mkdir -p "$OUTPUT_DIR_REL"
+  pushd "$OUTPUT_DIR_REL" >/dev/null 2>&1
+  for URL_PREFIX in ${URL_PREFIXES[@]}; do
+    curl -fsSL -O -J "${URL_PREFIX%/}/${URL_SUFFIX}" && break
+  done
+  popd >/dev/null 2>&1
 done
 popd >/dev/null 2>&1