From 96bbc24bdf2b8948a37b20d09dda34290b1608b7 Mon Sep 17 00:00:00 2001
From: Henry Lindeman <hmlindeman@yahoo.com>
Date: Thu, 11 Apr 2024 11:21:19 -0700
Subject: [PATCH 01/24] build containers in runtests script

Signed-off-by: Henry Lindeman <hmlindeman@yahoo.com>
---
 .../integration/automation/runtests.sh        | 49 ++++++++++++++++++-
 1 file changed, 48 insertions(+), 1 deletion(-)

diff --git a/apps/integration/integration/automation/runtests.sh b/apps/integration/integration/automation/runtests.sh
index 34a9ce9a4..2744c8472 100755
--- a/apps/integration/integration/automation/runtests.sh
+++ b/apps/integration/integration/automation/runtests.sh
@@ -1,10 +1,14 @@
 #!/bin/bash
 
+TAG="$1"
+[[ -z "${TAG}" ]] && TAG="latest_rc"
+
 main() {
   if [[ ! -d ".git" ]]; then
     echo "Error: please run this script from sycamore root!" >&2
     exit 1
   fi
+  echo "Building/testing tag ${TAG}" >&2
   echo "Get the newest git commits" >&2
   checkout_main_if_new
   local should_run=$?
@@ -33,6 +37,13 @@ checkout_main_if_new() {
 
 build_containers() {
   echo "Yep, definitely building containers. That's what this function does" >&2
+  docker-build-hub apps/crawler/crawler/http/Dockerfile
+  docker-build-hub apps/crawler/crawler/s3/Dockerfile
+  docker-build-hub apps/importer/Dockerfile.buildx
+  docker-build-hub apps/opensearch/Dockerfile
+  docker-build-hub apps/jupyter/Dockerfile.buildx --build-arg=TAG="${TAG}"
+  docker-build-hub apps/demo-ui/Dockerfile.buildx
+  docker-build-hub apps/remote-processor-service/Dockerfile.buildx
 }
 
 handle_outputs() {
@@ -42,11 +53,47 @@ handle_outputs() {
 runtests() {
   docker system prune -f --volumes
   docker compose up reset
-  poetry run pytest apps/integration/ -p integration.conftest --noconftest --docker-tag latest_rc
+  poetry run pytest apps/integration/ -p integration.conftest --noconftest --docker-tag "${TAG}"
   # this is a complicated command, so: ^                        ^            ^ test against containers tagged latest_rc
   #                                    |                     don't load conftest at pytest runtime; it's already loaded
   #                                     load conftest with plugins, to capture the custom command line arg --docker-tag
 }
 
+docker-build-hub() {
+  local docker_file="$1"
+  [[ -n "${docker_file}" ]] || error "missing ${docker_file}"
+  local repo_name="$(_docker-repo-name)"
+  [[ -n "${repo_name}" ]] || error "empty repo name"
+  shift
+
+  local platform=linux/amd64,linux/arm64
+  echo
+  echo "Building in sycamore and pushing to docker hub with repo name '${repo_name}'"
+  docker buildx build "$(_docker-build-args)" -t "${repo_name}:${TAG}" -f "${docker_file}" --platform ${platform} . \
+    || error "buildx failed"
+  echo "Successfully built in $dir using docker file $docker_file"
+}
+
+_docker-repo-name() {
+  local repo_name="$(grep '^# Repo name: ' "${docker_file}" | awk '{print $4}')"
+  if [[ $(echo "${repo_name}" | wc -w) != 1 ]]; then
+    echo "Unable to find repo name in ${docker_file}" 1>&2
+    exit 1
+  fi
+  echo "${repo_name}"
+}
+
+_docker-build-args() {
+  local branch="$(git status | head -1 | grep 'On branch ' | awk '{print $3}')"
+  local rev="$(git rev-parse --short HEAD)"
+  local date="$(git show -s --format=%ci HEAD | sed 's/ /_/g')"
+  local diff=unknown
+  if [[ $(git status | grep -c 'nothing to commit, working tree clean') = 1 ]]; then
+    diff=clean
+  else
+    diff="pending_changes_$(git diff HEAD | shasum | awk '{print $1}')"
+  fi
+  echo "--build-arg=GIT_BRANCH=${branch} --build-arg=GIT_COMMIT=${rev}--${date} --build-arg=GIT_DIFF=${diff}"
+}
 
 main

From 42683d3ab72ca158fa8d8496fcb39b5170c5dfb6 Mon Sep 17 00:00:00 2001
From: Henry Lindeman <hmlindeman@yahoo.com>
Date: Thu, 11 Apr 2024 12:57:50 -0700
Subject: [PATCH 02/24] fix buildscript wc weirdness

Signed-off-by: Henry Lindeman <hmlindeman@yahoo.com>
---
 .../integration/integration/automation/runtests.sh | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/apps/integration/integration/automation/runtests.sh b/apps/integration/integration/automation/runtests.sh
index 2744c8472..1ab934366 100755
--- a/apps/integration/integration/automation/runtests.sh
+++ b/apps/integration/integration/automation/runtests.sh
@@ -16,6 +16,7 @@ main() {
     echo "Changes detected. Running Tests" >&2
     poetry install
     build_containers
+    exit 0
     runtests
     handle_outputs
   else
@@ -23,12 +24,17 @@ main() {
   fi
 }
 
+error() {
+  echo "ERROR: $@"
+  exit 1
+}
+
 checkout_main_if_new() {
   old_sha="$(git rev-parse HEAD)"
   git fetch origin main >&2
   new_sha="$(git rev-parse FETCH_HEAD)"
   if [[ "${old_sha}" != "${new_sha}" ]]; then
-    git pull origin main >&2
+    git pull --rebase origin main >&2
     return 0
   else
     return 1
@@ -62,7 +68,7 @@ runtests() {
 docker-build-hub() {
   local docker_file="$1"
   [[ -n "${docker_file}" ]] || error "missing ${docker_file}"
-  local repo_name="$(_docker-repo-name)"
+  local repo_name="$(_docker-repo-name "${docker_file}")"
   [[ -n "${repo_name}" ]] || error "empty repo name"
   shift
 
@@ -75,8 +81,10 @@ docker-build-hub() {
 }
 
 _docker-repo-name() {
+  local docker_file="$1"
+  echo "Finding repo name in: ${docker_file}" >&2
   local repo_name="$(grep '^# Repo name: ' "${docker_file}" | awk '{print $4}')"
-  if [[ $(echo "${repo_name}" | wc -w) != 1 ]]; then
+  if (( $(wc -w <<< ${repo_name}) != 1 )); then
     echo "Unable to find repo name in ${docker_file}" 1>&2
     exit 1
   fi

From 36facda5797fd9182d1e62850b1ed5f1f3aaced2 Mon Sep 17 00:00:00 2001
From: Henry Lindeman <hmlindeman@yahoo.com>
Date: Thu, 11 Apr 2024 16:08:45 -0700
Subject: [PATCH 03/24] use docker registry caching for image builds

Signed-off-by: Henry Lindeman <hmlindeman@yahoo.com>
---
 apps/integration/integration/automation/runtests.sh | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/apps/integration/integration/automation/runtests.sh b/apps/integration/integration/automation/runtests.sh
index 1ab934366..9862b3171 100755
--- a/apps/integration/integration/automation/runtests.sh
+++ b/apps/integration/integration/automation/runtests.sh
@@ -75,8 +75,10 @@ docker-build-hub() {
   local platform=linux/amd64,linux/arm64
   echo
   echo "Building in sycamore and pushing to docker hub with repo name '${repo_name}'"
-  docker buildx build "$(_docker-build-args)" -t "${repo_name}:${TAG}" -f "${docker_file}" --platform ${platform} . \
-    || error "buildx failed"
+  docker buildx build "$(_docker-build-args)" -t "${repo_name}:${TAG}" -f "${docker_file}" --platform ${platform} \
+     --cache-to type=registry,ref="${repo_name}:build-cache",mode=max \
+     --cache-from type=registry,ref="${repo_name}:build-cache" \
+     "$@" --push . || error "buildx failed"
   echo "Successfully built in $dir using docker file $docker_file"
 }
 

From 79cb6d6435cb0f2206e87aa8cdb02f02c041cd4f Mon Sep 17 00:00:00 2001
From: Henry Lindeman <hmlindeman@yahoo.com>
Date: Fri, 12 Apr 2024 10:18:48 -0700
Subject: [PATCH 04/24] remove post-build exit

Signed-off-by: Henry Lindeman <hmlindeman@yahoo.com>
---
 apps/integration/integration/automation/runtests.sh | 1 -
 1 file changed, 1 deletion(-)

diff --git a/apps/integration/integration/automation/runtests.sh b/apps/integration/integration/automation/runtests.sh
index 9862b3171..f0def436a 100755
--- a/apps/integration/integration/automation/runtests.sh
+++ b/apps/integration/integration/automation/runtests.sh
@@ -16,7 +16,6 @@ main() {
     echo "Changes detected. Running Tests" >&2
     poetry install
     build_containers
-    exit 0
     runtests
     handle_outputs
   else

From f9d266f3466393defa364cf9b93bdd2adcbebcc0 Mon Sep 17 00:00:00 2001
From: Henry Lindeman <hmlindeman@yahoo.com>
Date: Mon, 15 Apr 2024 10:18:20 -0700
Subject: [PATCH 05/24] put test logs in s3

Signed-off-by: Henry Lindeman <hmlindeman@yahoo.com>
---
 .../integration/automation/runtests.sh        | 28 +++++++++++++++----
 1 file changed, 22 insertions(+), 6 deletions(-)

diff --git a/apps/integration/integration/automation/runtests.sh b/apps/integration/integration/automation/runtests.sh
index f0def436a..292133074 100755
--- a/apps/integration/integration/automation/runtests.sh
+++ b/apps/integration/integration/automation/runtests.sh
@@ -3,21 +3,31 @@
 TAG="$1"
 [[ -z "${TAG}" ]] && TAG="latest_rc"
 
+NOW="$(date +"%Y-%m-%d_%H_%M")"
+RUNDIR="apps/integration/runs/${NOW}"
+GIT_LOGFILE="${RUNDIR}/git.log"
+DOCKER_LOGFILE="${RUNDIR}/docker.log"
+POETRY_LOGFILE="${RUNDIR}/poetry.log"
+PYTEST_LOGFILE="${RUNDIR}/pytest.log"
+QUERY_LOGFILE="${RUNDIR}/test_queries.log"
+
 main() {
   if [[ ! -d ".git" ]]; then
     echo "Error: please run this script from sycamore root!" >&2
     exit 1
   fi
+  mkdir -p "${RUNDIR}"
   echo "Building/testing tag ${TAG}" >&2
   echo "Get the newest git commits" >&2
   checkout_main_if_new
   local should_run=$?
   if [[ $should_run ]]; then
     echo "Changes detected. Running Tests" >&2
-    poetry install
-    build_containers
-    runtests
-    handle_outputs
+    poetry install > "${POETRY_LOGFILE}" 2>&1
+    build_containers > "${DOCKER_LOGFILE}" 2>&1
+    runtests > "${PYTEST_LOGFILE}" 2>&1
+    local passed_tests=$?
+    handle_outputs passed_tests
   else
     echo "No changes detected. Skipping integration tests" >&2
   fi
@@ -30,10 +40,10 @@ error() {
 
 checkout_main_if_new() {
   old_sha="$(git rev-parse HEAD)"
-  git fetch origin main >&2
+  git fetch origin main > "${GIT_LOGFILE}"
   new_sha="$(git rev-parse FETCH_HEAD)"
   if [[ "${old_sha}" != "${new_sha}" ]]; then
-    git pull --rebase origin main >&2
+    git pull --rebase origin main >> "${GIT_LOGFILE}"
     return 0
   else
     return 1
@@ -53,6 +63,11 @@ build_containers() {
 
 handle_outputs() {
   echo "Yep, definitely handling test outputs. That's what this function does" >&2
+  local passed_tests="$1"
+  mv test-output.log "${QUERY_LOGFILE}"
+  [[ ${passed_tests} = 0 ]] && touch "${RUNDIR}/passed"
+  [[ ${passed_tests} != 0 ]] && touch "${RUNDIR}/failed"
+  aws s3 cp -r "${RUNDIR}" s3://sycamore-ci
 }
 
 runtests() {
@@ -62,6 +77,7 @@ runtests() {
   # this is a complicated command, so: ^                        ^            ^ test against containers tagged latest_rc
   #                                    |                     don't load conftest at pytest runtime; it's already loaded
   #                                     load conftest with plugins, to capture the custom command line arg --docker-tag
+  return $?
 }
 
 docker-build-hub() {

From 6e5122dbe459d397d0d08e7715fc85cdcd60e746 Mon Sep 17 00:00:00 2001
From: Henry Lindeman <hmlindeman@yahoo.com>
Date: Mon, 15 Apr 2024 10:27:02 -0700
Subject: [PATCH 06/24] specify arch in s3 path for results

Signed-off-by: Henry Lindeman <hmlindeman@yahoo.com>
---
 apps/integration/integration/automation/runtests.sh | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/apps/integration/integration/automation/runtests.sh b/apps/integration/integration/automation/runtests.sh
index 292133074..853d0b408 100755
--- a/apps/integration/integration/automation/runtests.sh
+++ b/apps/integration/integration/automation/runtests.sh
@@ -4,6 +4,9 @@ TAG="$1"
 [[ -z "${TAG}" ]] && TAG="latest_rc"
 
 NOW="$(date +"%Y-%m-%d_%H_%M")"
+ARCH="amd64"
+[[ "$(uname -m)" = "arm64" ]] && ARCH="arm64"
+
 RUNDIR="apps/integration/runs/${NOW}"
 GIT_LOGFILE="${RUNDIR}/git.log"
 DOCKER_LOGFILE="${RUNDIR}/docker.log"
@@ -67,7 +70,7 @@ handle_outputs() {
   mv test-output.log "${QUERY_LOGFILE}"
   [[ ${passed_tests} = 0 ]] && touch "${RUNDIR}/passed"
   [[ ${passed_tests} != 0 ]] && touch "${RUNDIR}/failed"
-  aws s3 cp -r "${RUNDIR}" s3://sycamore-ci
+  aws s3 cp -r "${RUNDIR}" "s3://sycamore-ci/${ARCH}"
 }
 
 runtests() {

From 49536db2c33d3a3a0cf64616d4300b3e5299d4ae Mon Sep 17 00:00:00 2001
From: Henry Lindeman <hmlindeman@yahoo.com>
Date: Thu, 18 Apr 2024 09:31:43 -0700
Subject: [PATCH 07/24] address pr comments. only build images for machines
 arch, and push after testing

Signed-off-by: Henry Lindeman <hmlindeman@yahoo.com>
---
 .../integration/automation/runtests.sh        | 98 ++++++++++++-------
 1 file changed, 61 insertions(+), 37 deletions(-)

diff --git a/apps/integration/integration/automation/runtests.sh b/apps/integration/integration/automation/runtests.sh
index 853d0b408..8717e736a 100755
--- a/apps/integration/integration/automation/runtests.sh
+++ b/apps/integration/integration/automation/runtests.sh
@@ -1,9 +1,9 @@
 #!/bin/bash
 
 TAG="$1"
-[[ -z "${TAG}" ]] && TAG="latest_rc"
+[[ -z "${TAG}" ]] && TAG="integration_tests"
 
-NOW="$(date +"%Y-%m-%d_%H_%M")"
+NOW="$(date +"%Y-%m-%d_%H_%M_%S")"
 ARCH="amd64"
 [[ "$(uname -m)" = "arm64" ]] && ARCH="arm64"
 
@@ -15,57 +15,56 @@ PYTEST_LOGFILE="${RUNDIR}/pytest.log"
 QUERY_LOGFILE="${RUNDIR}/test_queries.log"
 
 main() {
-  if [[ ! -d ".git" ]]; then
-    echo "Error: please run this script from sycamore root!" >&2
-    exit 1
-  fi
+  [[ ! -d ".git" ]] && die "Please run this script from sycamore root!"
   mkdir -p "${RUNDIR}"
   echo "Building/testing tag ${TAG}" >&2
   echo "Get the newest git commits" >&2
-  checkout_main_if_new
-  local should_run=$?
-  if [[ $should_run ]]; then
+  if [[ $(checkout_main_if_new) ]]; then
     echo "Changes detected. Running Tests" >&2
-    poetry install > "${POETRY_LOGFILE}" 2>&1
-    build_containers > "${DOCKER_LOGFILE}" 2>&1
-    runtests > "${PYTEST_LOGFILE}" 2>&1
+    poetry install > "${POETRY_LOGFILE}" 2>&1 \
+    && build_images > "${DOCKER_LOGFILE}" 2>&1 \
+    && runtests > "${PYTEST_LOGFILE}" 2>&1
     local passed_tests=$?
-    handle_outputs passed_tests
+    push_images >> "${DOCKER_LOGFILE}" 2>&1
+    handle_outputs $passed_tests
   else
     echo "No changes detected. Skipping integration tests" >&2
   fi
 }
 
 error() {
-  echo "ERROR: $@"
-  exit 1
+  echo "ERROR: $@" >&2
 }
 
 checkout_main_if_new() {
   old_sha="$(git rev-parse HEAD)"
-  git fetch origin main > "${GIT_LOGFILE}"
+  git fetch origin main > "${GIT_LOGFILE}" 2>&1
   new_sha="$(git rev-parse FETCH_HEAD)"
   if [[ "${old_sha}" != "${new_sha}" ]]; then
-    git pull --rebase origin main >> "${GIT_LOGFILE}"
+    [[ $(git status | grep -c 'nothing to commit, working tree clean') = 1 ]] \
+      || { echo "Working tree not clean" > "${GIT_LOGFILE}" && return 1; }
+    git pull --rebase origin main >> "${GIT_LOGFILE}" 2>&1
     return 0
   else
     return 1
   fi
 }
 
-build_containers() {
-  echo "Yep, definitely building containers. That's what this function does" >&2
-  docker-build-hub apps/crawler/crawler/http/Dockerfile
-  docker-build-hub apps/crawler/crawler/s3/Dockerfile
-  docker-build-hub apps/importer/Dockerfile.buildx
-  docker-build-hub apps/opensearch/Dockerfile
-  docker-build-hub apps/jupyter/Dockerfile.buildx --build-arg=TAG="${TAG}"
-  docker-build-hub apps/demo-ui/Dockerfile.buildx
-  docker-build-hub apps/remote-processor-service/Dockerfile.buildx
+build_images() {
+  echo "Building all images" >&2
+  docker-build-hub apps/crawler/crawler/http/Dockerfile \
+  && docker-build-hub apps/crawler/crawler/s3/Dockerfile \
+  && docker-build-hub apps/importer/Dockerfile.buildx \
+  && docker-build-hub apps/opensearch/Dockerfile \
+  && docker-build-hub apps/jupyter/Dockerfile.buildx --build-arg=TAG="${TAG}" \
+  && docker-build-hub apps/demo-ui/Dockerfile.buildx \
+  && docker-build-hub apps/remote-processor-service/Dockerfile.buildx \
+  && return 0
+  return 1
 }
 
 handle_outputs() {
-  echo "Yep, definitely handling test outputs. That's what this function does" >&2
+  echo "Handling test outputs" >&2
   local passed_tests="$1"
   mv test-output.log "${QUERY_LOGFILE}"
   [[ ${passed_tests} = 0 ]] && touch "${RUNDIR}/passed"
@@ -73,31 +72,56 @@ handle_outputs() {
   aws s3 cp -r "${RUNDIR}" "s3://sycamore-ci/${ARCH}"
 }
 
+push_images() {
+  echo "Pushing tested images to dockerhub" >&2
+  docker-push-hub apps/crawler/crawler/http/Dockerfile \
+  && docker-push-hub apps/crawler/crawler/s3/Dockerfile \
+  && docker-push-hub apps/importer/Dockerfile.buildx \
+  && docker-push-hub apps/opensearch/Dockerfile \
+  && docker-push-hub apps/jupyter/Dockerfile.buildx \
+  && docker-push-hub apps/demo-ui/Dockerfile.buildx \
+  && docker-push-hub apps/remote-processor-service/Dockerfile.buildx \
+  && return 0
+  return 1
+}
+
 runtests() {
   docker system prune -f --volumes
   docker compose up reset
   poetry run pytest apps/integration/ -p integration.conftest --noconftest --docker-tag "${TAG}"
-  # this is a complicated command, so: ^                        ^            ^ test against containers tagged latest_rc
-  #                                    |                     don't load conftest at pytest runtime; it's already loaded
-  #                                     load conftest with plugins, to capture the custom command line arg --docker-tag
+  # this is a complicated command, so:
+  # -p integration.conftest - load conftest with plugins, to capture the custom command line arg (--docker-tag)
+  # --noconftest            - don't load conftest at pytest runtime; it's already loaded
+  # --docker-tag            - specify tag of containers to test
   return $?
 }
 
 docker-build-hub() {
   local docker_file="$1"
-  [[ -n "${docker_file}" ]] || error "missing ${docker_file}"
+  [[ -n "${docker_file}" ]] || { error "missing ${docker_file}" && return 1;}
   local repo_name="$(_docker-repo-name "${docker_file}")"
-  [[ -n "${repo_name}" ]] || error "empty repo name"
+  [[ -n "${repo_name}" ]] || { error "empty repo name" && return 1;}
   shift
 
-  local platform=linux/amd64,linux/arm64
   echo
   echo "Building in sycamore and pushing to docker hub with repo name '${repo_name}'"
-  docker buildx build "$(_docker-build-args)" -t "${repo_name}:${TAG}" -f "${docker_file}" --platform ${platform} \
+  docker buildx build "$(_docker-build-args)" -t "${repo_name}:${TAG}" -f "${docker_file}" \
      --cache-to type=registry,ref="${repo_name}:build-cache",mode=max \
      --cache-from type=registry,ref="${repo_name}:build-cache" \
-     "$@" --push . || error "buildx failed"
-  echo "Successfully built in $dir using docker file $docker_file"
+     "$@" --load . || { error "buildx failed" && return 1;}
+  echo "Successfully built using docker file $docker_file"
+}
+
+docker-push-hub() {
+  local docker_file="$1"
+  [[ -n "${docker_file}" ]] || { error "missing ${docker_file}" && return 1;}
+  local repo_name="$(_docker-repo-name "${docker_file}")"
+  [[ -n "${repo_name}" ]] || { error "empty repo name" && return 1;}
+
+  echo
+  echo "Pushing image to docker hub for repo '${repo_name}"
+  docker push "${repo_name}:${TAG}" || { error "docker push failed" && return 1;}
+  echo "Successfully pushed image previously built from dockerfile ${docker_file}"
 }
 
 _docker-repo-name() {
@@ -112,7 +136,7 @@ _docker-repo-name() {
 }
 
 _docker-build-args() {
-  local branch="$(git status | head -1 | grep 'On branch ' | awk '{print $3}')"
+  local branch="$(git status | head -n1 | grep -i 'On branch ' | awk '{print $3}')"
   local rev="$(git rev-parse --short HEAD)"
   local date="$(git show -s --format=%ci HEAD | sed 's/ /_/g')"
   local diff=unknown

From bb69c94b7b3c67f7f7b4d14cf96771e1d52a0d80 Mon Sep 17 00:00:00 2001
From: Henry Lindeman <hmlindeman@yahoo.com>
Date: Thu, 18 Apr 2024 09:45:18 -0700
Subject: [PATCH 08/24] allow untracked files when deciding whether to checkout

Signed-off-by: Henry Lindeman <hmlindeman@yahoo.com>
---
 apps/integration/integration/automation/runtests.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/apps/integration/integration/automation/runtests.sh b/apps/integration/integration/automation/runtests.sh
index 8717e736a..f650af5d6 100755
--- a/apps/integration/integration/automation/runtests.sh
+++ b/apps/integration/integration/automation/runtests.sh
@@ -41,7 +41,8 @@ checkout_main_if_new() {
   git fetch origin main > "${GIT_LOGFILE}" 2>&1
   new_sha="$(git rev-parse FETCH_HEAD)"
   if [[ "${old_sha}" != "${new_sha}" ]]; then
-    [[ $(git status | grep -c 'nothing to commit, working tree clean') = 1 ]] \
+    [[ $(git status | grep -c -e 'nothing to commit, working tree clean' \
+        -e 'nothing added to commit but untracked files present') = 1 ]] \
       || { echo "Working tree not clean" > "${GIT_LOGFILE}" && return 1; }
     git pull --rebase origin main >> "${GIT_LOGFILE}" 2>&1
     return 0

From db8288342b6c4156d11b465aac63a659b5551fdb Mon Sep 17 00:00:00 2001
From: Henry Lindeman <hmlindeman@yahoo.com>
Date: Thu, 18 Apr 2024 09:54:15 -0700
Subject: [PATCH 09/24] use exit code for checkout_main_if_new

Signed-off-by: Henry Lindeman <hmlindeman@yahoo.com>
---
 apps/integration/integration/automation/runtests.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/integration/integration/automation/runtests.sh b/apps/integration/integration/automation/runtests.sh
index f650af5d6..6a99245c4 100755
--- a/apps/integration/integration/automation/runtests.sh
+++ b/apps/integration/integration/automation/runtests.sh
@@ -19,7 +19,7 @@ main() {
   mkdir -p "${RUNDIR}"
   echo "Building/testing tag ${TAG}" >&2
   echo "Get the newest git commits" >&2
-  if [[ $(checkout_main_if_new) ]]; then
+  if [[ checkout_main_if_new ]]; then
     echo "Changes detected. Running Tests" >&2
     poetry install > "${POETRY_LOGFILE}" 2>&1 \
     && build_images > "${DOCKER_LOGFILE}" 2>&1 \

From fe3ad44a5a14a865a3f4ac8ed29eca5ddf0ce9e2 Mon Sep 17 00:00:00 2001
From: Henry Lindeman <hmlindeman@yahoo.com>
Date: Mon, 29 Apr 2024 09:41:34 -0700
Subject: [PATCH 10/24] make git status checks more robust with --porcelain

Signed-off-by: Henry Lindeman <hmlindeman@yahoo.com>
---
 .gitignore                                            |  4 ++++
 .../integration/automation/{runtests.sh => runtests}  | 11 +++++------
 2 files changed, 9 insertions(+), 6 deletions(-)
 rename apps/integration/integration/automation/{runtests.sh => runtests} (91%)

diff --git a/.gitignore b/.gitignore
index 8ae4ffd2b..03624c6b2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -33,3 +33,7 @@ pyvenv.cfg
 apps/jupyter/bind_dir/redirect.html
 .venv
 lib/remote-processors/remote_processors/*pb2*
+poetry.toml
+apps/integration/runs/
+notebooks/data/
+test-output.log
diff --git a/apps/integration/integration/automation/runtests.sh b/apps/integration/integration/automation/runtests
similarity index 91%
rename from apps/integration/integration/automation/runtests.sh
rename to apps/integration/integration/automation/runtests
index 6a99245c4..21d27a805 100755
--- a/apps/integration/integration/automation/runtests.sh
+++ b/apps/integration/integration/automation/runtests
@@ -21,7 +21,7 @@ main() {
   echo "Get the newest git commits" >&2
   if [[ checkout_main_if_new ]]; then
     echo "Changes detected. Running Tests" >&2
-    poetry install > "${POETRY_LOGFILE}" 2>&1 \
+    poetry install --no-root > "${POETRY_LOGFILE}" 2>&1 \
     && build_images > "${DOCKER_LOGFILE}" 2>&1 \
     && runtests > "${PYTEST_LOGFILE}" 2>&1
     local passed_tests=$?
@@ -41,8 +41,7 @@ checkout_main_if_new() {
   git fetch origin main > "${GIT_LOGFILE}" 2>&1
   new_sha="$(git rev-parse FETCH_HEAD)"
   if [[ "${old_sha}" != "${new_sha}" ]]; then
-    [[ $(git status | grep -c -e 'nothing to commit, working tree clean' \
-        -e 'nothing added to commit but untracked files present') = 1 ]] \
+    [[ -z $(git status --porcelain) ]] \
       || { echo "Working tree not clean" > "${GIT_LOGFILE}" && return 1; }
     git pull --rebase origin main >> "${GIT_LOGFILE}" 2>&1
     return 0
@@ -137,11 +136,11 @@ _docker-repo-name() {
 }
 
 _docker-build-args() {
-  local branch="$(git status | head -n1 | grep -i 'On branch ' | awk '{print $3}')"
+  local branch="$(git branch --show-current)"
   local rev="$(git rev-parse --short HEAD)"
-  local date="$(git show -s --format=%ci HEAD | sed 's/ /_/g')"
+  local date="$(git show -s --format=%ci HEAD | sed -e 's/ /_/g')"
   local diff=unknown
-  if [[ $(git status | grep -c 'nothing to commit, working tree clean') = 1 ]]; then
+  if [[ -z $(git status --porcelain) ]]; then
     diff=clean
   else
     diff="pending_changes_$(git diff HEAD | shasum | awk '{print $1}')"

From 6122bdec66d8d87f5119f6f0291f8e3ace4a2c7b Mon Sep 17 00:00:00 2001
From: Henry Lindeman <hmlindeman@yahoo.com>
Date: Fri, 3 May 2024 09:09:57 -0700
Subject: [PATCH 11/24] address style comments (mostly)

Signed-off-by: Henry Lindeman <hmlindeman@yahoo.com>
---
 apps/integration/integration/automation/runtests | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/apps/integration/integration/automation/runtests b/apps/integration/integration/automation/runtests
index 21d27a805..ce08ae9e4 100755
--- a/apps/integration/integration/automation/runtests
+++ b/apps/integration/integration/automation/runtests
@@ -19,11 +19,11 @@ main() {
   mkdir -p "${RUNDIR}"
   echo "Building/testing tag ${TAG}" >&2
   echo "Get the newest git commits" >&2
-  if [[ checkout_main_if_new ]]; then
+  if checkout_main_if_new; then
     echo "Changes detected. Running Tests" >&2
     poetry install --no-root > "${POETRY_LOGFILE}" 2>&1 \
-    && build_images > "${DOCKER_LOGFILE}" 2>&1 \
-    && runtests > "${PYTEST_LOGFILE}" 2>&1
+        && build_images > "${DOCKER_LOGFILE}" 2>&1 \
+        && runtests > "${PYTEST_LOGFILE}" 2>&1
     local passed_tests=$?
     push_images >> "${DOCKER_LOGFILE}" 2>&1
     handle_outputs $passed_tests
@@ -67,8 +67,11 @@ handle_outputs() {
   echo "Handling test outputs" >&2
   local passed_tests="$1"
   mv test-output.log "${QUERY_LOGFILE}"
-  [[ ${passed_tests} = 0 ]] && touch "${RUNDIR}/passed"
-  [[ ${passed_tests} != 0 ]] && touch "${RUNDIR}/failed"
+  if [[ ${passed_tests} = 0 ]]; then
+    touch "${RUNDIR}/passed"
+  else
+    touch "${RUNDIR}/failed"
+  fi
   aws s3 cp -r "${RUNDIR}" "s3://sycamore-ci/${ARCH}"
 }
 

From 82fb8e6fccf99032df087d8509367b423034d340 Mon Sep 17 00:00:00 2001
From: Henry Lindeman <hmlindeman@yahoo.com>
Date: Fri, 3 May 2024 10:22:51 -0700
Subject: [PATCH 12/24] add args to specify which parts of integration script
 to run. also rename it

Signed-off-by: Henry Lindeman <hmlindeman@yahoo.com>
---
 .../automation/{runtests => integrate}        | 58 +++++++++++++++++--
 1 file changed, 53 insertions(+), 5 deletions(-)
 rename apps/integration/integration/automation/{runtests => integrate} (75%)

diff --git a/apps/integration/integration/automation/runtests b/apps/integration/integration/automation/integrate
similarity index 75%
rename from apps/integration/integration/automation/runtests
rename to apps/integration/integration/automation/integrate
index ce08ae9e4..7ec144651 100755
--- a/apps/integration/integration/automation/runtests
+++ b/apps/integration/integration/automation/integrate
@@ -1,7 +1,50 @@
 #!/bin/bash
 
-TAG="$1"
-[[ -z "${TAG}" ]] && TAG="integration_tests"
+# Parse args
+SKIP_BUILD=0
+SKIP_TESTS=0
+SKIP_PUSH=0
+TAG="integration_tests"
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --help)
+      echo "Utility script for building containers, running integration tests, and pushing images"
+      echo "Make sure to run this from the sycamore root directory."
+      echo "-------------------------------------------------------"
+      echo "Arguments:"
+      echo "  --help                Display this message"
+      echo "  --build               Build images"
+      echo "  --tests               Run integration tests"
+      echo "  --push                Push images"
+      echo "  --tag     [TAG]       When building, running, and/or pushing, use this docker tag."
+      echo "                        Default is 'integration_tests'"
+      exit 0
+      ;;
+    --build)
+      SKIP_BUILD=1
+      echo "Will build images"
+      shift
+      ;;
+    --tests)
+      SKIP_TESTS=1
+      echo "Will run integration tests"
+      shift
+      ;;
+    --push)
+      SKIP_PUSH=1
+      echo "Will push images"
+      shift
+      ;;
+    --tag)
+      [[ -z $2 ]] && die "A tag must be speicified when using the --tag arg; e.g. --tag my-tag"
+      [[ $2 == "--*" ]] && die "Detected tag was $2. Tags should not begin with --"
+      TAG="$2"
+      echo "Using tag ${TAG}"
+      shift
+      shift
+      ;;
+  esac
+done
 
 NOW="$(date +"%Y-%m-%d_%H_%M_%S")"
 ARCH="amd64"
@@ -22,10 +65,10 @@ main() {
   if checkout_main_if_new; then
     echo "Changes detected. Running Tests" >&2
     poetry install --no-root > "${POETRY_LOGFILE}" 2>&1 \
-        && build_images > "${DOCKER_LOGFILE}" 2>&1 \
-        && runtests > "${PYTEST_LOGFILE}" 2>&1
+        && { [[ $SKIP_BUILD ]] || build_images > "${DOCKER_LOGFILE}" 2>&1; } \
+        && { [[ $SKIP_TESTS ]] || runtests > "${PYTEST_LOGFILE}" 2>&1; }
     local passed_tests=$?
-    push_images >> "${DOCKER_LOGFILE}" 2>&1
+    [[ $SKIP_PUSH ]] || push_images >> "${DOCKER_LOGFILE}" 2>&1
     handle_outputs $passed_tests
   else
     echo "No changes detected. Skipping integration tests" >&2
@@ -36,6 +79,11 @@ error() {
   echo "ERROR: $@" >&2
 }
 
+die() {
+  error "$1"
+  exit 1
+}
+
 checkout_main_if_new() {
   old_sha="$(git rev-parse HEAD)"
   git fetch origin main > "${GIT_LOGFILE}" 2>&1

From 1666e46d569dd25a986269fe28ba83c028a80ced Mon Sep 17 00:00:00 2001
From: Henry Lindeman <hmlindeman@yahoo.com>
Date: Tue, 7 May 2024 10:25:56 -0700
Subject: [PATCH 13/24] address more pr comments

Signed-off-by: Henry Lindeman <hmlindeman@yahoo.com>
---
 .../integration/automation/integrate          | 87 ++++++++++---------
 1 file changed, 46 insertions(+), 41 deletions(-)

diff --git a/apps/integration/integration/automation/integrate b/apps/integration/integration/automation/integrate
index 7ec144651..6898955b9 100755
--- a/apps/integration/integration/automation/integrate
+++ b/apps/integration/integration/automation/integrate
@@ -1,13 +1,25 @@
 #!/bin/bash
 
+NOW="$(date +"%Y-%m-%d_%H_%M_%S")"
+ARCH="amd64"
+[[ "$(uname -m)" = "arm64" ]] && ARCH="arm64"
+
+RUNDIR="apps/integration/runs/${NOW}"
+GIT_LOGFILE="${RUNDIR}/git.log"
+DOCKER_LOGFILE="${RUNDIR}/docker.log"
+POETRY_LOGFILE="${RUNDIR}/poetry.log"
+PYTEST_LOGFILE="${RUNDIR}/pytest.log"
+QUERY_LOGFILE="${RUNDIR}/test_queries.log"
+
 # Parse args
-SKIP_BUILD=0
-SKIP_TESTS=0
-SKIP_PUSH=0
+DO_BUILD=1
+DO_TESTS=1
+DO_PUSH=1
+DO_CLEAN=1
 TAG="integration_tests"
 while [[ $# -gt 0 ]]; do
   case "$1" in
-    --help)
+    --help|-h)
       echo "Utility script for building containers, running integration tests, and pushing images"
       echo "Make sure to run this from the sycamore root directory."
       echo "-------------------------------------------------------"
@@ -21,55 +33,50 @@ while [[ $# -gt 0 ]]; do
       exit 0
       ;;
     --build)
-      SKIP_BUILD=1
+      DO_BUILD=0
       echo "Will build images"
       shift
       ;;
     --tests)
-      SKIP_TESTS=1
+      DO_TESTS=0
       echo "Will run integration tests"
       shift
       ;;
     --push)
-      SKIP_PUSH=1
+      DO_PUSH=0
       echo "Will push images"
       shift
       ;;
     --tag)
       [[ -z $2 ]] && die "A tag must be speicified when using the --tag arg; e.g. --tag my-tag"
-      [[ $2 == "--*" ]] && die "Detected tag was $2. Tags should not begin with --"
+      [[ $2 =~ "[a-z]*" ]] && die "Detected tag was $2. Tags should begin with lowercase letters"
       TAG="$2"
       echo "Using tag ${TAG}"
       shift
       shift
       ;;
+    --clean)
+      DO_CLEAN=0
+      echo "Will clean ${RUNDIR} before running anything"
+      shift
+      ;;
   esac
 done
 
-NOW="$(date +"%Y-%m-%d_%H_%M_%S")"
-ARCH="amd64"
-[[ "$(uname -m)" = "arm64" ]] && ARCH="arm64"
-
-RUNDIR="apps/integration/runs/${NOW}"
-GIT_LOGFILE="${RUNDIR}/git.log"
-DOCKER_LOGFILE="${RUNDIR}/docker.log"
-POETRY_LOGFILE="${RUNDIR}/poetry.log"
-PYTEST_LOGFILE="${RUNDIR}/pytest.log"
-QUERY_LOGFILE="${RUNDIR}/test_queries.log"
-
 main() {
-  [[ ! -d ".git" ]] && die "Please run this script from sycamore root!"
+  [[ -d ".git" ]] || die "Please run this script from sycamore root!"
+  [[ $DO_CLEAN ]] && rm -rf "${RUNDIR}"
   mkdir -p "${RUNDIR}"
   echo "Building/testing tag ${TAG}" >&2
   echo "Get the newest git commits" >&2
   if checkout_main_if_new; then
     echo "Changes detected. Running Tests" >&2
     poetry install --no-root > "${POETRY_LOGFILE}" 2>&1 \
-        && { [[ $SKIP_BUILD ]] || build_images > "${DOCKER_LOGFILE}" 2>&1; } \
-        && { [[ $SKIP_TESTS ]] || runtests > "${PYTEST_LOGFILE}" 2>&1; }
-    local passed_tests=$?
-    [[ $SKIP_PUSH ]] || push_images >> "${DOCKER_LOGFILE}" 2>&1
-    handle_outputs $passed_tests
+        && { [[ $DO_BUILD ]] && build_images > "${DOCKER_LOGFILE}" 2>&1; } \
+        && { [[ $DO_TESTS ]] && runtests > "${PYTEST_LOGFILE}" 2>&1; } \
+        && touch "${RUNDIR}/passed"
+    [[ $DO_PUSH ]] && push_images >> "${DOCKER_LOGFILE}" 2>&1
+    handle_outputs
   else
     echo "No changes detected. Skipping integration tests" >&2
   fi
@@ -80,7 +87,7 @@ error() {
 }
 
 die() {
-  error "$1"
+  error "$@"
   exit 1
 }
 
@@ -89,9 +96,10 @@ checkout_main_if_new() {
   git fetch origin main > "${GIT_LOGFILE}" 2>&1
   new_sha="$(git rev-parse FETCH_HEAD)"
   if [[ "${old_sha}" != "${new_sha}" ]]; then
-    [[ -z $(git status --porcelain) ]] \
-      || { echo "Working tree not clean" > "${GIT_LOGFILE}" && return 1; }
+    [[ -z $(git status --porcelain) ]] || die "Working tree not clean"
     git pull --rebase origin main >> "${GIT_LOGFILE}" 2>&1
+    echo "==================" >> "${GIT_LOGFILE}"
+    echo "Using git rev ${new_sha}" >> "${GIT_LOGFILE}"
     return 0
   else
     return 1
@@ -113,14 +121,9 @@ build_images() {
 
 handle_outputs() {
   echo "Handling test outputs" >&2
-  local passed_tests="$1"
   mv test-output.log "${QUERY_LOGFILE}"
-  if [[ ${passed_tests} = 0 ]]; then
-    touch "${RUNDIR}/passed"
-  else
-    touch "${RUNDIR}/failed"
-  fi
-  aws s3 cp -r "${RUNDIR}" "s3://sycamore-ci/${ARCH}"
+  [[ -f "${RUNDIR}/passed" ]] || touch "${RUNDIR}/failed"
+  aws s3 cp -r "${RUNDIR}/*" "s3://sycamore-ci/${NOW}/${ARCH}"
 }
 
 push_images() {
@@ -137,7 +140,8 @@ push_images() {
 }
 
 runtests() {
-  docker system prune -f --volumes
+  docker volume rm sycamore_crawl_data sycamore_jupyter_data sycamore_opensearch_data
+  docker network prune
   docker compose up reset
   poetry run pytest apps/integration/ -p integration.conftest --noconftest --docker-tag "${TAG}"
   # this is a complicated command, so:
@@ -149,9 +153,9 @@ runtests() {
 
 docker-build-hub() {
   local docker_file="$1"
-  [[ -n "${docker_file}" ]] || { error "missing ${docker_file}" && return 1;}
+  [[ -n "${docker_file}" ]] || { error "missing ${docker_file}"; return 1;}
   local repo_name="$(_docker-repo-name "${docker_file}")"
-  [[ -n "${repo_name}" ]] || { error "empty repo name" && return 1;}
+  [[ -n "${repo_name}" ]] || { error "empty repo name"; return 1;}
   shift
 
   echo
@@ -165,13 +169,13 @@ docker-build-hub() {
 
 docker-push-hub() {
   local docker_file="$1"
-  [[ -n "${docker_file}" ]] || { error "missing ${docker_file}" && return 1;}
+  [[ -n "${docker_file}" ]] || { error "missing ${docker_file}"; return 1;}
   local repo_name="$(_docker-repo-name "${docker_file}")"
-  [[ -n "${repo_name}" ]] || { error "empty repo name" && return 1;}
+  [[ -n "${repo_name}" ]] || { error "empty repo name"; return 1;}
 
   echo
   echo "Pushing image to docker hub for repo '${repo_name}"
-  docker push "${repo_name}:${TAG}" || { error "docker push failed" && return 1;}
+  docker push "${repo_name}:${TAG}" || { error "docker push failed"; return 1;}
   echo "Successfully pushed image previously built from dockerfile ${docker_file}"
 }
 
@@ -179,6 +183,7 @@ _docker-repo-name() {
   local docker_file="$1"
   echo "Finding repo name in: ${docker_file}" >&2
   local repo_name="$(grep '^# Repo name: ' "${docker_file}" | awk '{print $4}')"
+  [[ "${repo_name}" = *private* ]] && die "Private repo ${repo_name} disallowed"
   if (( $(wc -w <<< ${repo_name}) != 1 )); then
     echo "Unable to find repo name in ${docker_file}" 1>&2
     exit 1

From fdd90b95cdb538fea94f3e1106a9cdc62f2ed191 Mon Sep 17 00:00:00 2001
From: Henry Lindeman <hmlindeman@yahoo.com>
Date: Wed, 8 May 2024 11:10:30 -0700
Subject: [PATCH 14/24] change -r to --recursive because aws cli is verbose

Signed-off-by: Henry Lindeman <hmlindeman@yahoo.com>
---
 apps/integration/integration/automation/integrate | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/integration/integration/automation/integrate b/apps/integration/integration/automation/integrate
index 6898955b9..5d6960a3a 100755
--- a/apps/integration/integration/automation/integrate
+++ b/apps/integration/integration/automation/integrate
@@ -123,7 +123,7 @@ handle_outputs() {
   echo "Handling test outputs" >&2
   mv test-output.log "${QUERY_LOGFILE}"
   [[ -f "${RUNDIR}/passed" ]] || touch "${RUNDIR}/failed"
-  aws s3 cp -r "${RUNDIR}/*" "s3://sycamore-ci/${NOW}/${ARCH}"
+  aws s3 cp --recursive "${RUNDIR}/*" "s3://sycamore-ci/${NOW}/${ARCH}"
 }
 
 push_images() {

From bbbb1c177b58a7a2759e98effef29980f2833680 Mon Sep 17 00:00:00 2001
From: Henry Lindeman <hmlindeman@yahoo.com>
Date: Wed, 8 May 2024 11:13:58 -0700
Subject: [PATCH 15/24] check that test-output exists before trying to move it

Signed-off-by: Henry Lindeman <hmlindeman@yahoo.com>
---
 apps/integration/integration/automation/integrate | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/integration/integration/automation/integrate b/apps/integration/integration/automation/integrate
index 5d6960a3a..143d0e07e 100755
--- a/apps/integration/integration/automation/integrate
+++ b/apps/integration/integration/automation/integrate
@@ -121,7 +121,7 @@ build_images() {
 
 handle_outputs() {
   echo "Handling test outputs" >&2
-  mv test-output.log "${QUERY_LOGFILE}"
+  [[ -f test-output.log ]] && mv test-output.log "${QUERY_LOGFILE}"
   [[ -f "${RUNDIR}/passed" ]] || touch "${RUNDIR}/failed"
   aws s3 cp --recursive "${RUNDIR}/*" "s3://sycamore-ci/${NOW}/${ARCH}"
 }

From f1247c7eb6523de65daf31cf339fd3951019cd9f Mon Sep 17 00:00:00 2001
From: Henry Lindeman <hmlindeman@yahoo.com>
Date: Wed, 8 May 2024 11:53:54 -0700
Subject: [PATCH 16/24] aws cli doesnt do globs either

Signed-off-by: Henry Lindeman <hmlindeman@yahoo.com>
---
 apps/integration/integration/automation/integrate | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/integration/integration/automation/integrate b/apps/integration/integration/automation/integrate
index 143d0e07e..9933a3d84 100755
--- a/apps/integration/integration/automation/integrate
+++ b/apps/integration/integration/automation/integrate
@@ -123,7 +123,7 @@ handle_outputs() {
   echo "Handling test outputs" >&2
   [[ -f test-output.log ]] && mv test-output.log "${QUERY_LOGFILE}"
   [[ -f "${RUNDIR}/passed" ]] || touch "${RUNDIR}/failed"
-  aws s3 cp --recursive "${RUNDIR}/*" "s3://sycamore-ci/${NOW}/${ARCH}"
+  aws s3 cp --recursive "${RUNDIR}/" "s3://sycamore-ci/${NOW}/${ARCH}"
 }
 
 push_images() {

From 67f958f201e69fd8b33613bd43c8fc8ccbb79298 Mon Sep 17 00:00:00 2001
From: Henry Lindeman <hmlindeman@yahoo.com>
Date: Fri, 10 May 2024 10:16:11 -0700
Subject: [PATCH 17/24] add ssh orchestration for building and testing on
 multiple arches

Signed-off-by: Henry Lindeman <hmlindeman@yahoo.com>
---
 .../integration/automation/integrate          | 77 +++++++++++++++----
 1 file changed, 63 insertions(+), 14 deletions(-)

diff --git a/apps/integration/integration/automation/integrate b/apps/integration/integration/automation/integrate
index 9933a3d84..3f6590c7d 100755
--- a/apps/integration/integration/automation/integrate
+++ b/apps/integration/integration/automation/integrate
@@ -2,7 +2,8 @@
 
 NOW="$(date +"%Y-%m-%d_%H_%M_%S")"
 ARCH="amd64"
-[[ "$(uname -m)" = "arm64" ]] && ARCH="arm64"
+[[ "$(uname -m)" = "arm64" || "$(uname -m)" = "aarch64" ]] && ARCH="arm64"
+ARCH="$(archname "$(uname -m)")"
 
 RUNDIR="apps/integration/runs/${NOW}"
 GIT_LOGFILE="${RUNDIR}/git.log"
@@ -12,11 +13,12 @@ PYTEST_LOGFILE="${RUNDIR}/pytest.log"
 QUERY_LOGFILE="${RUNDIR}/test_queries.log"
 
 # Parse args
-DO_BUILD=1
-DO_TESTS=1
-DO_PUSH=1
-DO_CLEAN=1
+SKIP_BUILD=0
+SKIP_TESTS=0
+SKIP_PUSH=0
+DO_CLEAN=0
 TAG="integration_tests"
+declare SSH_TARGET
 while [[ $# -gt 0 ]]; do
   case "$1" in
     --help|-h)
@@ -28,28 +30,31 @@ while [[ $# -gt 0 ]]; do
       echo "  --build               Build images"
       echo "  --tests               Run integration tests"
       echo "  --push                Push images"
+      echo "  --clean               Remove logs from previous runs before doing anything."
       echo "  --tag     [TAG]       When building, running, and/or pushing, use this docker tag."
       echo "                        Default is 'integration_tests'"
+      echo "  --ssh     [TARGET]    When building and running tests, also build and run on this host."
+      echo "                        Useful for multi-arch builds and tests, e.g. --ssh my-arm-box"
       exit 0
       ;;
     --build)
-      DO_BUILD=0
+      SKIP_BUILD=1
       echo "Will build images"
       shift
       ;;
     --tests)
-      DO_TESTS=0
+      SKIP_TESTS=1
       echo "Will run integration tests"
       shift
       ;;
     --push)
-      DO_PUSH=0
+      SKIP_PUSH=1
       echo "Will push images"
       shift
       ;;
     --tag)
-      [[ -z $2 ]] && die "A tag must be speicified when using the --tag arg; e.g. --tag my-tag"
-      [[ $2 =~ "[a-z]*" ]] && die "Detected tag was $2. Tags should begin with lowercase letters"
+      [[ -z $2 ]] && die "A tag must be specified when using the --tag arg; e.g. --tag my-tag"
+      [[ $2 =~ "[a-z]*" ]] || die "Detected tag was $2. Tags should begin with lowercase letters"
       TAG="$2"
       echo "Using tag ${TAG}"
       shift
@@ -60,22 +65,35 @@ while [[ $# -gt 0 ]]; do
       echo "Will clean ${RUNDIR} before running anything"
       shift
       ;;
+    --ssh)
+      [[ -z $2 ]] && die "A configured ssh target must be specified when using the --ssh arg; e.g. --ssh my-host"
+      [[ $2 =~ "[a-z]*" ]] || die "Detected ssh target was $2. ssh tartgets should begin with lowercase letters"
+      SSH_TARGET="$2"
+      SSH_BUILDX_PORT=18460 # Selected by googling 5d10 and rerolling until small enough
+      echo "Using ssh target ${SSH_TARGET}"
+      shift
+      shift
+      ;;
   esac
 done
 
 main() {
   [[ -d ".git" ]] || die "Please run this script from sycamore root!"
   [[ $DO_CLEAN ]] && rm -rf "${RUNDIR}"
+  if [[ -n $SSH_TARGET && ! $SKIP_BUILD ]]; then
+    create-dual-builder
+    trap cleanup-dual-builder 0 1 2 3 6
+  fi
   mkdir -p "${RUNDIR}"
   echo "Building/testing tag ${TAG}" >&2
   echo "Get the newest git commits" >&2
   if checkout_main_if_new; then
     echo "Changes detected. Running Tests" >&2
     poetry install --no-root > "${POETRY_LOGFILE}" 2>&1 \
-        && { [[ $DO_BUILD ]] && build_images > "${DOCKER_LOGFILE}" 2>&1; } \
-        && { [[ $DO_TESTS ]] && runtests > "${PYTEST_LOGFILE}" 2>&1; } \
+        && { [[ $SKIP_BUILD ]] || build_images > "${DOCKER_LOGFILE}" 2>&1; } \
+        && { [[ $SKIP_TESTS ]] || runtests > "${PYTEST_LOGFILE}" 2>&1; } \
         && touch "${RUNDIR}/passed"
-    [[ $DO_PUSH ]] && push_images >> "${DOCKER_LOGFILE}" 2>&1
+    [[ $SKIP_PUSH ]] || push_images >> "${DOCKER_LOGFILE}" 2>&1
     handle_outputs
   else
     echo "No changes detected. Skipping integration tests" >&2
@@ -140,6 +158,9 @@ push_images() {
 }
 
 runtests() {
+  if [[ -n $SSH_TARGET ]]; then
+    ssh "${SSH_TARGET}" "cd sycamore && ./apps/integration/integration/automation/integrate --test --clean --tag ${TAG}"
+  fi
   docker volume rm sycamore_crawl_data sycamore_jupyter_data sycamore_opensearch_data
   docker network prune
   docker compose up reset
@@ -163,7 +184,7 @@ docker-build-hub() {
   docker buildx build "$(_docker-build-args)" -t "${repo_name}:${TAG}" -f "${docker_file}" \
      --cache-to type=registry,ref="${repo_name}:build-cache",mode=max \
      --cache-from type=registry,ref="${repo_name}:build-cache" \
-     "$@" --load . || { error "buildx failed" && return 1;}
+     "$@" --push . || { error "buildx failed" && return 1;}
   echo "Successfully built using docker file $docker_file"
 }
 
@@ -204,4 +225,32 @@ _docker-build-args() {
   echo "--build-arg=GIT_BRANCH=${branch} --build-arg=GIT_COMMIT=${rev}--${date} --build-arg=GIT_DIFF=${diff}"
 }
 
+create-dual-builder() {
+  # Over ssh, start a buildkit container on the target, and use port forwarding
+  # to talk to it. Also start a local buildkit container, and then create a buildx
+  # remote driver that talks to both of them.
+  ssh -N -L "${SSH_BUILDX_PORT}":localhost:"${SSH_BUILDX_PORT}" "${SSH_TARGET}" &
+  REMOTE_ARCH="$(archname "$(ssh "${SSH_TARGET}" uname -m)")"
+  ssh "${SSH_TARGET}" docker run -d --name=remote-buildkitd --privileged -p "${SSH_BUILDX_PORT}":"${SSH_BUILDX_PORT}" \
+        moby/buildkit:latest --addr "tcp://0.0.0.0:${SSH_BUILDX_PORT}"
+  docker run -d --name=remote-buildkitd --privileged -p "$((SSH_BUILDX_PORT - 1))":"$((SSH_BUILDX_PORT - 1))" \
+        moby/buildkit:latest --addr "tcp://0.0.0.0:$((SSH_BUILDX_PORT - 1))"
+  docker buildx create --name dual-builder --platform "linux/${ARCH}" --driver=remote "tcp://localhost:$((SSH_BUILDX_PORT - 1))"
+  docker buildx create --append --name dual-builder --platform "linux/${REMOTE_ARCH}" --driver=remote "tcp://localhost:${SSH_BUILDX_PORT}"
+  docker buildx use dual-builder
+}
+
+cleanup-dual-builder() {
+  docker buildx rm dual-builder
+  ssh "${SSH_TARGET}" "docker stop remote-buildkitd && docker rm remote-buildkitd"
+  docker stop remote-buildkitd && docker rm remote-buildkitd
+  pgrep -f "${PORT}:localhost:${PORT}" | xargs kill
+}
+
+archname() {
+  local unamearchname="$1"
+  local arch="amd64"
+  [[ "$unamearchname" = "arm64" || "$unamearchname" = "aarch64" ]] && arch="arm64"
+  echo "${arch}"
+
 main

From cab5ebb8fc7ac4b7b636d7e1fd72f1387e0f2da4 Mon Sep 17 00:00:00 2001
From: Henry Lindeman <hmlindeman@yahoo.com>
Date: Fri, 10 May 2024 10:19:54 -0700
Subject: [PATCH 18/24] syntax

Signed-off-by: Henry Lindeman <hmlindeman@yahoo.com>
---
 apps/integration/integration/automation/integrate | 1 +
 1 file changed, 1 insertion(+)

diff --git a/apps/integration/integration/automation/integrate b/apps/integration/integration/automation/integrate
index 3f6590c7d..f7be97c43 100755
--- a/apps/integration/integration/automation/integrate
+++ b/apps/integration/integration/automation/integrate
@@ -252,5 +252,6 @@ archname() {
   local arch="amd64"
   [[ "$unamearchname" = "arm64" || "$unamearchname" = "aarch64" ]] && arch="arm64"
   echo "${arch}"
+}
 
 main

From bc6d094c3d235230f6a278b44117d682167d0e45 Mon Sep 17 00:00:00 2001
From: Henry Lindeman <hmlindeman@yahoo.com>
Date: Fri, 10 May 2024 11:17:27 -0700
Subject: [PATCH 19/24] test flags correctly

Signed-off-by: Henry Lindeman <hmlindeman@yahoo.com>
---
 .../integration/automation/integrate          | 39 ++++++++++---------
 1 file changed, 20 insertions(+), 19 deletions(-)

diff --git a/apps/integration/integration/automation/integrate b/apps/integration/integration/automation/integrate
index f7be97c43..b50740829 100755
--- a/apps/integration/integration/automation/integrate
+++ b/apps/integration/integration/automation/integrate
@@ -1,5 +1,21 @@
 #!/bin/bash
 
+archname() {
+  local unamearchname="$1"
+  local arch="amd64"
+  [[ "$unamearchname" = "arm64" || "$unamearchname" = "aarch64" ]] && arch="arm64"
+  echo "${arch}"
+}
+
+error() {
+  echo "ERROR: $@" >&2
+}
+
+die() {
+  error "$@"
+  exit 1
+}
+
 NOW="$(date +"%Y-%m-%d_%H_%M_%S")"
 ARCH="amd64"
 [[ "$(uname -m)" = "arm64" || "$(uname -m)" = "aarch64" ]] && ARCH="arm64"
@@ -80,7 +96,7 @@ done
 main() {
   [[ -d ".git" ]] || die "Please run this script from sycamore root!"
   [[ $DO_CLEAN ]] && rm -rf "${RUNDIR}"
-  if [[ -n $SSH_TARGET && ! $SKIP_BUILD ]]; then
+  if [[ -n $SSH_TARGET && $SKIP_BUILD -ne 0 ]]; then
     create-dual-builder
     trap cleanup-dual-builder 0 1 2 3 6
   fi
@@ -90,24 +106,16 @@ main() {
   if checkout_main_if_new; then
     echo "Changes detected. Running Tests" >&2
     poetry install --no-root > "${POETRY_LOGFILE}" 2>&1 \
-        && { [[ $SKIP_BUILD ]] || build_images > "${DOCKER_LOGFILE}" 2>&1; } \
-        && { [[ $SKIP_TESTS ]] || runtests > "${PYTEST_LOGFILE}" 2>&1; } \
+        && { [[ $SKIP_BUILD -eq 0 ]] || build_images > "${DOCKER_LOGFILE}" 2>&1; } \
+        && { [[ $SKIP_TESTS -eq 0 ]] || runtests > "${PYTEST_LOGFILE}" 2>&1; } \
         && touch "${RUNDIR}/passed"
-    [[ $SKIP_PUSH ]] || push_images >> "${DOCKER_LOGFILE}" 2>&1
+    [[ $SKIP_PUSH -eq 0 ]] || push_images >> "${DOCKER_LOGFILE}" 2>&1
     handle_outputs
   else
     echo "No changes detected. Skipping integration tests" >&2
   fi
 }
 
-error() {
-  echo "ERROR: $@" >&2
-}
-
-die() {
-  error "$@"
-  exit 1
-}
 
 checkout_main_if_new() {
   old_sha="$(git rev-parse HEAD)"
@@ -247,11 +255,4 @@ cleanup-dual-builder() {
   pgrep -f "${PORT}:localhost:${PORT}" | xargs kill
 }
 
-archname() {
-  local unamearchname="$1"
-  local arch="amd64"
-  [[ "$unamearchname" = "arm64" || "$unamearchname" = "aarch64" ]] && arch="arm64"
-  echo "${arch}"
-}
-
 main

From 4b3484410305f135ecd214143abc6aa83e8fbbb6 Mon Sep 17 00:00:00 2001
From: Henry Lindeman <hmlindeman@yahoo.com>
Date: Fri, 10 May 2024 11:28:01 -0700
Subject: [PATCH 20/24] fix regex match

Signed-off-by: Henry Lindeman <hmlindeman@yahoo.com>
---
 apps/integration/integration/automation/integrate | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/apps/integration/integration/automation/integrate b/apps/integration/integration/automation/integrate
index b50740829..ad498d909 100755
--- a/apps/integration/integration/automation/integrate
+++ b/apps/integration/integration/automation/integrate
@@ -70,7 +70,7 @@ while [[ $# -gt 0 ]]; do
       ;;
     --tag)
       [[ -z $2 ]] && die "A tag must be specified when using the --tag arg; e.g. --tag my-tag"
-      [[ $2 =~ "[a-z]*" ]] || die "Detected tag was $2. Tags should begin with lowercase letters"
+      [[ $2 =~ [a-z]* ]] || die "Detected tag was $2. Tags should begin with lowercase letters"
       TAG="$2"
       echo "Using tag ${TAG}"
       shift
@@ -83,7 +83,7 @@ while [[ $# -gt 0 ]]; do
       ;;
     --ssh)
       [[ -z $2 ]] && die "A configured ssh target must be specified when using the --ssh arg; e.g. --ssh my-host"
-      [[ $2 =~ "[a-z]*" ]] || die "Detected ssh target was $2. ssh tartgets should begin with lowercase letters"
+      [[ $2 =~ [a-z]* ]] || die "Detected ssh target was $2. ssh tartgets should begin with lowercase letters"
       SSH_TARGET="$2"
       SSH_BUILDX_PORT=18460 # Selected by googling 5d10 and rerolling until small enough
       echo "Using ssh target ${SSH_TARGET}"

From 2cb85e106bf4d80d05e706815cceb9c8d6fbbe21 Mon Sep 17 00:00:00 2001
From: Henry Lindeman <hmlindeman@yahoo.com>
Date: Fri, 10 May 2024 11:43:09 -0700
Subject: [PATCH 21/24] tell docker to build for both platforms

Signed-off-by: Henry Lindeman <hmlindeman@yahoo.com>
---
 .../integration/automation/integrate            | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/apps/integration/integration/automation/integrate b/apps/integration/integration/automation/integrate
index ad498d909..87ce2ab57 100755
--- a/apps/integration/integration/automation/integrate
+++ b/apps/integration/integration/automation/integrate
@@ -17,8 +17,6 @@ die() {
 }
 
 NOW="$(date +"%Y-%m-%d_%H_%M_%S")"
-ARCH="amd64"
-[[ "$(uname -m)" = "arm64" || "$(uname -m)" = "aarch64" ]] && ARCH="arm64"
 ARCH="$(archname "$(uname -m)")"
 
 RUNDIR="apps/integration/runs/${NOW}"
@@ -192,7 +190,8 @@ docker-build-hub() {
   docker buildx build "$(_docker-build-args)" -t "${repo_name}:${TAG}" -f "${docker_file}" \
      --cache-to type=registry,ref="${repo_name}:build-cache",mode=max \
      --cache-from type=registry,ref="${repo_name}:build-cache" \
-     "$@" --push . || { error "buildx failed" && return 1;}
+     --platform="$(_docker-platforms)" "$@" --push . \
+     || { error "buildx failed" && return 1;}
   echo "Successfully built using docker file $docker_file"
 }
 
@@ -233,6 +232,18 @@ _docker-build-args() {
   echo "--build-arg=GIT_BRANCH=${branch} --build-arg=GIT_COMMIT=${rev}--${date} --build-arg=GIT_DIFF=${diff}"
 }
 
+_docker-platforms() {
+  local remotearch="${ARCH}"
+  if [[ -n $SSH_TARGET ]]; then
+    remotearch="$(archname "$(ssh "${SSH_TARGET}" uname -m)")"
+  fi
+  if [[ $ARCH != $remotearch ]]; then
+    echo "linux/${ARCH},linux/${remotearch}"
+  else
+    echo "linux/${ARCH}"
+  fi
+}
+
 create-dual-builder() {
   # Over ssh, start a buildkit container on the target, and use port forwarding
   # to talk to it. Also start a local buildkit container, and then create a buildx

From 6091289499509c6e3fc43fee43413ce9f3eb153b Mon Sep 17 00:00:00 2001
From: Henry Lindeman <hmlindeman@yahoo.com>
Date: Fri, 10 May 2024 11:47:40 -0700
Subject: [PATCH 22/24] kill the right port forward process

Signed-off-by: Henry Lindeman <hmlindeman@yahoo.com>
---
 apps/integration/integration/automation/integrate | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/integration/integration/automation/integrate b/apps/integration/integration/automation/integrate
index 87ce2ab57..e5c87a028 100755
--- a/apps/integration/integration/automation/integrate
+++ b/apps/integration/integration/automation/integrate
@@ -263,7 +263,7 @@ cleanup-dual-builder() {
   docker buildx rm dual-builder
   ssh "${SSH_TARGET}" "docker stop remote-buildkitd && docker rm remote-buildkitd"
   docker stop remote-buildkitd && docker rm remote-buildkitd
-  pgrep -f "${PORT}:localhost:${PORT}" | xargs kill
+  pgrep -f "${SSH_BUILDX_PORT}:localhost:${SSH_BUILDX_PORT}" | xargs kill
 }
 
 main

From fed8b44ac033eab5c5966f314bfd3fd0b129a622 Mon Sep 17 00:00:00 2001
From: Henry Lindeman <hmlindeman@yahoo.com>
Date: Fri, 10 May 2024 16:51:24 -0700
Subject: [PATCH 23/24] run remote tests asynchronously

Signed-off-by: Henry Lindeman <hmlindeman@yahoo.com>
---
 apps/integration/integration/automation/integrate | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/integration/integration/automation/integrate b/apps/integration/integration/automation/integrate
index f7be97c43..fd602166f 100755
--- a/apps/integration/integration/automation/integrate
+++ b/apps/integration/integration/automation/integrate
@@ -159,7 +159,7 @@ push_images() {
 
 runtests() {
   if [[ -n $SSH_TARGET ]]; then
-    ssh "${SSH_TARGET}" "cd sycamore && ./apps/integration/integration/automation/integrate --test --clean --tag ${TAG}"
+    ssh "${SSH_TARGET}" "cd sycamore && ./apps/integration/integration/automation/integrate --test --clean --tag ${TAG}" &
   fi
   docker volume rm sycamore_crawl_data sycamore_jupyter_data sycamore_opensearch_data
   docker network prune

From a9eba5c4fba5c68f716deb38f194137f5bfa154c Mon Sep 17 00:00:00 2001
From: Henry Lindeman <hmlindeman@yahoo.com>
Date: Fri, 10 May 2024 17:28:07 -0700
Subject: [PATCH 24/24] dont prompt about whether to prune networks, just prune
 them

Signed-off-by: Henry Lindeman <hmlindeman@yahoo.com>
---
 apps/integration/integration/automation/integrate | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/apps/integration/integration/automation/integrate b/apps/integration/integration/automation/integrate
index 34596862b..6b96d2aec 100755
--- a/apps/integration/integration/automation/integrate
+++ b/apps/integration/integration/automation/integrate
@@ -168,7 +168,7 @@ runtests() {
     ssh "${SSH_TARGET}" "cd sycamore && ./apps/integration/integration/automation/integrate --test --clean --tag ${TAG}" &
   fi
   docker volume rm sycamore_crawl_data sycamore_jupyter_data sycamore_opensearch_data
-  docker network prune
+  docker network prune -f
   docker compose up reset
   poetry run pytest apps/integration/ -p integration.conftest --noconftest --docker-tag "${TAG}"
   # this is a complicated command, so: