[docker] Add script for mirroring dockerhub images

build.yaml

@@ -2684,14 +2684,14 @@ steps:
       - create_ci_test_repo
   - kind: buildImage2
     name: hailgenetics_hail_image
-    dockerFile: /io/docker/hail/Dockerfile
-    contextPath: /io/docker/hail/
+    dockerFile: /io/docker/hailgenetics/hail/Dockerfile
+    contextPath: /io/docker/hailgenetics/hail
     publishAs: hailgenetics/hail
     inputs:
-      - from: /repo/docker/hail
-        to: /io/docker/hail
+      - from: /repo/docker/hailgenetics/hail
+        to: /io/docker/hailgenetics/hail
       - from: /just-wheel/wheel-container.tar
-        to: /io/docker/hail/wheel-container.tar
+        to: /io/docker/hailgenetics/hail/wheel-container.tar
     dependsOn:
       - merge_code
       - build_hail_jar_and_wheel_only

docker/copy_image.sh

@@ -0,0 +1,13 @@
+if command -v skopeo
+then
+    copy_image() {
+        skopeo copy --override-os linux --override-arch amd64 docker://docker.io/$1 docker://$2
+    }
+else
+    echo Could not find skopeo, falling back to docker which will be slower.
+    copy_image() {
+        docker pull $1
+        docker tag $1 $2
+        docker push $2
+    }
+fi

docker/hailgenetics/Makefile

@@ -0,0 +1,11 @@
+include ../../config.mk
+
+.PHONY: publish-python-dill
+
+publish-python-dill:
+	DOCKER_PREFIX=$(DOCKER_PREFIX) bash python-dill/push.sh
+
+mirror-dockerhub-images:
+	DOCKER_PREFIX=$(DOCKER_PREFIX) \
+	HAIL_PIP_VERSION=$(shell cat ../../hail/python/hail/hail_pip_version) \
+	./mirror_images.sh

docker/hailgenetics/mirror_images.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+set -ex
+
+source ../copy_image.sh
+
+if [[ -z "${DOCKER_PREFIX}" ]];
+then
+    echo "Env variable DOCKER_PREFIX must be set"
+    exit 1
+fi
+
+if [[ -z "${HAIL_PIP_VERSION}" ]];
+then
+    echo "Env variable HAIL_PIP_VERSION must be set"
+    exit 1
+fi
+
+python_dill_images=(
+    "python-dill:3.7"
+    "python-dill:3.7-slim"
+    "python-dill:3.8"
+    "python-dill:3.8-slim"
+    "python-dill:3.9"
+    "python-dill:3.9-slim"
+    "python-dill:3.10"
+    "python-dill:3.10-slim"
+)
+
+for image in "${python_dill_images[@]}"
+do
+    copy_image "hailgenetics/${image}" "${DOCKER_PREFIX}/hailgenetics/${image}"
+done
+
+pip_release_images=(
+    "hail:${HAIL_PIP_VERSION}"
+    "genetics:${HAIL_PIP_VERSION}"
+)
+for image in "${pip_release_images[@]}"
+do
+    copy_image "hailgenetics/${image}" "${DOCKER_PREFIX}/hailgenetics/${image}"
+done

docker/python-dill/Dockerfile → docker/hailgenetics/python-dill/Dockerfile

@@ -1,4 +1,5 @@
-FROM python:@PYTHON_VERSION@
+ARG PYTHON_VERSION
+FROM python:${PYTHON_VERSION}
 RUN pip install --upgrade --no-cache-dir dill numpy scipy sklearn && \
     python3 -m pip check && \
     apt-get update && \

docker/hailgenetics/python-dill/push.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -ex
+
+for version in 3.7 3.7-slim 3.8 3.8-slim 3.9 3.9-slim 3.10 3.10-slim
+do
+    public=hailgenetics/python-dill:$version
+
+    DOCKER_BUILDKIT=1 docker build \
+        --build-arg PYTHON_VERSION=$version \
+        --file Dockerfile.out \
+        --build-arg BUILDKIT_INLINE_CACHE=1 \
+        --tag ${public} \
+        .
+
+    time DOCKER_BUILDKIT=1 docker push ${public}
+done

docker/third-party/copy_images.sh

@@ -2,6 +2,8 @@
 
 set -ex
 
+source ../copy_image.sh
+
 images=$(cat images.txt)
 
 if [ -z "${DOCKER_PREFIX}" ]
@@ -10,21 +12,6 @@ then
     exit 1
 fi
 
-if command -v skopeo
-then
-    copy_image() {
-        skopeo copy --override-os linux --override-arch amd64 docker://docker.io/$1 docker://$2
-    }
-else
-    echo Could not find skopeo, falling back to docker which will be slower.
-    copy_image() {
-        docker pull $1
-        docker tag $1 $2
-        docker push $2
-    }
-fi
-
-
 for image in ${images}
 do
     dest="${DOCKER_PREFIX}/${image}"

infra/bootstrap_utils.sh

@@ -10,12 +10,10 @@ fi
 source $HAIL/devbin/functions.sh
 
 copy_images() {
-    cd $HAIL/docker/third-party
-    DOCKER_PREFIX=$(get_global_config_field docker_prefix)
-    DOCKER_PREFIX=$DOCKER_PREFIX ./copy_images.sh
-    cd -
+    make -C $HAIL/docker/third-party copy
 
-    make -C $HAIL/docker/python-dill push DOCKER_PREFIX=$DOCKER_PREFIX
+    make -C $HAIL/hail python/hail/hail_pip_version
+    make -C $HAIL/docker/hailgenetics mirror-dockerhub-images
 }
 
 generate_ssl_certs() {