[docker] Add config support for docker prefix and root image (#10347)

* Switch to Artifact Registry (#29)

* config.mk: use AR docker image

* build.yaml: use AR for images

* Gitignore artifacts

* fetch-image.sh: gcloud auth for AR

* image-fetcher dockerfile: base on AR

* image fetcher make file: use AR

* build-batch-worker-image-startup.sh: add gcloud auth configure-docker for AR, user AR for images

* batch worker: ue AR for public_gcr_images

* Worker dockerfile: use AR

* ci build.py: add gcloud auth configure-docker for AR, use AR base image

* Makefiles: use AR docker images

* stat-permissions.sh: fix for macOS gnu-grep

* third-party dockers: use AR

* ci dockerFile create_namespace: use AR image

* Docker hail-ubuntu: use AR image

* ci build.py: add region into the jinja render config for Dockerfiles

* CI deploy config: add region

* Python-dill: use AR image

* Auth deployment.yaml: add REGION

* Use hail-dvc default ci image pointer

* Add Artifact Registry in main.tf

* Add AR in dev-docs

* Add AR for ukbb-rg

* Add AR in more makefiles

* Fix AR usage for image-fetcher

* Specify "region" in the "hail-ubuntu-stmp" target.

* Fix zone assignment in startup script.

* Allow ci to read from Artifact Registry.

* Makefiles: parametrise `DOCKER_ROOT_IMAGE` to support both GCR and AR

* Parametrise `DOCKER_PREFIX` to support both GCR and AR

* Un-gitignore "vlad"

* Fix

* batch: pass DOCKER_PREFIX to worker

* DOCKER_PREFIX in build-batch-worker-image-startup.sh

* build.yaml: add ubuntu tags back

* ci/ci/build.py: parametrise docker prefix

* infra README: configure-docker AR

Co-authored-by: Leonhard Gruenschloss <leonhard.gruenschloss@populationgenomics.org.au>

* Fix building worker image (#40)

* Add Artifact Registry permissions for the batch-agent service account.

* Gitignore */hail.jar and *.dylib

* Comment in gitignore on dylib

* Default use_artifact_registry to false

* Use DOCKER_ROOT_IMAGE to pass to batch worker image

* Use DOCKER_ROOT_IMAGE wherever makes sense

* Render build-batch-worker-image-startup.sh with Jinja to set up docker root image

* Fix jinja params

* Add phony rule to configure gcloud and Docker

* Add comments on HAIL_DOCKER_PREFIX

* Make sure the env variable HAIL_DOCKER_PREFIX available to the batch worker instance

* Add missing docker_root_image

* Infra README: correct env var name

* Fix env var case

* remove other instances of gcr.io in favor of DOCKER_PREFIX

* add hail_version to gitignore once and for all

* debug

* add back yaml changes after rebase

* remove gcloud target from hail/Makefile

Co-authored-by: Vlad Savelyev <vladislav.sav@gmail.com>
Co-authored-by: Leonhard Gruenschloss <leonhard.gruenschloss@populationgenomics.org.au>
Co-authored-by: vladsaveliev <vladislav.savelyev@populationgenomics.org.au>

.gitignore

@@ -26,6 +26,10 @@ GRTAGS
 GTAGS
 hail/python/hail/docs/experimental/hail.experimental.DB.rst
 hail/python/hailtop/batch/docs/api/
+**/hail_version
 web_common/web_common/static/css/
 website/docs.tar.gz
 website/website/static/css/
+*.dylib  # macOS dynamic libraries
+*/hail.jar
+infra/.terraform.lock.hcl

address/Makefile

@@ -1,7 +1,7 @@
 include ../config.mk
 
-ADDRESS_LATEST = gcr.io/$(PROJECT)/address:latest
-ADDRESS_IMAGE = gcr.io/$(PROJECT)/address:$(shell docker images -q --no-trunc address | sed -e 's,[^:]*:,,')
+ADDRESS_LATEST = $(DOCKER_PREFIX)/address:latest
+ADDRESS_IMAGE = $(DOCKER_PREFIX)/address:$(shell docker images -q --no-trunc address | sed -e 's,[^:]*:,,')
 
 PYTHONPATH := $${PYTHONPATH:+$${PYTHONPATH}:}../hail/python:../gear:../web_common
 PYTHON := PYTHONPATH=$(PYTHONPATH) python3

admin-pod/Makefile

@@ -1,6 +1,6 @@
 include ../config.mk
 
-SERVICE_BASE_IMAGE = gcr.io/$(PROJECT)/service-base:$(shell docker images -q --no-trunc service-base:latest | sed -e 's,[^:]*:,,')
+SERVICE_BASE_IMAGE = $(DOCKER_PREFIX)/service-base:$(shell docker images -q --no-trunc service-base:latest | sed -e 's,[^:]*:,,')
 
 .PHONY: deploy
 deploy:

auth/Makefile

@@ -1,7 +1,7 @@
 include ../config.mk
 
-AUTH_LATEST = gcr.io/$(PROJECT)/auth:latest
-AUTH_IMAGE = gcr.io/$(PROJECT)/auth:$(shell docker images -q --no-trunc auth:latest | sed -e 's,[^:]*:,,')
+AUTH_LATEST = $(DOCKER_PREFIX)/auth:latest
+AUTH_IMAGE = $(DOCKER_PREFIX)/auth:$(shell docker images -q --no-trunc auth:latest | sed -e 's,[^:]*:,,')
 
 EXTRA_PYTHONPATH := ../hail/python:../gear:../web_common
 PYTHON := PYTHONPATH=$${PYTHONPATH:+$${PYTHONPATH}:}$(EXTRA_PYTHONPATH) python3

batch/Dockerfile.worker

@@ -1,4 +1,4 @@
-FROM gcr.io/{{ global.project }}/python:3.7-slim-stretch
+FROM {{ global.docker_prefix }}/python:3.7-slim-stretch
 
 COPY docker/hail-ubuntu/retry /bin/retry
 COPY docker/hail-ubuntu/hail-apt-get-install /bin/hail-apt-get-install

batch/Makefile

@@ -1,10 +1,10 @@
 include ../config.mk
 
-BATCH_LATEST = gcr.io/$(PROJECT)/batch:latest
-BATCH_IMAGE = gcr.io/$(PROJECT)/batch:$(shell docker images -q --no-trunc batch | sed -e 's,[^:]*:,,')
+BATCH_LATEST = $(DOCKER_PREFIX)/batch:latest
+BATCH_IMAGE = $(DOCKER_PREFIX)/batch:$(shell docker images -q --no-trunc batch | sed -e 's,[^:]*:,,')
 
-BATCH_WORKER_LATEST = gcr.io/$(PROJECT)/batch-worker:latest
-BATCH_WORKER_IMAGE = gcr.io/$(PROJECT)/batch-worker:$(shell docker images -q --no-trunc batch-worker | sed -e 's,[^:]*:,,')
+BATCH_WORKER_LATEST = $(DOCKER_PREFIX)/batch-worker:latest
+BATCH_WORKER_IMAGE = $(DOCKER_PREFIX)/batch-worker:$(shell docker images -q --no-trunc batch-worker | sed -e 's,[^:]*:,,')
 
 EXTRA_PYTHONPATH := ../hail/python:../gear:../web_common
 PYTHON := PYTHONPATH=$${PYTHONPATH:+$${PYTHONPATH}:}$(EXTRA_PYTHONPATH) python3
@@ -32,7 +32,7 @@ build-worker: build-prereqs
 # janky
 	cp ../hail/build/libs/hail-all-spark.jar ./hail.jar
 	-docker pull $(BATCH_WORKER_LATEST)
-	python3 ../ci/jinja2_render.py '{"global":{"project":"$(PROJECT)"}, "service_base_image":{"image":"service-base"}}' Dockerfile.worker Dockerfile.worker.out
+	python3 ../ci/jinja2_render.py '{"global":{"docker_prefix":"$(DOCKER_PREFIX)"}}' Dockerfile.worker Dockerfile.worker.out
 	docker build -t batch-worker -f Dockerfile.worker.out --cache-from batch-worker,$(BATCH_WORKER_LATEST),service-base ..
 
 .PHONY: build
@@ -49,7 +49,7 @@ push: build
 	docker tag batch-worker $(BATCH_WORKER_IMAGE)
 	docker push $(BATCH_WORKER_IMAGE)
 
-JINJA_ENVIRONMENT = '{"code":{"sha":"$(shell git rev-parse --short=12 HEAD)"},"deploy":$(DEPLOY),"batch_image":{"image":"$(BATCH_IMAGE)"},"batch_worker_image":{"image":"$(BATCH_WORKER_IMAGE)"},"default_ns":{"name":"$(NAMESPACE)"},"batch_database":{"user_secret_name":"sql-batch-user-config"},"global":{"project":"$(PROJECT)","domain":"$(DOMAIN)","k8s_server_url":"$(KUBERNETES_SERVER_URL)"},"scope":"$(SCOPE)"}'
+JINJA_ENVIRONMENT = '{"code":{"sha":"$(shell git rev-parse --short=12 HEAD)"},"deploy":$(DEPLOY),"batch_image":{"image":"$(BATCH_IMAGE)"},"batch_worker_image":{"image":"$(BATCH_WORKER_IMAGE)"},"default_ns":{"name":"$(NAMESPACE)"},"batch_database":{"user_secret_name":"sql-batch-user-config"},"global":{"project":"$(PROJECT)","domain":"$(DOMAIN)","k8s_server_url":"$(KUBERNETES_SERVER_URL)","docker_prefix":"$(DOCKER_PREFIX)","docker_root_image":"$(DOCKER_ROOT_IMAGE)"},"scope":"$(SCOPE)"}'
 
 .PHONY: deploy
 deploy: push
@@ -63,7 +63,8 @@ deploy: push
 .PHONY: create-build-worker-image-instance
 create-build-worker-image-instance:
 	-gcloud -q compute --project $(PROJECT) instances delete --zone=$(ZONE) build-batch-worker-image
-	gcloud -q compute --project $(PROJECT) instances create --zone=$(ZONE) build-batch-worker-image --machine-type=n1-standard-1 --network=default --network-tier=PREMIUM --metadata-from-file startup-script=build-batch-worker-image-startup.sh --no-restart-on-failure --maintenance-policy=MIGRATE --scopes=https://www.googleapis.com/auth/cloud-platform --image=$$(gcloud compute images list --standard-images --filter 'family="ubuntu-minimal-2004-lts"' --format='value(name)') --image-project=ubuntu-os-cloud --boot-disk-size=10GB --boot-disk-type=pd-ssd
+	python3 ../ci/jinja2_render.py '{"global":{"docker_root_image":"$(DOCKER_ROOT_IMAGE)"}}' build-batch-worker-image-startup.sh build-batch-worker-image-startup.sh.out
+	gcloud -q compute --project $(PROJECT) instances create --zone=$(ZONE) build-batch-worker-image --machine-type=n1-standard-1 --network=default --network-tier=PREMIUM --metadata-from-file startup-script=build-batch-worker-image-startup.sh.out --no-restart-on-failure --maintenance-policy=MIGRATE --scopes=https://www.googleapis.com/auth/cloud-platform --image=$$(gcloud compute images list --standard-images --filter 'family="ubuntu-minimal-2004-lts"' --format='value(name)') --image-project=ubuntu-os-cloud --boot-disk-size=10GB --boot-disk-type=pd-ssd
 
 .PHONY: create-worker-image
 create-worker-image:

batch/batch/batch_configuration.py

@@ -8,6 +8,8 @@
 
 GCP_REGION = os.environ['HAIL_GCP_REGION']
 GCP_ZONE = os.environ['HAIL_GCP_ZONE']
+DOCKER_ROOT_IMAGE = os.environ['HAIL_DOCKER_ROOT_IMAGE']
+DOCKER_PREFIX = os.environ['HAIL_DOCKER_PREFIX']
 
 BATCH_GCP_REGIONS = set(json.loads(os.environ['HAIL_BATCH_GCP_REGIONS']))
 BATCH_GCP_REGIONS.add(GCP_REGION)

batch/batch/driver/create_instance.py

@@ -5,7 +5,7 @@
 
 from hailtop import aiogoogle
 
-from ..batch_configuration import PROJECT, DEFAULT_NAMESPACE
+from ..batch_configuration import PROJECT, DOCKER_ROOT_IMAGE, DOCKER_PREFIX, DEFAULT_NAMESPACE
 from ..inst_coll_config import machine_type_to_dict
 from ..worker_config import WorkerConfig
 from ..log_store import LogStore
@@ -186,6 +186,8 @@ async def create_instance(app, zone, machine_name, machine_type, activation_toke
 ZONE=$(curl -s http://metadata.google.internal/computeMetadata/v1/instance/zone -H 'Metadata-Flavor: Google')
 
 BATCH_WORKER_IMAGE=$(curl -s -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/attributes/batch_worker_image")
+DOCKER_ROOT_IMAGE=$(curl -s -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/attributes/docker_root_image")
+DOCKER_PREFIX=$(curl -s -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/attributes/docker_prefix")
 
 # Setup fluentd
 touch /worker.log
@@ -263,6 +265,9 @@ async def create_instance(app, zone, machine_name, machine_type, activation_toke
 -e INSTANCE_ID=$INSTANCE_ID \
 -e PROJECT=$PROJECT \
 -e ZONE=$ZONE \
+-e DOCKER_PREFIX=$DOCKER_PREFIX \
+-e DOCKER_ROOT_IMAGE=$DOCKER_ROOT_IMAGE \
+-e DOCKER_PREFIX=$DOCKER_PREFIX \
 -e WORKER_CONFIG=$WORKER_CONFIG \
 -e MAX_IDLE_TIME_MSECS=$MAX_IDLE_TIME_MSECS \
 -e WORKER_DATA_DISK_MOUNT=/mnt/disks/$WORKER_DATA_DISK_NAME \
@@ -311,6 +316,12 @@ async def create_instance(app, zone, machine_name, machine_type, activation_toke
             }, {
                 'key': 'batch_worker_image',
                 'value': BATCH_WORKER_IMAGE
+            }, {
+                'key': 'docker_root_image',
+                'value': DOCKER_ROOT_IMAGE
+            }, {
+                'key': 'docker_prefix',
+                'value': DOCKER_PREFIX
             }, {
                 'key': 'namespace',
                 'value': DEFAULT_NAMESPACE

batch/batch/public_gcr_images.py

@@ -1,7 +1,7 @@
 from typing import List
 
 
-def public_gcr_images(project: str) -> List[str]:
+def public_gcr_images(docker_prefix: str) -> List[str]:
     # the worker cannot import batch_configuration because it does not have all the environment
     # variables
-    return [f'gcr.io/{project}/{name}' for name in ('query', 'hail', 'python-dill', 'batch-worker')]
+    return [f'{docker_prefix}/{name}' for name in ('query', 'hail', 'python-dill', 'batch-worker')]

batch/batch/worker/worker.py

@@ -68,6 +68,7 @@
 PROJECT = os.environ['PROJECT']
 ZONE = os.environ['ZONE'].rsplit('/', 1)[1]
 PUBLIC_GCR_IMAGES = public_gcr_images(PROJECT)
+DOCKER_PREFIX = os.environ['DOCKER_PREFIX']
 WORKER_CONFIG = json.loads(base64.b64decode(os.environ['WORKER_CONFIG']).decode())
 MAX_IDLE_TIME_MSECS = int(os.environ['MAX_IDLE_TIME_MSECS'])
 WORKER_DATA_DISK_MOUNT = os.environ['WORKER_DATA_DISK_MOUNT']
@@ -84,6 +85,7 @@
 log.info(f'INSTANCE_ID {INSTANCE_ID}')
 log.info(f'PROJECT {PROJECT}')
 log.info(f'ZONE {ZONE}')
+log.info(f'DOCKER_PREFIX {DOCKER_PREFIX}')
 log.info(f'WORKER_CONFIG {WORKER_CONFIG}')
 log.info(f'MAX_IDLE_TIME_MSECS {MAX_IDLE_TIME_MSECS}')
 log.info(f'WORKER_DATA_DISK_MOUNT {WORKER_DATA_DISK_MOUNT}')
@@ -166,7 +168,7 @@ async def handle_error(e):
                     if eget.status == 404:
                         await handle_error(eget)
                         continue
-            # No such image: gcr.io/...
+            # No such image: {DOCKER_PREFIX}/...
             if e.status == 404 and 'No such image' in e.message:
                 await handle_error(e)
                 continue
@@ -270,7 +272,7 @@ def __init__(self, job, name, spec):
             image_ref.tag = 'latest'
 
         if image_ref.name() in HAIL_GENETICS_IMAGES:
-            image_ref.domain = 'gcr.io'
+            image_ref.domain = DOCKER_PREFIX.split('/', maxsplit=1)[0]
             image_ref.path = f'{PROJECT}/{image_ref.name()}'
 
         self.image_ref = image_ref

batch/build-batch-worker-image-startup.sh

@@ -29,7 +29,7 @@ rm -rf /var/lib/apt/lists/*
 
 [ -f /etc/docker/daemon.json ] || echo "{}" > /etc/docker/daemon.json
 
-VERSION=1.5.0
+VERSION=2.0.4
 OS=linux
 ARCH=amd64
 
@@ -39,10 +39,9 @@ curl -fsSL "https://github.com/GoogleCloudPlatform/docker-credential-gcr/release
 
 # avoid "unable to get current user home directory: os/user lookup failed"
 export HOME=/root
-docker-credential-gcr configure-docker
 
-GCP_PROJECT=$(curl -s -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/project/project-id")
-docker pull gcr.io/$GCP_PROJECT/ubuntu:18.04
+docker-credential-gcr configure-docker --include-artifact-registry
+docker pull {{ global.docker_root_image }}
 
 # add docker daemon debug logging
 jq '.debug = true' /etc/docker/daemon.json > daemon.json.tmp

batch/deployment.yaml

@@ -59,6 +59,10 @@ spec:
            value: "{{ default_ns.name }}"
          - name: PROJECT
            value: "{{ global.project }}"
+         - name: HAIL_DOCKER_ROOT_IMAGE
+           value: "{{ global.docker_root_image }}"
+         - name: HAIL_DOCKER_PREFIX
+           value: "{{ global.docker_prefix }}"
          - name: HAIL_GCP_REGION
            valueFrom:
              secretKeyRef:
@@ -195,6 +199,10 @@ spec:
            value: "{{ default_ns.name }}"
          - name: PROJECT
            value: "{{ global.project }}"
+         - name: HAIL_DOCKER_ROOT_IMAGE
+           value: "{{ global.docker_root_image }}"
+         - name: HAIL_DOCKER_PREFIX
+           value: "{{ global.docker_prefix }}"
          - name: HAIL_GCP_REGION
            valueFrom:
              secretKeyRef:

benchmark-service/Makefile

@@ -1,7 +1,7 @@
 include ../config.mk
 
-BENCHMARK_LATEST = gcr.io/$(PROJECT)/benchmark:latest
-BENCHMARK_IMAGE = gcr.io/$(PROJECT)/benchmark:$(shell docker images -q --no-trunc benchmark:latest | sed -e 's,[^:]*:,,')
+BENCHMARK_LATEST = $(DOCKER_PREFIX)/benchmark:latest
+BENCHMARK_IMAGE = $(DOCKER_PREFIX)/benchmark:$(shell docker images -q --no-trunc benchmark:latest | sed -e 's,[^:]*:,,')
 
 EXTRA_PYTHONPATH := ../hail/python:../gear:../web_common
 PYTHON := PYTHONPATH=$${PYTHONPATH:+$${PYTHONPATH}:}$(EXTRA_PYTHONPATH) python3

benchmark/Dockerfile

@@ -1,4 +1,4 @@
-FROM gcr.io/{{ global.project }}/ubuntu:18.04
+FROM {{ global.docker_root_image }}
 
 ENV LANG C.UTF-8
 

benchmark/Makefile

@@ -30,7 +30,7 @@ cleanup_image:
 
 PROJECT := hail-vdc
 BENCHMARK_DOCKER_TAG := benchmark_$(shell whoami)
-BENCHMARK_REPO_BASE = gcr.io/$(PROJECT)/$(BENCHMARK_DOCKER_TAG)
+BENCHMARK_REPO_BASE = $(DOCKER_PREFIX)/$(BENCHMARK_DOCKER_TAG)
 
 ifndef HAIL_WHEEL
 image_sha:
@@ -42,7 +42,7 @@ image_sha: wheel cleanup_image
 	cp $(BENCHMARK_WHEEL) .
 	# it's possible that the HAIL_WHEEL installs different dependencies, but this generally creates less work for docker
 	cp ../hail/python/requirements.txt .
-	python3 ../ci/jinja2_render.py '{"global":{"project":"$(PROJECT)"}}' Dockerfile Dockerfile.out
+	python3 ../ci/jinja2_render.py '{"global":{"docker_root_image":"$(DOCKER_ROOT_IMAGE)"}' Dockerfile Dockerfile.out
 	docker build -f Dockerfile.out -t $(BENCHMARK_DOCKER_TAG) . --build-arg HAIL_WHEEL=$(notdir $(HAIL_WHEEL)) --build-arg BENCHMARK_WHEEL=$(notdir $(BENCHMARK_WHEEL))
 	@printf $$(docker images -q --no-trunc $(BENCHMARK_DOCKER_TAG) | sed -e 's,[^:]*:,,') > image_sha
 	@echo Image sha is `cat image_sha`
@@ -72,4 +72,3 @@ clean:
 	rm -f $(notdir $(HAIL_WHEEL))
 	rm -f $(notdir $(BENCHMARK_WHEEL))
 	rm -f requirements.txt
-

bootstrap-gateway/Makefile

@@ -2,8 +2,8 @@ include ../config.mk
 
 .PHONY: build push deploy
 
-GATEWAY_LATEST = gcr.io/$(PROJECT)/gateway:latest
-GATEWAY_IMAGE = gcr.io/$(PROJECT)/gateway:$(shell docker images -q --no-trunc gateway | sed -e 's,[^:]*:,,')
+GATEWAY_LATEST = $(DOCKER_PREFIX)/gateway:latest
+GATEWAY_IMAGE = $(DOCKER_PREFIX)/gateway:$(shell docker images -q --no-trunc gateway | sed -e 's,[^:]*:,,')
 
 build:
 	$(MAKE) -C ../docker hail-ubuntu

build.yaml

@@ -2435,7 +2435,8 @@ steps:
       export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
       export HAIL_NETCAT_UBUNTU_IMAGE={{ netcat_ubuntu_image.image }}
       export HAIL_HAIL_BASE_IMAGE={{ hail_base_image.image }}
-      export DOCKER_ROOT_IMAGE="gcr.io/{{ global.project }}/ubuntu:18.04"
+      export DOCKER_PREFIX="{{ global.docker_prefix }}"
+      export DOCKER_ROOT_IMAGE="{{ global.docker_root_image }}"
       export HAIL_TEST_TOKEN_FILE=/user-tokens/tokens.json
       export HAIL_TEST_DEV_TOKEN_FILE=/dev-tokens/tokens.json
       export HAIL_TOKEN="{{ token }}"
@@ -2504,7 +2505,8 @@ steps:
       export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
       export HAIL_NETCAT_UBUNTU_IMAGE={{ netcat_ubuntu_image.image }}
       export HAIL_HAIL_BASE_IMAGE={{ hail_base_image.image }}
-      export DOCKER_ROOT_IMAGE="gcr.io/{{ global.project }}/ubuntu:18.04"
+      export DOCKER_PREFIX="{{ global.docker_prefix }}"
+      export DOCKER_ROOT_IMAGE="{{ global.docker_root_image }}"
       export HAIL_TEST_TOKEN_FILE=/user-tokens/tokens.json
       export HAIL_TEST_DEV_TOKEN_FILE=/dev-tokens/tokens.json
       export HAIL_TOKEN="{{ token }}"
@@ -2573,7 +2575,8 @@ steps:
       export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
       export HAIL_NETCAT_UBUNTU_IMAGE={{ netcat_ubuntu_image.image }}
       export HAIL_HAIL_BASE_IMAGE={{ hail_base_image.image }}
-      export DOCKER_ROOT_IMAGE="gcr.io/{{ global.project }}/ubuntu:18.04"
+      export DOCKER_PREFIX="{{ global.docker_prefix }}"
+      export DOCKER_ROOT_IMAGE="{{ global.docker_root_image }}"
       export HAIL_TEST_TOKEN_FILE=/user-tokens/tokens.json
       export HAIL_TEST_DEV_TOKEN_FILE=/dev-tokens/tokens.json
       export HAIL_TOKEN="{{ token }}"
@@ -2642,7 +2645,8 @@ steps:
       export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
       export HAIL_NETCAT_UBUNTU_IMAGE={{ netcat_ubuntu_image.image }}
       export HAIL_HAIL_BASE_IMAGE={{ hail_base_image.image }}
-      export DOCKER_ROOT_IMAGE="gcr.io/{{ global.project }}/ubuntu:18.04"
+      export DOCKER_PREFIX="{{ global.docker_prefix }}"
+      export DOCKER_ROOT_IMAGE="{{ global.docker_root_image }}"
       export HAIL_TEST_TOKEN_FILE=/user-tokens/tokens.json
       export HAIL_TEST_DEV_TOKEN_FILE=/dev-tokens/tokens.json
       export HAIL_TOKEN="{{ token }}"
@@ -2711,7 +2715,8 @@ steps:
       export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
       export HAIL_NETCAT_UBUNTU_IMAGE={{ netcat_ubuntu_image.image }}
       export HAIL_HAIL_BASE_IMAGE={{ hail_base_image.image }}
-      export DOCKER_ROOT_IMAGE="gcr.io/{{ global.project }}/ubuntu:18.04"
+      export DOCKER_PREFIX="{{ global.docker_prefix }}"
+      export DOCKER_ROOT_IMAGE="{{ global.docker_root_image }}"
       export HAIL_TEST_TOKEN_FILE=/user-tokens/tokens.json
       export HAIL_TEST_DEV_TOKEN_FILE=/dev-tokens/tokens.json
       export HAIL_TOKEN="{{ token }}"
@@ -2933,8 +2938,9 @@ steps:
       export HAIL_GSA_KEY_FILE=/test-gsa-key/key.json
       export PYTEST_SPLITS=5
       export PYTEST_SPLIT_INDEX=0
-      export DOCKER_ROOT_IMAGE="gcr.io/{{ global.project }}/ubuntu:18.04"
-      export PYTHON_DILL_IMAGE="gcr.io/{{ global.project }}/python-dill:3.7-slim"
+      export DOCKER_PREFIX="{{ global.docker_prefix }}"
+      export DOCKER_ROOT_IMAGE="{{ global.docker_root_image }}"
+      export PYTHON_DILL_IMAGE="{{ global.docker_prefix }}/python-dill:3.7-slim"
       hailctl config set batch/billing_project test
       hailctl config set batch/bucket hail-test-dmk9z
       python3 -m pytest \
@@ -2984,8 +2990,9 @@ steps:
       export HAIL_GSA_KEY_FILE=/test-gsa-key/key.json
       export PYTEST_SPLITS=5
       export PYTEST_SPLIT_INDEX=1
-      export DOCKER_ROOT_IMAGE="gcr.io/{{ global.project }}/ubuntu:18.04"
-      export PYTHON_DILL_IMAGE="gcr.io/{{ global.project }}/python-dill:3.7-slim"
+      export DOCKER_PREFIX="{{ global.docker_prefix }}"
+      export DOCKER_ROOT_IMAGE="{{ global.docker_root_image }}"
+      export PYTHON_DILL_IMAGE="{{ global.docker_prefix }}/python-dill:3.7-slim"
       hailctl config set batch/billing_project test
       hailctl config set batch/bucket hail-test-dmk9z
       python3 -m pytest \
@@ -3035,8 +3042,9 @@ steps:
       export HAIL_GSA_KEY_FILE=/test-gsa-key/key.json
       export PYTEST_SPLITS=5
       export PYTEST_SPLIT_INDEX=2
-      export DOCKER_ROOT_IMAGE="gcr.io/{{ global.project }}/ubuntu:18.04"
-      export PYTHON_DILL_IMAGE="gcr.io/{{ global.project }}/python-dill:3.7-slim"
+      export DOCKER_PREFIX="{{ global.docker_prefix }}"
+      export DOCKER_ROOT_IMAGE="{{ global.docker_root_image }}"
+      export PYTHON_DILL_IMAGE="{{ global.docker_prefix }}/python-dill:3.7-slim"
       hailctl config set batch/billing_project test
       hailctl config set batch/bucket hail-test-dmk9z
       python3 -m pytest \
@@ -3086,8 +3094,9 @@ steps:
       export HAIL_GSA_KEY_FILE=/test-gsa-key/key.json
       export PYTEST_SPLITS=5
       export PYTEST_SPLIT_INDEX=3
-      export DOCKER_ROOT_IMAGE="gcr.io/{{ global.project }}/ubuntu:18.04"
-      export PYTHON_DILL_IMAGE="gcr.io/{{ global.project }}/python-dill:3.7-slim"
+      export DOCKER_PREFIX="{{ global.docker_prefix }}"
+      export DOCKER_ROOT_IMAGE="{{ global.docker_root_image }}"
+      export PYTHON_DILL_IMAGE="{{ global.docker_prefix }}/python-dill:3.7-slim"
       hailctl config set batch/billing_project test
       hailctl config set batch/bucket hail-test-dmk9z
       python3 -m pytest \
@@ -3137,8 +3146,9 @@ steps:
       export HAIL_GSA_KEY_FILE=/test-gsa-key/key.json
       export PYTEST_SPLITS=5
       export PYTEST_SPLIT_INDEX=4
-      export DOCKER_ROOT_IMAGE="gcr.io/{{ global.project }}/ubuntu:18.04"
-      export PYTHON_DILL_IMAGE="gcr.io/{{ global.project }}/python-dill:3.7-slim"
+      export DOCKER_PREFIX="{{ global.docker_prefix }}"
+      export DOCKER_ROOT_IMAGE="{{ global.docker_root_image }}"
+      export PYTHON_DILL_IMAGE="{{ global.docker_prefix }}/python-dill:3.7-slim"
       hailctl config set batch/billing_project test
       hailctl config set batch/bucket hail-test-dmk9z
       python3 -m pytest \