Merge pull request #53 from populationgenomics/ar-changes

Artifact Registry: address upstream comments
daniel-goldstein · Feb 26, 2021 · 32452b0 · 32452b0
2 parents 20d8c90 + 6697465
commit 32452b0
Show file tree

Hide file tree

Showing 14 changed files with 37 additions and 32 deletions.
diff --git a/batch/Makefile b/batch/Makefile
@@ -48,7 +48,7 @@ push: build
 	docker tag batch-worker $(BATCH_WORKER_IMAGE)
 	docker push $(BATCH_WORKER_IMAGE)
 
-JINJA_ENVIRONMENT = '{"code":{"sha":"$(shell git rev-parse --short=12 HEAD)"},"deploy":$(DEPLOY),"batch_image":{"image":"$(BATCH_IMAGE)"},"batch_worker_image":{"image":"$(BATCH_WORKER_IMAGE)"},"default_ns":{"name":"$(NAMESPACE)"},"batch_database":{"user_secret_name":"sql-batch-user-config"},"global":{"project":"$(PROJECT)","domain":"$(DOMAIN)","k8s_server_url":"$(KUBERNETES_SERVER_URL)","docker_prefix":"$(DOCKER_PREFIX)"},"scope":"$(SCOPE)"}'
+JINJA_ENVIRONMENT = '{"code":{"sha":"$(shell git rev-parse --short=12 HEAD)"},"deploy":$(DEPLOY),"batch_image":{"image":"$(BATCH_IMAGE)"},"batch_worker_image":{"image":"$(BATCH_WORKER_IMAGE)"},"default_ns":{"name":"$(NAMESPACE)"},"batch_database":{"user_secret_name":"sql-batch-user-config"},"global":{"project":"$(PROJECT)","domain":"$(DOMAIN)","k8s_server_url":"$(KUBERNETES_SERVER_URL)","docker_prefix":"$(DOCKER_PREFIX)"}},"scope":"$(SCOPE)"}'
 
 .PHONY: deploy
 deploy: push
@@ -62,7 +62,8 @@ deploy: push
 .PHONY: create-build-worker-image-instance
 create-build-worker-image-instance:
 	-gcloud -q compute --project $(PROJECT) instances delete --zone=$(ZONE) build-batch-worker-image
-	gcloud -q compute --project $(PROJECT) instances create --zone=$(ZONE) build-batch-worker-image --machine-type=n1-standard-1 --network=default --network-tier=PREMIUM --metadata-from-file startup-script=build-batch-worker-image-startup.sh --no-restart-on-failure --maintenance-policy=MIGRATE --scopes=https://www.googleapis.com/auth/cloud-platform --image=$$(gcloud compute images list --standard-images --filter 'family="ubuntu-minimal-2004-lts"' --format='value(name)') --image-project=ubuntu-os-cloud --boot-disk-size=10GB --boot-disk-type=pd-ssd
+	python3 ../ci/jinja2_render.py '{"global":{"docker_root_image":"$(DOCKER_ROOT_IMAGE)"}}' build-batch-worker-image-startup.sh build-batch-worker-image-startup.sh.out
+	gcloud -q compute --project $(PROJECT) instances create --zone=$(ZONE) build-batch-worker-image --machine-type=n1-standard-1 --network=default --network-tier=PREMIUM --metadata-from-file startup-script=build-batch-worker-image-startup.sh.out --no-restart-on-failure --maintenance-policy=MIGRATE --scopes=https://www.googleapis.com/auth/cloud-platform --image=$$(gcloud compute images list --standard-images --filter 'family="ubuntu-minimal-2004-lts"' --format='value(name)') --image-project=ubuntu-os-cloud --boot-disk-size=10GB --boot-disk-type=pd-ssd
 
 .PHONY: create-worker-image
 create-worker-image:

diff --git a/batch/batch/batch_configuration.py b/batch/batch/batch_configuration.py
@@ -8,7 +8,7 @@
 
 GCP_REGION = os.environ['HAIL_GCP_REGION']
 GCP_ZONE = os.environ['HAIL_GCP_ZONE']
-DOCKER_PREFIX = os.environ['HAIL_DOCKER_PREFIX']
+DOCKER_ROOT_IMAGE = os.environ['HAIL_DOCKER_ROOT_IMAGE']
 
 BATCH_GCP_REGIONS = set(json.loads(os.environ['HAIL_BATCH_GCP_REGIONS']))
 BATCH_GCP_REGIONS.add(GCP_REGION)

diff --git a/batch/batch/driver/create_instance.py b/batch/batch/driver/create_instance.py
@@ -5,7 +5,7 @@
 
 from hailtop import aiogoogle
 
-from ..batch_configuration import PROJECT, DOCKER_PREFIX, DEFAULT_NAMESPACE
+from ..batch_configuration import PROJECT, DOCKER_ROOT_IMAGE, DEFAULT_NAMESPACE
 from ..worker_config import WorkerConfig
 from ..log_store import LogStore
 
@@ -173,7 +173,7 @@ async def create_instance(app, zone, machine_name, machine_type, activation_toke
 ZONE=$(curl -s http://metadata.google.internal/computeMetadata/v1/instance/zone -H 'Metadata-Flavor: Google')
 
 BATCH_WORKER_IMAGE=$(curl -s -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/attributes/batch_worker_image")
-DOCKER_PREFIX=$(curl -s -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/attributes/docker_prefix")
+DOCKER_ROOT_IMAGE=$(curl -s -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/attributes/docker_root_image")
 
 # Setup fluentd
 touch /worker.log
@@ -250,7 +250,7 @@ async def create_instance(app, zone, machine_name, machine_type, activation_toke
 -e BATCH_LOGS_BUCKET_NAME=$BATCH_LOGS_BUCKET_NAME \
 -e INSTANCE_ID=$INSTANCE_ID \
 -e PROJECT=$PROJECT \
--e DOCKER_PREFIX=$DOCKER_PREFIX \
+-e DOCKER_ROOT_IMAGE=$DOCKER_ROOT_IMAGE \
 -e WORKER_CONFIG=$WORKER_CONFIG \
 -e MAX_IDLE_TIME_MSECS=$MAX_IDLE_TIME_MSECS \
 -e WORKER_DATA_DISK_MOUNT=/mnt/disks/$WORKER_DATA_DISK_NAME \
@@ -295,8 +295,8 @@ async def create_instance(app, zone, machine_name, machine_type, activation_toke
                 'key': 'batch_worker_image',
                 'value': BATCH_WORKER_IMAGE
             }, {
-                'key': 'docker_prefix',
-                'value': DOCKER_PREFIX
+                'key': 'docker_root_image',
+                'value': DOCKER_ROOT_IMAGE
             }, {
                 'key': 'namespace',
                 'value': DEFAULT_NAMESPACE

diff --git a/batch/build-batch-worker-image-startup.sh b/batch/build-batch-worker-image-startup.sh
@@ -40,11 +40,8 @@ curl -fsSL "https://github.com/GoogleCloudPlatform/docker-credential-gcr/release
 # avoid "unable to get current user home directory: os/user lookup failed"
 export HOME=/root
 
-DOCKER_PREFIX=$(curl -s -H "Metadata-Flavor: Google" "http://metadata.google.internal/computeMetadata/v1/instance/attributes/docker_prefix")
-
 docker-credential-gcr configure-docker --include-artifact-registry
-
-docker pull ${DOCKER_PREFIX}/ubuntu:18.04
+docker pull {{ global.docker_root_image }}
 docker pull gcr.io/google.com/cloudsdktool/cloud-sdk:310.0.0-alpine
 
 # add docker daemon debug logging

diff --git a/batch/deployment.yaml b/batch/deployment.yaml
@@ -62,8 +62,8 @@ spec:
            value: "{{ default_ns.name }}"
          - name: PROJECT
            value: "{{ global.project }}"
-         - name: HAIL_DOCKER_PREFIX
-           value: "{{ global.docker_prefix }}"
+         - name: HAIL_DOCKER_ROOT_IMAGE
+           value: "{{ global.docker_root_image }}"
          - name: HAIL_GCP_REGION
            valueFrom:
              secretKeyRef:
@@ -200,8 +200,8 @@ spec:
            value: "{{ default_ns.name }}"
          - name: PROJECT
            value: "{{ global.project }}"
-         - name: HAIL_DOCKER_PREFIX
-           value: "{{ global.docker_prefix }}"
+         - name: HAIL_DOCKER_ROOT_IMAGE
+           value: "{{ global.docker_root_image }}"
          - name: HAIL_GCP_REGION
            valueFrom:
              secretKeyRef:

diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile
@@ -1,4 +1,4 @@
-FROM {{ global.docker_prefix }}/ubuntu:18.04
+FROM {{ global.docker_root_image }}
 
 ENV LANG C.UTF-8
 

diff --git a/benchmark/Makefile b/benchmark/Makefile
@@ -41,7 +41,7 @@ image_sha: wheel cleanup_image
 	cp $(BENCHMARK_WHEEL) .
 	# it's possible that the HAIL_WHEEL installs different dependencies, but this generally creates less work for docker
 	cp ../hail/python/requirements.txt .
-	python3 ../ci/jinja2_render.py '{"global":{"docker_prefix":"$(DOCKER_PREFIX)"}' Dockerfile Dockerfile.out
+	python3 ../ci/jinja2_render.py '{"global":{"docker_root_image":"$(DOCKER_ROOT_IMAGE)"}' Dockerfile Dockerfile.out
 	docker build -f Dockerfile.out -t $(BENCHMARK_DOCKER_TAG) . --build-arg HAIL_WHEEL=$(notdir $(HAIL_WHEEL)) --build-arg BENCHMARK_WHEEL=$(notdir $(BENCHMARK_WHEEL))
 	@printf $$(docker images -q --no-trunc $(BENCHMARK_DOCKER_TAG) | sed -e 's,[^:]*:,,') > image_sha
 	@echo Image sha is `cat image_sha`

diff --git a/build.yaml b/build.yaml
@@ -2296,7 +2296,7 @@ steps:
      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
      export HAIL_NETCAT_UBUNTU_IMAGE={{ netcat_ubuntu_image.image }}
      export HAIL_HAIL_BASE_IMAGE={{ hail_base_image.image }}
-     export DOCKER_ROOT_IMAGE="{{ global.docker_prefix }}/ubuntu:18.04"
+     export DOCKER_ROOT_IMAGE="{{ global.docker_root_image }}"
      export HAIL_TEST_TOKEN_FILE=/user-tokens/tokens.json
      export HAIL_TEST_DEV_TOKEN_FILE=/dev-tokens/tokens.json
      export HAIL_TOKEN="{{ token }}"
@@ -2364,7 +2364,7 @@ steps:
      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
      export HAIL_NETCAT_UBUNTU_IMAGE={{ netcat_ubuntu_image.image }}
      export HAIL_HAIL_BASE_IMAGE={{ hail_base_image.image }}
-     export DOCKER_ROOT_IMAGE="{{ global.docker_prefix }}/ubuntu:18.04"
+     export DOCKER_ROOT_IMAGE="{{ global.docker_root_image }}"
      export HAIL_TEST_TOKEN_FILE=/user-tokens/tokens.json
      export HAIL_TEST_DEV_TOKEN_FILE=/dev-tokens/tokens.json
      export HAIL_TOKEN="{{ token }}"
@@ -2432,7 +2432,7 @@ steps:
      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
      export HAIL_NETCAT_UBUNTU_IMAGE={{ netcat_ubuntu_image.image }}
      export HAIL_HAIL_BASE_IMAGE={{ hail_base_image.image }}
-     export DOCKER_ROOT_IMAGE="{{ global.docker_prefix }}/ubuntu:18.04"
+     export DOCKER_ROOT_IMAGE="{{ global.docker_root_image }}"
      export HAIL_TEST_TOKEN_FILE=/user-tokens/tokens.json
      export HAIL_TEST_DEV_TOKEN_FILE=/dev-tokens/tokens.json
      export HAIL_TOKEN="{{ token }}"
@@ -2500,7 +2500,7 @@ steps:
      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
      export HAIL_NETCAT_UBUNTU_IMAGE={{ netcat_ubuntu_image.image }}
      export HAIL_HAIL_BASE_IMAGE={{ hail_base_image.image }}
-     export DOCKER_ROOT_IMAGE="{{ global.docker_prefix }}/ubuntu:18.04"
+     export DOCKER_ROOT_IMAGE="{{ global.docker_root_image }}"
      export HAIL_TEST_TOKEN_FILE=/user-tokens/tokens.json
      export HAIL_TEST_DEV_TOKEN_FILE=/dev-tokens/tokens.json
      export HAIL_TOKEN="{{ token }}"
@@ -2568,7 +2568,7 @@ steps:
      export HAIL_DEFAULT_NAMESPACE={{ default_ns.name }}
      export HAIL_NETCAT_UBUNTU_IMAGE={{ netcat_ubuntu_image.image }}
      export HAIL_HAIL_BASE_IMAGE={{ hail_base_image.image }}
-     export DOCKER_ROOT_IMAGE="{{ global.docker_prefix }}/ubuntu:18.04"
+     export DOCKER_ROOT_IMAGE="{{ global.docker_root_image }}"
      export HAIL_TEST_TOKEN_FILE=/user-tokens/tokens.json
      export HAIL_TEST_DEV_TOKEN_FILE=/dev-tokens/tokens.json
      export HAIL_TOKEN="{{ token }}"
@@ -2785,7 +2785,7 @@ steps:
      export HAIL_GSA_KEY_FILE=/test-gsa-key/key.json
      export PYTEST_SPLITS=5
      export PYTEST_SPLIT_INDEX=0
-     export DOCKER_ROOT_IMAGE="{{ global.docker_prefix }}/ubuntu:18.04"
+     export DOCKER_ROOT_IMAGE="{{ global.docker_root_image }}"
      export PYTHON_DILL_IMAGE="{{ global.docker_prefix }}/python-dill:3.7-slim"
      hailctl config set batch/billing_project test
      hailctl config set batch/bucket cpg-hail-test
@@ -2836,7 +2836,7 @@ steps:
      export HAIL_GSA_KEY_FILE=/test-gsa-key/key.json
      export PYTEST_SPLITS=5
      export PYTEST_SPLIT_INDEX=1
-     export DOCKER_ROOT_IMAGE="{{ global.docker_prefix }}/ubuntu:18.04"
+     export DOCKER_ROOT_IMAGE="{{ global.docker_root_image }}"
      export PYTHON_DILL_IMAGE="{{ global.docker_prefix }}/python-dill:3.7-slim"
      hailctl config set batch/billing_project test
      hailctl config set batch/bucket cpg-hail-test
@@ -2887,7 +2887,7 @@ steps:
      export HAIL_GSA_KEY_FILE=/test-gsa-key/key.json
      export PYTEST_SPLITS=5
      export PYTEST_SPLIT_INDEX=2
-     export DOCKER_ROOT_IMAGE="{{ global.docker_prefix }}/ubuntu:18.04"
+     export DOCKER_ROOT_IMAGE="{{ global.docker_root_image }}"
      export PYTHON_DILL_IMAGE="{{ global.docker_prefix }}/python-dill:3.7-slim"
      hailctl config set batch/billing_project test
      hailctl config set batch/bucket cpg-hail-test
@@ -2938,7 +2938,7 @@ steps:
      export HAIL_GSA_KEY_FILE=/test-gsa-key/key.json
      export PYTEST_SPLITS=5
      export PYTEST_SPLIT_INDEX=3
-     export DOCKER_ROOT_IMAGE="{{ global.docker_prefix }}/ubuntu:18.04"
+     export DOCKER_ROOT_IMAGE="{{ global.docker_root_image }}"
      export PYTHON_DILL_IMAGE="{{ global.docker_prefix }}/python-dill:3.7-slim"
      hailctl config set batch/billing_project test
      hailctl config set batch/bucket cpg-hail-test
@@ -2989,7 +2989,7 @@ steps:
      export HAIL_GSA_KEY_FILE=/test-gsa-key/key.json
      export PYTEST_SPLITS=5
      export PYTEST_SPLIT_INDEX=4
-     export DOCKER_ROOT_IMAGE="{{ global.docker_prefix }}/ubuntu:18.04"
+     export DOCKER_ROOT_IMAGE="{{ global.docker_root_image }}"
      export PYTHON_DILL_IMAGE="{{ global.docker_prefix }}/python-dill:3.7-slim"
      hailctl config set batch/billing_project test
      hailctl config set batch/bucket cpg-hail-test

diff --git a/ci/Makefile b/ci/Makefile
@@ -47,5 +47,5 @@ push: build
 .PHONY: deploy
 deploy: push push-ci-utils
 	! [ -z $(NAMESPACE) ]  # call this like: make deploy NAMESPACE=default
-	python3 jinja2_render.py '{"code":{"sha":"$(shell git rev-parse --short=12 HEAD)"},"deploy":$(DEPLOY),"default_ns":{"name":"$(NAMESPACE)"},"ci_image":{"image":"$(CI_IMAGE)"},"global":{"project":"$(PROJECT)","zone":"$(ZONE)","docker_prefix":"$(DOCKER_PREFIX)","ip":"$(IP)","domain":"$(DOMAIN)", "k8s_server_url":"$(KUBERNETES_SERVER_URL)"},"ci_utils_image":{"image":"$(CI_UTILS_IMAGE)"},"ci_database":{"user_secret_name":"sql-ci-user-config"}}' deployment.yaml deployment.yaml.out
+	python3 jinja2_render.py '{"code":{"sha":"$(shell git rev-parse --short=12 HEAD)"},"deploy":$(DEPLOY),"default_ns":{"name":"$(NAMESPACE)"},"ci_image":{"image":"$(CI_IMAGE)"},"global":{"project":"$(PROJECT)","zone":"$(ZONE)","docker_prefix":"$(DOCKER_PREFIX)","docker_root_image":"$(DOCKER_ROOT_IMAGE)","ip":"$(IP)","domain":"$(DOMAIN)", "k8s_server_url":"$(KUBERNETES_SERVER_URL)"},"ci_utils_image":{"image":"$(CI_UTILS_IMAGE)"},"ci_database":{"user_secret_name":"sql-ci-user-config"}}' deployment.yaml deployment.yaml.out
 	kubectl -n $(NAMESPACE) apply -f deployment.yaml.out
diff --git a/ci/bootstrap.py b/ci/bootstrap.py
@@ -17,6 +17,8 @@
 from batch.driver.k8s_cache import K8sCache
 
 KUBERNETES_SERVER_URL = os.environ['KUBERNETES_SERVER_URL']
+DOCKER_PREFIX = os.environ['DOCKER_PREFIX']
+DOCKER_ROOT_IMAGE = f'{DOCKER_PREFIX}/ubuntu:18.04'
 
 
 def populate_secret_host_path(host_path: str, secret_data: Union[str, bytes]):
@@ -144,7 +146,7 @@ async def run(self):
                     f'{root}/shared:/shared',
                     '-v',
                     f'{job_root}/io:/io',
-                    'ubuntu:18.04',
+                    DOCKER_ROOT_IMAGE,
                     '/bin/bash',
                     '-c',
                     copy_script,
@@ -282,7 +284,7 @@ async def run(self):
                         f'{root}/shared:/shared',
                         '-v',
                         f'{job_root}/io:/io',
-                        'ubuntu:18.04',
+                        DOCKER_ROOT_IMAGE,
                         '/bin/bash',
                         '-c',
                         copy_script,

diff --git a/ci/ci/build.py b/ci/ci/build.py
@@ -12,6 +12,7 @@
     GCP_PROJECT,
     GCP_ZONE,
     DOCKER_PREFIX,
+    DOCKER_ROOT_IMAGE,
     DOMAIN,
     IP,
     CI_UTILS_IMAGE,
@@ -169,6 +170,7 @@ def input_config(self, code, scope):
             'project': GCP_PROJECT,
             'zone': GCP_ZONE,
             'docker_prefix': DOCKER_PREFIX,
+            'docker_root_image': DOCKER_ROOT_IMAGE,
             'domain': DOMAIN,
             'ip': IP,
             'k8s_server_url': KUBERNETES_SERVER_URL,

diff --git a/ci/ci/environment.py b/ci/ci/environment.py
@@ -7,6 +7,7 @@
 GCP_REGION = '-'.join(GCP_ZONE.split('-')[:-1])  # us-west1-a -> us-west1
 DOCKER_PREFIX = os.environ.get('HAIL_DOCKER_PREFIX', f'gcr.io/{GCP_REGION}')
 assert DOCKER_PREFIX != ''
+DOCKER_ROOT_IMAGE = os.environ['HAIL_DOCKER_ROOT_IMAGE']
 DOMAIN = os.environ['HAIL_DOMAIN']
 assert DOMAIN != ''
 IP = os.environ.get('HAIL_IP')

diff --git a/ci/deployment.yaml b/ci/deployment.yaml
@@ -54,6 +54,8 @@ spec:
              value: "{{ global.zone }}"
            - name: HAIL_DOCKER_PREFIX
              value: "{{ global.docker_prefix }}"
+           - name: HAIL_DOCKER_ROOT_IMAGE
+             value: "{{ global.docker_root_image }}"
            - name: HAIL_CI_UTILS_IMAGE
              value: "{{ ci_utils_image.image }}"
            - name: HAIL_IP

diff --git a/ci/test/resources/build.yaml b/ci/test/resources/build.yaml
@@ -7,7 +7,7 @@ steps:
    name: inline_image
    dockerFile:
      inline: |
-       FROM {{ global.region }}-docker.pkg.dev/{{ global.project }}/hail/ubuntu:18.04
+       FROM {{ global.docker_root_image }}
        RUN apt-get update && apt-get install -y git
    contextPath: .
  - kind: runImage