[gcr] eliminate GCR

batch/test/test_batch.py

@@ -361,15 +361,15 @@ def test_unknown_image(client: BatchClient):
 
 
 @skip_in_azure
-def test_invalid_gcr(client: BatchClient):
+def test_invalid_gar(client: BatchClient):
     bb = create_batch(client)
     # GCP projects can't be strictly numeric
-    j = bb.create_job('gcr.io/1/does-not-exist', ['echo', 'test'])
+    j = bb.create_job('us-docker.pkg.dev/1/does-not-exist', ['echo', 'test'])
     b = bb.submit()
     status = j.wait()
     try:
         assert j._get_exit_code(status, 'main') is None
-        assert status['status']['container_statuses']['main']['short_error'] == 'image repository is invalid', str(
+        assert status['status']['container_statuses']['main']['short_error'] == 'image cannot be pulled', str(
             (status, b.debug_info())
         )
     except Exception as e:

benchmark/Makefile

@@ -30,7 +30,7 @@ cleanup_image:
 
 BENCHMARK_PROJECT ?= hail-vdc
 BENCHMARK_DOCKER_TAG ?= benchmark_$(shell whoami)
-BENCHMARK_REPO_BASE = gcr.io/$(BENCHMARK_PROJECT)/$(BENCHMARK_DOCKER_TAG)
+BENCHMARK_REPO_BASE = us-docker.pkg.dev/$(BENCHMARK_PROJECT)/$(BENCHMARK_DOCKER_TAG)
 DOCKER_ROOT_IMAGE := ubuntu:20.04
 
 ifndef HAIL_WHEEL

dev-docs/development_process.md

@@ -66,7 +66,7 @@ Install and configure tools necessary for working on the Hail Services:
 gcloud auth login
 gcloud config set project hail-vdc
 gcloud container clusters get-credentials vdc --zone=us-central1-a
-gcloud auth -q configure-docker gcr.io
+gcloud auth -q configure-docker us-docker.pkg.dev
 ```
 
 5. Add these lines to `~/.zshrc` or `~/.bashrc` to configure your shell and environment for Hail:

hail/Makefile

@@ -367,16 +367,16 @@ test-dataproc-38: install-hailctl
 # set GITHUB_OAUTH_HEADER_FILE to that filename
 #
 # create the hailgenetics/hail image (see /docker) and place it somewhere skopeo can access it
-# set HAIL_GENETICS_HAIL_IMAGE to that image's full name including the protocol, e.g. docker://gcr.io/hail-vdc/hailgenetics/hail:1234abcd
+# set HAIL_GENETICS_HAIL_IMAGE to that image's full name including the protocol, e.g. docker://us-docker.pkg.dev/hail-vdc/hailgenetics/hail:1234abcd
 #
 # create the hailgenetics/hailtop image (see /docker) and place it somewhere skopeo can access it
-# set HAIL_GENETICS_HAILTOP_IMAGE to that image's full name including the protocol, e.g. docker://gcr.io/hail-vdc/hailgenetics/hailtop:1234abcd
+# set HAIL_GENETICS_HAILTOP_IMAGE to that image's full name including the protocol, e.g. docker://us-docker.pkg.dev/hail-vdc/hailgenetics/hailtop:1234abcd
 #
 # create the hailgenetics/vep-grch37-85 image (see /docker) and place it somewhere skopeo can access it
-# set HAIL_GENETICS_VEP_GRCH37_85_IMAGE to that image's full name including the protocol, e.g. docker://gcr.io/hail-vdc/hailgenetics/vep-grch37-85:1234abcd
+# set HAIL_GENETICS_VEP_GRCH37_85_IMAGE to that image's full name including the protocol, e.g. docker://us-docker.pkg.dev/hail-vdc/hailgenetics/vep-grch37-85:1234abcd
 #
 # create the hailgenetics/vep-grch38-95 image (see /docker) and place it somewhere skopeo can access it
-# set HAIL_GENETICS_VEP_GRCH38_95_IMAGE to that image's full name including the protocol, e.g. docker://gcr.io/hail-vdc/hailgenetics/vep-grch38-95:1234abcd
+# set HAIL_GENETICS_VEP_GRCH38_95_IMAGE to that image's full name including the protocol, e.g. docker://us-docker.pkg.dev/hail-vdc/hailgenetics/vep-grch38-95:1234abcd
 #
 # build a Azure-HDInsight-compatible wheel file (see build.yaml "build_wheel_for_azure" or start a
 # cluster to find the correct Scala and Spark versions because the version webpage does not include

hail/python/hailtop/batch/docs/cookbook/clumping.rst

@@ -96,8 +96,8 @@ The following Docker command pushes the image to GCR:
 
 .. code-block:: sh
 
-    docker tag 1kg-gwas gcr.io/<MY_PROJECT>/1kg-gwas
-    docker push gcr.io/<MY_PROJECT>/1kg-gwas
+    docker tag 1kg-gwas us-docker.pkg.dev/<MY_PROJECT>/1kg-gwas
+    docker push us-docker.pkg.dev/<MY_PROJECT>/1kg-gwas
 
 Replace ``<MY_PROJECT>`` with the name of your Google project. Ensure your Batch service account
 :ref:`can access images in GCR <service-accounts>`.
@@ -129,7 +129,7 @@ access the binary PLINK file output and association results in downstream jobs.
         """
         cores = 2
         g = batch.new_job(name='run-gwas')
-        g.image('gcr.io/<MY_PROJECT>/1kg-gwas:latest')
+        g.image('us-docker.pkg.dev/<MY_PROJECT>/1kg-gwas:latest')
         g.cpu(cores)
         g.declare_resource_group(ofile={
             'bed': '{root}.bed',

hail/python/hailtop/batch/docs/cookbook/files/batch_clumping.py

@@ -7,7 +7,7 @@ def gwas(batch, vcf, phenotypes):
     """
     cores = 2
     g = batch.new_job(name='run-gwas')
-    g.image('gcr.io/<MY_PROJECT>/1kg-gwas:latest')
+    g.image('us-docker.pkg.dev/<MY_PROJECT>/1kg-gwas:latest')
     g.cpu(cores)
     g.declare_resource_group(ofile={
         'bed': '{root}.bed',

hail/python/hailtop/batch/docs/cookbook/random_forest.rst

@@ -129,12 +129,12 @@ along with any desired Python packages.
 
 For running the random forest, we need both the `sklearn` and `pandas` Python
 packages installed in the image. We use :func:`.docker.build_python_image` to build
-an image and push it automatically to the location specified (ex: `gcr.io/hail-vdc/random-forest`).
+an image and push it automatically to the location specified (ex: `us-docker.pkg.dev/hail-vdc/random-forest`).
 
 .. code-block:: python
 
-    image = hb.build_python_image('gcr.io/hail-vdc/random-forest',
-                              requirements=['sklearn', 'pandas'])
+    image = hb.build_python_image('us-docker.pkg.dev/hail-vdc/random-forest',
+                                  requirements=['sklearn', 'pandas'])
 
 ~~~~~~~~~~~~
 Control Code

hail/python/hailtop/batch/docs/docker_resources.rst

@@ -75,21 +75,21 @@ To create a Docker image, use
 
 .. code-block:: sh
 
-    docker build -t gcr.io/<my-project>/<my-image>:<tag> -f Dockerfile .
+    docker build -t us-docker.pkg.dev/<my-project>/<my-image>:<tag> -f Dockerfile .
 
     * `<dir>` is the context directory, `.` means the current working directory,
     * `-t <name>` specifies the image name, and
     * `-f <dockerfile>` specifies the Dockerfile file.
     * A more complete description may be found `here: <https://docs.docker.com/engine/reference/commandline/build/>`__.
 
-For example, we can build an image named gcr.io/<my-project>/<my-image> based on the Dockerfile named Dockerfile, using the current working directory as the context:
+For example, we can build an image named us-docker.pkg.dev/<my-project>/<my-image> based on the Dockerfile named Dockerfile, using the current working directory as the context:
 
 .. code-block:: sh
 
-    docker build -t gcr.io/<my-project>/<my-image>:<tag> -f Dockerfile .
+    docker build -t us-docker.pkg.dev/<my-project>/<my-image>:<tag> -f Dockerfile .
 
 
-In this example we prepend the image name with `gcr.io/<my-project>/` so that it may be pushed to the Google Container Registry, in the next step.
+In this example we prepend the image name with `us-docker.pkg.dev/<my-project>/` so that it may be pushed to the Google Container Registry, in the next step.
 
 Pushing Images
 --------------
@@ -100,8 +100,8 @@ Docker Hub. Below is an example of pushing the image to the Google Container Reg
 
 .. code-block:: sh
 
-    docker push gcr.io/<my-project>/<my-image>:<tag>
+    docker push us-docker.pkg.dev/<my-project>/<my-image>:<tag>
 
 
 Now you can use your Docker image with Batch to run your code with the method :meth:`.BashJob.image`
-specifying the image as `gcr.io/<my-project>/<my-image>:<tag>`!
+specifying the image as `us-docker.pkg.dev/<my-project>/<my-image>:<tag>`!

infra/gcp/README.md

@@ -208,11 +208,12 @@ You can now install Hail:
   for Docker can be applied). The following steps should be completed from
   the $HAIL/infra/gcp directory, unless otherwise stated.
 
-- Run the following to authenticate docker and kubectl with the new
-  container registry and kubernetes cluster, respectively.
+- Run the following to authenticate docker and kubectl with the new artifact
+  registry and kubernetes cluster, respectively. The `GKE_ZONE` is the zone of
+  the GKE cluster and the `GAR_REGION` is the region of the artifact registry.
 
   ```
-  ./bootstrap.sh configure_gcloud <ZONE>
+  ./bootstrap.sh configure_gcloud <GKE_ZONE> <GAR_REGION>
   ```
 
 - Edit `$HAIL/letsencrypt/subdomains.txt` to include just the services you plan

infra/gcp/bootstrap.sh

@@ -4,10 +4,11 @@ source ../bootstrap_utils.sh
 
 function configure_gcloud() {
     ZONE=${1:-"us-central1-a"}
+    REGION=${2:-"us"}
 
-    gcloud -q auth configure-docker
-    # If you are using the Artifact Registry:
-    # gcloud -q auth configure-docker $REGION-docker.pkg.dev
+    # If you are using the Container Registry:
+    # gcloud -q auth configure-docker
+    gcloud -q auth configure-docker $REGION-docker.pkg.dev
     gcloud container clusters get-credentials --zone $ZONE vdc
 }