From 3c406b60ef76200ae391b7a92391b98d522ccc63 Mon Sep 17 00:00:00 2001 From: Dan King Date: Tue, 2 May 2023 11:35:58 -0400 Subject: [PATCH 1/4] [gcr] eliminate GCR --- batch/test/test_batch.py | 2 +- benchmark/Makefile | 2 +- dev-docs/development_process.md | 2 +- hail/Makefile | 8 ++++---- hail/python/hailtop/batch/docs/cookbook/clumping.rst | 6 +++--- .../batch/docs/cookbook/files/batch_clumping.py | 2 +- .../hailtop/batch/docs/cookbook/random_forest.rst | 6 +++--- hail/python/hailtop/batch/docs/docker_resources.rst | 12 ++++++------ 8 files changed, 20 insertions(+), 20 deletions(-) diff --git a/batch/test/test_batch.py b/batch/test/test_batch.py index 7080ca0cea6..90a99290dab 100644 --- a/batch/test/test_batch.py +++ b/batch/test/test_batch.py @@ -364,7 +364,7 @@ def test_unknown_image(client: BatchClient): def test_invalid_gcr(client: BatchClient): bb = create_batch(client) # GCP projects can't be strictly numeric - j = bb.create_job('gcr.io/1/does-not-exist', ['echo', 'test']) + j = bb.create_job('us-docker.pkg.dev/1/does-not-exist', ['echo', 'test']) b = bb.submit() status = j.wait() try: diff --git a/benchmark/Makefile b/benchmark/Makefile index 5f6d7dba674..0891843d0e5 100644 --- a/benchmark/Makefile +++ b/benchmark/Makefile @@ -30,7 +30,7 @@ cleanup_image: BENCHMARK_PROJECT ?= hail-vdc BENCHMARK_DOCKER_TAG ?= benchmark_$(shell whoami) -BENCHMARK_REPO_BASE = gcr.io/$(BENCHMARK_PROJECT)/$(BENCHMARK_DOCKER_TAG) +BENCHMARK_REPO_BASE = us-docker.pkg.dev/$(BENCHMARK_PROJECT)/$(BENCHMARK_DOCKER_TAG) DOCKER_ROOT_IMAGE := ubuntu:20.04 ifndef HAIL_WHEEL diff --git a/dev-docs/development_process.md b/dev-docs/development_process.md index ced29f6f48c..79dec6874d8 100644 --- a/dev-docs/development_process.md +++ b/dev-docs/development_process.md @@ -66,7 +66,7 @@ Install and configure tools necessary for working on the Hail Services: gcloud auth login gcloud config set project hail-vdc gcloud container clusters get-credentials vdc --zone=us-central1-a -gcloud auth -q configure-docker gcr.io +gcloud auth -q configure-docker us-docker.pkg.dev ``` 5. Add these lines to `~/.zshrc` or `~/.bashrc` to configure your shell and environment for Hail: diff --git a/hail/Makefile b/hail/Makefile index 04366e98977..d91f9817a42 100644 --- a/hail/Makefile +++ b/hail/Makefile @@ -367,16 +367,16 @@ test-dataproc-38: install-hailctl # set GITHUB_OAUTH_HEADER_FILE to that filename # # create the hailgenetics/hail image (see /docker) and place it somewhere skopeo can access it -# set HAIL_GENETICS_HAIL_IMAGE to that image's full name including the protocol, e.g. docker://gcr.io/hail-vdc/hailgenetics/hail:1234abcd +# set HAIL_GENETICS_HAIL_IMAGE to that image's full name including the protocol, e.g. docker://us-docker.pkg.dev/hail-vdc/hailgenetics/hail:1234abcd # # create the hailgenetics/hailtop image (see /docker) and place it somewhere skopeo can access it -# set HAIL_GENETICS_HAILTOP_IMAGE to that image's full name including the protocol, e.g. docker://gcr.io/hail-vdc/hailgenetics/hailtop:1234abcd +# set HAIL_GENETICS_HAILTOP_IMAGE to that image's full name including the protocol, e.g. docker://us-docker.pkg.dev/hail-vdc/hailgenetics/hailtop:1234abcd # # create the hailgenetics/vep-grch37-85 image (see /docker) and place it somewhere skopeo can access it -# set HAIL_GENETICS_VEP_GRCH37_85_IMAGE to that image's full name including the protocol, e.g. docker://gcr.io/hail-vdc/hailgenetics/vep-grch37-85:1234abcd +# set HAIL_GENETICS_VEP_GRCH37_85_IMAGE to that image's full name including the protocol, e.g. docker://us-docker.pkg.dev/hail-vdc/hailgenetics/vep-grch37-85:1234abcd # # create the hailgenetics/vep-grch38-95 image (see /docker) and place it somewhere skopeo can access it -# set HAIL_GENETICS_VEP_GRCH38_95_IMAGE to that image's full name including the protocol, e.g. docker://gcr.io/hail-vdc/hailgenetics/vep-grch38-95:1234abcd +# set HAIL_GENETICS_VEP_GRCH38_95_IMAGE to that image's full name including the protocol, e.g. docker://us-docker.pkg.dev/hail-vdc/hailgenetics/vep-grch38-95:1234abcd # # build a Azure-HDInsight-compatible wheel file (see build.yaml "build_wheel_for_azure" or start a # cluster to find the correct Scala and Spark versions because the version webpage does not include diff --git a/hail/python/hailtop/batch/docs/cookbook/clumping.rst b/hail/python/hailtop/batch/docs/cookbook/clumping.rst index 902ff689198..db34d54d11d 100644 --- a/hail/python/hailtop/batch/docs/cookbook/clumping.rst +++ b/hail/python/hailtop/batch/docs/cookbook/clumping.rst @@ -96,8 +96,8 @@ The following Docker command pushes the image to GCR: .. code-block:: sh - docker tag 1kg-gwas gcr.io//1kg-gwas - docker push gcr.io//1kg-gwas + docker tag 1kg-gwas us-docker.pkg.dev//1kg-gwas + docker push us-docker.pkg.dev//1kg-gwas Replace ```` with the name of your Google project. Ensure your Batch service account :ref:`can access images in GCR `. @@ -129,7 +129,7 @@ access the binary PLINK file output and association results in downstream jobs. """ cores = 2 g = batch.new_job(name='run-gwas') - g.image('gcr.io//1kg-gwas:latest') + g.image('us-docker.pkg.dev//1kg-gwas:latest') g.cpu(cores) g.declare_resource_group(ofile={ 'bed': '{root}.bed', diff --git a/hail/python/hailtop/batch/docs/cookbook/files/batch_clumping.py b/hail/python/hailtop/batch/docs/cookbook/files/batch_clumping.py index d0e539b815d..f3e0bb4eddf 100644 --- a/hail/python/hailtop/batch/docs/cookbook/files/batch_clumping.py +++ b/hail/python/hailtop/batch/docs/cookbook/files/batch_clumping.py @@ -7,7 +7,7 @@ def gwas(batch, vcf, phenotypes): """ cores = 2 g = batch.new_job(name='run-gwas') - g.image('gcr.io//1kg-gwas:latest') + g.image('us-docker.pkg.dev//1kg-gwas:latest') g.cpu(cores) g.declare_resource_group(ofile={ 'bed': '{root}.bed', diff --git a/hail/python/hailtop/batch/docs/cookbook/random_forest.rst b/hail/python/hailtop/batch/docs/cookbook/random_forest.rst index 144297bcb35..41d9766fd62 100644 --- a/hail/python/hailtop/batch/docs/cookbook/random_forest.rst +++ b/hail/python/hailtop/batch/docs/cookbook/random_forest.rst @@ -129,12 +129,12 @@ along with any desired Python packages. For running the random forest, we need both the `sklearn` and `pandas` Python packages installed in the image. We use :func:`.docker.build_python_image` to build -an image and push it automatically to the location specified (ex: `gcr.io/hail-vdc/random-forest`). +an image and push it automatically to the location specified (ex: `us-docker.pkg.dev/hail-vdc/random-forest`). .. code-block:: python - image = hb.build_python_image('gcr.io/hail-vdc/random-forest', - requirements=['sklearn', 'pandas']) + image = hb.build_python_image('us-docker.pkg.dev/hail-vdc/random-forest', + requirements=['sklearn', 'pandas']) ~~~~~~~~~~~~ Control Code diff --git a/hail/python/hailtop/batch/docs/docker_resources.rst b/hail/python/hailtop/batch/docs/docker_resources.rst index 9b3b9d28c2d..a7573aed938 100644 --- a/hail/python/hailtop/batch/docs/docker_resources.rst +++ b/hail/python/hailtop/batch/docs/docker_resources.rst @@ -75,21 +75,21 @@ To create a Docker image, use .. code-block:: sh - docker build -t gcr.io//: -f Dockerfile . + docker build -t us-docker.pkg.dev//: -f Dockerfile . * `` is the context directory, `.` means the current working directory, * `-t ` specifies the image name, and * `-f ` specifies the Dockerfile file. * A more complete description may be found `here: `__. -For example, we can build an image named gcr.io// based on the Dockerfile named Dockerfile, using the current working directory as the context: +For example, we can build an image named us-docker.pkg.dev// based on the Dockerfile named Dockerfile, using the current working directory as the context: .. code-block:: sh - docker build -t gcr.io//: -f Dockerfile . + docker build -t us-docker.pkg.dev//: -f Dockerfile . -In this example we prepend the image name with `gcr.io//` so that it may be pushed to the Google Container Registry, in the next step. +In this example we prepend the image name with `us-docker.pkg.dev//` so that it may be pushed to the Google Container Registry, in the next step. Pushing Images -------------- @@ -100,8 +100,8 @@ Docker Hub. Below is an example of pushing the image to the Google Container Reg .. code-block:: sh - docker push gcr.io//: + docker push us-docker.pkg.dev//: Now you can use your Docker image with Batch to run your code with the method :meth:`.BashJob.image` -specifying the image as `gcr.io//:`! +specifying the image as `us-docker.pkg.dev//:`! From b74578694e3569e2a56e89d670a2668fa8caa73a Mon Sep 17 00:00:00 2001 From: Dan King Date: Tue, 2 May 2023 11:45:53 -0400 Subject: [PATCH 2/4] removed a few more refs to container registry --- infra/gcp/README.md | 2 +- infra/gcp/bootstrap.sh | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/infra/gcp/README.md b/infra/gcp/README.md index 4f8c71d0c1d..4f8fc2a4b9f 100644 --- a/infra/gcp/README.md +++ b/infra/gcp/README.md @@ -209,7 +209,7 @@ You can now install Hail: the $HAIL/infra/gcp directory, unless otherwise stated. - Run the following to authenticate docker and kubectl with the new - container registry and kubernetes cluster, respectively. + artifact registry and kubernetes cluster, respectively. ``` ./bootstrap.sh configure_gcloud diff --git a/infra/gcp/bootstrap.sh b/infra/gcp/bootstrap.sh index 87009ec73d8..42b53f1dc2e 100755 --- a/infra/gcp/bootstrap.sh +++ b/infra/gcp/bootstrap.sh @@ -5,9 +5,9 @@ source ../bootstrap_utils.sh function configure_gcloud() { ZONE=${1:-"us-central1-a"} - gcloud -q auth configure-docker - # If you are using the Artifact Registry: - # gcloud -q auth configure-docker $REGION-docker.pkg.dev + # If you are using the Container Registry: + # gcloud -q auth configure-docker + gcloud -q auth configure-docker $REGION-docker.pkg.dev gcloud container clusters get-credentials --zone $ZONE vdc } From e503a0f1309f633ee0d71bcd620ef9775fb8f104 Mon Sep 17 00:00:00 2001 From: Dan King Date: Tue, 2 May 2023 16:03:51 -0400 Subject: [PATCH 3/4] fix test --- batch/test/test_batch.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/batch/test/test_batch.py b/batch/test/test_batch.py index 90a99290dab..6fd0cded712 100644 --- a/batch/test/test_batch.py +++ b/batch/test/test_batch.py @@ -361,7 +361,7 @@ def test_unknown_image(client: BatchClient): @skip_in_azure -def test_invalid_gcr(client: BatchClient): +def test_invalid_gar(client: BatchClient): bb = create_batch(client) # GCP projects can't be strictly numeric j = bb.create_job('us-docker.pkg.dev/1/does-not-exist', ['echo', 'test']) @@ -369,7 +369,7 @@ def test_invalid_gcr(client: BatchClient): status = j.wait() try: assert j._get_exit_code(status, 'main') is None - assert status['status']['container_statuses']['main']['short_error'] == 'image repository is invalid', str( + assert status['status']['container_statuses']['main']['short_error'] == 'image cannot be pulled', str( (status, b.debug_info()) ) except Exception as e: From f227bbc54997b42a6841eeaa3997e95b91558652 Mon Sep 17 00:00:00 2001 From: Dan King Date: Tue, 2 May 2023 16:06:34 -0400 Subject: [PATCH 4/4] fixes --- infra/gcp/README.md | 7 ++++--- infra/gcp/bootstrap.sh | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/infra/gcp/README.md b/infra/gcp/README.md index 4f8fc2a4b9f..fb17d2ee635 100644 --- a/infra/gcp/README.md +++ b/infra/gcp/README.md @@ -208,11 +208,12 @@ You can now install Hail: for Docker can be applied). The following steps should be completed from the $HAIL/infra/gcp directory, unless otherwise stated. -- Run the following to authenticate docker and kubectl with the new - artifact registry and kubernetes cluster, respectively. +- Run the following to authenticate docker and kubectl with the new artifact + registry and kubernetes cluster, respectively. The `GKE_ZONE` is the zone of + the GKE cluster and the `GAR_REGION` is the region of the artifact registry. ``` - ./bootstrap.sh configure_gcloud + ./bootstrap.sh configure_gcloud ``` - Edit `$HAIL/letsencrypt/subdomains.txt` to include just the services you plan diff --git a/infra/gcp/bootstrap.sh b/infra/gcp/bootstrap.sh index 42b53f1dc2e..85ffd30946d 100755 --- a/infra/gcp/bootstrap.sh +++ b/infra/gcp/bootstrap.sh @@ -4,6 +4,7 @@ source ../bootstrap_utils.sh function configure_gcloud() { ZONE=${1:-"us-central1-a"} + REGION=${2:-"us"} # If you are using the Container Registry: # gcloud -q auth configure-docker