WX-1828 Private Docker Hub repos in GCP Batch (#7515)

Co-authored-by: Adam Nichols <aednichols@gmail.com>
broadinstitute · Sep 27, 2024 · d3ded6f · d3ded6f
1 parent 74885ef
commit d3ded6f
Show file tree

Hide file tree

Showing 6 changed files with 52 additions and 3 deletions.
diff --git a/...es/standardTestCases/docker_hash/gcpbatch_docker_hash_dockerhub_private_usa_dockerhub.wdl b/...es/standardTestCases/docker_hash/gcpbatch_docker_hash_dockerhub_private_usa_dockerhub.wdl
@@ -0,0 +1,13 @@
+task dockerhub {
+    command {
+        echo "hello"
+    }
+    runtime {
+        docker: "broadinstitute/cloud-cromwell:dev"
+        backend: "GCPBATCHUSADockerhub"
+    }
+}
+
+workflow docker_hash_dockerhub_private {
+    call dockerhub
+}
diff --git a/centaur/src/main/resources/standardTestCases/docker_hash_dockerhub_private.test b/centaur/src/main/resources/standardTestCases/docker_hash_dockerhub_private.test
@@ -1,7 +1,7 @@
 name: docker_hash_dockerhub_private
 testFormat: workflowsuccess
-# see https://github.com/broadinstitute/cromwell/pull/7515
-backends: [Papi, GCPBATCH_FAIL]
+backendsMode: any
+backends: [Papi, GCPBATCH]
 
 files {
   workflow: docker_hash/docker_hash_dockerhub_private.wdl

diff --git a/centaur/src/main/resources/standardTestCases/docker_hash_dockerhub_private_usa.test b/centaur/src/main/resources/standardTestCases/docker_hash_dockerhub_private_usa.test
@@ -1,7 +1,7 @@
 name: docker_hash_dockerhub_private_config_usa_wf_options
 testFormat: workflowsuccess
 # see https://github.com/broadinstitute/cromwell/pull/7515
-backends: [Papiv2USADockerhub, GCPBATCH_FAIL, GCPBATCH_NEEDS_ALT]
+backends: [Papiv2USADockerhub, GCPBATCH_ALT]
 
 files {
   workflow: docker_hash/docker_hash_dockerhub_private_usa_dockerhub.wdl

diff --git a/centaur/src/main/resources/standardTestCases/gcpbatch_docker_hash_dockerhub_private_usa.test b/centaur/src/main/resources/standardTestCases/gcpbatch_docker_hash_dockerhub_private_usa.test
@@ -0,0 +1,18 @@
+name: gcpbatch_docker_hash_dockerhub_private_config_usa_wf_options
+testFormat: workflowsuccess
+# see https://github.com/broadinstitute/cromwell/pull/7515
+backends: [GCPBATCHUSADockerhub]
+
+files {
+  workflow: docker_hash/gcpbatch_docker_hash_dockerhub_private_usa_dockerhub.wdl
+  # Updated the options to read_from_cache: false for
+  # https://github.com/broadinstitute/cromwell/issues/3998
+  options-dir: "Error: BA-6546 The environment variable CROMWELL_BUILD_RESOURCES_DIRECTORY must be set/export pointing to a valid path such as '${YOUR_CROMWELL_DIR}/target/ci/resources'"
+  options-dir: ${?CROMWELL_BUILD_RESOURCES_DIRECTORY}
+  options: ${files.options-dir}/private_docker_papi_v2_usa.options
+}
+
+metadata {
+  "calls.docker_hash_dockerhub_private.dockerhub.runtimeAttributes.docker": "broadinstitute/cloud-cromwell:dev",
+  "calls.docker_hash_dockerhub_private.dockerhub.dockerImageUsed": "broadinstitute/cloud-cromwell@sha256:0d51f90e1dd6a449d4587004c945e43f2a7bbf615151308cff40c15998cc3ad4"
+}
diff --git a/src/ci/resources/gcp_batch_application.conf b/src/ci/resources/gcp_batch_application.conf
@@ -41,5 +41,11 @@ backend {
         include "gcp_batch_provider_config.inc.conf"
       }
     }
+    GCPBATCHUSADockerhub {
+      actor-factory = "cromwell.backend.google.batch.GcpBatchBackendLifecycleActorFactory"
+      config {
+        include "gcp_batch_provider_config.inc.conf"
+      }
+    }
   }
 }
diff --git a/src/ci/resources/gcp_batch_shared_application.inc.conf b/src/ci/resources/gcp_batch_shared_application.inc.conf
@@ -113,5 +113,17 @@ backend {
         include "dockerhub_provider_config_v2.inc.conf"
       }
     }
+    GCPBATCHUSADockerhub {
+      actor-factory = "REPLACEME!"
+      config {
+        # When importing: Remember to also include an appropriate provider_config.inc.conf here.
+
+        include "dockerhub_provider_config_v2_usa.inc.conf"
+        # This SA does not have permission to bill this project when accessing RP buckets.
+        # This is on purpose so that we can assert the failure (see requester_pays_localization_negative)
+        genomics.compute-service-account = "centaur@broad-dsde-cromwell-dev.iam.gserviceaccount.com"
+        filesystems.http {}
+      }
+    }
   }
 }