[WX-1345] Automatic Token Acquisition for TES Config

.gitignore

@@ -58,6 +58,10 @@ tesk_application.conf
 **/venv/
 exome_germline_single_sample_v1.3/
 **/*.pyc
+src/ci/resources/*.temp
 
 # GHA credentials
 gha-creds-*.json
+
+# jenv
+.java-version

project/ContinuousIntegration.scala

@@ -9,6 +9,8 @@ object ContinuousIntegration {
   lazy val ciSettings: Seq[Setting[_]] = List(
     srcCiResources := sourceDirectory.value / "ci" / "resources",
     targetCiResources := target.value / "ci" / "resources",
+    envFile := srcCiResources.value / "env.temp", //generated by resources/acquire_b2c_token.sh
+
     vaultToken := userHome / ".vault-token",
     copyCiResources := {
       IO.copyDirectory(srcCiResources.value, targetCiResources.value)
@@ -26,7 +28,12 @@ object ContinuousIntegration {
       if (vaultToken.value.isDirectory) {
         sys.error(s"""The vault token file "${vaultToken.value}" should not be a directory.""")
       }
-      val cmd = List(
+
+      // Only include the local file argument if the file exists (local development w/ acquire_b2c_token.sh)
+      // Don't include it otherwise (CI/CD and other development)
+      val localEnvFileArgs = if(envFile.value.exists()) List("-e", s"ENV_FILE=${envFile.value}") else List()
+
+      val cmd: List[String] = List.concat(List(
         "docker",
         "run",
         "--rm",
@@ -35,7 +42,9 @@ object ContinuousIntegration {
         "-v",
         s"${srcCiResources.value}:${srcCiResources.value}",
         "-v",
-        s"${targetCiResources.value}:${targetCiResources.value}",
+        s"${targetCiResources.value}:${targetCiResources.value}"),
+        localEnvFileArgs,
+        List(
         "-e",
         "ENVIRONMENT=not_used",
         "-e",
@@ -44,7 +53,7 @@ object ContinuousIntegration {
         s"OUT_PATH=${targetCiResources.value}",
         "broadinstitute/dsde-toolbox:dev",
         "render-templates.sh"
-      )
+      ))
       val result = cmd ! log
       if (result != 0) {
         sys.error(
@@ -70,6 +79,7 @@ object ContinuousIntegration {
   private val srcCiResources: SettingKey[File] = settingKey[File]("Source directory for CI resources")
   private val targetCiResources: SettingKey[File] = settingKey[File]("Target directory for CI resources")
   private val vaultToken: SettingKey[File] = settingKey[File]("File with the vault token")
+  private val envFile: SettingKey[File] = settingKey[File]("File with the environment variables needed to render CI resources.")
 
   /**
     * For "reasons" these projects are excluded from the root aggregation in build.sbt.

runConfigurations/Repo template_ Cromwell server TES.run.xml

@@ -19,4 +19,24 @@
       <option name="RunConfigurationTask" enabled="true" run_configuration_name="renderCiResources" run_configuration_type="SbtRunConfiguration" />
     </method>
   </configuration>
+  <configuration default="false" name="Repo template: Cromwell server TES" type="Application" factoryName="Application">
+    <option name="ALTERNATIVE_JRE_PATH" value="$PROJECT_DIR$/../.sdkman/candidates/java/current" />
+    <envs>
+      <env name="CROMWELL_BUILD_CENTAUR_SLICK_PROFILE" value="slick.jdbc.MySQLProfile$" />
+      <env name="CROMWELL_BUILD_CENTAUR_JDBC_DRIVER" value="com.mysql.cj.jdbc.Driver" />
+      <env name="CROMWELL_BUILD_CENTAUR_JDBC_URL" value="jdbc:mysql://localhost:3306/cromwell_test?allowPublicKeyRetrieval=true&amp;useSSL=false&amp;rewriteBatchedStatements=true&amp;serverTimezone=UTC&amp;useInformationSchema=true" />
+      <env name="CROMWELL_BUILD_RESOURCES_DIRECTORY" value="target/ci/resources" />
+      <env name="CROMWELL_BUILD_PAPI_JSON_FILE" value="target/ci/resources/cromwell-centaur-service-account.json" />
+      <env name="CROMWELL_BUILD_CENTAUR_READ_LINES_LIMIT" value="128000" />
+      <env name="CROMWELL_BUILD_CENTAUR_256_BITS_KEY" value="AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" />
+    </envs>
+    <option name="MAIN_CLASS_NAME" value="cromwell.CromwellApp" />
+    <module name="cromwell" />
+    <option name="PROGRAM_PARAMETERS" value="server" />
+    <option name="VM_PARAMETERS" value="-Dconfig.file=target/ci/resources/tes_application.conf" />
+    <method v="2">
+      <option name="Make" enabled="true" />
+      <option name="RunConfigurationTask" enabled="false" run_configuration_name="renderCiResources" run_configuration_type="SbtRunConfiguration" />
+    </method>
+  </configuration>
 </component>

runConfigurations/Repo template_ Cromwell server Terra TES.run.xml

@@ -0,0 +1,23 @@
+<component name="ProjectRunConfigurationManager">
+  <configuration default="false" name="Repo template: Cromwell server Terra TES" type="Application" factoryName="Application">
+    <option name="ALTERNATIVE_JRE_PATH" value="$PROJECT_DIR$/../.sdkman/candidates/java/current" />
+    <envs>
+      <env name="CROMWELL_BUILD_CENTAUR_SLICK_PROFILE" value="slick.jdbc.MySQLProfile$" />
+      <env name="CROMWELL_BUILD_CENTAUR_JDBC_DRIVER" value="com.mysql.cj.jdbc.Driver" />
+      <env name="CROMWELL_BUILD_CENTAUR_JDBC_URL" value="jdbc:mysql://localhost:3306/cromwell_test?allowPublicKeyRetrieval=true&amp;useSSL=false&amp;rewriteBatchedStatements=true&amp;serverTimezone=UTC&amp;useInformationSchema=true" />
+      <env name="CROMWELL_BUILD_RESOURCES_DIRECTORY" value="target/ci/resources" />
+      <env name="CROMWELL_BUILD_PAPI_JSON_FILE" value="target/ci/resources/cromwell-centaur-service-account.json" />
+      <env name="CROMWELL_BUILD_CENTAUR_READ_LINES_LIMIT" value="128000" />
+      <env name="CROMWELL_BUILD_CENTAUR_256_BITS_KEY" value="AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" />
+    </envs>
+    <option name="MAIN_CLASS_NAME" value="cromwell.CromwellApp" />
+    <module name="cromwell" />
+    <option name="PROGRAM_PARAMETERS" value="server" />
+    <option name="VM_PARAMETERS" value="-Dconfig.file=target/ci/resources/terra_tes_application.conf" />
+    <method v="2">
+      <option name="Make" enabled="true" />
+      <option name="ToolBeforeRunTask" enabled="true" actionId="Tool_External Tools_Acquire Google B2C Token" />
+      <option name="RunConfigurationTask" enabled="true" run_configuration_name="renderCiResources" run_configuration_type="SbtRunConfiguration" />
+    </method>
+  </configuration>
+</component>

src/ci/resources/acquire_b2c_token.sh

@@ -0,0 +1,42 @@
+#!/bin/bash
+
+# This script acquires a b2c token using gcloud auth and writes it to a file.
+# This file is used by renderCiResources to populate the CROMWELL_B2C_TOKEN in tes_application.conf.
+# It should be run as a "Before Launch" task in IntelliJ so the token is refreshed regularly.
+
+### Setting up this script for the first time in IntelliJ? 
+### 1. Open the "Run/Debug Configurations" dialog for the TES repo template.
+### 2. Add a new "Before Launch" task, selecting "Run External Tool". 
+###    This task should run BEFORE the renderCiResources task.
+### 3. Configure the new task:
+###   - Name: "Acquire b2c token"
+###   - Program: /full/path/to/src/ci/resources/acquire_b2c_token.sh
+###   - Arguments: ./env.temp
+###   - Working directory: /full/path/to/src/ci/resources
+###   - Advanced Options > Uncheck "Synchronize files after execution"
+### 4. Click "OK" to save the new task. Apply and close the "Run/Debug Configurations" dialog.
+
+# User must provide an output file path as an argument.
+# This can be provided by IntelliJ as a program argument in the run configuration.
+if [ -z "$1" ]; then
+    echo "Error: Must specify an output file path for the environment file."
+    exit 1
+fi
+
+# Acquire a b2c token using gcloud auth.
+# Send stderr to /dev/null so that any gcloud auth errors/update prompts don't pollute the token output.
+echo "Using local gcloud auth to acquire a b2c token..."
+B2C_TOKEN=$(gcloud auth print-access-token 2>/dev/null)
+if [ $? -eq 0 ]; then
+    echo "Acquired b2c token: ${B2C_TOKEN:0:4}****"
+else
+    echo "Failed to acquire b2c token. Is your local shell logged into gcloud?"
+fi
+
+# Create the output directory if it doesn't exist
+output_dir=$(dirname "$1")
+mkdir -p "$output_dir"
+
+# Write the token to the output file. dsde-toolbox will call `source` on this file that we're writing, 
+# so it's expected that this file contains valid bash syntax for setting an environment variable.
+echo "export CROMWELL_B2C_TOKEN=${B2C_TOKEN}" > "$1"

src/ci/resources/terra_tes_application.conf.ctmpl

@@ -0,0 +1,80 @@
+include required(classpath("application.conf"))
+include "build_application.inc.conf"
+
+filesystems {
+    blob {
+        class = "cromwell.filesystems.blob.BlobPathBuilderFactory"
+        global {
+            class = "cromwell.filesystems.blob.BlobFileSystemManager"
+            config {
+                workspace-manager: {
+                    # static: Where is WSM Endpoint
+                    url: "https://workspace.dsde-dev.broadinstitute.org"
+                    # See comment in src/ci/resources/acquire_b2c_token.sh for the one-time setup required to automatically acquire this.
+                    # This token can also be manually acquired with gcloud auth print-access-token myAccount@gmail.com
+                    b2cToken: "{{ env "CROMWELL_B2C_TOKEN" }}"
+                }
+            }
+        }
+    }
+}
+
+engine {
+  filesystems {
+    local {
+      enabled: true
+    }
+    http {
+      enabled: true
+    }
+    blob {
+      enabled: true
+    }
+  }
+}
+
+
+backend {
+  default = "TES"
+  providers {
+    TES {
+      actor-factory = "cromwell.backend.impl.tes.TesBackendLifecycleActorFactory"
+      config {
+        # Root of a blob storage container to use
+        # Might look something like: https://lz813a3d637adefec2c6e88f.blob.core.windows.net/sc-d8143fd8-aa07-446d-9ba0-af72203f1794/some/path/"
+        root = ""
+        dockerRoot = "/cromwell-executions"
+        # TES Endpoint for cromwell to use. Might look something like: "https://lz7388ada396994bb48ea5c87a02eed673689c82c2af423d03.servicebus.windows.net/something/tes/v1/tasks
+        endpoint = ""
+        concurrent-job-limit = 1000
+        # Identity to execute the workflow as. Might look something like: "pet-2676571657071603a9eab"
+        workflow-execution-identity = ""
+        # See comment in src/ci/resources/acquire_b2c_token.sh for the one-time setup required to automatically acquire this.
+        # This token can also be manually acquired with gcloud auth print-access-token myAccount@gmail.com
+        bearer-token="Bearer {{ env "CROMWELL_B2C_TOKEN" }}"
+        # Override default polling to make it faster for speedy tests
+        poll-backoff {
+          min: "10 seconds"
+          max: "5 minutes"
+          multiplier: 1.1
+          randomization-factor: 0.5
+        }
+
+        filesystems {
+          blob {
+            enabled: true
+          }
+          local {
+            enabled: false
+          }
+          http {
+            enabled: true
+          }
+        }
+      }
+
+      # Have the engine (maybe) authenticate to docker.io. See BT-141 for more info.
+      include "dockerhub_provider_config_v1.inc.conf"
+    }
+  }
+}

src/ci/resources/tes_application.conf

@@ -2,26 +2,12 @@ include required(classpath("application.conf"))
 include "build_application.inc.conf"
 
 filesystems {
-  blob {
-    class = "cromwell.filesystems.blob.BlobPathBuilderFactory"
-    global {
-      # One BFSM is shared across all BlobPathBuilders
-      class = "cromwell.filesystems.blob.BlobFileSystemManager"
-      config {
-        container: "cromwell"
-        endpoint: "https://<storage-account>.blob.core.windows.net"
-        subscription: "00001111-2222-3333-aaaa-bbbbccccdddd"
-        # WSM config is needed for accessing WSM-managed blob containers
-        # created in Terra workspaces.
-        workspace-manager {
-          url: "https://workspace.dsde-dev.broadinstitute.org"
-          workspace-id: "00001111-2222-3333-aaaa-bbbbccccdddd"
-          container-resource-id: "00001111-2222-3333-aaaa-bbbbccccdddd"
-          b2cToken: "Zardoz"
+    blob {
+        class = "cromwell.filesystems.blob.BlobPathBuilderFactory"
+        global {
+            class = "cromwell.filesystems.blob.BlobFileSystemManager"
         }
-      }
     }
-  }
 }
 
 engine {
@@ -33,7 +19,7 @@ engine {
       enabled: true
     }
     blob {
-      enabled: false
+      enabled: true
     }
   }
 }
@@ -45,27 +31,29 @@ backend {
     TES {
       actor-factory = "cromwell.backend.impl.tes.TesBackendLifecycleActorFactory"
       config {
-        # Use for running on blob storage
-        #root = "https://<storage-account>.blob.core.windows.net/cromwell/cromwell-executions"
-        root = "cromwell-executions"
+        # Root of a blob storage container to use
+        # Might look something like: https://lz813a3d637adefec2c6e88f.blob.core.windows.net/sc-d8143fd8-aa07-446d-9ba0-af72203f1794/some/path/"
+        root = ""
         dockerRoot = "/cromwell-executions"
+        # TES Endpoint for cromwell to use. Might look something like: "https://lz7388ada396994bb48ea5c87a02eed673689c82c2af423d03.servicebus.windows.net/something/tes/v1/tasks
         endpoint = "http://127.0.0.1:9000/v1/tasks"
         concurrent-job-limit = 1000
-
+        # Identity to execute the workflow as. Might look something like: "pet-2676571657071603a9eab"
+        workflow-execution-identity = ""
         # Override default polling to make it faster for speedy tests
         poll-backoff {
-          min: "1 seconds"
+          min: "10 seconds"
           max: "5 minutes"
           multiplier: 1.1
           randomization-factor: 0.5
         }
-        
+
         filesystems {
           blob {
-            enabled: false
+            enabled: true
           }
           local {
-            enabled: true
+            enabled: false
           }
           http {
             enabled: true