Merge remote-tracking branch 'origin/gcp-batch' into gcp-batch

# Conflicts:
#	project/Dependencies.scala

CHANGELOG.md

@@ -6,9 +6,9 @@
 
 WDL `size` engine function now works for HTTP files. 
 
-### Azure ApplicationInsights Logging Support
-Cromwell can now send logs to Azure Application Insights. To enable, set environment 
-variable `APPLICATIONINSIGHTS_INSTRUMENTATIONKEY` to your account's key. [See here for information.](https://learn.microsoft.com/en-us/azure/azure-monitor/app/sdk-connection-string)
+### Azure Instrumentation Support
+Cromwell can now send logs and process information to Azure Application Insights. To enable, set environment 
+variable `APPLICATIONINSIGHTS_CONNECTION_STRING` to your connection string. [See here for information.](https://learn.microsoft.com/en-us/azure/azure-monitor/app/sdk-connection-string)
 
 
 ## 85 Release Notes

filesystems/blob/src/test/scala/cromwell/filesystems/blob/BlobPathBuilderSpec.scala

@@ -90,31 +90,30 @@ class BlobPathBuilderSpec extends AnyFlatSpec with Matchers with MockSugar {
   }
 
   //// The below tests are IGNORED because they depend on Azure auth information being present in the environment ////
-  private val subscriptionId: SubscriptionId = SubscriptionId(UUID.fromString("62b22893-6bc1-46d9-8a90-806bb3cce3c9"))
-  private val endpoint: EndpointURL = BlobPathBuilderSpec.buildEndpoint("coaexternalstorage")
-  private val store: BlobContainerName = BlobContainerName("inputs")
-
-  def makeBlobPathBuilder(blobEndpoint: EndpointURL, container: BlobContainerName): BlobPathBuilder = {
-    val blobTokenGenerator = NativeBlobSasTokenGenerator(container, blobEndpoint, Some(subscriptionId))
-    val fsm = new BlobFileSystemManager(container, blobEndpoint, 10, blobTokenGenerator)
-    new BlobPathBuilder(store, endpoint)(fsm)
-  }
+  val subscriptionId = SubscriptionId(UUID.fromString("62b22893-6bc1-46d9-8a90-806bb3cce3c9"))
 
   ignore should "resolve an absolute path string correctly to a path" in {
-    val builder = makeBlobPathBuilder(endpoint, store)
+    val endpoint = BlobPathBuilderSpec.buildEndpoint("coaexternalstorage")
+    val store = BlobContainerName("inputs")
+    val blobTokenGenerator = NativeBlobSasTokenGenerator(store, endpoint, Some(subscriptionId))
+    val fsm: BlobFileSystemManager = new BlobFileSystemManager(store, endpoint, 10, blobTokenGenerator)
+
     val rootString = s"${endpoint.value}/${store.value}/cromwell-execution"
-    val blobRoot: BlobPath = builder build rootString getOrElse fail()
+    val blobRoot: BlobPath = new BlobPathBuilder(store, endpoint)(fsm) build rootString getOrElse fail()
     blobRoot.toAbsolutePath.pathAsString should equal ("https://coaexternalstorage.blob.core.windows.net/inputs/cromwell-execution")
     val otherFile = blobRoot.resolve("https://coaexternalstorage.blob.core.windows.net/inputs/cromwell-execution/test/inputFile.txt")
     otherFile.toAbsolutePath.pathAsString should equal ("https://coaexternalstorage.blob.core.windows.net/inputs/cromwell-execution/test/inputFile.txt")
   }
 
   ignore should "build a blob path from a test string and read a file" in {
-    val builder = makeBlobPathBuilder(endpoint, store)
+    val endpoint = BlobPathBuilderSpec.buildEndpoint("coaexternalstorage")
     val endpointHost = BlobPathBuilder.parseURI(endpoint.value).map(_.getHost).getOrElse(fail("Could not parse URI"))
+    val store = BlobContainerName("inputs")
     val evalPath = "/test/inputFile.txt"
+    val blobTokenGenerator = NativeBlobSasTokenGenerator(store, endpoint, Some(subscriptionId))
+    val fsm: BlobFileSystemManager = new BlobFileSystemManager(store, endpoint, 10L, blobTokenGenerator)
     val testString = endpoint.value + "/" + store + evalPath
-    val blobPath: BlobPath = builder build testString getOrElse fail()
+    val blobPath: BlobPath = new BlobPathBuilder(store, endpoint)(fsm) build testString getOrElse fail()
 
     blobPath.container should equal(store)
     blobPath.endpoint should equal(endpoint)
@@ -126,46 +125,31 @@ class BlobPathBuilderSpec extends AnyFlatSpec with Matchers with MockSugar {
   }
 
   ignore should "build duplicate blob paths in the same filesystem" in {
-    val builder = makeBlobPathBuilder(endpoint, store)
+    val endpoint = BlobPathBuilderSpec.buildEndpoint("coaexternalstorage")
+    val store = BlobContainerName("inputs")
     val evalPath = "/test/inputFile.txt"
+    val blobTokenGenerator = NativeBlobSasTokenGenerator(store, endpoint, Some(subscriptionId))
+    val fsm: BlobFileSystemManager = new BlobFileSystemManager(store, endpoint, 10, blobTokenGenerator)
     val testString = endpoint.value + "/" + store + evalPath
-    val blobPath1: BlobPath = builder build testString getOrElse fail()
+    val blobPath1: BlobPath = new BlobPathBuilder(store, endpoint)(fsm) build testString getOrElse fail()
     blobPath1.nioPath.getFileSystem.close()
-    val blobPath2: BlobPath = builder build testString getOrElse fail()
+    val blobPath2: BlobPath = new BlobPathBuilder(store, endpoint)(fsm) build testString getOrElse fail()
     blobPath1 should equal(blobPath2)
     val is = blobPath1.newInputStream()
     val fileText = (is.readAllBytes.map(_.toChar)).mkString
     fileText should include ("This is my test file!!!! Did it work?")
   }
 
   ignore should "resolve a path without duplicating container name" in {
-    val builder = makeBlobPathBuilder(endpoint, store)
+    val endpoint = BlobPathBuilderSpec.buildEndpoint("coaexternalstorage")
+    val store = BlobContainerName("inputs")
+    val blobTokenGenerator = NativeBlobSasTokenGenerator(store, endpoint, Some(subscriptionId))
+    val fsm: BlobFileSystemManager = new BlobFileSystemManager(store, endpoint, 10, blobTokenGenerator)
+
     val rootString = s"${endpoint.value}/${store.value}/cromwell-execution"
-    val blobRoot: BlobPath = builder build rootString getOrElse fail()
+    val blobRoot: BlobPath = new BlobPathBuilder(store, endpoint)(fsm) build rootString getOrElse fail()
     blobRoot.toAbsolutePath.pathAsString should equal ("https://coaexternalstorage.blob.core.windows.net/inputs/cromwell-execution")
     val otherFile = blobRoot.resolve("test/inputFile.txt")
     otherFile.toAbsolutePath.pathAsString should equal ("https://coaexternalstorage.blob.core.windows.net/inputs/cromwell-execution/test/inputFile.txt")
   }
-
-  ignore should "correctly remove a prefix from the blob path" in {
-    val builder = makeBlobPathBuilder(endpoint, store)
-    val rootString = s"${endpoint.value}/${store.value}/cromwell-execution/"
-    val execDirString = s"${endpoint.value}/${store.value}/cromwell-execution/abc123/myworkflow/task1/def4356/execution/"
-    val fileString = s"${endpoint.value}/${store.value}/cromwell-execution/abc123/myworkflow/task1/def4356/execution/stdout"
-    val blobRoot: BlobPath = builder build rootString getOrElse fail()
-    val execDir: BlobPath = builder build execDirString getOrElse fail()
-    val blobFile: BlobPath = builder build fileString getOrElse fail()
-    blobFile.pathStringWithoutPrefix(blobRoot) should equal ("abc123/myworkflow/task1/def4356/execution/stdout")
-    blobFile.pathStringWithoutPrefix(execDir) should equal ("stdout")
-    blobFile.pathStringWithoutPrefix(blobFile) should equal ("")
-  }
-
-  ignore should "not change a path if it doesn't start with a prefix" in {
-    val builder = makeBlobPathBuilder(endpoint, store)
-    val otherRootString = s"${endpoint.value}/${store.value}/foobar/"
-    val fileString = s"${endpoint.value}/${store.value}/cromwell-execution/abc123/myworkflow/task1/def4356/execution/stdout"
-    val otherBlobRoot: BlobPath = builder build otherRootString getOrElse fail()
-    val blobFile: BlobPath = builder build fileString getOrElse fail()
-    blobFile.pathStringWithoutPrefix(otherBlobRoot) should equal ("/cromwell-execution/abc123/myworkflow/task1/def4356/execution/stdout")
-  }
 }

languageFactories/language-factory-core/src/main/scala/cromwell/languages/util/LanguageFactoryUtil.scala

@@ -94,7 +94,7 @@ object LanguageFactoryUtil {
     }
 
     val firstCodeLine = fileWithoutInitialWhitespace.headOption.map(_.dropWhile(_.isWhitespace))
-    firstCodeLine.exists { line => startsWithOptions.contains(line.trim) }
+    firstCodeLine.exists { line => startsWithOptions.contains(line) }
   }
 
   def chooseFactory(workflowSource: WorkflowSource, wsfc: WorkflowSourceFilesCollection): ErrorOr[LanguageFactory] = {

server/src/main/scala/cromwell/CromwellEntryPoint.scala

@@ -9,6 +9,7 @@ import cats.data.Validated._
 import cats.effect.{ContextShift, IO}
 import cats.syntax.apply._
 import cats.syntax.validated._
+import com.microsoft.applicationinsights.attach.ApplicationInsights
 import com.typesafe.config.{Config, ConfigFactory}
 import common.exception.MessageAggregation
 import common.validation.ErrorOr._
@@ -45,6 +46,11 @@ object CromwellEntryPoint extends GracefulStopSupport {
   private val dnsCacheTtl = config.getOrElse("system.dns-cache-ttl", 3 minutes)
   java.security.Security.setProperty("networkaddress.cache.ttl", dnsCacheTtl.toSeconds.toString)
 
+  // The presence of this env var tells us that the user is trying to send instrumentation data and
+  // logs to Azure Application Insights. If it's present, we'll attach the ApplicationInsights agent.
+  // To configure the behavior of this agent, see server/src/main/resources/applicationinsights.json
+  private lazy val useAzureInstrumentation = sys.env.contains("APPLICATIONINSIGHTS_CONNECTION_STRING")
+
   /**
     * Run Cromwell in server mode.
     */
@@ -145,6 +151,12 @@ object CromwellEntryPoint extends GracefulStopSupport {
     */
   private def initLogging(command: Command): Unit = {
 
+    // Enable Azure instrumentation and log-slurping if desired. Running ApplicationInsights.attach()
+    // without checking for the presence of the relevant env var doesn't cause any failures, but does
+    // print an error that would be confusing for non-Azure users.
+    if (useAzureInstrumentation)
+      ApplicationInsights.attach()
+
     val logbackSetting = command match {
       case Server => "STANDARD"
       case Submit => "PRETTY"