diff --git a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/actors/GcpBatchAsyncBackendJobExecutionActorSpec.scala b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/actors/GcpBatchAsyncBackendJobExecutionActorSpec.scala index f8d6e071b31..676a4d025e6 100644 --- a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/actors/GcpBatchAsyncBackendJobExecutionActorSpec.scala +++ b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/actors/GcpBatchAsyncBackendJobExecutionActorSpec.scala @@ -6,8 +6,11 @@ import _root_.wdl.draft2.model._ import akka.actor.{ActorRef, Props} import akka.testkit.{ImplicitSender, TestActorRef, TestDuration, TestProbe} import cats.data.NonEmptyList +import cloud.nio.impl.drs.DrsCloudNioFileProvider.DrsReadInterpreter +import cloud.nio.impl.drs.{DrsCloudNioFileSystemProvider, GoogleOauthDrsCredentials} import com.google.cloud.NoCredentials import com.google.cloud.batch.v1.{Job, JobName} +import com.typesafe.config.{Config, ConfigFactory} import common.collections.EnhancedCollections._ import cromwell.backend.BackendJobExecutionActor.BackendJobExecutionResponse import cromwell.backend._ @@ -56,6 +59,7 @@ import scala.concurrent.duration._ import scala.concurrent.{Await, ExecutionContext, Future, Promise} import scala.language.postfixOps import common.mock.MockSugar +import cromwell.filesystems.drs.DrsPathBuilder import org.mockito.Mockito._ class GcpBatchAsyncBackendJobExecutionActorSpec @@ -396,6 +400,55 @@ class GcpBatchAsyncBackendJobExecutionActorSpec GcpBatchAsyncBackendJobExecutionActor.groupParametersByGcsBucket(inputs) shouldEqual expected } + it should "generate a CSV manifest for DRS inputs, ignoring non-DRS inputs" in { + def makeDrsPathBuilder: DrsPathBuilder = { + val drsResolverConfig: Config = ConfigFactory.parseString( + """resolver { + | url = "http://drshub-url" + |} + |""".stripMargin + ) + + val fakeCredentials = NoCredentials.getInstance + + val drsReadInterpreter: DrsReadInterpreter = (_, _) => + throw new UnsupportedOperationException( + "PipelinesApiAsyncBackendJobExecutionActorSpec doesn't need to use drs read interpreter." + ) + + DrsPathBuilder( + new DrsCloudNioFileSystemProvider(drsResolverConfig, + GoogleOauthDrsCredentials(fakeCredentials, 1.minutes), + drsReadInterpreter + ), + None + ) + } + + val mount = GcpBatchWorkingDisk(DiskType.LOCAL, 1) + + def makeDrsInput(name: String, drsUri: String, containerPath: String): GcpBatchFileInput = { + val drsPath = makeDrsPathBuilder.build(drsUri).get + val containerRelativePath = DefaultPathBuilder.get(containerPath) + GcpBatchFileInput(name, drsPath, containerRelativePath, mount) + } + + val nonDrsInput: GcpBatchFileInput = GcpBatchFileInput("nnn", + DefaultPathBuilder.get("/local/nnn.bai"), + DefaultPathBuilder.get("/path/to/nnn.bai"), + mount + ) + + val inputs = List( + makeDrsInput("aaa", "drs://drs.example.org/aaa", "path/to/aaa.bai"), + nonDrsInput, + makeDrsInput("bbb", "drs://drs.example.org/bbb", "path/to/bbb.bai") + ) + + GcpBatchAsyncBackendJobExecutionActor.generateDrsLocalizerManifest(inputs) shouldEqual + "drs://drs.example.org/aaa,/mnt/disks/cromwell_root/path/to/aaa.bai\r\ndrs://drs.example.org/bbb,/mnt/disks/cromwell_root/path/to/bbb.bai\r\n" + } + it should "send proper value for \"number of reference files used gauge\" metric, or don't send anything if reference disks feature is disabled" in { val expectedInput1 = GcpBatchFileInput(name = "testfile1", diff --git a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchConfigurationSpec.scala b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchConfigurationSpec.scala index 42ae487b5fb..b65a153fb34 100644 --- a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchConfigurationSpec.scala +++ b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchConfigurationSpec.scala @@ -124,9 +124,14 @@ class GcpBatchConfigurationSpec ).root shouldBe "gs://my-cromwell-workflows-bucket" } - // it should "have correct docker" in { - // val dockerConf = new GcpBatchConfiguration(BackendConfigurationDescriptor(backendConfig, globalConfig), googleConfiguration, batchAttributes).dockerCredentials - // dockerConf shouldBe defined - // dockerConf.get.token shouldBe "dockerToken" - // } + it should "have correct docker" in { + val dockerConf = new GcpBatchConfiguration( + BackendConfigurationDescriptor(backendConfig, globalConfig), + googleConfiguration, + batchAttributes + ).dockerCredentials + + dockerConf shouldBe defined + dockerConf.get.token shouldBe "dockerToken" + } } diff --git a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributesSpec.scala b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributesSpec.scala index beb9b865c1b..110dca0e348 100644 --- a/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributesSpec.scala +++ b/supportedBackends/google/batch/src/test/scala/cromwell/backend/google/batch/models/GcpBatchRuntimeAttributesSpec.scala @@ -2,10 +2,9 @@ package cromwell.backend.google.batch.models import cats.data.NonEmptyList import cromwell.backend.RuntimeAttributeDefinition +import cromwell.backend.google.batch.io.{DiskType, GcpBatchAttachedDisk, GcpBatchWorkingDisk} import cromwell.backend.google.batch.models.GcpBatchTestConfig._ import cromwell.backend.validation.ContinueOnReturnCodeSet -//import cromwell.backend.google.batch.io.{DiskType, GcpBatchAttachedDisk} -import cromwell.backend.google.batch.io.{DiskType, GcpBatchWorkingDisk} import cromwell.core.WorkflowOptions import eu.timepit.refined.refineMV import org.scalatest.TestSuite @@ -48,9 +47,8 @@ final class GcpBatchRuntimeAttributesSpec } "fail to validate an invalid Docker entry" in { - pending val runtimeAttributes = Map("docker" -> WomInteger(1)) - assertBatchRuntimeAttributesFailedCreation(runtimeAttributes, "Expecting docker runtime attribute to be String") + assertBatchRuntimeAttributesFailedCreation(runtimeAttributes, "Expecting docker runtime attribute to be a String") } "validate a valid failOnStderr entry" in { @@ -152,21 +150,31 @@ final class GcpBatchRuntimeAttributesSpec ) } - // "validate a valid disks entry" in { - // val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "disks" -> WomString("local-disk 20 SSD")) - // val expectedRuntimeAttributes = expectedDefaults.copy(disks = Seq(GcpBatchAttachedDisk.parse("local-disk 20 SSD").get)) - // assertBatchRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) - // } - - // "fail to validate an invalid disks entry" in { - // val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "disks" -> WomInteger(10)) - // assertBatchRuntimeAttributesFailedCreation(runtimeAttributes, "Expecting disks runtime attribute to be a comma separated String or Array[String]") - // } - - // "fail to validate a valid disks array entry" in { - // val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "disks" -> WomArray(WomArrayType(WomStringType), List(WomString("blah"), WomString("blah blah")))) - // assertBatchRuntimeAttributesFailedCreation(runtimeAttributes, "Disk strings should be of the format 'local-disk SIZE TYPE' or '/mount/point SIZE TYPE'") - // } + "validate a valid disks entry" in { + val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "disks" -> WomString("local-disk 20 SSD")) + val expectedRuntimeAttributes = + expectedDefaults.copy(disks = Seq(GcpBatchAttachedDisk.parse("local-disk 20 SSD").get)) + assertBatchRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) + } + + "fail to validate an invalid disks entry" in { + val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "disks" -> WomInteger(10)) + assertBatchRuntimeAttributesFailedCreation( + runtimeAttributes, + "Expecting disks runtime attribute to be a comma separated String or Array[String]" + ) + } + + "fail to validate a valid disks array entry" in { + val runtimeAttributes = + Map("docker" -> WomString("ubuntu:latest"), + "disks" -> WomArray(WomArrayType(WomStringType), List(WomString("blah"), WomString("blah blah"))) + ) + assertBatchRuntimeAttributesFailedCreation( + runtimeAttributes, + "Disk strings should be of the format 'local-disk SIZE TYPE' or '/mount/point SIZE TYPE'" + ) + } "validate a valid memory entry" in { val runtimeAttributes = Map("docker" -> WomString("ubuntu:latest"), "memory" -> WomString("1 GB"))