Skip to content

Commit

Permalink
WX-927 call caching prefixes removed from GCP Batch backend (#7559)
Browse files Browse the repository at this point in the history
  • Loading branch information
mcovarr authored Sep 27, 2024
1 parent 9853b52 commit edc8918
Show file tree
Hide file tree
Showing 5 changed files with 3 additions and 18 deletions.
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# A list of call cache hint prefixes is explicitly specified but empty.
name: call_cache_hit_prefixes_empty_hint_papi
testFormat: runtwiceexpectingcallcaching
backends: [Papi, GCPBATCH]
backends: [Papi, GCPBATCH_CALL_CACHE_PREFIXES_NOT_IMPLEMENTED]

files {
workflow: call_cache_hit_prefixes/call_cache_hit_prefixes.wdl
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
name: call_cache_hit_prefixes_two_roots_empty_hint_cache_hit_papi
testFormat: runthriceexpectingcallcaching
# don't know
backends: [Papi, GCPBATCH_FAIL]
backends: [Papi, GCPBATCH_CALL_CACHE_PREFIXES_NOT_IMPLEMENTED]

files {
workflow: call_cache_hit_prefixes/call_cache_hit_prefixes.wdl
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ testFormat: runtwiceexpectingnocallcaching
# should NOT call cache the second run of call_cache_hit_prefixes_two_roots_empty_hint_cache_miss_papi *** FAILED *** (11 minutes, 11 seconds)
# centaur.test.CentaurTestException: Found unexpected cache hits for call_cache_hit_prefixes_two_roots_empty_hint_cache_miss_papi:
# calls.call_cache_hit_prefixes.yo.callCaching.result: Cache Hit: 19e522ed-685e-4c53-9d49-949d8b05a2a9:call_cache_hit_prefixes.yo:-1
backends: [Papi, GCPBATCH_FAIL]
backends: [Papi, GCPBATCH_CALL_CACHE_PREFIXES_NOT_IMPLEMENTED]

files {
workflow: call_cache_hit_prefixes/call_cache_hit_prefixes.wdl
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package cromwell.backend.google.batch.models

import com.google.auth.Credentials
import com.typesafe.config.Config
import cromwell.backend.google.batch.models.GcpBatchWorkflowPaths.callCachePathPrefixFromExecutionRoot
import cromwell.backend.google.batch.runnable.WorkflowOptionKeys
import cromwell.backend.io.WorkflowPaths
import cromwell.backend.{BackendJobDescriptorKey, BackendWorkflowDescriptor}
Expand All @@ -14,11 +13,6 @@ import scala.concurrent.ExecutionContext

object GcpBatchWorkflowPaths {
val GcsRootOptionKey = "gcp_batch_gcs_root"
private val GcsPrefix = "gs://"

private[models] def callCachePathPrefixFromExecutionRoot(executionRoot: String): String =
// If the root looks like gs://bucket/stuff-under-bucket this should return gs://bucket
GcsPrefix + executionRoot.substring(GcsPrefix.length).takeWhile(_ != '/')
}
case class GcpBatchWorkflowPaths(workflowDescriptor: BackendWorkflowDescriptor,
gcsCredentials: Credentials,
Expand All @@ -31,9 +25,6 @@ case class GcpBatchWorkflowPaths(workflowDescriptor: BackendWorkflowDescriptor,

override lazy val executionRootString: String =
workflowDescriptor.workflowOptions.getOrElse(GcpBatchWorkflowPaths.GcsRootOptionKey, gcpBatchConfiguration.root)
override lazy val callCacheRootPrefix: Option[String] = Option(
callCachePathPrefixFromExecutionRoot(executionRootString)
)

private val workflowOptions: WorkflowOptions = workflowDescriptor.workflowOptions

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,10 +40,4 @@ class GcpBatchWorkflowPathsSpec extends TestKitSuite with AnyFlatSpecLike with M
workflowPaths.workflowRoot.pathAsString should
be(s"gs://my-cromwell-workflows-bucket/wf_hello/${workflowDescriptor.id}/")
}

it should "calculate the call cache path prefix from the workflow execution root correctly" in {
val WorkspaceBucket = "gs://workspace-id"
val ExecutionRoot = WorkspaceBucket + "/submission-id"
GcpBatchWorkflowPaths.callCachePathPrefixFromExecutionRoot(ExecutionRoot) shouldBe WorkspaceBucket
}
}

0 comments on commit edc8918

Please sign in to comment.