From aff0349e6feb4b2b2f197c5d8749591e1625f7d2 Mon Sep 17 00:00:00 2001 From: quekx Date: Mon, 3 Jun 2024 10:26:36 -0700 Subject: [PATCH] fix issue with path - need refinement --- ...wsBatchAsyncBackendJobExecutionActor.scala | 22 ++++++++++--------- .../impl/aws/io/AwsBatchGlobFunctions.scala | 9 ++++---- 2 files changed, 17 insertions(+), 14 deletions(-) diff --git a/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsBatchAsyncBackendJobExecutionActor.scala b/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsBatchAsyncBackendJobExecutionActor.scala index d9fca384add..f4988b55906 100755 --- a/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsBatchAsyncBackendJobExecutionActor.scala +++ b/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/AwsBatchAsyncBackendJobExecutionActor.scala @@ -428,12 +428,14 @@ class AwsBatchAsyncBackendJobExecutionActor(override val standardParams: Standar val wfid = standardParams.jobDescriptor.toString.split(":")(0) val globName = GlobFunctions.globName(s"${womFile.value}-${wfid}") val globbedDir = Paths.get(womFile.value).getParent match { - case path: Path => path.toString - case _ => "./" + // remove ./ so it does not appear on s3 path + case path: Path => path.toString.stripPrefix("./") + case _ => "" } // generalize folder and list file - val globDirectory = DefaultPathBuilder.get(globbedDir + "/." + globName + "/") - val globListFile = DefaultPathBuilder.get(globbedDir + "/." + globName + ".list") + val globDirPrefix = s"${globbedDir}/.${globName}".stripPrefix("/"); + val globDirectory = DefaultPathBuilder.get(globDirPrefix + "/") + val globListFile = DefaultPathBuilder.get(globDirPrefix + ".list") // locate the disk where the globbed data resides val (_, globDirectoryDisk) = relativePathAndVolume(womFile.value, runtimeAttributes.disks) @@ -461,14 +463,14 @@ class AwsBatchAsyncBackendJobExecutionActor(override val standardParams: Standar private def generateAwsBatchGlobFileOutputs(womFile: WomGlobFile): List[AwsBatchFileOutput] = { val (globName, globbedDir, globDirectoryDisk, globDirectoryDestinationPath, globListFileDestinationPath) = generateGlobPaths(womFile) - val (relpathDir,_) = relativePathAndVolume(DefaultPathBuilder.get(globbedDir + "/." + globName + "/" + "*").toString,runtimeAttributes.disks) - val (relpathList,_) = relativePathAndVolume(DefaultPathBuilder.get(globbedDir + "/." + globName + ".list").toString,runtimeAttributes.disks) + val (relpathDir,_) = relativePathAndVolume(DefaultPathBuilder.get(globbedDir + "/." + globName + "/" + "*").toString.stripPrefix("/"),runtimeAttributes.disks) + val (relpathList,_) = relativePathAndVolume(DefaultPathBuilder.get(globbedDir + "/." + globName + ".list").toString.stripPrefix("/"),runtimeAttributes.disks) // We need both the glob directory and the glob list: List( // The glob directory:. - AwsBatchFileOutput(DefaultPathBuilder.get(globbedDir.toString + "/." + globName + "/" + "*").toString,globDirectoryDestinationPath, relpathDir, globDirectoryDisk), + AwsBatchFileOutput(DefaultPathBuilder.get(globbedDir.toString + "/." + globName + "/" + "*").toString.stripPrefix("/"),globDirectoryDestinationPath, relpathDir, globDirectoryDisk), // The glob list file: - AwsBatchFileOutput(DefaultPathBuilder.get(globbedDir.toString + "/." + globName + ".list").toString, globListFileDestinationPath, relpathList, globDirectoryDisk) + AwsBatchFileOutput(DefaultPathBuilder.get(globbedDir.toString + "/." + globName + ".list").toString.stripPrefix("/"), globListFileDestinationPath, relpathList, globDirectoryDisk) ) } @@ -865,8 +867,8 @@ class AwsBatchAsyncBackendJobExecutionActor(override val standardParams: Standar val (globName, globbedDir, _, _, _) = generateGlobPaths(globFile) val controlFileName = "cromwell_glob_control_file" val absoluteGlobValue = commandDirectory.resolve(globFile.value).pathAsString - val globDirectory = globbedDir + "/." + globName + "/" - val globList = globbedDir + "/." + globName + ".list" + val globDirectory = (globbedDir + "/." + globName + "/").stripPrefix("/") + val globList = (globbedDir + "/." + globName + ".list").stripPrefix("/") val globLinkCommand: String = (if (configuration.globLinkCommand.isDefined) { "( " + configuration.globLinkCommand.getOrElse("").toString + " )" diff --git a/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/io/AwsBatchGlobFunctions.scala b/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/io/AwsBatchGlobFunctions.scala index f44946031c5..ff099e46e29 100644 --- a/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/io/AwsBatchGlobFunctions.scala +++ b/supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/io/AwsBatchGlobFunctions.scala @@ -69,8 +69,9 @@ trait AwsBatchGlobFunctions extends GlobFunctions { val wfid = callContext.root.toString.split("/").toList.filter(element => wfid_regex.pattern.matcher(element).matches()).lastOption.getOrElse("") val globPatternName = globName(s"${pattern}-${wfid}") val globbedDir = Paths.get(pattern).getParent match { - case x: Path => x.toString - case _ => "./" + // remove "./" to avoid it from appearing in s3 path + case x: Path => x.toString.stripPrefix("./") + case _ => "" } val listFilePath = if (pattern.startsWith("/mnt/efs/")) { DefaultPathBuilder.get(globbedDir + "/." + globPatternName + ".list") @@ -81,9 +82,9 @@ trait AwsBatchGlobFunctions extends GlobFunctions { lines.toList map { fileName => // again : this should be config based... if (pattern.startsWith("/mnt/efs/")) { - s"${globbedDir}/.${globPatternName}/${fileName}" + s"${globbedDir}/.${globPatternName}/${fileName}".stripPrefix("/") } else { - callContext.root.resolve(s"${globbedDir}/.${globPatternName}/${fileName}".stripPrefix("/")).pathAsString + callContext.root.resolve(s"${globbedDir}/.${globPatternName}/${fileName}".stripPrefix("/")).pathAsString.stripPrefix("/") } } }