From 40f76965a817d29ebb02b70d46f726b3e2f28e2f Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Tue, 1 Mar 2022 18:52:03 -0500 Subject: [PATCH 01/45] bad stdout location tests --- .../standardTestCases/bad_stdout_input.test | 12 ++++++++++++ .../bad_stdout_mid_workflow.test | 12 ++++++++++++ .../standardTestCases/bad_stdout_outputs.test | 12 ++++++++++++ .../standardTestCases/bad_stdout_subworkflow.test | 15 +++++++++++++++ .../bad_stdouts/bad_stdout_input.wdl | 10 ++++++++++ .../bad_stdouts/bad_stdout_mid_workflow.wdl | 9 +++++++++ .../bad_stdouts/bad_stdout_output.wdl | 7 +++++++ .../bad_stdouts/bad_stdout_subworkflow.wdl | 9 +++++++++ 8 files changed, 86 insertions(+) create mode 100644 centaur/src/main/resources/standardTestCases/bad_stdout_input.test create mode 100644 centaur/src/main/resources/standardTestCases/bad_stdout_mid_workflow.test create mode 100644 centaur/src/main/resources/standardTestCases/bad_stdout_outputs.test create mode 100644 centaur/src/main/resources/standardTestCases/bad_stdout_subworkflow.test create mode 100644 centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_input.wdl create mode 100644 centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_mid_workflow.wdl create mode 100644 centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_output.wdl create mode 100644 centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_subworkflow.wdl diff --git a/centaur/src/main/resources/standardTestCases/bad_stdout_input.test b/centaur/src/main/resources/standardTestCases/bad_stdout_input.test new file mode 100644 index 00000000000..ca48fe736f1 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stdout_input.test @@ -0,0 +1,12 @@ +# Verifies that if we use stdout in a workflow level, the workflow fails properly. +name: bad_stdout_input +testFormat: workflowfailure + +files { + workflow: bad_stdouts/bad_stdout_input.wdl +} + +metadata { + workflowName: bad_stdout + status: Failed +} diff --git a/centaur/src/main/resources/standardTestCases/bad_stdout_mid_workflow.test b/centaur/src/main/resources/standardTestCases/bad_stdout_mid_workflow.test new file mode 100644 index 00000000000..1f66c347ae4 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stdout_mid_workflow.test @@ -0,0 +1,12 @@ +# Verifies that if we use stdout in a workflow level, the workflow fails properly. +name: bad_stdout_mid_workflow +testFormat: workflowfailure + +files { + workflow: bad_stdouts/bad_stdout_mid_workflow.wdl +} + +metadata { + workflowName: bad_stdout + status: Failed +} diff --git a/centaur/src/main/resources/standardTestCases/bad_stdout_outputs.test b/centaur/src/main/resources/standardTestCases/bad_stdout_outputs.test new file mode 100644 index 00000000000..d00c2a45850 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stdout_outputs.test @@ -0,0 +1,12 @@ +# Verifies that if we use stdout in a workflow level, the workflow fails properly. +name: bad_stdout_output +testFormat: workflowfailure + +files { + workflow: bad_stdouts/bad_stdout_output.wdl +} + +metadata { + workflowName: bad_stdout + status: Failed +} diff --git a/centaur/src/main/resources/standardTestCases/bad_stdout_subworkflow.test b/centaur/src/main/resources/standardTestCases/bad_stdout_subworkflow.test new file mode 100644 index 00000000000..b5ea413bf18 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stdout_subworkflow.test @@ -0,0 +1,15 @@ +# Verifies that if we use stdout in a workflow level, the workflow fails properly. +name: bad_stdout_subworkflow +testFormat: workflowfailure + +files { + workflow: bad_stdouts/bad_stdout_subworkflow.wdl + imports: [ + bad_stdouts/bad_stdout_mid_workflow.wdl + ] +} + +metadata { + workflowName: calls_bad_stdout_subworkflow + status: Failed +} diff --git a/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_input.wdl b/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_input.wdl new file mode 100644 index 00000000000..c68f0ebffd3 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_input.wdl @@ -0,0 +1,10 @@ +version 1.0 + +workflow bad_stdout { + input { + File f = stdout() + } + output { + String s = read_string(f) + } +} diff --git a/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_mid_workflow.wdl b/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_mid_workflow.wdl new file mode 100644 index 00000000000..d4f57bfd125 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_mid_workflow.wdl @@ -0,0 +1,9 @@ +version 1.0 + +workflow bad_stdout { + File f = stdout() + + output { + String s = read_string(f) + } +} diff --git a/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_output.wdl b/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_output.wdl new file mode 100644 index 00000000000..ad2bc9a24e4 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_output.wdl @@ -0,0 +1,7 @@ +version 1.0 + +workflow bad_stdout { + output { + File f = stdout() + } +} diff --git a/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_subworkflow.wdl b/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_subworkflow.wdl new file mode 100644 index 00000000000..7134825b827 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_subworkflow.wdl @@ -0,0 +1,9 @@ +version 1.0 + +import "bad_stdout_mid_workflow.wdl" + +workflow calls_bad_stdout_subworkflow { + scatter(i in range(1000)) { + call bad_stdout_mid_workflow.bad_stdout + } +} From 1048541e8dad023d7c32410c2addadfea9456921 Mon Sep 17 00:00:00 2001 From: Katrina Pierre Date: Mon, 4 Apr 2022 16:32:39 -0400 Subject: [PATCH 02/45] just a draft --- .../AstToWorkflowDefinitionElement.scala | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala index ad4c28f064a..a91ab1e83d1 100644 --- a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala +++ b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala @@ -8,6 +8,7 @@ import common.transforms.CheckedAtoB import common.validation.ErrorOr._ import wdl.model.draft3.elements._ import wom.SourceFileLocation +import wdl.model.draft3.elements.ExpressionElement.StdoutElement object AstToWorkflowDefinitionElement { @@ -40,12 +41,32 @@ object AstToWorkflowDefinitionElement { } } + private def checkStdout(inputSectionValidation: WorkflowBodyElement, b: outputsSectionValidation[WorkflowBodyElement]): Unit = { + inputsSectionValidation.map(x => x match { + case Some(value) => val here = value.inputDeclarations.flatMap(_.expression).exists(_.isInstanceOf[StdoutElement.type]) + s"Workflow cannot have more than $numExpected $sectionName sections, found ${elements.size}.".invalidNel + case None => _ + }) + } + private def validateSize[A](elements: Vector[A], sectionName: String, numExpected: Int): ErrorOr[Option[A]] = { val sectionValidation: ErrorOr[Option[A]] = if (elements.size > numExpected) { s"Workflow cannot have more than $numExpected $sectionName sections, found ${elements.size}.".invalidNel } else { elements.headOption.validNel } + sectionValidation } + + /*for(validate<-AstToWorkflowDefinitionElement){ + validate match { + case check input{} + case check p + + } + */ + } + +//def validateStdout(a:) \ No newline at end of file From 3733cd810cc3e56ecd7eabdb7da63afd8b2693bb Mon Sep 17 00:00:00 2001 From: Saloni Shah Date: Mon, 4 Apr 2022 17:04:54 -0400 Subject: [PATCH 03/45] checkStdout helper function --- .../AstToWorkflowDefinitionElement.scala | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala index a91ab1e83d1..f43bdbfaf92 100644 --- a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala +++ b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala @@ -30,23 +30,27 @@ object AstToWorkflowDefinitionElement { val inputsSectionValidation: ErrorOr[Option[InputsSectionElement]] = validateSize(bodyElements.filterByType[InputsSectionElement], "inputs", 1) val outputsSectionValidation: ErrorOr[Option[OutputsSectionElement]] = validateSize(bodyElements.filterByType[OutputsSectionElement], "outputs", 1) + val inputsSectionStdoutValidation: ErrorOr[ErrorOr[Option[String]]] = inputsSectionValidation.map(checkIfStdoutExist) + val graphSections: Vector[WorkflowGraphElement] = bodyElements.filterByType[WorkflowGraphElement] val metaSectionValidation: ErrorOr[Option[MetaSectionElement]] = validateSize(bodyElements.filterByType[MetaSectionElement], "meta", 1) val parameterMetaSectionValidation: ErrorOr[Option[ParameterMetaSectionElement]] = validateSize(bodyElements.filterByType[ParameterMetaSectionElement], "parameterMeta", 1) - (inputsSectionValidation, outputsSectionValidation, metaSectionValidation, parameterMetaSectionValidation) mapN { - (validInputs, validOutputs, meta, parameterMeta) => + (inputsSectionValidation, outputsSectionValidation, metaSectionValidation, parameterMetaSectionValidation, inputsSectionStdoutValidation) mapN { + (validInputs, validOutputs, meta, parameterMeta, _) => WorkflowDefinitionElement(name, validInputs, graphSections.toSet, validOutputs, meta, parameterMeta, sourceLocation) } } - private def checkStdout(inputSectionValidation: WorkflowBodyElement, b: outputsSectionValidation[WorkflowBodyElement]): Unit = { - inputsSectionValidation.map(x => x match { - case Some(value) => val here = value.inputDeclarations.flatMap(_.expression).exists(_.isInstanceOf[StdoutElement.type]) - s"Workflow cannot have more than $numExpected $sectionName sections, found ${elements.size}.".invalidNel - case None => _ - }) + private def checkIfStdoutExist(inputSection: Option[InputsSectionElement]): ErrorOr[Option[String]] = { + inputSection match { + case Some(section) => + if (section.inputDeclarations.flatMap(_.expression).exists(_.isInstanceOf[StdoutElement.type])) { + s"Workflow cannot have stdout expression in input section at workflow-level.".invalidNel + } else None.validNel + case None => None.validNel + } } private def validateSize[A](elements: Vector[A], sectionName: String, numExpected: Int): ErrorOr[Option[A]] = { @@ -69,4 +73,4 @@ object AstToWorkflowDefinitionElement { } -//def validateStdout(a:) \ No newline at end of file +//def validateStdout(a:) From 5c412519c7d913d44c3fe626be32ef20214b6fb1 Mon Sep 17 00:00:00 2001 From: Katrina Pierre Date: Thu, 14 Apr 2022 02:14:25 -0400 Subject: [PATCH 04/45] Adding stderr wdl and test files, refactoring functions and adding functions to handle stderr --- .../standardTestCases/bad_stderr_input.test | 12 +++ .../bad_stderr_mid_workflow.test | 12 +++ .../standardTestCases/bad_stderr_outputs.test | 12 +++ .../bad_stderr_subworkflow.test | 15 ++++ .../bad_stderrs/bad_stderr_input.wdl | 11 +++ .../bad_stderrs/bad_stderr_mid_workflow.wdl | 9 +++ .../bad_stderrs/bad_stderr_output.wdl | 7 ++ .../bad_stderrs/bad_stderr_subworkflow.wdl | 9 +++ .../AstToWorkflowDefinitionElement.scala | 77 ++++++++++++++----- 9 files changed, 146 insertions(+), 18 deletions(-) create mode 100644 centaur/src/main/resources/standardTestCases/bad_stderr_input.test create mode 100644 centaur/src/main/resources/standardTestCases/bad_stderr_mid_workflow.test create mode 100644 centaur/src/main/resources/standardTestCases/bad_stderr_outputs.test create mode 100644 centaur/src/main/resources/standardTestCases/bad_stderr_subworkflow.test create mode 100644 centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_input.wdl create mode 100644 centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_mid_workflow.wdl create mode 100644 centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_output.wdl create mode 100644 centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_subworkflow.wdl diff --git a/centaur/src/main/resources/standardTestCases/bad_stderr_input.test b/centaur/src/main/resources/standardTestCases/bad_stderr_input.test new file mode 100644 index 00000000000..57e999dcd89 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stderr_input.test @@ -0,0 +1,12 @@ +# Verifies that if we use stderr in a workflow level, the workflow fails properly. +name: bad_stderr_input +testFormat: workflowfailure + +files { + workflow: bad_stderrs/bad_stderr_input.wdl +} + +metadata { + workflowName: bad_stderr + status: Failed +} \ No newline at end of file diff --git a/centaur/src/main/resources/standardTestCases/bad_stderr_mid_workflow.test b/centaur/src/main/resources/standardTestCases/bad_stderr_mid_workflow.test new file mode 100644 index 00000000000..5ab4668814a --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stderr_mid_workflow.test @@ -0,0 +1,12 @@ +# Verifies that if we use stderr in a workflow level, the workflow fails properly. +name: bad_stderr_mid_workflow +testFormat: workflowfailure + +files { + workflow: bad_stderrs/bad_stderr_mid_workflow.wdl +} + +metadata { + workflowName: bad_stderr + status: Failed +} diff --git a/centaur/src/main/resources/standardTestCases/bad_stderr_outputs.test b/centaur/src/main/resources/standardTestCases/bad_stderr_outputs.test new file mode 100644 index 00000000000..126bbc2fd11 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stderr_outputs.test @@ -0,0 +1,12 @@ +# Verifies that if we use stderr in a workflow level, the workflow fails properly. +name: bad_stderr_output +testFormat: workflowfailure + +files { + workflow: bad_stderrs/bad_stderr_output.wdl +} + +metadata { + workflowName: bad_stderr + status: Failed +} diff --git a/centaur/src/main/resources/standardTestCases/bad_stderr_subworkflow.test b/centaur/src/main/resources/standardTestCases/bad_stderr_subworkflow.test new file mode 100644 index 00000000000..cf22aca78fe --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stderr_subworkflow.test @@ -0,0 +1,15 @@ +# Verifies that if we use stderr in a workflow level, the workflow fails properly. +name: bad_stderr_subworkflow +testFormat: workflowfailure + +files { + workflow: bad_stderrs/bad_stderr_subworkflow.wdl + imports: [ + bad_stderrs/bad_stderr_mid_workflow.wdl + ] +} + +metadata { + workflowName: calls_bad_stderr_subworkflow + status: Failed +} diff --git a/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_input.wdl b/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_input.wdl new file mode 100644 index 00000000000..ef7e2b7ff82 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_input.wdl @@ -0,0 +1,11 @@ +version 1.0 + +workflow bad_stderr { + + input { + File f = stderr() + } + output { + String s = read_string(f) + } +} diff --git a/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_mid_workflow.wdl b/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_mid_workflow.wdl new file mode 100644 index 00000000000..eb6331ff8c5 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_mid_workflow.wdl @@ -0,0 +1,9 @@ +version 1.0 + +workflow bad_stderr { + File f = stderr() + + output { + String s = read_string(f) + } +} diff --git a/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_output.wdl b/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_output.wdl new file mode 100644 index 00000000000..65e99949a52 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_output.wdl @@ -0,0 +1,7 @@ +version 1.0 + +workflow bad_stderr { + output { + File f = stderr() + } +} diff --git a/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_subworkflow.wdl b/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_subworkflow.wdl new file mode 100644 index 00000000000..b1249f8dd9d --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_subworkflow.wdl @@ -0,0 +1,9 @@ +version 1.0 + +import "bad_stderr_mid_workflow.wdl" + +workflow calls_bad_stderr_subworkflow { + scatter(i in range(1000)) { + call bad_stderr_mid_workflow.bad_stderr + } +} diff --git a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala index f43bdbfaf92..82513fdeb12 100644 --- a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala +++ b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala @@ -8,7 +8,7 @@ import common.transforms.CheckedAtoB import common.validation.ErrorOr._ import wdl.model.draft3.elements._ import wom.SourceFileLocation -import wdl.model.draft3.elements.ExpressionElement.StdoutElement +import wdl.model.draft3.elements.ExpressionElement._ object AstToWorkflowDefinitionElement { @@ -27,23 +27,33 @@ object AstToWorkflowDefinitionElement { sourceLocation: Option[SourceFileLocation], bodyElements: Vector[WorkflowBodyElement]) = { - val inputsSectionValidation: ErrorOr[Option[InputsSectionElement]] = validateSize(bodyElements.filterByType[InputsSectionElement], "inputs", 1) - val outputsSectionValidation: ErrorOr[Option[OutputsSectionElement]] = validateSize(bodyElements.filterByType[OutputsSectionElement], "outputs", 1) + val inputsSectionValidation: ErrorOr[Option[InputsSectionElement]] = for { + inputValidateElement <- validateSize(bodyElements.filterByType[InputsSectionElement], "inputs", 1): ErrorOr[Option[InputsSectionElement]] + _ <- checkIfStdoutInputExist(inputValidateElement) + _ <- checkIfStderrInputExist(inputValidateElement) + } yield inputValidateElement - val inputsSectionStdoutValidation: ErrorOr[ErrorOr[Option[String]]] = inputsSectionValidation.map(checkIfStdoutExist) + val intermediateValueDeclarationStdoutCheck: ErrorOr[Option[String]] = checkStdoutIntermediates(bodyElements.filterByType[IntermediateValueDeclarationElement]) + val intermediateValueDeclarationStderrCheck: ErrorOr[Option[String]] = checkStderrIntermediates(bodyElements.filterByType[IntermediateValueDeclarationElement]) + + val outputsSectionValidation: ErrorOr[Option[OutputsSectionElement]] = for { + outputValidateElement <- validateSize(bodyElements.filterByType[OutputsSectionElement], "outputs", 1): ErrorOr[Option[OutputsSectionElement]] + _ <- checkIfStdoutOutputExists(outputValidateElement) + _ <- checkIfStderrOutputExists(outputValidateElement) + } yield outputValidateElement val graphSections: Vector[WorkflowGraphElement] = bodyElements.filterByType[WorkflowGraphElement] val metaSectionValidation: ErrorOr[Option[MetaSectionElement]] = validateSize(bodyElements.filterByType[MetaSectionElement], "meta", 1) val parameterMetaSectionValidation: ErrorOr[Option[ParameterMetaSectionElement]] = validateSize(bodyElements.filterByType[ParameterMetaSectionElement], "parameterMeta", 1) - (inputsSectionValidation, outputsSectionValidation, metaSectionValidation, parameterMetaSectionValidation, inputsSectionStdoutValidation) mapN { - (validInputs, validOutputs, meta, parameterMeta, _) => + (inputsSectionValidation, outputsSectionValidation, metaSectionValidation, parameterMetaSectionValidation, intermediateValueDeclarationStdoutCheck, intermediateValueDeclarationStderrCheck) mapN { + (validInputs, validOutputs, meta, parameterMeta, _, _) => WorkflowDefinitionElement(name, validInputs, graphSections.toSet, validOutputs, meta, parameterMeta, sourceLocation) } } - private def checkIfStdoutExist(inputSection: Option[InputsSectionElement]): ErrorOr[Option[String]] = { + private def checkIfStdoutInputExist(inputSection: Option[InputsSectionElement]): ErrorOr[Option[String]] = { inputSection match { case Some(section) => if (section.inputDeclarations.flatMap(_.expression).exists(_.isInstanceOf[StdoutElement.type])) { @@ -53,6 +63,48 @@ object AstToWorkflowDefinitionElement { } } + private def checkIfStdoutOutputExists(outputSection: Option[OutputsSectionElement]): ErrorOr[Option[String]] = { + outputSection match { + case Some(section) => + if (section.outputs.map(_.expression).exists(_.isInstanceOf[StdoutElement.type])) { + s"Workflow cannot have stdout expression in output section at workflow-level.".invalidNel + } else None.validNel + case None => None.validNel + } + } + + private def checkStdoutIntermediates(intermediate: Vector[IntermediateValueDeclarationElement]): ErrorOr[Option[String]] = { + if (intermediate.map(_.expression).exists(_.isInstanceOf[StdoutElement.type])) { + s"Workflow cannot have stderr expression at intermediate declaration section at workflow-level.".invalidNel + } else None.validNel + } + + private def checkIfStderrInputExist(inputs: Option[InputsSectionElement]): ErrorOr[Option[String]] = { + inputs match { + case Some(section) => + if (section.inputDeclarations.flatMap(_.expression).exists(_.isInstanceOf[StderrElement.type])) { + s"Workflow cannot have stderr expression in input section at workflow-level.".invalidNel + } else None.validNel + case None => None.validNel + } + } + + private def checkIfStderrOutputExists(outputs: Option[OutputsSectionElement]): ErrorOr[Option[String]] = { + outputs match { + case Some(section) => + if (section.outputs.map(_.expression).exists(_.isInstanceOf[StderrElement.type])) { + s"Workflow cannot have stderr expression in output section at workflow-level.".invalidNel + } else None.validNel + case None => None.validNel + } + } + + private def checkStderrIntermediates(intermediates: Vector[IntermediateValueDeclarationElement]): ErrorOr[Option[String]] = { + if (intermediates.map(_.expression).exists(_.isInstanceOf[StderrElement.type])) { + s"Workflow cannot have stdout expression at intermediate declaration section at workflow-level.".invalidNel + } else None.validNel + } + private def validateSize[A](elements: Vector[A], sectionName: String, numExpected: Int): ErrorOr[Option[A]] = { val sectionValidation: ErrorOr[Option[A]] = if (elements.size > numExpected) { s"Workflow cannot have more than $numExpected $sectionName sections, found ${elements.size}.".invalidNel @@ -62,15 +114,4 @@ object AstToWorkflowDefinitionElement { sectionValidation } - - /*for(validate<-AstToWorkflowDefinitionElement){ - validate match { - case check input{} - case check p - - } - */ - } - -//def validateStdout(a:) From ef48af45aa0574448a33857af438c250c4a2114b Mon Sep 17 00:00:00 2001 From: Katrina Pierre Date: Thu, 14 Apr 2022 02:16:40 -0400 Subject: [PATCH 05/45] Fixing error messages --- .../base/ast2wdlom/AstToWorkflowDefinitionElement.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala index 82513fdeb12..9b27dce772f 100644 --- a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala +++ b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala @@ -75,7 +75,7 @@ object AstToWorkflowDefinitionElement { private def checkStdoutIntermediates(intermediate: Vector[IntermediateValueDeclarationElement]): ErrorOr[Option[String]] = { if (intermediate.map(_.expression).exists(_.isInstanceOf[StdoutElement.type])) { - s"Workflow cannot have stderr expression at intermediate declaration section at workflow-level.".invalidNel + s"Workflow cannot have stdout expression at intermediate declaration section at workflow-level.".invalidNel } else None.validNel } @@ -101,7 +101,7 @@ object AstToWorkflowDefinitionElement { private def checkStderrIntermediates(intermediates: Vector[IntermediateValueDeclarationElement]): ErrorOr[Option[String]] = { if (intermediates.map(_.expression).exists(_.isInstanceOf[StderrElement.type])) { - s"Workflow cannot have stdout expression at intermediate declaration section at workflow-level.".invalidNel + s"Workflow cannot have stderr expression at intermediate declaration section at workflow-level.".invalidNel } else None.validNel } From db173f69b4b4ce5aed5388377b646ae2d2296dd0 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Tue, 1 Mar 2022 18:52:03 -0500 Subject: [PATCH 06/45] bad stdout location tests --- .../standardTestCases/bad_stdout_input.test | 12 ++++++++++++ .../bad_stdout_mid_workflow.test | 12 ++++++++++++ .../standardTestCases/bad_stdout_outputs.test | 12 ++++++++++++ .../standardTestCases/bad_stdout_subworkflow.test | 15 +++++++++++++++ .../bad_stdouts/bad_stdout_input.wdl | 10 ++++++++++ .../bad_stdouts/bad_stdout_mid_workflow.wdl | 9 +++++++++ .../bad_stdouts/bad_stdout_output.wdl | 7 +++++++ .../bad_stdouts/bad_stdout_subworkflow.wdl | 9 +++++++++ 8 files changed, 86 insertions(+) create mode 100644 centaur/src/main/resources/standardTestCases/bad_stdout_input.test create mode 100644 centaur/src/main/resources/standardTestCases/bad_stdout_mid_workflow.test create mode 100644 centaur/src/main/resources/standardTestCases/bad_stdout_outputs.test create mode 100644 centaur/src/main/resources/standardTestCases/bad_stdout_subworkflow.test create mode 100644 centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_input.wdl create mode 100644 centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_mid_workflow.wdl create mode 100644 centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_output.wdl create mode 100644 centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_subworkflow.wdl diff --git a/centaur/src/main/resources/standardTestCases/bad_stdout_input.test b/centaur/src/main/resources/standardTestCases/bad_stdout_input.test new file mode 100644 index 00000000000..ca48fe736f1 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stdout_input.test @@ -0,0 +1,12 @@ +# Verifies that if we use stdout in a workflow level, the workflow fails properly. +name: bad_stdout_input +testFormat: workflowfailure + +files { + workflow: bad_stdouts/bad_stdout_input.wdl +} + +metadata { + workflowName: bad_stdout + status: Failed +} diff --git a/centaur/src/main/resources/standardTestCases/bad_stdout_mid_workflow.test b/centaur/src/main/resources/standardTestCases/bad_stdout_mid_workflow.test new file mode 100644 index 00000000000..1f66c347ae4 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stdout_mid_workflow.test @@ -0,0 +1,12 @@ +# Verifies that if we use stdout in a workflow level, the workflow fails properly. +name: bad_stdout_mid_workflow +testFormat: workflowfailure + +files { + workflow: bad_stdouts/bad_stdout_mid_workflow.wdl +} + +metadata { + workflowName: bad_stdout + status: Failed +} diff --git a/centaur/src/main/resources/standardTestCases/bad_stdout_outputs.test b/centaur/src/main/resources/standardTestCases/bad_stdout_outputs.test new file mode 100644 index 00000000000..d00c2a45850 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stdout_outputs.test @@ -0,0 +1,12 @@ +# Verifies that if we use stdout in a workflow level, the workflow fails properly. +name: bad_stdout_output +testFormat: workflowfailure + +files { + workflow: bad_stdouts/bad_stdout_output.wdl +} + +metadata { + workflowName: bad_stdout + status: Failed +} diff --git a/centaur/src/main/resources/standardTestCases/bad_stdout_subworkflow.test b/centaur/src/main/resources/standardTestCases/bad_stdout_subworkflow.test new file mode 100644 index 00000000000..b5ea413bf18 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stdout_subworkflow.test @@ -0,0 +1,15 @@ +# Verifies that if we use stdout in a workflow level, the workflow fails properly. +name: bad_stdout_subworkflow +testFormat: workflowfailure + +files { + workflow: bad_stdouts/bad_stdout_subworkflow.wdl + imports: [ + bad_stdouts/bad_stdout_mid_workflow.wdl + ] +} + +metadata { + workflowName: calls_bad_stdout_subworkflow + status: Failed +} diff --git a/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_input.wdl b/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_input.wdl new file mode 100644 index 00000000000..c68f0ebffd3 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_input.wdl @@ -0,0 +1,10 @@ +version 1.0 + +workflow bad_stdout { + input { + File f = stdout() + } + output { + String s = read_string(f) + } +} diff --git a/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_mid_workflow.wdl b/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_mid_workflow.wdl new file mode 100644 index 00000000000..d4f57bfd125 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_mid_workflow.wdl @@ -0,0 +1,9 @@ +version 1.0 + +workflow bad_stdout { + File f = stdout() + + output { + String s = read_string(f) + } +} diff --git a/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_output.wdl b/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_output.wdl new file mode 100644 index 00000000000..ad2bc9a24e4 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_output.wdl @@ -0,0 +1,7 @@ +version 1.0 + +workflow bad_stdout { + output { + File f = stdout() + } +} diff --git a/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_subworkflow.wdl b/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_subworkflow.wdl new file mode 100644 index 00000000000..7134825b827 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_subworkflow.wdl @@ -0,0 +1,9 @@ +version 1.0 + +import "bad_stdout_mid_workflow.wdl" + +workflow calls_bad_stdout_subworkflow { + scatter(i in range(1000)) { + call bad_stdout_mid_workflow.bad_stdout + } +} From c874b8faf0146464dc5fe82bb1a47f85c4bb70ad Mon Sep 17 00:00:00 2001 From: Katrina Pierre Date: Mon, 4 Apr 2022 16:32:39 -0400 Subject: [PATCH 07/45] just a draft --- .../AstToWorkflowDefinitionElement.scala | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala index ad4c28f064a..a91ab1e83d1 100644 --- a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala +++ b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala @@ -8,6 +8,7 @@ import common.transforms.CheckedAtoB import common.validation.ErrorOr._ import wdl.model.draft3.elements._ import wom.SourceFileLocation +import wdl.model.draft3.elements.ExpressionElement.StdoutElement object AstToWorkflowDefinitionElement { @@ -40,12 +41,32 @@ object AstToWorkflowDefinitionElement { } } + private def checkStdout(inputSectionValidation: WorkflowBodyElement, b: outputsSectionValidation[WorkflowBodyElement]): Unit = { + inputsSectionValidation.map(x => x match { + case Some(value) => val here = value.inputDeclarations.flatMap(_.expression).exists(_.isInstanceOf[StdoutElement.type]) + s"Workflow cannot have more than $numExpected $sectionName sections, found ${elements.size}.".invalidNel + case None => _ + }) + } + private def validateSize[A](elements: Vector[A], sectionName: String, numExpected: Int): ErrorOr[Option[A]] = { val sectionValidation: ErrorOr[Option[A]] = if (elements.size > numExpected) { s"Workflow cannot have more than $numExpected $sectionName sections, found ${elements.size}.".invalidNel } else { elements.headOption.validNel } + sectionValidation } + + /*for(validate<-AstToWorkflowDefinitionElement){ + validate match { + case check input{} + case check p + + } + */ + } + +//def validateStdout(a:) \ No newline at end of file From 8707f50275477e08590e147af3cbf11ef9063dc1 Mon Sep 17 00:00:00 2001 From: Saloni Shah Date: Mon, 4 Apr 2022 17:04:54 -0400 Subject: [PATCH 08/45] checkStdout helper function --- .../AstToWorkflowDefinitionElement.scala | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala index a91ab1e83d1..f43bdbfaf92 100644 --- a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala +++ b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala @@ -30,23 +30,27 @@ object AstToWorkflowDefinitionElement { val inputsSectionValidation: ErrorOr[Option[InputsSectionElement]] = validateSize(bodyElements.filterByType[InputsSectionElement], "inputs", 1) val outputsSectionValidation: ErrorOr[Option[OutputsSectionElement]] = validateSize(bodyElements.filterByType[OutputsSectionElement], "outputs", 1) + val inputsSectionStdoutValidation: ErrorOr[ErrorOr[Option[String]]] = inputsSectionValidation.map(checkIfStdoutExist) + val graphSections: Vector[WorkflowGraphElement] = bodyElements.filterByType[WorkflowGraphElement] val metaSectionValidation: ErrorOr[Option[MetaSectionElement]] = validateSize(bodyElements.filterByType[MetaSectionElement], "meta", 1) val parameterMetaSectionValidation: ErrorOr[Option[ParameterMetaSectionElement]] = validateSize(bodyElements.filterByType[ParameterMetaSectionElement], "parameterMeta", 1) - (inputsSectionValidation, outputsSectionValidation, metaSectionValidation, parameterMetaSectionValidation) mapN { - (validInputs, validOutputs, meta, parameterMeta) => + (inputsSectionValidation, outputsSectionValidation, metaSectionValidation, parameterMetaSectionValidation, inputsSectionStdoutValidation) mapN { + (validInputs, validOutputs, meta, parameterMeta, _) => WorkflowDefinitionElement(name, validInputs, graphSections.toSet, validOutputs, meta, parameterMeta, sourceLocation) } } - private def checkStdout(inputSectionValidation: WorkflowBodyElement, b: outputsSectionValidation[WorkflowBodyElement]): Unit = { - inputsSectionValidation.map(x => x match { - case Some(value) => val here = value.inputDeclarations.flatMap(_.expression).exists(_.isInstanceOf[StdoutElement.type]) - s"Workflow cannot have more than $numExpected $sectionName sections, found ${elements.size}.".invalidNel - case None => _ - }) + private def checkIfStdoutExist(inputSection: Option[InputsSectionElement]): ErrorOr[Option[String]] = { + inputSection match { + case Some(section) => + if (section.inputDeclarations.flatMap(_.expression).exists(_.isInstanceOf[StdoutElement.type])) { + s"Workflow cannot have stdout expression in input section at workflow-level.".invalidNel + } else None.validNel + case None => None.validNel + } } private def validateSize[A](elements: Vector[A], sectionName: String, numExpected: Int): ErrorOr[Option[A]] = { @@ -69,4 +73,4 @@ object AstToWorkflowDefinitionElement { } -//def validateStdout(a:) \ No newline at end of file +//def validateStdout(a:) From 4a6583dde09583cb0c8238a6162bb726e2d8050e Mon Sep 17 00:00:00 2001 From: Katrina Pierre Date: Thu, 14 Apr 2022 02:14:25 -0400 Subject: [PATCH 09/45] Adding stderr wdl and test files, refactoring functions and adding functions to handle stderr --- .../standardTestCases/bad_stderr_input.test | 12 +++ .../bad_stderr_mid_workflow.test | 12 +++ .../standardTestCases/bad_stderr_outputs.test | 12 +++ .../bad_stderr_subworkflow.test | 15 ++++ .../bad_stderrs/bad_stderr_input.wdl | 11 +++ .../bad_stderrs/bad_stderr_mid_workflow.wdl | 9 +++ .../bad_stderrs/bad_stderr_output.wdl | 7 ++ .../bad_stderrs/bad_stderr_subworkflow.wdl | 9 +++ .../AstToWorkflowDefinitionElement.scala | 77 ++++++++++++++----- 9 files changed, 146 insertions(+), 18 deletions(-) create mode 100644 centaur/src/main/resources/standardTestCases/bad_stderr_input.test create mode 100644 centaur/src/main/resources/standardTestCases/bad_stderr_mid_workflow.test create mode 100644 centaur/src/main/resources/standardTestCases/bad_stderr_outputs.test create mode 100644 centaur/src/main/resources/standardTestCases/bad_stderr_subworkflow.test create mode 100644 centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_input.wdl create mode 100644 centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_mid_workflow.wdl create mode 100644 centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_output.wdl create mode 100644 centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_subworkflow.wdl diff --git a/centaur/src/main/resources/standardTestCases/bad_stderr_input.test b/centaur/src/main/resources/standardTestCases/bad_stderr_input.test new file mode 100644 index 00000000000..57e999dcd89 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stderr_input.test @@ -0,0 +1,12 @@ +# Verifies that if we use stderr in a workflow level, the workflow fails properly. +name: bad_stderr_input +testFormat: workflowfailure + +files { + workflow: bad_stderrs/bad_stderr_input.wdl +} + +metadata { + workflowName: bad_stderr + status: Failed +} \ No newline at end of file diff --git a/centaur/src/main/resources/standardTestCases/bad_stderr_mid_workflow.test b/centaur/src/main/resources/standardTestCases/bad_stderr_mid_workflow.test new file mode 100644 index 00000000000..5ab4668814a --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stderr_mid_workflow.test @@ -0,0 +1,12 @@ +# Verifies that if we use stderr in a workflow level, the workflow fails properly. +name: bad_stderr_mid_workflow +testFormat: workflowfailure + +files { + workflow: bad_stderrs/bad_stderr_mid_workflow.wdl +} + +metadata { + workflowName: bad_stderr + status: Failed +} diff --git a/centaur/src/main/resources/standardTestCases/bad_stderr_outputs.test b/centaur/src/main/resources/standardTestCases/bad_stderr_outputs.test new file mode 100644 index 00000000000..126bbc2fd11 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stderr_outputs.test @@ -0,0 +1,12 @@ +# Verifies that if we use stderr in a workflow level, the workflow fails properly. +name: bad_stderr_output +testFormat: workflowfailure + +files { + workflow: bad_stderrs/bad_stderr_output.wdl +} + +metadata { + workflowName: bad_stderr + status: Failed +} diff --git a/centaur/src/main/resources/standardTestCases/bad_stderr_subworkflow.test b/centaur/src/main/resources/standardTestCases/bad_stderr_subworkflow.test new file mode 100644 index 00000000000..cf22aca78fe --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stderr_subworkflow.test @@ -0,0 +1,15 @@ +# Verifies that if we use stderr in a workflow level, the workflow fails properly. +name: bad_stderr_subworkflow +testFormat: workflowfailure + +files { + workflow: bad_stderrs/bad_stderr_subworkflow.wdl + imports: [ + bad_stderrs/bad_stderr_mid_workflow.wdl + ] +} + +metadata { + workflowName: calls_bad_stderr_subworkflow + status: Failed +} diff --git a/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_input.wdl b/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_input.wdl new file mode 100644 index 00000000000..ef7e2b7ff82 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_input.wdl @@ -0,0 +1,11 @@ +version 1.0 + +workflow bad_stderr { + + input { + File f = stderr() + } + output { + String s = read_string(f) + } +} diff --git a/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_mid_workflow.wdl b/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_mid_workflow.wdl new file mode 100644 index 00000000000..eb6331ff8c5 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_mid_workflow.wdl @@ -0,0 +1,9 @@ +version 1.0 + +workflow bad_stderr { + File f = stderr() + + output { + String s = read_string(f) + } +} diff --git a/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_output.wdl b/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_output.wdl new file mode 100644 index 00000000000..65e99949a52 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_output.wdl @@ -0,0 +1,7 @@ +version 1.0 + +workflow bad_stderr { + output { + File f = stderr() + } +} diff --git a/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_subworkflow.wdl b/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_subworkflow.wdl new file mode 100644 index 00000000000..b1249f8dd9d --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_subworkflow.wdl @@ -0,0 +1,9 @@ +version 1.0 + +import "bad_stderr_mid_workflow.wdl" + +workflow calls_bad_stderr_subworkflow { + scatter(i in range(1000)) { + call bad_stderr_mid_workflow.bad_stderr + } +} diff --git a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala index f43bdbfaf92..82513fdeb12 100644 --- a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala +++ b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala @@ -8,7 +8,7 @@ import common.transforms.CheckedAtoB import common.validation.ErrorOr._ import wdl.model.draft3.elements._ import wom.SourceFileLocation -import wdl.model.draft3.elements.ExpressionElement.StdoutElement +import wdl.model.draft3.elements.ExpressionElement._ object AstToWorkflowDefinitionElement { @@ -27,23 +27,33 @@ object AstToWorkflowDefinitionElement { sourceLocation: Option[SourceFileLocation], bodyElements: Vector[WorkflowBodyElement]) = { - val inputsSectionValidation: ErrorOr[Option[InputsSectionElement]] = validateSize(bodyElements.filterByType[InputsSectionElement], "inputs", 1) - val outputsSectionValidation: ErrorOr[Option[OutputsSectionElement]] = validateSize(bodyElements.filterByType[OutputsSectionElement], "outputs", 1) + val inputsSectionValidation: ErrorOr[Option[InputsSectionElement]] = for { + inputValidateElement <- validateSize(bodyElements.filterByType[InputsSectionElement], "inputs", 1): ErrorOr[Option[InputsSectionElement]] + _ <- checkIfStdoutInputExist(inputValidateElement) + _ <- checkIfStderrInputExist(inputValidateElement) + } yield inputValidateElement - val inputsSectionStdoutValidation: ErrorOr[ErrorOr[Option[String]]] = inputsSectionValidation.map(checkIfStdoutExist) + val intermediateValueDeclarationStdoutCheck: ErrorOr[Option[String]] = checkStdoutIntermediates(bodyElements.filterByType[IntermediateValueDeclarationElement]) + val intermediateValueDeclarationStderrCheck: ErrorOr[Option[String]] = checkStderrIntermediates(bodyElements.filterByType[IntermediateValueDeclarationElement]) + + val outputsSectionValidation: ErrorOr[Option[OutputsSectionElement]] = for { + outputValidateElement <- validateSize(bodyElements.filterByType[OutputsSectionElement], "outputs", 1): ErrorOr[Option[OutputsSectionElement]] + _ <- checkIfStdoutOutputExists(outputValidateElement) + _ <- checkIfStderrOutputExists(outputValidateElement) + } yield outputValidateElement val graphSections: Vector[WorkflowGraphElement] = bodyElements.filterByType[WorkflowGraphElement] val metaSectionValidation: ErrorOr[Option[MetaSectionElement]] = validateSize(bodyElements.filterByType[MetaSectionElement], "meta", 1) val parameterMetaSectionValidation: ErrorOr[Option[ParameterMetaSectionElement]] = validateSize(bodyElements.filterByType[ParameterMetaSectionElement], "parameterMeta", 1) - (inputsSectionValidation, outputsSectionValidation, metaSectionValidation, parameterMetaSectionValidation, inputsSectionStdoutValidation) mapN { - (validInputs, validOutputs, meta, parameterMeta, _) => + (inputsSectionValidation, outputsSectionValidation, metaSectionValidation, parameterMetaSectionValidation, intermediateValueDeclarationStdoutCheck, intermediateValueDeclarationStderrCheck) mapN { + (validInputs, validOutputs, meta, parameterMeta, _, _) => WorkflowDefinitionElement(name, validInputs, graphSections.toSet, validOutputs, meta, parameterMeta, sourceLocation) } } - private def checkIfStdoutExist(inputSection: Option[InputsSectionElement]): ErrorOr[Option[String]] = { + private def checkIfStdoutInputExist(inputSection: Option[InputsSectionElement]): ErrorOr[Option[String]] = { inputSection match { case Some(section) => if (section.inputDeclarations.flatMap(_.expression).exists(_.isInstanceOf[StdoutElement.type])) { @@ -53,6 +63,48 @@ object AstToWorkflowDefinitionElement { } } + private def checkIfStdoutOutputExists(outputSection: Option[OutputsSectionElement]): ErrorOr[Option[String]] = { + outputSection match { + case Some(section) => + if (section.outputs.map(_.expression).exists(_.isInstanceOf[StdoutElement.type])) { + s"Workflow cannot have stdout expression in output section at workflow-level.".invalidNel + } else None.validNel + case None => None.validNel + } + } + + private def checkStdoutIntermediates(intermediate: Vector[IntermediateValueDeclarationElement]): ErrorOr[Option[String]] = { + if (intermediate.map(_.expression).exists(_.isInstanceOf[StdoutElement.type])) { + s"Workflow cannot have stderr expression at intermediate declaration section at workflow-level.".invalidNel + } else None.validNel + } + + private def checkIfStderrInputExist(inputs: Option[InputsSectionElement]): ErrorOr[Option[String]] = { + inputs match { + case Some(section) => + if (section.inputDeclarations.flatMap(_.expression).exists(_.isInstanceOf[StderrElement.type])) { + s"Workflow cannot have stderr expression in input section at workflow-level.".invalidNel + } else None.validNel + case None => None.validNel + } + } + + private def checkIfStderrOutputExists(outputs: Option[OutputsSectionElement]): ErrorOr[Option[String]] = { + outputs match { + case Some(section) => + if (section.outputs.map(_.expression).exists(_.isInstanceOf[StderrElement.type])) { + s"Workflow cannot have stderr expression in output section at workflow-level.".invalidNel + } else None.validNel + case None => None.validNel + } + } + + private def checkStderrIntermediates(intermediates: Vector[IntermediateValueDeclarationElement]): ErrorOr[Option[String]] = { + if (intermediates.map(_.expression).exists(_.isInstanceOf[StderrElement.type])) { + s"Workflow cannot have stdout expression at intermediate declaration section at workflow-level.".invalidNel + } else None.validNel + } + private def validateSize[A](elements: Vector[A], sectionName: String, numExpected: Int): ErrorOr[Option[A]] = { val sectionValidation: ErrorOr[Option[A]] = if (elements.size > numExpected) { s"Workflow cannot have more than $numExpected $sectionName sections, found ${elements.size}.".invalidNel @@ -62,15 +114,4 @@ object AstToWorkflowDefinitionElement { sectionValidation } - - /*for(validate<-AstToWorkflowDefinitionElement){ - validate match { - case check input{} - case check p - - } - */ - } - -//def validateStdout(a:) From cd3e3997638b8ef135251d40448b7d9e3a808f3c Mon Sep 17 00:00:00 2001 From: Katrina Pierre Date: Thu, 14 Apr 2022 02:16:40 -0400 Subject: [PATCH 10/45] Fixing error messages --- .../base/ast2wdlom/AstToWorkflowDefinitionElement.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala index 82513fdeb12..9b27dce772f 100644 --- a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala +++ b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala @@ -75,7 +75,7 @@ object AstToWorkflowDefinitionElement { private def checkStdoutIntermediates(intermediate: Vector[IntermediateValueDeclarationElement]): ErrorOr[Option[String]] = { if (intermediate.map(_.expression).exists(_.isInstanceOf[StdoutElement.type])) { - s"Workflow cannot have stderr expression at intermediate declaration section at workflow-level.".invalidNel + s"Workflow cannot have stdout expression at intermediate declaration section at workflow-level.".invalidNel } else None.validNel } @@ -101,7 +101,7 @@ object AstToWorkflowDefinitionElement { private def checkStderrIntermediates(intermediates: Vector[IntermediateValueDeclarationElement]): ErrorOr[Option[String]] = { if (intermediates.map(_.expression).exists(_.isInstanceOf[StderrElement.type])) { - s"Workflow cannot have stdout expression at intermediate declaration section at workflow-level.".invalidNel + s"Workflow cannot have stderr expression at intermediate declaration section at workflow-level.".invalidNel } else None.validNel } From ce37aa77c0f2dce72ee596eca82cf8e792a51bb0 Mon Sep 17 00:00:00 2001 From: Katrina Pierre Date: Thu, 14 Apr 2022 14:18:24 -0400 Subject: [PATCH 11/45] Rebasing --- .../src/main/resources/standardTestCases/bad_stderr_input.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/centaur/src/main/resources/standardTestCases/bad_stderr_input.test b/centaur/src/main/resources/standardTestCases/bad_stderr_input.test index 57e999dcd89..648f06d4c6d 100644 --- a/centaur/src/main/resources/standardTestCases/bad_stderr_input.test +++ b/centaur/src/main/resources/standardTestCases/bad_stderr_input.test @@ -9,4 +9,4 @@ files { metadata { workflowName: bad_stderr status: Failed -} \ No newline at end of file +} From f7ce919d50be26ced9c1a24ba537e9bc11b197e0 Mon Sep 17 00:00:00 2001 From: Katrina Pierre Date: Thu, 14 Apr 2022 14:19:32 -0400 Subject: [PATCH 12/45] Rebasing --- .../src/main/resources/standardTestCases/bad_stderr_input.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/centaur/src/main/resources/standardTestCases/bad_stderr_input.test b/centaur/src/main/resources/standardTestCases/bad_stderr_input.test index 57e999dcd89..648f06d4c6d 100644 --- a/centaur/src/main/resources/standardTestCases/bad_stderr_input.test +++ b/centaur/src/main/resources/standardTestCases/bad_stderr_input.test @@ -9,4 +9,4 @@ files { metadata { workflowName: bad_stderr status: Failed -} \ No newline at end of file +} From 18a9016c35969bda05022b18326d3bca2da62e0b Mon Sep 17 00:00:00 2001 From: Katrina Pierre Date: Fri, 15 Apr 2022 11:30:54 -0400 Subject: [PATCH 13/45] Omitting the metadata section for Travis builds to pass --- .../main/resources/standardTestCases/bad_stderr_input.test | 5 ----- .../resources/standardTestCases/bad_stderr_mid_workflow.test | 5 ----- .../main/resources/standardTestCases/bad_stderr_outputs.test | 5 ----- .../resources/standardTestCases/bad_stderr_subworkflow.test | 5 ----- .../main/resources/standardTestCases/bad_stdout_input.test | 5 ----- .../resources/standardTestCases/bad_stdout_mid_workflow.test | 5 ----- .../main/resources/standardTestCases/bad_stdout_outputs.test | 5 ----- .../resources/standardTestCases/bad_stdout_subworkflow.test | 5 ----- 8 files changed, 40 deletions(-) diff --git a/centaur/src/main/resources/standardTestCases/bad_stderr_input.test b/centaur/src/main/resources/standardTestCases/bad_stderr_input.test index 648f06d4c6d..6775db336cf 100644 --- a/centaur/src/main/resources/standardTestCases/bad_stderr_input.test +++ b/centaur/src/main/resources/standardTestCases/bad_stderr_input.test @@ -5,8 +5,3 @@ testFormat: workflowfailure files { workflow: bad_stderrs/bad_stderr_input.wdl } - -metadata { - workflowName: bad_stderr - status: Failed -} diff --git a/centaur/src/main/resources/standardTestCases/bad_stderr_mid_workflow.test b/centaur/src/main/resources/standardTestCases/bad_stderr_mid_workflow.test index 5ab4668814a..af7e62d6401 100644 --- a/centaur/src/main/resources/standardTestCases/bad_stderr_mid_workflow.test +++ b/centaur/src/main/resources/standardTestCases/bad_stderr_mid_workflow.test @@ -5,8 +5,3 @@ testFormat: workflowfailure files { workflow: bad_stderrs/bad_stderr_mid_workflow.wdl } - -metadata { - workflowName: bad_stderr - status: Failed -} diff --git a/centaur/src/main/resources/standardTestCases/bad_stderr_outputs.test b/centaur/src/main/resources/standardTestCases/bad_stderr_outputs.test index 126bbc2fd11..9c2b02aa8b1 100644 --- a/centaur/src/main/resources/standardTestCases/bad_stderr_outputs.test +++ b/centaur/src/main/resources/standardTestCases/bad_stderr_outputs.test @@ -5,8 +5,3 @@ testFormat: workflowfailure files { workflow: bad_stderrs/bad_stderr_output.wdl } - -metadata { - workflowName: bad_stderr - status: Failed -} diff --git a/centaur/src/main/resources/standardTestCases/bad_stderr_subworkflow.test b/centaur/src/main/resources/standardTestCases/bad_stderr_subworkflow.test index cf22aca78fe..408f17664a5 100644 --- a/centaur/src/main/resources/standardTestCases/bad_stderr_subworkflow.test +++ b/centaur/src/main/resources/standardTestCases/bad_stderr_subworkflow.test @@ -8,8 +8,3 @@ files { bad_stderrs/bad_stderr_mid_workflow.wdl ] } - -metadata { - workflowName: calls_bad_stderr_subworkflow - status: Failed -} diff --git a/centaur/src/main/resources/standardTestCases/bad_stdout_input.test b/centaur/src/main/resources/standardTestCases/bad_stdout_input.test index ca48fe736f1..bf7c9bf6a65 100644 --- a/centaur/src/main/resources/standardTestCases/bad_stdout_input.test +++ b/centaur/src/main/resources/standardTestCases/bad_stdout_input.test @@ -5,8 +5,3 @@ testFormat: workflowfailure files { workflow: bad_stdouts/bad_stdout_input.wdl } - -metadata { - workflowName: bad_stdout - status: Failed -} diff --git a/centaur/src/main/resources/standardTestCases/bad_stdout_mid_workflow.test b/centaur/src/main/resources/standardTestCases/bad_stdout_mid_workflow.test index 1f66c347ae4..ae73adeeb29 100644 --- a/centaur/src/main/resources/standardTestCases/bad_stdout_mid_workflow.test +++ b/centaur/src/main/resources/standardTestCases/bad_stdout_mid_workflow.test @@ -5,8 +5,3 @@ testFormat: workflowfailure files { workflow: bad_stdouts/bad_stdout_mid_workflow.wdl } - -metadata { - workflowName: bad_stdout - status: Failed -} diff --git a/centaur/src/main/resources/standardTestCases/bad_stdout_outputs.test b/centaur/src/main/resources/standardTestCases/bad_stdout_outputs.test index d00c2a45850..67b51cd2bf0 100644 --- a/centaur/src/main/resources/standardTestCases/bad_stdout_outputs.test +++ b/centaur/src/main/resources/standardTestCases/bad_stdout_outputs.test @@ -5,8 +5,3 @@ testFormat: workflowfailure files { workflow: bad_stdouts/bad_stdout_output.wdl } - -metadata { - workflowName: bad_stdout - status: Failed -} diff --git a/centaur/src/main/resources/standardTestCases/bad_stdout_subworkflow.test b/centaur/src/main/resources/standardTestCases/bad_stdout_subworkflow.test index b5ea413bf18..28a24d8864f 100644 --- a/centaur/src/main/resources/standardTestCases/bad_stdout_subworkflow.test +++ b/centaur/src/main/resources/standardTestCases/bad_stdout_subworkflow.test @@ -8,8 +8,3 @@ files { bad_stdouts/bad_stdout_mid_workflow.wdl ] } - -metadata { - workflowName: calls_bad_stdout_subworkflow - status: Failed -} From 40ce41a0cbc81f1207da3d1c61570c1546bb8a80 Mon Sep 17 00:00:00 2001 From: Katrina P <68349264+kpierre13@users.noreply.github.com> Date: Wed, 27 Apr 2022 09:52:09 -0400 Subject: [PATCH 14/45] Bw 1126 workflow level stdout / stderr expressions fail, hang execution forever (#6745) * Combining similar-use functions * All tests created * Saving changes * Adding unit tests --- .../AstToWorkflowDefinitionElementSpec.scala | 97 +++++++++++++++++++ .../AstToWorkflowDefinitionElement.scala | 56 +++-------- 2 files changed, 112 insertions(+), 41 deletions(-) create mode 100644 wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala diff --git a/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala b/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala new file mode 100644 index 00000000000..0c13d23f6a5 --- /dev/null +++ b/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala @@ -0,0 +1,97 @@ +import cats.data.Validated._ +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers._ +import wdl.model.draft3.elements.ExpressionElement._ +import wdl.model.draft3.elements.{InputDeclarationElement, InputsSectionElement, IntermediateValueDeclarationElement, OutputDeclarationElement, OutputsSectionElement, PrimitiveTypeElement} +import wom.types.{WomSingleFileType, WomStringType} +import wdl.transforms.base.ast2wdlom.AstToWorkflowDefinitionElement._ + + +class AstToWorkflowDefinitionElementSpec extends AnyFlatSpec{ + behavior of "Check Stdouts and Stderrs" + + val mockInputSectionStdout = Option(InputsSectionElement(Vector(InputDeclarationElement(PrimitiveTypeElement(WomSingleFileType), "i", Some(StdoutElement))))) + val mockInputSectionStderr= Option(InputsSectionElement(Vector(InputDeclarationElement(PrimitiveTypeElement(WomSingleFileType), "i", Some(StderrElement))))) + val mockInputSectionNonStd= Option(InputsSectionElement(Vector(InputDeclarationElement(PrimitiveTypeElement(WomStringType), "more", Some(StringLiteral("more")))))) + + + val mockIntermediatesStdout = Vector(IntermediateValueDeclarationElement(PrimitiveTypeElement(WomSingleFileType), "y", StdoutElement)) + val mockIntermediatesStderr = Vector(IntermediateValueDeclarationElement(PrimitiveTypeElement(WomSingleFileType), "y", StderrElement)) + val mockIntermediatesNonStd = Vector(IntermediateValueDeclarationElement(PrimitiveTypeElement(WomStringType), "here", StringLiteral("here"))) + + val mockOutputSectionStdout = Option(OutputsSectionElement(Vector(OutputDeclarationElement(PrimitiveTypeElement(WomSingleFileType), "s", StdoutElement)))) + val mockOutputSectionStderr = Option(OutputsSectionElement(Vector(OutputDeclarationElement(PrimitiveTypeElement(WomSingleFileType), "s", StderrElement)))) + val mockOutputSectionNonStd = Option(OutputsSectionElement(Vector(OutputDeclarationElement(PrimitiveTypeElement(WomStringType), "more", StringLiteral("more"))))) + + + it should "return an error when there is an stdout in input section" in { + val testInputs = checkIfStdInputExist(mockInputSectionStdout, StdoutElement, "stdout") + testInputs match { + case Valid(_) => fail("Input section contained stdout. Should have failed.") + case Invalid(e) => e.head should be("Workflow cannot have stdout expression in input section at workflow-level.") + } + } + + it should "return an error when there is an stderr in input section" in { + val testInputs = checkIfStdInputExist(mockInputSectionStderr, StderrElement, "stderr") + testInputs match { + case Valid(_) => fail("Input section contained stderr. Should have failed.") + case Invalid(e) => e.head should be("Workflow cannot have stderr expression in input section at workflow-level.") + } + } + + it should "not throw an error for non-stdout/stderr in the inputs section" in { + val testInputs = checkIfStdInputExist(mockInputSectionNonStd, StdoutElement, "non-stdout/stderr") + testInputs match { + case Valid(_) => "Input section contained a non-stdout/stderr element." + case Invalid(e) => e.head should be("Workflow failed at input section.") + } + } + + it should "return an error when there is an stdout in output section" in { + val testOutputs = checkIfStdOutputExists(mockOutputSectionStdout, StdoutElement, "stdout") + testOutputs match { + case Valid(_) => fail("Output section contained stdout. Should have failed.") + case Invalid(e) => e.head should be("Workflow cannot have stdout expression in output section at workflow-level.") + } + } + + it should "return an error when there is an stderr in output section" in { + val testOutputs = checkIfStdOutputExists(mockOutputSectionStderr, StderrElement, "stderr") + testOutputs match { + case Valid(_) => fail("Output section contained stderr. Should have failed.") + case Invalid(e) => e.head should be("Workflow cannot have stderr expression in output section at workflow-level.") + } + } + it should "not throw an error for non-stdout/stderr in the outputs section" in { + val testOutputs = checkIfStdOutputExists(mockOutputSectionNonStd, StdoutElement, "non-stdout/stderr") + testOutputs match { + case Valid(_) => "Output section contained a non-stdout/stderr element." + case Invalid(e) => e.head should be("Workflow failed at output section.") + } + } + + it should "return an error when there is an stdout at intermediate declaration section" in { + val testIntermediates = checkStdIntermediates(mockIntermediatesStdout, StdoutElement, "stdout") + testIntermediates match { + case Valid(_) => fail("Intermediate section contained stdout. Should have failed.") + case Invalid(e) => e.head should be("Workflow cannot have stdout expression at intermediate declaration section at workflow-level.") + } + } + it should "return an error when there is an stderr at intermediate declaration section" in { + val testIntermediates = checkStdIntermediates(mockIntermediatesStderr, StderrElement, "stderr") + testIntermediates match { + case Valid(_) => fail("Intermediate section contained stderr. Should have failed.") + case Invalid(e) => e.head should be("Workflow cannot have stderr expression at intermediate declaration section at workflow-level.") + } + } + + it should "not throw an error for non-stdout/stderr in the intermediates section" in { + val testIntermediates = checkStdIntermediates(mockIntermediatesNonStd, StdoutElement, "non-stdout/stderr") + testIntermediates match { + case Valid(_) => "Intermediate section contained a non-stdout/stderr element." + case Invalid(e) => e.head should be("Workflow failed at intermediate section.") + } + } + +} diff --git a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala index 9b27dce772f..5b9d56c1e22 100644 --- a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala +++ b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala @@ -29,17 +29,17 @@ object AstToWorkflowDefinitionElement { val inputsSectionValidation: ErrorOr[Option[InputsSectionElement]] = for { inputValidateElement <- validateSize(bodyElements.filterByType[InputsSectionElement], "inputs", 1): ErrorOr[Option[InputsSectionElement]] - _ <- checkIfStdoutInputExist(inputValidateElement) - _ <- checkIfStderrInputExist(inputValidateElement) + _ <- checkIfStdInputExist(inputValidateElement, StdoutElement, "stdout") + _ <- checkIfStdInputExist(inputValidateElement, StderrElement, "stderr") } yield inputValidateElement - val intermediateValueDeclarationStdoutCheck: ErrorOr[Option[String]] = checkStdoutIntermediates(bodyElements.filterByType[IntermediateValueDeclarationElement]) - val intermediateValueDeclarationStderrCheck: ErrorOr[Option[String]] = checkStderrIntermediates(bodyElements.filterByType[IntermediateValueDeclarationElement]) + val intermediateValueDeclarationStdoutCheck = checkStdIntermediates(bodyElements.filterByType[IntermediateValueDeclarationElement], StdoutElement, "stdout") + val intermediateValueDeclarationStderrCheck: ErrorOr[Option[String]] = checkStdIntermediates(bodyElements.filterByType[IntermediateValueDeclarationElement], StderrElement, "stderr") val outputsSectionValidation: ErrorOr[Option[OutputsSectionElement]] = for { outputValidateElement <- validateSize(bodyElements.filterByType[OutputsSectionElement], "outputs", 1): ErrorOr[Option[OutputsSectionElement]] - _ <- checkIfStdoutOutputExists(outputValidateElement) - _ <- checkIfStderrOutputExists(outputValidateElement) + _ <- checkIfStdOutputExists(outputValidateElement, StdoutElement, "stdout") + _ <- checkIfStdOutputExists(outputValidateElement, StderrElement, "stderr") } yield outputValidateElement val graphSections: Vector[WorkflowGraphElement] = bodyElements.filterByType[WorkflowGraphElement] @@ -53,55 +53,29 @@ object AstToWorkflowDefinitionElement { } } - private def checkIfStdoutInputExist(inputSection: Option[InputsSectionElement]): ErrorOr[Option[String]] = { + def checkIfStdInputExist(inputSection: Option[InputsSectionElement], expressionType: FunctionCallElement, expressionName: String): ErrorOr[Option[String]] = { inputSection match { case Some(section) => - if (section.inputDeclarations.flatMap(_.expression).exists(_.isInstanceOf[StdoutElement.type])) { - s"Workflow cannot have stdout expression in input section at workflow-level.".invalidNel + if (section.inputDeclarations.flatMap(_.expression).exists(_.isInstanceOf[expressionType.type])) { + s"Workflow cannot have $expressionName expression in input section at workflow-level.".invalidNel } else None.validNel case None => None.validNel } } - private def checkIfStdoutOutputExists(outputSection: Option[OutputsSectionElement]): ErrorOr[Option[String]] = { + def checkIfStdOutputExists[A](outputSection: Option[OutputsSectionElement], expressionType: FunctionCallElement, expressionName: String): ErrorOr[Option[String]] = { outputSection match { case Some(section) => - if (section.outputs.map(_.expression).exists(_.isInstanceOf[StdoutElement.type])) { - s"Workflow cannot have stdout expression in output section at workflow-level.".invalidNel + if (section.outputs.map(_.expression).exists(_.isInstanceOf[expressionType.type])) { + s"Workflow cannot have $expressionName expression in output section at workflow-level.".invalidNel } else None.validNel case None => None.validNel } } - private def checkStdoutIntermediates(intermediate: Vector[IntermediateValueDeclarationElement]): ErrorOr[Option[String]] = { - if (intermediate.map(_.expression).exists(_.isInstanceOf[StdoutElement.type])) { - s"Workflow cannot have stdout expression at intermediate declaration section at workflow-level.".invalidNel - } else None.validNel - } - - private def checkIfStderrInputExist(inputs: Option[InputsSectionElement]): ErrorOr[Option[String]] = { - inputs match { - case Some(section) => - if (section.inputDeclarations.flatMap(_.expression).exists(_.isInstanceOf[StderrElement.type])) { - s"Workflow cannot have stderr expression in input section at workflow-level.".invalidNel - } else None.validNel - case None => None.validNel - } - } - - private def checkIfStderrOutputExists(outputs: Option[OutputsSectionElement]): ErrorOr[Option[String]] = { - outputs match { - case Some(section) => - if (section.outputs.map(_.expression).exists(_.isInstanceOf[StderrElement.type])) { - s"Workflow cannot have stderr expression in output section at workflow-level.".invalidNel - } else None.validNel - case None => None.validNel - } - } - - private def checkStderrIntermediates(intermediates: Vector[IntermediateValueDeclarationElement]): ErrorOr[Option[String]] = { - if (intermediates.map(_.expression).exists(_.isInstanceOf[StderrElement.type])) { - s"Workflow cannot have stderr expression at intermediate declaration section at workflow-level.".invalidNel + def checkStdIntermediates[A](intermediate: Vector[IntermediateValueDeclarationElement], expressionType: FunctionCallElement, expressionName: String): ErrorOr[Option[String]] = { + if (intermediate.map(_.expression).exists(_.isInstanceOf[expressionType.type])) { + s"Workflow cannot have $expressionName expression at intermediate declaration section at workflow-level.".invalidNel } else None.validNel } From 33cbb5915f93c5d0f0187633ebf9d44e74d2bd96 Mon Sep 17 00:00:00 2001 From: Katrina P <68349264+kpierre13@users.noreply.github.com> Date: Wed, 27 Apr 2022 13:39:18 -0400 Subject: [PATCH 15/45] Update wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala Co-authored-by: Chris Llanwarne --- .../src/test/scala/AstToWorkflowDefinitionElementSpec.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala b/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala index 0c13d23f6a5..9b32ded8d26 100644 --- a/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala +++ b/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala @@ -11,8 +11,8 @@ class AstToWorkflowDefinitionElementSpec extends AnyFlatSpec{ behavior of "Check Stdouts and Stderrs" val mockInputSectionStdout = Option(InputsSectionElement(Vector(InputDeclarationElement(PrimitiveTypeElement(WomSingleFileType), "i", Some(StdoutElement))))) - val mockInputSectionStderr= Option(InputsSectionElement(Vector(InputDeclarationElement(PrimitiveTypeElement(WomSingleFileType), "i", Some(StderrElement))))) - val mockInputSectionNonStd= Option(InputsSectionElement(Vector(InputDeclarationElement(PrimitiveTypeElement(WomStringType), "more", Some(StringLiteral("more")))))) + val mockInputSectionStderr = Option(InputsSectionElement(Vector(InputDeclarationElement(PrimitiveTypeElement(WomSingleFileType), "i", Some(StderrElement))))) + val mockInputSectionNonStd = Option(InputsSectionElement(Vector(InputDeclarationElement(PrimitiveTypeElement(WomStringType), "more", Some(StringLiteral("more")))))) val mockIntermediatesStdout = Vector(IntermediateValueDeclarationElement(PrimitiveTypeElement(WomSingleFileType), "y", StdoutElement)) From 5eecccfdba11ee30f66e93acf001196b3c89b310 Mon Sep 17 00:00:00 2001 From: Katrina P <68349264+kpierre13@users.noreply.github.com> Date: Wed, 27 Apr 2022 13:39:32 -0400 Subject: [PATCH 16/45] Update wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala Co-authored-by: Chris Llanwarne --- .../src/test/scala/AstToWorkflowDefinitionElementSpec.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala b/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala index 9b32ded8d26..6ca549625aa 100644 --- a/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala +++ b/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala @@ -40,7 +40,7 @@ class AstToWorkflowDefinitionElementSpec extends AnyFlatSpec{ } } - it should "not throw an error for non-stdout/stderr in the inputs section" in { + it should "not return an error for non-stdout/stderr in the inputs section" in { val testInputs = checkIfStdInputExist(mockInputSectionNonStd, StdoutElement, "non-stdout/stderr") testInputs match { case Valid(_) => "Input section contained a non-stdout/stderr element." From f1e23d9625518f94e4110633a8a59c6f077ae571 Mon Sep 17 00:00:00 2001 From: Katrina P <68349264+kpierre13@users.noreply.github.com> Date: Wed, 27 Apr 2022 13:40:13 -0400 Subject: [PATCH 17/45] Update wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala Co-authored-by: Chris Llanwarne --- .../src/test/scala/AstToWorkflowDefinitionElementSpec.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala b/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala index 6ca549625aa..6b493905cc7 100644 --- a/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala +++ b/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala @@ -63,7 +63,7 @@ class AstToWorkflowDefinitionElementSpec extends AnyFlatSpec{ case Invalid(e) => e.head should be("Workflow cannot have stderr expression in output section at workflow-level.") } } - it should "not throw an error for non-stdout/stderr in the outputs section" in { + it should "not return an error for non-stdout/stderr in the outputs section" in { val testOutputs = checkIfStdOutputExists(mockOutputSectionNonStd, StdoutElement, "non-stdout/stderr") testOutputs match { case Valid(_) => "Output section contained a non-stdout/stderr element." From 72c8d97208e2200271ce525cb35eb7ce13b46a99 Mon Sep 17 00:00:00 2001 From: Katrina P <68349264+kpierre13@users.noreply.github.com> Date: Wed, 27 Apr 2022 13:40:18 -0400 Subject: [PATCH 18/45] Update wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala Co-authored-by: Chris Llanwarne --- .../src/test/scala/AstToWorkflowDefinitionElementSpec.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala b/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala index 6b493905cc7..df2d29a178b 100644 --- a/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala +++ b/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala @@ -86,7 +86,7 @@ class AstToWorkflowDefinitionElementSpec extends AnyFlatSpec{ } } - it should "not throw an error for non-stdout/stderr in the intermediates section" in { + it should "not return an error for non-stdout/stderr in the intermediates section" in { val testIntermediates = checkStdIntermediates(mockIntermediatesNonStd, StdoutElement, "non-stdout/stderr") testIntermediates match { case Valid(_) => "Intermediate section contained a non-stdout/stderr element." From 251bfb6bc8c5195c3f653ca1874240acd768fc13 Mon Sep 17 00:00:00 2001 From: Katrina Pierre Date: Wed, 27 Apr 2022 14:43:36 -0400 Subject: [PATCH 19/45] Changing to more general function naming --- .../AstToWorkflowDefinitionElementSpec.scala | 18 +++++++++--------- .../AstToWorkflowDefinitionElement.scala | 18 +++++++++--------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala b/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala index df2d29a178b..adb4ecc6b5c 100644 --- a/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala +++ b/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala @@ -25,7 +25,7 @@ class AstToWorkflowDefinitionElementSpec extends AnyFlatSpec{ it should "return an error when there is an stdout in input section" in { - val testInputs = checkIfStdInputExist(mockInputSectionStdout, StdoutElement, "stdout") + val testInputs = checkDisallowedInputElement(mockInputSectionStdout, StdoutElement, "stdout") testInputs match { case Valid(_) => fail("Input section contained stdout. Should have failed.") case Invalid(e) => e.head should be("Workflow cannot have stdout expression in input section at workflow-level.") @@ -33,7 +33,7 @@ class AstToWorkflowDefinitionElementSpec extends AnyFlatSpec{ } it should "return an error when there is an stderr in input section" in { - val testInputs = checkIfStdInputExist(mockInputSectionStderr, StderrElement, "stderr") + val testInputs = checkDisallowedInputElement(mockInputSectionStderr, StderrElement, "stderr") testInputs match { case Valid(_) => fail("Input section contained stderr. Should have failed.") case Invalid(e) => e.head should be("Workflow cannot have stderr expression in input section at workflow-level.") @@ -41,7 +41,7 @@ class AstToWorkflowDefinitionElementSpec extends AnyFlatSpec{ } it should "not return an error for non-stdout/stderr in the inputs section" in { - val testInputs = checkIfStdInputExist(mockInputSectionNonStd, StdoutElement, "non-stdout/stderr") + val testInputs = checkDisallowedInputElement(mockInputSectionNonStd, StdoutElement, "non-stdout/stderr") testInputs match { case Valid(_) => "Input section contained a non-stdout/stderr element." case Invalid(e) => e.head should be("Workflow failed at input section.") @@ -49,7 +49,7 @@ class AstToWorkflowDefinitionElementSpec extends AnyFlatSpec{ } it should "return an error when there is an stdout in output section" in { - val testOutputs = checkIfStdOutputExists(mockOutputSectionStdout, StdoutElement, "stdout") + val testOutputs = checkDisallowedOutputElement(mockOutputSectionStdout, StdoutElement, "stdout") testOutputs match { case Valid(_) => fail("Output section contained stdout. Should have failed.") case Invalid(e) => e.head should be("Workflow cannot have stdout expression in output section at workflow-level.") @@ -57,14 +57,14 @@ class AstToWorkflowDefinitionElementSpec extends AnyFlatSpec{ } it should "return an error when there is an stderr in output section" in { - val testOutputs = checkIfStdOutputExists(mockOutputSectionStderr, StderrElement, "stderr") + val testOutputs = checkDisallowedOutputElement(mockOutputSectionStderr, StderrElement, "stderr") testOutputs match { case Valid(_) => fail("Output section contained stderr. Should have failed.") case Invalid(e) => e.head should be("Workflow cannot have stderr expression in output section at workflow-level.") } } it should "not return an error for non-stdout/stderr in the outputs section" in { - val testOutputs = checkIfStdOutputExists(mockOutputSectionNonStd, StdoutElement, "non-stdout/stderr") + val testOutputs = checkDisallowedOutputElement(mockOutputSectionNonStd, StdoutElement, "non-stdout/stderr") testOutputs match { case Valid(_) => "Output section contained a non-stdout/stderr element." case Invalid(e) => e.head should be("Workflow failed at output section.") @@ -72,14 +72,14 @@ class AstToWorkflowDefinitionElementSpec extends AnyFlatSpec{ } it should "return an error when there is an stdout at intermediate declaration section" in { - val testIntermediates = checkStdIntermediates(mockIntermediatesStdout, StdoutElement, "stdout") + val testIntermediates = checkDisallowedIntermediates(mockIntermediatesStdout, StdoutElement, "stdout") testIntermediates match { case Valid(_) => fail("Intermediate section contained stdout. Should have failed.") case Invalid(e) => e.head should be("Workflow cannot have stdout expression at intermediate declaration section at workflow-level.") } } it should "return an error when there is an stderr at intermediate declaration section" in { - val testIntermediates = checkStdIntermediates(mockIntermediatesStderr, StderrElement, "stderr") + val testIntermediates = checkDisallowedIntermediates(mockIntermediatesStderr, StderrElement, "stderr") testIntermediates match { case Valid(_) => fail("Intermediate section contained stderr. Should have failed.") case Invalid(e) => e.head should be("Workflow cannot have stderr expression at intermediate declaration section at workflow-level.") @@ -87,7 +87,7 @@ class AstToWorkflowDefinitionElementSpec extends AnyFlatSpec{ } it should "not return an error for non-stdout/stderr in the intermediates section" in { - val testIntermediates = checkStdIntermediates(mockIntermediatesNonStd, StdoutElement, "non-stdout/stderr") + val testIntermediates = checkDisallowedIntermediates(mockIntermediatesNonStd, StdoutElement, "non-stdout/stderr") testIntermediates match { case Valid(_) => "Intermediate section contained a non-stdout/stderr element." case Invalid(e) => e.head should be("Workflow failed at intermediate section.") diff --git a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala index 5b9d56c1e22..a7175b23990 100644 --- a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala +++ b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala @@ -29,17 +29,17 @@ object AstToWorkflowDefinitionElement { val inputsSectionValidation: ErrorOr[Option[InputsSectionElement]] = for { inputValidateElement <- validateSize(bodyElements.filterByType[InputsSectionElement], "inputs", 1): ErrorOr[Option[InputsSectionElement]] - _ <- checkIfStdInputExist(inputValidateElement, StdoutElement, "stdout") - _ <- checkIfStdInputExist(inputValidateElement, StderrElement, "stderr") + _ <- checkDisallowedInputElement(inputValidateElement, StdoutElement, "stdout") + _ <- checkDisallowedInputElement(inputValidateElement, StderrElement, "stderr") } yield inputValidateElement - val intermediateValueDeclarationStdoutCheck = checkStdIntermediates(bodyElements.filterByType[IntermediateValueDeclarationElement], StdoutElement, "stdout") - val intermediateValueDeclarationStderrCheck: ErrorOr[Option[String]] = checkStdIntermediates(bodyElements.filterByType[IntermediateValueDeclarationElement], StderrElement, "stderr") + val intermediateValueDeclarationStdoutCheck = checkDisallowedIntermediates(bodyElements.filterByType[IntermediateValueDeclarationElement], StdoutElement, "stdout") + val intermediateValueDeclarationStderrCheck: ErrorOr[Option[String]] = checkDisallowedIntermediates(bodyElements.filterByType[IntermediateValueDeclarationElement], StderrElement, "stderr") val outputsSectionValidation: ErrorOr[Option[OutputsSectionElement]] = for { outputValidateElement <- validateSize(bodyElements.filterByType[OutputsSectionElement], "outputs", 1): ErrorOr[Option[OutputsSectionElement]] - _ <- checkIfStdOutputExists(outputValidateElement, StdoutElement, "stdout") - _ <- checkIfStdOutputExists(outputValidateElement, StderrElement, "stderr") + _ <- checkDisallowedOutputElement(outputValidateElement, StdoutElement, "stdout") + _ <- checkDisallowedOutputElement(outputValidateElement, StderrElement, "stderr") } yield outputValidateElement val graphSections: Vector[WorkflowGraphElement] = bodyElements.filterByType[WorkflowGraphElement] @@ -53,7 +53,7 @@ object AstToWorkflowDefinitionElement { } } - def checkIfStdInputExist(inputSection: Option[InputsSectionElement], expressionType: FunctionCallElement, expressionName: String): ErrorOr[Option[String]] = { + def checkDisallowedInputElement(inputSection: Option[InputsSectionElement], expressionType: FunctionCallElement, expressionName: String): ErrorOr[Option[String]] = { inputSection match { case Some(section) => if (section.inputDeclarations.flatMap(_.expression).exists(_.isInstanceOf[expressionType.type])) { @@ -63,7 +63,7 @@ object AstToWorkflowDefinitionElement { } } - def checkIfStdOutputExists[A](outputSection: Option[OutputsSectionElement], expressionType: FunctionCallElement, expressionName: String): ErrorOr[Option[String]] = { + def checkDisallowedOutputElement(outputSection: Option[OutputsSectionElement], expressionType: FunctionCallElement, expressionName: String): ErrorOr[Option[String]] = { outputSection match { case Some(section) => if (section.outputs.map(_.expression).exists(_.isInstanceOf[expressionType.type])) { @@ -73,7 +73,7 @@ object AstToWorkflowDefinitionElement { } } - def checkStdIntermediates[A](intermediate: Vector[IntermediateValueDeclarationElement], expressionType: FunctionCallElement, expressionName: String): ErrorOr[Option[String]] = { + def checkDisallowedIntermediates(intermediate: Vector[IntermediateValueDeclarationElement], expressionType: FunctionCallElement, expressionName: String): ErrorOr[Option[String]] = { if (intermediate.map(_.expression).exists(_.isInstanceOf[expressionType.type])) { s"Workflow cannot have $expressionName expression at intermediate declaration section at workflow-level.".invalidNel } else None.validNel From 62c134bc2d25c984f65cfc808329be04f0159b49 Mon Sep 17 00:00:00 2001 From: Katrina Pierre Date: Wed, 27 Apr 2022 15:35:31 -0400 Subject: [PATCH 20/45] Return type change for functions --- .../AstToWorkflowDefinitionElement.scala | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala index a7175b23990..a43467adbe7 100644 --- a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala +++ b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala @@ -53,30 +53,30 @@ object AstToWorkflowDefinitionElement { } } - def checkDisallowedInputElement(inputSection: Option[InputsSectionElement], expressionType: FunctionCallElement, expressionName: String): ErrorOr[Option[String]] = { + def checkDisallowedInputElement(inputSection: Option[InputsSectionElement], expressionType: FunctionCallElement, expressionName: String): ErrorOr[Unit] = { inputSection match { case Some(section) => if (section.inputDeclarations.flatMap(_.expression).exists(_.isInstanceOf[expressionType.type])) { s"Workflow cannot have $expressionName expression in input section at workflow-level.".invalidNel - } else None.validNel - case None => None.validNel + } else ().validNel + case None => ().validNel } } - def checkDisallowedOutputElement(outputSection: Option[OutputsSectionElement], expressionType: FunctionCallElement, expressionName: String): ErrorOr[Option[String]] = { + def checkDisallowedOutputElement(outputSection: Option[OutputsSectionElement], expressionType: FunctionCallElement, expressionName: String): ErrorOr[Unit] = { outputSection match { case Some(section) => if (section.outputs.map(_.expression).exists(_.isInstanceOf[expressionType.type])) { s"Workflow cannot have $expressionName expression in output section at workflow-level.".invalidNel - } else None.validNel - case None => None.validNel + } else ().validNel + case None => ().validNel } } - def checkDisallowedIntermediates(intermediate: Vector[IntermediateValueDeclarationElement], expressionType: FunctionCallElement, expressionName: String): ErrorOr[Option[String]] = { + def checkDisallowedIntermediates(intermediate: Vector[IntermediateValueDeclarationElement], expressionType: FunctionCallElement, expressionName: String): ErrorOr[Unit] = { if (intermediate.map(_.expression).exists(_.isInstanceOf[expressionType.type])) { s"Workflow cannot have $expressionName expression at intermediate declaration section at workflow-level.".invalidNel - } else None.validNel + } else ().validNel } private def validateSize[A](elements: Vector[A], sectionName: String, numExpected: Int): ErrorOr[Option[A]] = { From 4040fe5c0c67403e8258b297948c3fa70bcb76e6 Mon Sep 17 00:00:00 2001 From: Katrina Pierre Date: Wed, 27 Apr 2022 15:36:44 -0400 Subject: [PATCH 21/45] Reverting Saloni newline addition --- .../base/ast2wdlom/AstToWorkflowDefinitionElement.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala index a43467adbe7..2331fe6a2bf 100644 --- a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala +++ b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala @@ -88,4 +88,4 @@ object AstToWorkflowDefinitionElement { sectionValidation } -} +} \ No newline at end of file From 7bf1774b79ebab495a132db2dc6187c690a05285 Mon Sep 17 00:00:00 2001 From: Katrina Pierre Date: Wed, 27 Apr 2022 15:38:05 -0400 Subject: [PATCH 22/45] Reverting Saloni newline addition and fixing return type --- .../base/ast2wdlom/AstToWorkflowDefinitionElement.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala index 2331fe6a2bf..6b8e7d08dd0 100644 --- a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala +++ b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala @@ -34,7 +34,7 @@ object AstToWorkflowDefinitionElement { } yield inputValidateElement val intermediateValueDeclarationStdoutCheck = checkDisallowedIntermediates(bodyElements.filterByType[IntermediateValueDeclarationElement], StdoutElement, "stdout") - val intermediateValueDeclarationStderrCheck: ErrorOr[Option[String]] = checkDisallowedIntermediates(bodyElements.filterByType[IntermediateValueDeclarationElement], StderrElement, "stderr") + val intermediateValueDeclarationStderrCheck = checkDisallowedIntermediates(bodyElements.filterByType[IntermediateValueDeclarationElement], StderrElement, "stderr") val outputsSectionValidation: ErrorOr[Option[OutputsSectionElement]] = for { outputValidateElement <- validateSize(bodyElements.filterByType[OutputsSectionElement], "outputs", 1): ErrorOr[Option[OutputsSectionElement]] From b40766ae243e845254baccef551f5123de6aa9a3 Mon Sep 17 00:00:00 2001 From: Chris Llanwarne Date: Tue, 1 Mar 2022 18:52:03 -0500 Subject: [PATCH 23/45] bad stdout location tests --- .../standardTestCases/bad_stdout_input.test | 12 ++++++++++++ .../bad_stdout_mid_workflow.test | 12 ++++++++++++ .../standardTestCases/bad_stdout_outputs.test | 12 ++++++++++++ .../standardTestCases/bad_stdout_subworkflow.test | 15 +++++++++++++++ .../bad_stdouts/bad_stdout_input.wdl | 10 ++++++++++ .../bad_stdouts/bad_stdout_mid_workflow.wdl | 9 +++++++++ .../bad_stdouts/bad_stdout_output.wdl | 7 +++++++ .../bad_stdouts/bad_stdout_subworkflow.wdl | 9 +++++++++ 8 files changed, 86 insertions(+) create mode 100644 centaur/src/main/resources/standardTestCases/bad_stdout_input.test create mode 100644 centaur/src/main/resources/standardTestCases/bad_stdout_mid_workflow.test create mode 100644 centaur/src/main/resources/standardTestCases/bad_stdout_outputs.test create mode 100644 centaur/src/main/resources/standardTestCases/bad_stdout_subworkflow.test create mode 100644 centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_input.wdl create mode 100644 centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_mid_workflow.wdl create mode 100644 centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_output.wdl create mode 100644 centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_subworkflow.wdl diff --git a/centaur/src/main/resources/standardTestCases/bad_stdout_input.test b/centaur/src/main/resources/standardTestCases/bad_stdout_input.test new file mode 100644 index 00000000000..ca48fe736f1 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stdout_input.test @@ -0,0 +1,12 @@ +# Verifies that if we use stdout in a workflow level, the workflow fails properly. +name: bad_stdout_input +testFormat: workflowfailure + +files { + workflow: bad_stdouts/bad_stdout_input.wdl +} + +metadata { + workflowName: bad_stdout + status: Failed +} diff --git a/centaur/src/main/resources/standardTestCases/bad_stdout_mid_workflow.test b/centaur/src/main/resources/standardTestCases/bad_stdout_mid_workflow.test new file mode 100644 index 00000000000..1f66c347ae4 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stdout_mid_workflow.test @@ -0,0 +1,12 @@ +# Verifies that if we use stdout in a workflow level, the workflow fails properly. +name: bad_stdout_mid_workflow +testFormat: workflowfailure + +files { + workflow: bad_stdouts/bad_stdout_mid_workflow.wdl +} + +metadata { + workflowName: bad_stdout + status: Failed +} diff --git a/centaur/src/main/resources/standardTestCases/bad_stdout_outputs.test b/centaur/src/main/resources/standardTestCases/bad_stdout_outputs.test new file mode 100644 index 00000000000..d00c2a45850 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stdout_outputs.test @@ -0,0 +1,12 @@ +# Verifies that if we use stdout in a workflow level, the workflow fails properly. +name: bad_stdout_output +testFormat: workflowfailure + +files { + workflow: bad_stdouts/bad_stdout_output.wdl +} + +metadata { + workflowName: bad_stdout + status: Failed +} diff --git a/centaur/src/main/resources/standardTestCases/bad_stdout_subworkflow.test b/centaur/src/main/resources/standardTestCases/bad_stdout_subworkflow.test new file mode 100644 index 00000000000..b5ea413bf18 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stdout_subworkflow.test @@ -0,0 +1,15 @@ +# Verifies that if we use stdout in a workflow level, the workflow fails properly. +name: bad_stdout_subworkflow +testFormat: workflowfailure + +files { + workflow: bad_stdouts/bad_stdout_subworkflow.wdl + imports: [ + bad_stdouts/bad_stdout_mid_workflow.wdl + ] +} + +metadata { + workflowName: calls_bad_stdout_subworkflow + status: Failed +} diff --git a/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_input.wdl b/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_input.wdl new file mode 100644 index 00000000000..c68f0ebffd3 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_input.wdl @@ -0,0 +1,10 @@ +version 1.0 + +workflow bad_stdout { + input { + File f = stdout() + } + output { + String s = read_string(f) + } +} diff --git a/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_mid_workflow.wdl b/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_mid_workflow.wdl new file mode 100644 index 00000000000..d4f57bfd125 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_mid_workflow.wdl @@ -0,0 +1,9 @@ +version 1.0 + +workflow bad_stdout { + File f = stdout() + + output { + String s = read_string(f) + } +} diff --git a/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_output.wdl b/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_output.wdl new file mode 100644 index 00000000000..ad2bc9a24e4 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_output.wdl @@ -0,0 +1,7 @@ +version 1.0 + +workflow bad_stdout { + output { + File f = stdout() + } +} diff --git a/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_subworkflow.wdl b/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_subworkflow.wdl new file mode 100644 index 00000000000..7134825b827 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_subworkflow.wdl @@ -0,0 +1,9 @@ +version 1.0 + +import "bad_stdout_mid_workflow.wdl" + +workflow calls_bad_stdout_subworkflow { + scatter(i in range(1000)) { + call bad_stdout_mid_workflow.bad_stdout + } +} From 2b531bc4f8ff16b727aa2f84f330ee55f54d04d9 Mon Sep 17 00:00:00 2001 From: Katrina Pierre Date: Mon, 4 Apr 2022 16:32:39 -0400 Subject: [PATCH 24/45] just a draft --- .../AstToWorkflowDefinitionElement.scala | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala index ad4c28f064a..a91ab1e83d1 100644 --- a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala +++ b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala @@ -8,6 +8,7 @@ import common.transforms.CheckedAtoB import common.validation.ErrorOr._ import wdl.model.draft3.elements._ import wom.SourceFileLocation +import wdl.model.draft3.elements.ExpressionElement.StdoutElement object AstToWorkflowDefinitionElement { @@ -40,12 +41,32 @@ object AstToWorkflowDefinitionElement { } } + private def checkStdout(inputSectionValidation: WorkflowBodyElement, b: outputsSectionValidation[WorkflowBodyElement]): Unit = { + inputsSectionValidation.map(x => x match { + case Some(value) => val here = value.inputDeclarations.flatMap(_.expression).exists(_.isInstanceOf[StdoutElement.type]) + s"Workflow cannot have more than $numExpected $sectionName sections, found ${elements.size}.".invalidNel + case None => _ + }) + } + private def validateSize[A](elements: Vector[A], sectionName: String, numExpected: Int): ErrorOr[Option[A]] = { val sectionValidation: ErrorOr[Option[A]] = if (elements.size > numExpected) { s"Workflow cannot have more than $numExpected $sectionName sections, found ${elements.size}.".invalidNel } else { elements.headOption.validNel } + sectionValidation } + + /*for(validate<-AstToWorkflowDefinitionElement){ + validate match { + case check input{} + case check p + + } + */ + } + +//def validateStdout(a:) \ No newline at end of file From 7f085014cd601a7c90696db5bd9bb2be8c25a3fc Mon Sep 17 00:00:00 2001 From: Saloni Shah Date: Mon, 4 Apr 2022 17:04:54 -0400 Subject: [PATCH 25/45] checkStdout helper function --- .../AstToWorkflowDefinitionElement.scala | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala index a91ab1e83d1..f43bdbfaf92 100644 --- a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala +++ b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala @@ -30,23 +30,27 @@ object AstToWorkflowDefinitionElement { val inputsSectionValidation: ErrorOr[Option[InputsSectionElement]] = validateSize(bodyElements.filterByType[InputsSectionElement], "inputs", 1) val outputsSectionValidation: ErrorOr[Option[OutputsSectionElement]] = validateSize(bodyElements.filterByType[OutputsSectionElement], "outputs", 1) + val inputsSectionStdoutValidation: ErrorOr[ErrorOr[Option[String]]] = inputsSectionValidation.map(checkIfStdoutExist) + val graphSections: Vector[WorkflowGraphElement] = bodyElements.filterByType[WorkflowGraphElement] val metaSectionValidation: ErrorOr[Option[MetaSectionElement]] = validateSize(bodyElements.filterByType[MetaSectionElement], "meta", 1) val parameterMetaSectionValidation: ErrorOr[Option[ParameterMetaSectionElement]] = validateSize(bodyElements.filterByType[ParameterMetaSectionElement], "parameterMeta", 1) - (inputsSectionValidation, outputsSectionValidation, metaSectionValidation, parameterMetaSectionValidation) mapN { - (validInputs, validOutputs, meta, parameterMeta) => + (inputsSectionValidation, outputsSectionValidation, metaSectionValidation, parameterMetaSectionValidation, inputsSectionStdoutValidation) mapN { + (validInputs, validOutputs, meta, parameterMeta, _) => WorkflowDefinitionElement(name, validInputs, graphSections.toSet, validOutputs, meta, parameterMeta, sourceLocation) } } - private def checkStdout(inputSectionValidation: WorkflowBodyElement, b: outputsSectionValidation[WorkflowBodyElement]): Unit = { - inputsSectionValidation.map(x => x match { - case Some(value) => val here = value.inputDeclarations.flatMap(_.expression).exists(_.isInstanceOf[StdoutElement.type]) - s"Workflow cannot have more than $numExpected $sectionName sections, found ${elements.size}.".invalidNel - case None => _ - }) + private def checkIfStdoutExist(inputSection: Option[InputsSectionElement]): ErrorOr[Option[String]] = { + inputSection match { + case Some(section) => + if (section.inputDeclarations.flatMap(_.expression).exists(_.isInstanceOf[StdoutElement.type])) { + s"Workflow cannot have stdout expression in input section at workflow-level.".invalidNel + } else None.validNel + case None => None.validNel + } } private def validateSize[A](elements: Vector[A], sectionName: String, numExpected: Int): ErrorOr[Option[A]] = { @@ -69,4 +73,4 @@ object AstToWorkflowDefinitionElement { } -//def validateStdout(a:) \ No newline at end of file +//def validateStdout(a:) From 7d64dc79b5a9a99acf2507572b09a5dc8a1cdbb0 Mon Sep 17 00:00:00 2001 From: Katrina Pierre Date: Thu, 14 Apr 2022 02:14:25 -0400 Subject: [PATCH 26/45] Adding stderr wdl and test files, refactoring functions and adding functions to handle stderr --- .../standardTestCases/bad_stderr_input.test | 12 +++ .../bad_stderr_mid_workflow.test | 12 +++ .../standardTestCases/bad_stderr_outputs.test | 12 +++ .../bad_stderr_subworkflow.test | 15 ++++ .../bad_stderrs/bad_stderr_input.wdl | 11 +++ .../bad_stderrs/bad_stderr_mid_workflow.wdl | 9 +++ .../bad_stderrs/bad_stderr_output.wdl | 7 ++ .../bad_stderrs/bad_stderr_subworkflow.wdl | 9 +++ .../AstToWorkflowDefinitionElement.scala | 77 ++++++++++++++----- 9 files changed, 146 insertions(+), 18 deletions(-) create mode 100644 centaur/src/main/resources/standardTestCases/bad_stderr_input.test create mode 100644 centaur/src/main/resources/standardTestCases/bad_stderr_mid_workflow.test create mode 100644 centaur/src/main/resources/standardTestCases/bad_stderr_outputs.test create mode 100644 centaur/src/main/resources/standardTestCases/bad_stderr_subworkflow.test create mode 100644 centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_input.wdl create mode 100644 centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_mid_workflow.wdl create mode 100644 centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_output.wdl create mode 100644 centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_subworkflow.wdl diff --git a/centaur/src/main/resources/standardTestCases/bad_stderr_input.test b/centaur/src/main/resources/standardTestCases/bad_stderr_input.test new file mode 100644 index 00000000000..57e999dcd89 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stderr_input.test @@ -0,0 +1,12 @@ +# Verifies that if we use stderr in a workflow level, the workflow fails properly. +name: bad_stderr_input +testFormat: workflowfailure + +files { + workflow: bad_stderrs/bad_stderr_input.wdl +} + +metadata { + workflowName: bad_stderr + status: Failed +} \ No newline at end of file diff --git a/centaur/src/main/resources/standardTestCases/bad_stderr_mid_workflow.test b/centaur/src/main/resources/standardTestCases/bad_stderr_mid_workflow.test new file mode 100644 index 00000000000..5ab4668814a --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stderr_mid_workflow.test @@ -0,0 +1,12 @@ +# Verifies that if we use stderr in a workflow level, the workflow fails properly. +name: bad_stderr_mid_workflow +testFormat: workflowfailure + +files { + workflow: bad_stderrs/bad_stderr_mid_workflow.wdl +} + +metadata { + workflowName: bad_stderr + status: Failed +} diff --git a/centaur/src/main/resources/standardTestCases/bad_stderr_outputs.test b/centaur/src/main/resources/standardTestCases/bad_stderr_outputs.test new file mode 100644 index 00000000000..126bbc2fd11 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stderr_outputs.test @@ -0,0 +1,12 @@ +# Verifies that if we use stderr in a workflow level, the workflow fails properly. +name: bad_stderr_output +testFormat: workflowfailure + +files { + workflow: bad_stderrs/bad_stderr_output.wdl +} + +metadata { + workflowName: bad_stderr + status: Failed +} diff --git a/centaur/src/main/resources/standardTestCases/bad_stderr_subworkflow.test b/centaur/src/main/resources/standardTestCases/bad_stderr_subworkflow.test new file mode 100644 index 00000000000..cf22aca78fe --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stderr_subworkflow.test @@ -0,0 +1,15 @@ +# Verifies that if we use stderr in a workflow level, the workflow fails properly. +name: bad_stderr_subworkflow +testFormat: workflowfailure + +files { + workflow: bad_stderrs/bad_stderr_subworkflow.wdl + imports: [ + bad_stderrs/bad_stderr_mid_workflow.wdl + ] +} + +metadata { + workflowName: calls_bad_stderr_subworkflow + status: Failed +} diff --git a/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_input.wdl b/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_input.wdl new file mode 100644 index 00000000000..ef7e2b7ff82 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_input.wdl @@ -0,0 +1,11 @@ +version 1.0 + +workflow bad_stderr { + + input { + File f = stderr() + } + output { + String s = read_string(f) + } +} diff --git a/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_mid_workflow.wdl b/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_mid_workflow.wdl new file mode 100644 index 00000000000..eb6331ff8c5 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_mid_workflow.wdl @@ -0,0 +1,9 @@ +version 1.0 + +workflow bad_stderr { + File f = stderr() + + output { + String s = read_string(f) + } +} diff --git a/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_output.wdl b/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_output.wdl new file mode 100644 index 00000000000..65e99949a52 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_output.wdl @@ -0,0 +1,7 @@ +version 1.0 + +workflow bad_stderr { + output { + File f = stderr() + } +} diff --git a/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_subworkflow.wdl b/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_subworkflow.wdl new file mode 100644 index 00000000000..b1249f8dd9d --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_subworkflow.wdl @@ -0,0 +1,9 @@ +version 1.0 + +import "bad_stderr_mid_workflow.wdl" + +workflow calls_bad_stderr_subworkflow { + scatter(i in range(1000)) { + call bad_stderr_mid_workflow.bad_stderr + } +} diff --git a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala index f43bdbfaf92..82513fdeb12 100644 --- a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala +++ b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala @@ -8,7 +8,7 @@ import common.transforms.CheckedAtoB import common.validation.ErrorOr._ import wdl.model.draft3.elements._ import wom.SourceFileLocation -import wdl.model.draft3.elements.ExpressionElement.StdoutElement +import wdl.model.draft3.elements.ExpressionElement._ object AstToWorkflowDefinitionElement { @@ -27,23 +27,33 @@ object AstToWorkflowDefinitionElement { sourceLocation: Option[SourceFileLocation], bodyElements: Vector[WorkflowBodyElement]) = { - val inputsSectionValidation: ErrorOr[Option[InputsSectionElement]] = validateSize(bodyElements.filterByType[InputsSectionElement], "inputs", 1) - val outputsSectionValidation: ErrorOr[Option[OutputsSectionElement]] = validateSize(bodyElements.filterByType[OutputsSectionElement], "outputs", 1) + val inputsSectionValidation: ErrorOr[Option[InputsSectionElement]] = for { + inputValidateElement <- validateSize(bodyElements.filterByType[InputsSectionElement], "inputs", 1): ErrorOr[Option[InputsSectionElement]] + _ <- checkIfStdoutInputExist(inputValidateElement) + _ <- checkIfStderrInputExist(inputValidateElement) + } yield inputValidateElement - val inputsSectionStdoutValidation: ErrorOr[ErrorOr[Option[String]]] = inputsSectionValidation.map(checkIfStdoutExist) + val intermediateValueDeclarationStdoutCheck: ErrorOr[Option[String]] = checkStdoutIntermediates(bodyElements.filterByType[IntermediateValueDeclarationElement]) + val intermediateValueDeclarationStderrCheck: ErrorOr[Option[String]] = checkStderrIntermediates(bodyElements.filterByType[IntermediateValueDeclarationElement]) + + val outputsSectionValidation: ErrorOr[Option[OutputsSectionElement]] = for { + outputValidateElement <- validateSize(bodyElements.filterByType[OutputsSectionElement], "outputs", 1): ErrorOr[Option[OutputsSectionElement]] + _ <- checkIfStdoutOutputExists(outputValidateElement) + _ <- checkIfStderrOutputExists(outputValidateElement) + } yield outputValidateElement val graphSections: Vector[WorkflowGraphElement] = bodyElements.filterByType[WorkflowGraphElement] val metaSectionValidation: ErrorOr[Option[MetaSectionElement]] = validateSize(bodyElements.filterByType[MetaSectionElement], "meta", 1) val parameterMetaSectionValidation: ErrorOr[Option[ParameterMetaSectionElement]] = validateSize(bodyElements.filterByType[ParameterMetaSectionElement], "parameterMeta", 1) - (inputsSectionValidation, outputsSectionValidation, metaSectionValidation, parameterMetaSectionValidation, inputsSectionStdoutValidation) mapN { - (validInputs, validOutputs, meta, parameterMeta, _) => + (inputsSectionValidation, outputsSectionValidation, metaSectionValidation, parameterMetaSectionValidation, intermediateValueDeclarationStdoutCheck, intermediateValueDeclarationStderrCheck) mapN { + (validInputs, validOutputs, meta, parameterMeta, _, _) => WorkflowDefinitionElement(name, validInputs, graphSections.toSet, validOutputs, meta, parameterMeta, sourceLocation) } } - private def checkIfStdoutExist(inputSection: Option[InputsSectionElement]): ErrorOr[Option[String]] = { + private def checkIfStdoutInputExist(inputSection: Option[InputsSectionElement]): ErrorOr[Option[String]] = { inputSection match { case Some(section) => if (section.inputDeclarations.flatMap(_.expression).exists(_.isInstanceOf[StdoutElement.type])) { @@ -53,6 +63,48 @@ object AstToWorkflowDefinitionElement { } } + private def checkIfStdoutOutputExists(outputSection: Option[OutputsSectionElement]): ErrorOr[Option[String]] = { + outputSection match { + case Some(section) => + if (section.outputs.map(_.expression).exists(_.isInstanceOf[StdoutElement.type])) { + s"Workflow cannot have stdout expression in output section at workflow-level.".invalidNel + } else None.validNel + case None => None.validNel + } + } + + private def checkStdoutIntermediates(intermediate: Vector[IntermediateValueDeclarationElement]): ErrorOr[Option[String]] = { + if (intermediate.map(_.expression).exists(_.isInstanceOf[StdoutElement.type])) { + s"Workflow cannot have stderr expression at intermediate declaration section at workflow-level.".invalidNel + } else None.validNel + } + + private def checkIfStderrInputExist(inputs: Option[InputsSectionElement]): ErrorOr[Option[String]] = { + inputs match { + case Some(section) => + if (section.inputDeclarations.flatMap(_.expression).exists(_.isInstanceOf[StderrElement.type])) { + s"Workflow cannot have stderr expression in input section at workflow-level.".invalidNel + } else None.validNel + case None => None.validNel + } + } + + private def checkIfStderrOutputExists(outputs: Option[OutputsSectionElement]): ErrorOr[Option[String]] = { + outputs match { + case Some(section) => + if (section.outputs.map(_.expression).exists(_.isInstanceOf[StderrElement.type])) { + s"Workflow cannot have stderr expression in output section at workflow-level.".invalidNel + } else None.validNel + case None => None.validNel + } + } + + private def checkStderrIntermediates(intermediates: Vector[IntermediateValueDeclarationElement]): ErrorOr[Option[String]] = { + if (intermediates.map(_.expression).exists(_.isInstanceOf[StderrElement.type])) { + s"Workflow cannot have stdout expression at intermediate declaration section at workflow-level.".invalidNel + } else None.validNel + } + private def validateSize[A](elements: Vector[A], sectionName: String, numExpected: Int): ErrorOr[Option[A]] = { val sectionValidation: ErrorOr[Option[A]] = if (elements.size > numExpected) { s"Workflow cannot have more than $numExpected $sectionName sections, found ${elements.size}.".invalidNel @@ -62,15 +114,4 @@ object AstToWorkflowDefinitionElement { sectionValidation } - - /*for(validate<-AstToWorkflowDefinitionElement){ - validate match { - case check input{} - case check p - - } - */ - } - -//def validateStdout(a:) From d85569f052c7f18845d1c11f60b7d07551e03c9a Mon Sep 17 00:00:00 2001 From: Katrina Pierre Date: Thu, 14 Apr 2022 02:16:40 -0400 Subject: [PATCH 27/45] Fixing error messages --- .../base/ast2wdlom/AstToWorkflowDefinitionElement.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala index 82513fdeb12..9b27dce772f 100644 --- a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala +++ b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala @@ -75,7 +75,7 @@ object AstToWorkflowDefinitionElement { private def checkStdoutIntermediates(intermediate: Vector[IntermediateValueDeclarationElement]): ErrorOr[Option[String]] = { if (intermediate.map(_.expression).exists(_.isInstanceOf[StdoutElement.type])) { - s"Workflow cannot have stderr expression at intermediate declaration section at workflow-level.".invalidNel + s"Workflow cannot have stdout expression at intermediate declaration section at workflow-level.".invalidNel } else None.validNel } @@ -101,7 +101,7 @@ object AstToWorkflowDefinitionElement { private def checkStderrIntermediates(intermediates: Vector[IntermediateValueDeclarationElement]): ErrorOr[Option[String]] = { if (intermediates.map(_.expression).exists(_.isInstanceOf[StderrElement.type])) { - s"Workflow cannot have stdout expression at intermediate declaration section at workflow-level.".invalidNel + s"Workflow cannot have stderr expression at intermediate declaration section at workflow-level.".invalidNel } else None.validNel } From b55cf3db109554bf86c40ec0a16c63b8fe3a24d2 Mon Sep 17 00:00:00 2001 From: Katrina Pierre Date: Thu, 14 Apr 2022 14:18:24 -0400 Subject: [PATCH 28/45] Rebasing --- .../src/main/resources/standardTestCases/bad_stderr_input.test | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/centaur/src/main/resources/standardTestCases/bad_stderr_input.test b/centaur/src/main/resources/standardTestCases/bad_stderr_input.test index 57e999dcd89..648f06d4c6d 100644 --- a/centaur/src/main/resources/standardTestCases/bad_stderr_input.test +++ b/centaur/src/main/resources/standardTestCases/bad_stderr_input.test @@ -9,4 +9,4 @@ files { metadata { workflowName: bad_stderr status: Failed -} \ No newline at end of file +} From f3bc4f9695b0a7b4aa029852a1967334d10f4834 Mon Sep 17 00:00:00 2001 From: Katrina Pierre Date: Fri, 15 Apr 2022 11:30:54 -0400 Subject: [PATCH 29/45] Omitting the metadata section for Travis builds to pass --- .../main/resources/standardTestCases/bad_stderr_input.test | 5 ----- .../resources/standardTestCases/bad_stderr_mid_workflow.test | 5 ----- .../main/resources/standardTestCases/bad_stderr_outputs.test | 5 ----- .../resources/standardTestCases/bad_stderr_subworkflow.test | 5 ----- .../main/resources/standardTestCases/bad_stdout_input.test | 5 ----- .../resources/standardTestCases/bad_stdout_mid_workflow.test | 5 ----- .../main/resources/standardTestCases/bad_stdout_outputs.test | 5 ----- .../resources/standardTestCases/bad_stdout_subworkflow.test | 5 ----- 8 files changed, 40 deletions(-) diff --git a/centaur/src/main/resources/standardTestCases/bad_stderr_input.test b/centaur/src/main/resources/standardTestCases/bad_stderr_input.test index 648f06d4c6d..6775db336cf 100644 --- a/centaur/src/main/resources/standardTestCases/bad_stderr_input.test +++ b/centaur/src/main/resources/standardTestCases/bad_stderr_input.test @@ -5,8 +5,3 @@ testFormat: workflowfailure files { workflow: bad_stderrs/bad_stderr_input.wdl } - -metadata { - workflowName: bad_stderr - status: Failed -} diff --git a/centaur/src/main/resources/standardTestCases/bad_stderr_mid_workflow.test b/centaur/src/main/resources/standardTestCases/bad_stderr_mid_workflow.test index 5ab4668814a..af7e62d6401 100644 --- a/centaur/src/main/resources/standardTestCases/bad_stderr_mid_workflow.test +++ b/centaur/src/main/resources/standardTestCases/bad_stderr_mid_workflow.test @@ -5,8 +5,3 @@ testFormat: workflowfailure files { workflow: bad_stderrs/bad_stderr_mid_workflow.wdl } - -metadata { - workflowName: bad_stderr - status: Failed -} diff --git a/centaur/src/main/resources/standardTestCases/bad_stderr_outputs.test b/centaur/src/main/resources/standardTestCases/bad_stderr_outputs.test index 126bbc2fd11..9c2b02aa8b1 100644 --- a/centaur/src/main/resources/standardTestCases/bad_stderr_outputs.test +++ b/centaur/src/main/resources/standardTestCases/bad_stderr_outputs.test @@ -5,8 +5,3 @@ testFormat: workflowfailure files { workflow: bad_stderrs/bad_stderr_output.wdl } - -metadata { - workflowName: bad_stderr - status: Failed -} diff --git a/centaur/src/main/resources/standardTestCases/bad_stderr_subworkflow.test b/centaur/src/main/resources/standardTestCases/bad_stderr_subworkflow.test index cf22aca78fe..408f17664a5 100644 --- a/centaur/src/main/resources/standardTestCases/bad_stderr_subworkflow.test +++ b/centaur/src/main/resources/standardTestCases/bad_stderr_subworkflow.test @@ -8,8 +8,3 @@ files { bad_stderrs/bad_stderr_mid_workflow.wdl ] } - -metadata { - workflowName: calls_bad_stderr_subworkflow - status: Failed -} diff --git a/centaur/src/main/resources/standardTestCases/bad_stdout_input.test b/centaur/src/main/resources/standardTestCases/bad_stdout_input.test index ca48fe736f1..bf7c9bf6a65 100644 --- a/centaur/src/main/resources/standardTestCases/bad_stdout_input.test +++ b/centaur/src/main/resources/standardTestCases/bad_stdout_input.test @@ -5,8 +5,3 @@ testFormat: workflowfailure files { workflow: bad_stdouts/bad_stdout_input.wdl } - -metadata { - workflowName: bad_stdout - status: Failed -} diff --git a/centaur/src/main/resources/standardTestCases/bad_stdout_mid_workflow.test b/centaur/src/main/resources/standardTestCases/bad_stdout_mid_workflow.test index 1f66c347ae4..ae73adeeb29 100644 --- a/centaur/src/main/resources/standardTestCases/bad_stdout_mid_workflow.test +++ b/centaur/src/main/resources/standardTestCases/bad_stdout_mid_workflow.test @@ -5,8 +5,3 @@ testFormat: workflowfailure files { workflow: bad_stdouts/bad_stdout_mid_workflow.wdl } - -metadata { - workflowName: bad_stdout - status: Failed -} diff --git a/centaur/src/main/resources/standardTestCases/bad_stdout_outputs.test b/centaur/src/main/resources/standardTestCases/bad_stdout_outputs.test index d00c2a45850..67b51cd2bf0 100644 --- a/centaur/src/main/resources/standardTestCases/bad_stdout_outputs.test +++ b/centaur/src/main/resources/standardTestCases/bad_stdout_outputs.test @@ -5,8 +5,3 @@ testFormat: workflowfailure files { workflow: bad_stdouts/bad_stdout_output.wdl } - -metadata { - workflowName: bad_stdout - status: Failed -} diff --git a/centaur/src/main/resources/standardTestCases/bad_stdout_subworkflow.test b/centaur/src/main/resources/standardTestCases/bad_stdout_subworkflow.test index b5ea413bf18..28a24d8864f 100644 --- a/centaur/src/main/resources/standardTestCases/bad_stdout_subworkflow.test +++ b/centaur/src/main/resources/standardTestCases/bad_stdout_subworkflow.test @@ -8,8 +8,3 @@ files { bad_stdouts/bad_stdout_mid_workflow.wdl ] } - -metadata { - workflowName: calls_bad_stdout_subworkflow - status: Failed -} From eaf281eba4a57a5aa982df8d0c3ba01ff07a9d2e Mon Sep 17 00:00:00 2001 From: Katrina P <68349264+kpierre13@users.noreply.github.com> Date: Wed, 27 Apr 2022 09:52:09 -0400 Subject: [PATCH 30/45] Bw 1126 workflow level stdout / stderr expressions fail, hang execution forever (#6745) * Combining similar-use functions * All tests created * Saving changes * Adding unit tests --- .../AstToWorkflowDefinitionElementSpec.scala | 97 +++++++++++++++++++ .../AstToWorkflowDefinitionElement.scala | 56 +++-------- 2 files changed, 112 insertions(+), 41 deletions(-) create mode 100644 wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala diff --git a/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala b/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala new file mode 100644 index 00000000000..0c13d23f6a5 --- /dev/null +++ b/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala @@ -0,0 +1,97 @@ +import cats.data.Validated._ +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers._ +import wdl.model.draft3.elements.ExpressionElement._ +import wdl.model.draft3.elements.{InputDeclarationElement, InputsSectionElement, IntermediateValueDeclarationElement, OutputDeclarationElement, OutputsSectionElement, PrimitiveTypeElement} +import wom.types.{WomSingleFileType, WomStringType} +import wdl.transforms.base.ast2wdlom.AstToWorkflowDefinitionElement._ + + +class AstToWorkflowDefinitionElementSpec extends AnyFlatSpec{ + behavior of "Check Stdouts and Stderrs" + + val mockInputSectionStdout = Option(InputsSectionElement(Vector(InputDeclarationElement(PrimitiveTypeElement(WomSingleFileType), "i", Some(StdoutElement))))) + val mockInputSectionStderr= Option(InputsSectionElement(Vector(InputDeclarationElement(PrimitiveTypeElement(WomSingleFileType), "i", Some(StderrElement))))) + val mockInputSectionNonStd= Option(InputsSectionElement(Vector(InputDeclarationElement(PrimitiveTypeElement(WomStringType), "more", Some(StringLiteral("more")))))) + + + val mockIntermediatesStdout = Vector(IntermediateValueDeclarationElement(PrimitiveTypeElement(WomSingleFileType), "y", StdoutElement)) + val mockIntermediatesStderr = Vector(IntermediateValueDeclarationElement(PrimitiveTypeElement(WomSingleFileType), "y", StderrElement)) + val mockIntermediatesNonStd = Vector(IntermediateValueDeclarationElement(PrimitiveTypeElement(WomStringType), "here", StringLiteral("here"))) + + val mockOutputSectionStdout = Option(OutputsSectionElement(Vector(OutputDeclarationElement(PrimitiveTypeElement(WomSingleFileType), "s", StdoutElement)))) + val mockOutputSectionStderr = Option(OutputsSectionElement(Vector(OutputDeclarationElement(PrimitiveTypeElement(WomSingleFileType), "s", StderrElement)))) + val mockOutputSectionNonStd = Option(OutputsSectionElement(Vector(OutputDeclarationElement(PrimitiveTypeElement(WomStringType), "more", StringLiteral("more"))))) + + + it should "return an error when there is an stdout in input section" in { + val testInputs = checkIfStdInputExist(mockInputSectionStdout, StdoutElement, "stdout") + testInputs match { + case Valid(_) => fail("Input section contained stdout. Should have failed.") + case Invalid(e) => e.head should be("Workflow cannot have stdout expression in input section at workflow-level.") + } + } + + it should "return an error when there is an stderr in input section" in { + val testInputs = checkIfStdInputExist(mockInputSectionStderr, StderrElement, "stderr") + testInputs match { + case Valid(_) => fail("Input section contained stderr. Should have failed.") + case Invalid(e) => e.head should be("Workflow cannot have stderr expression in input section at workflow-level.") + } + } + + it should "not throw an error for non-stdout/stderr in the inputs section" in { + val testInputs = checkIfStdInputExist(mockInputSectionNonStd, StdoutElement, "non-stdout/stderr") + testInputs match { + case Valid(_) => "Input section contained a non-stdout/stderr element." + case Invalid(e) => e.head should be("Workflow failed at input section.") + } + } + + it should "return an error when there is an stdout in output section" in { + val testOutputs = checkIfStdOutputExists(mockOutputSectionStdout, StdoutElement, "stdout") + testOutputs match { + case Valid(_) => fail("Output section contained stdout. Should have failed.") + case Invalid(e) => e.head should be("Workflow cannot have stdout expression in output section at workflow-level.") + } + } + + it should "return an error when there is an stderr in output section" in { + val testOutputs = checkIfStdOutputExists(mockOutputSectionStderr, StderrElement, "stderr") + testOutputs match { + case Valid(_) => fail("Output section contained stderr. Should have failed.") + case Invalid(e) => e.head should be("Workflow cannot have stderr expression in output section at workflow-level.") + } + } + it should "not throw an error for non-stdout/stderr in the outputs section" in { + val testOutputs = checkIfStdOutputExists(mockOutputSectionNonStd, StdoutElement, "non-stdout/stderr") + testOutputs match { + case Valid(_) => "Output section contained a non-stdout/stderr element." + case Invalid(e) => e.head should be("Workflow failed at output section.") + } + } + + it should "return an error when there is an stdout at intermediate declaration section" in { + val testIntermediates = checkStdIntermediates(mockIntermediatesStdout, StdoutElement, "stdout") + testIntermediates match { + case Valid(_) => fail("Intermediate section contained stdout. Should have failed.") + case Invalid(e) => e.head should be("Workflow cannot have stdout expression at intermediate declaration section at workflow-level.") + } + } + it should "return an error when there is an stderr at intermediate declaration section" in { + val testIntermediates = checkStdIntermediates(mockIntermediatesStderr, StderrElement, "stderr") + testIntermediates match { + case Valid(_) => fail("Intermediate section contained stderr. Should have failed.") + case Invalid(e) => e.head should be("Workflow cannot have stderr expression at intermediate declaration section at workflow-level.") + } + } + + it should "not throw an error for non-stdout/stderr in the intermediates section" in { + val testIntermediates = checkStdIntermediates(mockIntermediatesNonStd, StdoutElement, "non-stdout/stderr") + testIntermediates match { + case Valid(_) => "Intermediate section contained a non-stdout/stderr element." + case Invalid(e) => e.head should be("Workflow failed at intermediate section.") + } + } + +} diff --git a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala index 9b27dce772f..5b9d56c1e22 100644 --- a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala +++ b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala @@ -29,17 +29,17 @@ object AstToWorkflowDefinitionElement { val inputsSectionValidation: ErrorOr[Option[InputsSectionElement]] = for { inputValidateElement <- validateSize(bodyElements.filterByType[InputsSectionElement], "inputs", 1): ErrorOr[Option[InputsSectionElement]] - _ <- checkIfStdoutInputExist(inputValidateElement) - _ <- checkIfStderrInputExist(inputValidateElement) + _ <- checkIfStdInputExist(inputValidateElement, StdoutElement, "stdout") + _ <- checkIfStdInputExist(inputValidateElement, StderrElement, "stderr") } yield inputValidateElement - val intermediateValueDeclarationStdoutCheck: ErrorOr[Option[String]] = checkStdoutIntermediates(bodyElements.filterByType[IntermediateValueDeclarationElement]) - val intermediateValueDeclarationStderrCheck: ErrorOr[Option[String]] = checkStderrIntermediates(bodyElements.filterByType[IntermediateValueDeclarationElement]) + val intermediateValueDeclarationStdoutCheck = checkStdIntermediates(bodyElements.filterByType[IntermediateValueDeclarationElement], StdoutElement, "stdout") + val intermediateValueDeclarationStderrCheck: ErrorOr[Option[String]] = checkStdIntermediates(bodyElements.filterByType[IntermediateValueDeclarationElement], StderrElement, "stderr") val outputsSectionValidation: ErrorOr[Option[OutputsSectionElement]] = for { outputValidateElement <- validateSize(bodyElements.filterByType[OutputsSectionElement], "outputs", 1): ErrorOr[Option[OutputsSectionElement]] - _ <- checkIfStdoutOutputExists(outputValidateElement) - _ <- checkIfStderrOutputExists(outputValidateElement) + _ <- checkIfStdOutputExists(outputValidateElement, StdoutElement, "stdout") + _ <- checkIfStdOutputExists(outputValidateElement, StderrElement, "stderr") } yield outputValidateElement val graphSections: Vector[WorkflowGraphElement] = bodyElements.filterByType[WorkflowGraphElement] @@ -53,55 +53,29 @@ object AstToWorkflowDefinitionElement { } } - private def checkIfStdoutInputExist(inputSection: Option[InputsSectionElement]): ErrorOr[Option[String]] = { + def checkIfStdInputExist(inputSection: Option[InputsSectionElement], expressionType: FunctionCallElement, expressionName: String): ErrorOr[Option[String]] = { inputSection match { case Some(section) => - if (section.inputDeclarations.flatMap(_.expression).exists(_.isInstanceOf[StdoutElement.type])) { - s"Workflow cannot have stdout expression in input section at workflow-level.".invalidNel + if (section.inputDeclarations.flatMap(_.expression).exists(_.isInstanceOf[expressionType.type])) { + s"Workflow cannot have $expressionName expression in input section at workflow-level.".invalidNel } else None.validNel case None => None.validNel } } - private def checkIfStdoutOutputExists(outputSection: Option[OutputsSectionElement]): ErrorOr[Option[String]] = { + def checkIfStdOutputExists[A](outputSection: Option[OutputsSectionElement], expressionType: FunctionCallElement, expressionName: String): ErrorOr[Option[String]] = { outputSection match { case Some(section) => - if (section.outputs.map(_.expression).exists(_.isInstanceOf[StdoutElement.type])) { - s"Workflow cannot have stdout expression in output section at workflow-level.".invalidNel + if (section.outputs.map(_.expression).exists(_.isInstanceOf[expressionType.type])) { + s"Workflow cannot have $expressionName expression in output section at workflow-level.".invalidNel } else None.validNel case None => None.validNel } } - private def checkStdoutIntermediates(intermediate: Vector[IntermediateValueDeclarationElement]): ErrorOr[Option[String]] = { - if (intermediate.map(_.expression).exists(_.isInstanceOf[StdoutElement.type])) { - s"Workflow cannot have stdout expression at intermediate declaration section at workflow-level.".invalidNel - } else None.validNel - } - - private def checkIfStderrInputExist(inputs: Option[InputsSectionElement]): ErrorOr[Option[String]] = { - inputs match { - case Some(section) => - if (section.inputDeclarations.flatMap(_.expression).exists(_.isInstanceOf[StderrElement.type])) { - s"Workflow cannot have stderr expression in input section at workflow-level.".invalidNel - } else None.validNel - case None => None.validNel - } - } - - private def checkIfStderrOutputExists(outputs: Option[OutputsSectionElement]): ErrorOr[Option[String]] = { - outputs match { - case Some(section) => - if (section.outputs.map(_.expression).exists(_.isInstanceOf[StderrElement.type])) { - s"Workflow cannot have stderr expression in output section at workflow-level.".invalidNel - } else None.validNel - case None => None.validNel - } - } - - private def checkStderrIntermediates(intermediates: Vector[IntermediateValueDeclarationElement]): ErrorOr[Option[String]] = { - if (intermediates.map(_.expression).exists(_.isInstanceOf[StderrElement.type])) { - s"Workflow cannot have stderr expression at intermediate declaration section at workflow-level.".invalidNel + def checkStdIntermediates[A](intermediate: Vector[IntermediateValueDeclarationElement], expressionType: FunctionCallElement, expressionName: String): ErrorOr[Option[String]] = { + if (intermediate.map(_.expression).exists(_.isInstanceOf[expressionType.type])) { + s"Workflow cannot have $expressionName expression at intermediate declaration section at workflow-level.".invalidNel } else None.validNel } From f67be32d24d782e3b4f5c1af65bcf7a5fae3587e Mon Sep 17 00:00:00 2001 From: Katrina P <68349264+kpierre13@users.noreply.github.com> Date: Wed, 27 Apr 2022 13:39:18 -0400 Subject: [PATCH 31/45] Update wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala Co-authored-by: Chris Llanwarne --- .../src/test/scala/AstToWorkflowDefinitionElementSpec.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala b/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala index 0c13d23f6a5..9b32ded8d26 100644 --- a/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala +++ b/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala @@ -11,8 +11,8 @@ class AstToWorkflowDefinitionElementSpec extends AnyFlatSpec{ behavior of "Check Stdouts and Stderrs" val mockInputSectionStdout = Option(InputsSectionElement(Vector(InputDeclarationElement(PrimitiveTypeElement(WomSingleFileType), "i", Some(StdoutElement))))) - val mockInputSectionStderr= Option(InputsSectionElement(Vector(InputDeclarationElement(PrimitiveTypeElement(WomSingleFileType), "i", Some(StderrElement))))) - val mockInputSectionNonStd= Option(InputsSectionElement(Vector(InputDeclarationElement(PrimitiveTypeElement(WomStringType), "more", Some(StringLiteral("more")))))) + val mockInputSectionStderr = Option(InputsSectionElement(Vector(InputDeclarationElement(PrimitiveTypeElement(WomSingleFileType), "i", Some(StderrElement))))) + val mockInputSectionNonStd = Option(InputsSectionElement(Vector(InputDeclarationElement(PrimitiveTypeElement(WomStringType), "more", Some(StringLiteral("more")))))) val mockIntermediatesStdout = Vector(IntermediateValueDeclarationElement(PrimitiveTypeElement(WomSingleFileType), "y", StdoutElement)) From 6b662e1f0b5949d14c2078782cfe61354e971c64 Mon Sep 17 00:00:00 2001 From: Katrina P <68349264+kpierre13@users.noreply.github.com> Date: Wed, 27 Apr 2022 13:39:32 -0400 Subject: [PATCH 32/45] Update wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala Co-authored-by: Chris Llanwarne --- .../src/test/scala/AstToWorkflowDefinitionElementSpec.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala b/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala index 9b32ded8d26..6ca549625aa 100644 --- a/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala +++ b/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala @@ -40,7 +40,7 @@ class AstToWorkflowDefinitionElementSpec extends AnyFlatSpec{ } } - it should "not throw an error for non-stdout/stderr in the inputs section" in { + it should "not return an error for non-stdout/stderr in the inputs section" in { val testInputs = checkIfStdInputExist(mockInputSectionNonStd, StdoutElement, "non-stdout/stderr") testInputs match { case Valid(_) => "Input section contained a non-stdout/stderr element." From 1f611000244d9a6ee59f8a1ab8fb26652e53f5c3 Mon Sep 17 00:00:00 2001 From: Katrina P <68349264+kpierre13@users.noreply.github.com> Date: Wed, 27 Apr 2022 13:40:13 -0400 Subject: [PATCH 33/45] Update wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala Co-authored-by: Chris Llanwarne --- .../src/test/scala/AstToWorkflowDefinitionElementSpec.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala b/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala index 6ca549625aa..6b493905cc7 100644 --- a/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala +++ b/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala @@ -63,7 +63,7 @@ class AstToWorkflowDefinitionElementSpec extends AnyFlatSpec{ case Invalid(e) => e.head should be("Workflow cannot have stderr expression in output section at workflow-level.") } } - it should "not throw an error for non-stdout/stderr in the outputs section" in { + it should "not return an error for non-stdout/stderr in the outputs section" in { val testOutputs = checkIfStdOutputExists(mockOutputSectionNonStd, StdoutElement, "non-stdout/stderr") testOutputs match { case Valid(_) => "Output section contained a non-stdout/stderr element." From 6ba58e0cddbe30266149f12bcf06a73e07eaa011 Mon Sep 17 00:00:00 2001 From: Katrina P <68349264+kpierre13@users.noreply.github.com> Date: Wed, 27 Apr 2022 13:40:18 -0400 Subject: [PATCH 34/45] Update wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala Co-authored-by: Chris Llanwarne --- .../src/test/scala/AstToWorkflowDefinitionElementSpec.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala b/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala index 6b493905cc7..df2d29a178b 100644 --- a/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala +++ b/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala @@ -86,7 +86,7 @@ class AstToWorkflowDefinitionElementSpec extends AnyFlatSpec{ } } - it should "not throw an error for non-stdout/stderr in the intermediates section" in { + it should "not return an error for non-stdout/stderr in the intermediates section" in { val testIntermediates = checkStdIntermediates(mockIntermediatesNonStd, StdoutElement, "non-stdout/stderr") testIntermediates match { case Valid(_) => "Intermediate section contained a non-stdout/stderr element." From 0b8d3fe68fb76a2afdb9408b8e150c29937219b6 Mon Sep 17 00:00:00 2001 From: Katrina Pierre Date: Wed, 27 Apr 2022 14:43:36 -0400 Subject: [PATCH 35/45] Changing to more general function naming --- .../AstToWorkflowDefinitionElementSpec.scala | 18 +++++++++--------- .../AstToWorkflowDefinitionElement.scala | 18 +++++++++--------- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala b/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala index df2d29a178b..adb4ecc6b5c 100644 --- a/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala +++ b/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala @@ -25,7 +25,7 @@ class AstToWorkflowDefinitionElementSpec extends AnyFlatSpec{ it should "return an error when there is an stdout in input section" in { - val testInputs = checkIfStdInputExist(mockInputSectionStdout, StdoutElement, "stdout") + val testInputs = checkDisallowedInputElement(mockInputSectionStdout, StdoutElement, "stdout") testInputs match { case Valid(_) => fail("Input section contained stdout. Should have failed.") case Invalid(e) => e.head should be("Workflow cannot have stdout expression in input section at workflow-level.") @@ -33,7 +33,7 @@ class AstToWorkflowDefinitionElementSpec extends AnyFlatSpec{ } it should "return an error when there is an stderr in input section" in { - val testInputs = checkIfStdInputExist(mockInputSectionStderr, StderrElement, "stderr") + val testInputs = checkDisallowedInputElement(mockInputSectionStderr, StderrElement, "stderr") testInputs match { case Valid(_) => fail("Input section contained stderr. Should have failed.") case Invalid(e) => e.head should be("Workflow cannot have stderr expression in input section at workflow-level.") @@ -41,7 +41,7 @@ class AstToWorkflowDefinitionElementSpec extends AnyFlatSpec{ } it should "not return an error for non-stdout/stderr in the inputs section" in { - val testInputs = checkIfStdInputExist(mockInputSectionNonStd, StdoutElement, "non-stdout/stderr") + val testInputs = checkDisallowedInputElement(mockInputSectionNonStd, StdoutElement, "non-stdout/stderr") testInputs match { case Valid(_) => "Input section contained a non-stdout/stderr element." case Invalid(e) => e.head should be("Workflow failed at input section.") @@ -49,7 +49,7 @@ class AstToWorkflowDefinitionElementSpec extends AnyFlatSpec{ } it should "return an error when there is an stdout in output section" in { - val testOutputs = checkIfStdOutputExists(mockOutputSectionStdout, StdoutElement, "stdout") + val testOutputs = checkDisallowedOutputElement(mockOutputSectionStdout, StdoutElement, "stdout") testOutputs match { case Valid(_) => fail("Output section contained stdout. Should have failed.") case Invalid(e) => e.head should be("Workflow cannot have stdout expression in output section at workflow-level.") @@ -57,14 +57,14 @@ class AstToWorkflowDefinitionElementSpec extends AnyFlatSpec{ } it should "return an error when there is an stderr in output section" in { - val testOutputs = checkIfStdOutputExists(mockOutputSectionStderr, StderrElement, "stderr") + val testOutputs = checkDisallowedOutputElement(mockOutputSectionStderr, StderrElement, "stderr") testOutputs match { case Valid(_) => fail("Output section contained stderr. Should have failed.") case Invalid(e) => e.head should be("Workflow cannot have stderr expression in output section at workflow-level.") } } it should "not return an error for non-stdout/stderr in the outputs section" in { - val testOutputs = checkIfStdOutputExists(mockOutputSectionNonStd, StdoutElement, "non-stdout/stderr") + val testOutputs = checkDisallowedOutputElement(mockOutputSectionNonStd, StdoutElement, "non-stdout/stderr") testOutputs match { case Valid(_) => "Output section contained a non-stdout/stderr element." case Invalid(e) => e.head should be("Workflow failed at output section.") @@ -72,14 +72,14 @@ class AstToWorkflowDefinitionElementSpec extends AnyFlatSpec{ } it should "return an error when there is an stdout at intermediate declaration section" in { - val testIntermediates = checkStdIntermediates(mockIntermediatesStdout, StdoutElement, "stdout") + val testIntermediates = checkDisallowedIntermediates(mockIntermediatesStdout, StdoutElement, "stdout") testIntermediates match { case Valid(_) => fail("Intermediate section contained stdout. Should have failed.") case Invalid(e) => e.head should be("Workflow cannot have stdout expression at intermediate declaration section at workflow-level.") } } it should "return an error when there is an stderr at intermediate declaration section" in { - val testIntermediates = checkStdIntermediates(mockIntermediatesStderr, StderrElement, "stderr") + val testIntermediates = checkDisallowedIntermediates(mockIntermediatesStderr, StderrElement, "stderr") testIntermediates match { case Valid(_) => fail("Intermediate section contained stderr. Should have failed.") case Invalid(e) => e.head should be("Workflow cannot have stderr expression at intermediate declaration section at workflow-level.") @@ -87,7 +87,7 @@ class AstToWorkflowDefinitionElementSpec extends AnyFlatSpec{ } it should "not return an error for non-stdout/stderr in the intermediates section" in { - val testIntermediates = checkStdIntermediates(mockIntermediatesNonStd, StdoutElement, "non-stdout/stderr") + val testIntermediates = checkDisallowedIntermediates(mockIntermediatesNonStd, StdoutElement, "non-stdout/stderr") testIntermediates match { case Valid(_) => "Intermediate section contained a non-stdout/stderr element." case Invalid(e) => e.head should be("Workflow failed at intermediate section.") diff --git a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala index 5b9d56c1e22..a7175b23990 100644 --- a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala +++ b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala @@ -29,17 +29,17 @@ object AstToWorkflowDefinitionElement { val inputsSectionValidation: ErrorOr[Option[InputsSectionElement]] = for { inputValidateElement <- validateSize(bodyElements.filterByType[InputsSectionElement], "inputs", 1): ErrorOr[Option[InputsSectionElement]] - _ <- checkIfStdInputExist(inputValidateElement, StdoutElement, "stdout") - _ <- checkIfStdInputExist(inputValidateElement, StderrElement, "stderr") + _ <- checkDisallowedInputElement(inputValidateElement, StdoutElement, "stdout") + _ <- checkDisallowedInputElement(inputValidateElement, StderrElement, "stderr") } yield inputValidateElement - val intermediateValueDeclarationStdoutCheck = checkStdIntermediates(bodyElements.filterByType[IntermediateValueDeclarationElement], StdoutElement, "stdout") - val intermediateValueDeclarationStderrCheck: ErrorOr[Option[String]] = checkStdIntermediates(bodyElements.filterByType[IntermediateValueDeclarationElement], StderrElement, "stderr") + val intermediateValueDeclarationStdoutCheck = checkDisallowedIntermediates(bodyElements.filterByType[IntermediateValueDeclarationElement], StdoutElement, "stdout") + val intermediateValueDeclarationStderrCheck: ErrorOr[Option[String]] = checkDisallowedIntermediates(bodyElements.filterByType[IntermediateValueDeclarationElement], StderrElement, "stderr") val outputsSectionValidation: ErrorOr[Option[OutputsSectionElement]] = for { outputValidateElement <- validateSize(bodyElements.filterByType[OutputsSectionElement], "outputs", 1): ErrorOr[Option[OutputsSectionElement]] - _ <- checkIfStdOutputExists(outputValidateElement, StdoutElement, "stdout") - _ <- checkIfStdOutputExists(outputValidateElement, StderrElement, "stderr") + _ <- checkDisallowedOutputElement(outputValidateElement, StdoutElement, "stdout") + _ <- checkDisallowedOutputElement(outputValidateElement, StderrElement, "stderr") } yield outputValidateElement val graphSections: Vector[WorkflowGraphElement] = bodyElements.filterByType[WorkflowGraphElement] @@ -53,7 +53,7 @@ object AstToWorkflowDefinitionElement { } } - def checkIfStdInputExist(inputSection: Option[InputsSectionElement], expressionType: FunctionCallElement, expressionName: String): ErrorOr[Option[String]] = { + def checkDisallowedInputElement(inputSection: Option[InputsSectionElement], expressionType: FunctionCallElement, expressionName: String): ErrorOr[Option[String]] = { inputSection match { case Some(section) => if (section.inputDeclarations.flatMap(_.expression).exists(_.isInstanceOf[expressionType.type])) { @@ -63,7 +63,7 @@ object AstToWorkflowDefinitionElement { } } - def checkIfStdOutputExists[A](outputSection: Option[OutputsSectionElement], expressionType: FunctionCallElement, expressionName: String): ErrorOr[Option[String]] = { + def checkDisallowedOutputElement(outputSection: Option[OutputsSectionElement], expressionType: FunctionCallElement, expressionName: String): ErrorOr[Option[String]] = { outputSection match { case Some(section) => if (section.outputs.map(_.expression).exists(_.isInstanceOf[expressionType.type])) { @@ -73,7 +73,7 @@ object AstToWorkflowDefinitionElement { } } - def checkStdIntermediates[A](intermediate: Vector[IntermediateValueDeclarationElement], expressionType: FunctionCallElement, expressionName: String): ErrorOr[Option[String]] = { + def checkDisallowedIntermediates(intermediate: Vector[IntermediateValueDeclarationElement], expressionType: FunctionCallElement, expressionName: String): ErrorOr[Option[String]] = { if (intermediate.map(_.expression).exists(_.isInstanceOf[expressionType.type])) { s"Workflow cannot have $expressionName expression at intermediate declaration section at workflow-level.".invalidNel } else None.validNel From 775a51fbbf388852f169f8c5fa9f71401c7ad5b0 Mon Sep 17 00:00:00 2001 From: Katrina Pierre Date: Wed, 27 Apr 2022 15:35:31 -0400 Subject: [PATCH 36/45] Return type change for functions --- .../AstToWorkflowDefinitionElement.scala | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala index a7175b23990..a43467adbe7 100644 --- a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala +++ b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala @@ -53,30 +53,30 @@ object AstToWorkflowDefinitionElement { } } - def checkDisallowedInputElement(inputSection: Option[InputsSectionElement], expressionType: FunctionCallElement, expressionName: String): ErrorOr[Option[String]] = { + def checkDisallowedInputElement(inputSection: Option[InputsSectionElement], expressionType: FunctionCallElement, expressionName: String): ErrorOr[Unit] = { inputSection match { case Some(section) => if (section.inputDeclarations.flatMap(_.expression).exists(_.isInstanceOf[expressionType.type])) { s"Workflow cannot have $expressionName expression in input section at workflow-level.".invalidNel - } else None.validNel - case None => None.validNel + } else ().validNel + case None => ().validNel } } - def checkDisallowedOutputElement(outputSection: Option[OutputsSectionElement], expressionType: FunctionCallElement, expressionName: String): ErrorOr[Option[String]] = { + def checkDisallowedOutputElement(outputSection: Option[OutputsSectionElement], expressionType: FunctionCallElement, expressionName: String): ErrorOr[Unit] = { outputSection match { case Some(section) => if (section.outputs.map(_.expression).exists(_.isInstanceOf[expressionType.type])) { s"Workflow cannot have $expressionName expression in output section at workflow-level.".invalidNel - } else None.validNel - case None => None.validNel + } else ().validNel + case None => ().validNel } } - def checkDisallowedIntermediates(intermediate: Vector[IntermediateValueDeclarationElement], expressionType: FunctionCallElement, expressionName: String): ErrorOr[Option[String]] = { + def checkDisallowedIntermediates(intermediate: Vector[IntermediateValueDeclarationElement], expressionType: FunctionCallElement, expressionName: String): ErrorOr[Unit] = { if (intermediate.map(_.expression).exists(_.isInstanceOf[expressionType.type])) { s"Workflow cannot have $expressionName expression at intermediate declaration section at workflow-level.".invalidNel - } else None.validNel + } else ().validNel } private def validateSize[A](elements: Vector[A], sectionName: String, numExpected: Int): ErrorOr[Option[A]] = { From 63a31ae95d81be7478ec62e18a79156add87e34b Mon Sep 17 00:00:00 2001 From: Katrina Pierre Date: Wed, 27 Apr 2022 15:36:44 -0400 Subject: [PATCH 37/45] Reverting Saloni newline addition --- .../base/ast2wdlom/AstToWorkflowDefinitionElement.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala index a43467adbe7..2331fe6a2bf 100644 --- a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala +++ b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala @@ -88,4 +88,4 @@ object AstToWorkflowDefinitionElement { sectionValidation } -} +} \ No newline at end of file From 858214da2b8965119d78748b048c7eb1a9a4d68c Mon Sep 17 00:00:00 2001 From: Katrina Pierre Date: Wed, 27 Apr 2022 15:38:05 -0400 Subject: [PATCH 38/45] Reverting Saloni newline addition and fixing return type --- .../base/ast2wdlom/AstToWorkflowDefinitionElement.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala index 2331fe6a2bf..6b8e7d08dd0 100644 --- a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala +++ b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala @@ -34,7 +34,7 @@ object AstToWorkflowDefinitionElement { } yield inputValidateElement val intermediateValueDeclarationStdoutCheck = checkDisallowedIntermediates(bodyElements.filterByType[IntermediateValueDeclarationElement], StdoutElement, "stdout") - val intermediateValueDeclarationStderrCheck: ErrorOr[Option[String]] = checkDisallowedIntermediates(bodyElements.filterByType[IntermediateValueDeclarationElement], StderrElement, "stderr") + val intermediateValueDeclarationStderrCheck = checkDisallowedIntermediates(bodyElements.filterByType[IntermediateValueDeclarationElement], StderrElement, "stderr") val outputsSectionValidation: ErrorOr[Option[OutputsSectionElement]] = for { outputValidateElement <- validateSize(bodyElements.filterByType[OutputsSectionElement], "outputs", 1): ErrorOr[Option[OutputsSectionElement]] From d2ef48520eb134abfdae8164d887659e1bdebd1b Mon Sep 17 00:00:00 2001 From: Katrina Pierre Date: Thu, 28 Apr 2022 13:47:14 -0400 Subject: [PATCH 39/45] Revert "Reverting Saloni newline addition and fixing return type" This reverts commit 858214da2b8965119d78748b048c7eb1a9a4d68c. Reverting for Travis CI tests --- .../base/ast2wdlom/AstToWorkflowDefinitionElement.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala index 6b8e7d08dd0..2331fe6a2bf 100644 --- a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala +++ b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala @@ -34,7 +34,7 @@ object AstToWorkflowDefinitionElement { } yield inputValidateElement val intermediateValueDeclarationStdoutCheck = checkDisallowedIntermediates(bodyElements.filterByType[IntermediateValueDeclarationElement], StdoutElement, "stdout") - val intermediateValueDeclarationStderrCheck = checkDisallowedIntermediates(bodyElements.filterByType[IntermediateValueDeclarationElement], StderrElement, "stderr") + val intermediateValueDeclarationStderrCheck: ErrorOr[Option[String]] = checkDisallowedIntermediates(bodyElements.filterByType[IntermediateValueDeclarationElement], StderrElement, "stderr") val outputsSectionValidation: ErrorOr[Option[OutputsSectionElement]] = for { outputValidateElement <- validateSize(bodyElements.filterByType[OutputsSectionElement], "outputs", 1): ErrorOr[Option[OutputsSectionElement]] From 7821d38d41236bf228ea2398867cd7556785fc32 Mon Sep 17 00:00:00 2001 From: Katrina Pierre Date: Wed, 27 Apr 2022 15:38:05 -0400 Subject: [PATCH 40/45] Reverting Saloni newline addition and fixing return type --- .../base/ast2wdlom/AstToWorkflowDefinitionElement.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala index 2331fe6a2bf..6b8e7d08dd0 100644 --- a/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala +++ b/wdl/transforms/new-base/src/main/scala/wdl/transforms/base/ast2wdlom/AstToWorkflowDefinitionElement.scala @@ -34,7 +34,7 @@ object AstToWorkflowDefinitionElement { } yield inputValidateElement val intermediateValueDeclarationStdoutCheck = checkDisallowedIntermediates(bodyElements.filterByType[IntermediateValueDeclarationElement], StdoutElement, "stdout") - val intermediateValueDeclarationStderrCheck: ErrorOr[Option[String]] = checkDisallowedIntermediates(bodyElements.filterByType[IntermediateValueDeclarationElement], StderrElement, "stderr") + val intermediateValueDeclarationStderrCheck = checkDisallowedIntermediates(bodyElements.filterByType[IntermediateValueDeclarationElement], StderrElement, "stderr") val outputsSectionValidation: ErrorOr[Option[OutputsSectionElement]] = for { outputValidateElement <- validateSize(bodyElements.filterByType[OutputsSectionElement], "outputs", 1): ErrorOr[Option[OutputsSectionElement]] From 8e75ea8f8a4a3078ab272b78a0d90a5bd32fe426 Mon Sep 17 00:00:00 2001 From: Katrina Pierre Date: Fri, 29 Apr 2022 11:28:52 -0400 Subject: [PATCH 41/45] remove_tests --- .../resources/standardTestCases/bad_stderr_input.test | 7 ------- .../standardTestCases/bad_stderr_mid_workflow.test | 7 ------- .../standardTestCases/bad_stderr_outputs.test | 7 ------- .../standardTestCases/bad_stderr_subworkflow.test | 10 ---------- .../resources/standardTestCases/bad_stdout_input.test | 7 ------- .../standardTestCases/bad_stdout_mid_workflow.test | 7 ------- .../standardTestCases/bad_stdout_outputs.test | 7 ------- .../standardTestCases/bad_stdout_subworkflow.test | 10 ---------- 8 files changed, 62 deletions(-) delete mode 100644 centaur/src/main/resources/standardTestCases/bad_stderr_input.test delete mode 100644 centaur/src/main/resources/standardTestCases/bad_stderr_mid_workflow.test delete mode 100644 centaur/src/main/resources/standardTestCases/bad_stderr_outputs.test delete mode 100644 centaur/src/main/resources/standardTestCases/bad_stderr_subworkflow.test delete mode 100644 centaur/src/main/resources/standardTestCases/bad_stdout_input.test delete mode 100644 centaur/src/main/resources/standardTestCases/bad_stdout_mid_workflow.test delete mode 100644 centaur/src/main/resources/standardTestCases/bad_stdout_outputs.test delete mode 100644 centaur/src/main/resources/standardTestCases/bad_stdout_subworkflow.test diff --git a/centaur/src/main/resources/standardTestCases/bad_stderr_input.test b/centaur/src/main/resources/standardTestCases/bad_stderr_input.test deleted file mode 100644 index 6775db336cf..00000000000 --- a/centaur/src/main/resources/standardTestCases/bad_stderr_input.test +++ /dev/null @@ -1,7 +0,0 @@ -# Verifies that if we use stderr in a workflow level, the workflow fails properly. -name: bad_stderr_input -testFormat: workflowfailure - -files { - workflow: bad_stderrs/bad_stderr_input.wdl -} diff --git a/centaur/src/main/resources/standardTestCases/bad_stderr_mid_workflow.test b/centaur/src/main/resources/standardTestCases/bad_stderr_mid_workflow.test deleted file mode 100644 index af7e62d6401..00000000000 --- a/centaur/src/main/resources/standardTestCases/bad_stderr_mid_workflow.test +++ /dev/null @@ -1,7 +0,0 @@ -# Verifies that if we use stderr in a workflow level, the workflow fails properly. -name: bad_stderr_mid_workflow -testFormat: workflowfailure - -files { - workflow: bad_stderrs/bad_stderr_mid_workflow.wdl -} diff --git a/centaur/src/main/resources/standardTestCases/bad_stderr_outputs.test b/centaur/src/main/resources/standardTestCases/bad_stderr_outputs.test deleted file mode 100644 index 9c2b02aa8b1..00000000000 --- a/centaur/src/main/resources/standardTestCases/bad_stderr_outputs.test +++ /dev/null @@ -1,7 +0,0 @@ -# Verifies that if we use stderr in a workflow level, the workflow fails properly. -name: bad_stderr_output -testFormat: workflowfailure - -files { - workflow: bad_stderrs/bad_stderr_output.wdl -} diff --git a/centaur/src/main/resources/standardTestCases/bad_stderr_subworkflow.test b/centaur/src/main/resources/standardTestCases/bad_stderr_subworkflow.test deleted file mode 100644 index 408f17664a5..00000000000 --- a/centaur/src/main/resources/standardTestCases/bad_stderr_subworkflow.test +++ /dev/null @@ -1,10 +0,0 @@ -# Verifies that if we use stderr in a workflow level, the workflow fails properly. -name: bad_stderr_subworkflow -testFormat: workflowfailure - -files { - workflow: bad_stderrs/bad_stderr_subworkflow.wdl - imports: [ - bad_stderrs/bad_stderr_mid_workflow.wdl - ] -} diff --git a/centaur/src/main/resources/standardTestCases/bad_stdout_input.test b/centaur/src/main/resources/standardTestCases/bad_stdout_input.test deleted file mode 100644 index bf7c9bf6a65..00000000000 --- a/centaur/src/main/resources/standardTestCases/bad_stdout_input.test +++ /dev/null @@ -1,7 +0,0 @@ -# Verifies that if we use stdout in a workflow level, the workflow fails properly. -name: bad_stdout_input -testFormat: workflowfailure - -files { - workflow: bad_stdouts/bad_stdout_input.wdl -} diff --git a/centaur/src/main/resources/standardTestCases/bad_stdout_mid_workflow.test b/centaur/src/main/resources/standardTestCases/bad_stdout_mid_workflow.test deleted file mode 100644 index ae73adeeb29..00000000000 --- a/centaur/src/main/resources/standardTestCases/bad_stdout_mid_workflow.test +++ /dev/null @@ -1,7 +0,0 @@ -# Verifies that if we use stdout in a workflow level, the workflow fails properly. -name: bad_stdout_mid_workflow -testFormat: workflowfailure - -files { - workflow: bad_stdouts/bad_stdout_mid_workflow.wdl -} diff --git a/centaur/src/main/resources/standardTestCases/bad_stdout_outputs.test b/centaur/src/main/resources/standardTestCases/bad_stdout_outputs.test deleted file mode 100644 index 67b51cd2bf0..00000000000 --- a/centaur/src/main/resources/standardTestCases/bad_stdout_outputs.test +++ /dev/null @@ -1,7 +0,0 @@ -# Verifies that if we use stdout in a workflow level, the workflow fails properly. -name: bad_stdout_output -testFormat: workflowfailure - -files { - workflow: bad_stdouts/bad_stdout_output.wdl -} diff --git a/centaur/src/main/resources/standardTestCases/bad_stdout_subworkflow.test b/centaur/src/main/resources/standardTestCases/bad_stdout_subworkflow.test deleted file mode 100644 index 28a24d8864f..00000000000 --- a/centaur/src/main/resources/standardTestCases/bad_stdout_subworkflow.test +++ /dev/null @@ -1,10 +0,0 @@ -# Verifies that if we use stdout in a workflow level, the workflow fails properly. -name: bad_stdout_subworkflow -testFormat: workflowfailure - -files { - workflow: bad_stdouts/bad_stdout_subworkflow.wdl - imports: [ - bad_stdouts/bad_stdout_mid_workflow.wdl - ] -} From f5e54f14dad31c78e59cf59f46ec86a24c3fe917 Mon Sep 17 00:00:00 2001 From: Katrina Pierre Date: Fri, 29 Apr 2022 12:04:04 -0400 Subject: [PATCH 42/45] remove_corresponding_wdl_files --- .../bad_stderrs/bad_stderr_input.wdl | 11 ----------- .../bad_stderrs/bad_stderr_mid_workflow.wdl | 9 --------- .../bad_stderrs/bad_stderr_output.wdl | 7 ------- .../bad_stderrs/bad_stderr_subworkflow.wdl | 9 --------- .../bad_stdouts/bad_stdout_input.wdl | 10 ---------- .../bad_stdouts/bad_stdout_mid_workflow.wdl | 9 --------- .../bad_stdouts/bad_stdout_output.wdl | 7 ------- .../bad_stdouts/bad_stdout_subworkflow.wdl | 9 --------- 8 files changed, 71 deletions(-) delete mode 100644 centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_input.wdl delete mode 100644 centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_mid_workflow.wdl delete mode 100644 centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_output.wdl delete mode 100644 centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_subworkflow.wdl delete mode 100644 centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_input.wdl delete mode 100644 centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_mid_workflow.wdl delete mode 100644 centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_output.wdl delete mode 100644 centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_subworkflow.wdl diff --git a/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_input.wdl b/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_input.wdl deleted file mode 100644 index ef7e2b7ff82..00000000000 --- a/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_input.wdl +++ /dev/null @@ -1,11 +0,0 @@ -version 1.0 - -workflow bad_stderr { - - input { - File f = stderr() - } - output { - String s = read_string(f) - } -} diff --git a/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_mid_workflow.wdl b/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_mid_workflow.wdl deleted file mode 100644 index eb6331ff8c5..00000000000 --- a/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_mid_workflow.wdl +++ /dev/null @@ -1,9 +0,0 @@ -version 1.0 - -workflow bad_stderr { - File f = stderr() - - output { - String s = read_string(f) - } -} diff --git a/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_output.wdl b/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_output.wdl deleted file mode 100644 index 65e99949a52..00000000000 --- a/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_output.wdl +++ /dev/null @@ -1,7 +0,0 @@ -version 1.0 - -workflow bad_stderr { - output { - File f = stderr() - } -} diff --git a/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_subworkflow.wdl b/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_subworkflow.wdl deleted file mode 100644 index b1249f8dd9d..00000000000 --- a/centaur/src/main/resources/standardTestCases/bad_stderrs/bad_stderr_subworkflow.wdl +++ /dev/null @@ -1,9 +0,0 @@ -version 1.0 - -import "bad_stderr_mid_workflow.wdl" - -workflow calls_bad_stderr_subworkflow { - scatter(i in range(1000)) { - call bad_stderr_mid_workflow.bad_stderr - } -} diff --git a/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_input.wdl b/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_input.wdl deleted file mode 100644 index c68f0ebffd3..00000000000 --- a/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_input.wdl +++ /dev/null @@ -1,10 +0,0 @@ -version 1.0 - -workflow bad_stdout { - input { - File f = stdout() - } - output { - String s = read_string(f) - } -} diff --git a/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_mid_workflow.wdl b/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_mid_workflow.wdl deleted file mode 100644 index d4f57bfd125..00000000000 --- a/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_mid_workflow.wdl +++ /dev/null @@ -1,9 +0,0 @@ -version 1.0 - -workflow bad_stdout { - File f = stdout() - - output { - String s = read_string(f) - } -} diff --git a/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_output.wdl b/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_output.wdl deleted file mode 100644 index ad2bc9a24e4..00000000000 --- a/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_output.wdl +++ /dev/null @@ -1,7 +0,0 @@ -version 1.0 - -workflow bad_stdout { - output { - File f = stdout() - } -} diff --git a/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_subworkflow.wdl b/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_subworkflow.wdl deleted file mode 100644 index 7134825b827..00000000000 --- a/centaur/src/main/resources/standardTestCases/bad_stdouts/bad_stdout_subworkflow.wdl +++ /dev/null @@ -1,9 +0,0 @@ -version 1.0 - -import "bad_stdout_mid_workflow.wdl" - -workflow calls_bad_stdout_subworkflow { - scatter(i in range(1000)) { - call bad_stdout_mid_workflow.bad_stdout - } -} From 2c91101629dcef072cd3467750c35e2143bf3f83 Mon Sep 17 00:00:00 2001 From: Katrina Pierre Date: Sat, 30 Apr 2022 15:39:56 -0400 Subject: [PATCH 43/45] Updating some unit tests with more correct functionality --- .../scala/AstToWorkflowDefinitionElementSpec.scala | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala b/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala index adb4ecc6b5c..92c63a1f0cc 100644 --- a/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala +++ b/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala @@ -43,8 +43,8 @@ class AstToWorkflowDefinitionElementSpec extends AnyFlatSpec{ it should "not return an error for non-stdout/stderr in the inputs section" in { val testInputs = checkDisallowedInputElement(mockInputSectionNonStd, StdoutElement, "non-stdout/stderr") testInputs match { - case Valid(_) => "Input section contained a non-stdout/stderr element." - case Invalid(e) => e.head should be("Workflow failed at input section.") + case Valid(_) => // No action + case Invalid(e) => fail("Check shouldn't have returned error as input section had non-stdout/stderr inputs") } } @@ -66,8 +66,8 @@ class AstToWorkflowDefinitionElementSpec extends AnyFlatSpec{ it should "not return an error for non-stdout/stderr in the outputs section" in { val testOutputs = checkDisallowedOutputElement(mockOutputSectionNonStd, StdoutElement, "non-stdout/stderr") testOutputs match { - case Valid(_) => "Output section contained a non-stdout/stderr element." - case Invalid(e) => e.head should be("Workflow failed at output section.") + case Valid(_) => // No action + case Invalid(e) => fail("Check shouldn't have returned error as output section had non-stdout/stderr outputs") } } @@ -89,8 +89,8 @@ class AstToWorkflowDefinitionElementSpec extends AnyFlatSpec{ it should "not return an error for non-stdout/stderr in the intermediates section" in { val testIntermediates = checkDisallowedIntermediates(mockIntermediatesNonStd, StdoutElement, "non-stdout/stderr") testIntermediates match { - case Valid(_) => "Intermediate section contained a non-stdout/stderr element." - case Invalid(e) => e.head should be("Workflow failed at intermediate section.") + case Valid(_) => // No action + case Invalid(e) => fail("Check shouldn't have returned error as intermediate section had non-stdout/stderr intermediates.") } } From 5a59a7bddac1f0474e59e2ae7db95df54123264b Mon Sep 17 00:00:00 2001 From: Katrina Pierre Date: Sat, 30 Apr 2022 16:34:42 -0400 Subject: [PATCH 44/45] Fixing missed input for Invalid function --- .../src/test/scala/AstToWorkflowDefinitionElementSpec.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala b/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala index 92c63a1f0cc..b84c2e2502f 100644 --- a/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala +++ b/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala @@ -44,7 +44,7 @@ class AstToWorkflowDefinitionElementSpec extends AnyFlatSpec{ val testInputs = checkDisallowedInputElement(mockInputSectionNonStd, StdoutElement, "non-stdout/stderr") testInputs match { case Valid(_) => // No action - case Invalid(e) => fail("Check shouldn't have returned error as input section had non-stdout/stderr inputs") + case Invalid(_) => fail("Check shouldn't have returned error as input section had non-stdout/stderr inputs") } } @@ -67,7 +67,7 @@ class AstToWorkflowDefinitionElementSpec extends AnyFlatSpec{ val testOutputs = checkDisallowedOutputElement(mockOutputSectionNonStd, StdoutElement, "non-stdout/stderr") testOutputs match { case Valid(_) => // No action - case Invalid(e) => fail("Check shouldn't have returned error as output section had non-stdout/stderr outputs") + case Invalid(_) => fail("Check shouldn't have returned error as output section had non-stdout/stderr outputs") } } @@ -90,7 +90,7 @@ class AstToWorkflowDefinitionElementSpec extends AnyFlatSpec{ val testIntermediates = checkDisallowedIntermediates(mockIntermediatesNonStd, StdoutElement, "non-stdout/stderr") testIntermediates match { case Valid(_) => // No action - case Invalid(e) => fail("Check shouldn't have returned error as intermediate section had non-stdout/stderr intermediates.") + case Invalid(_) => fail("Check shouldn't have returned error as intermediate section had non-stdout/stderr intermediates.") } } From a19b33dea71fc3cad0b8a810434e5cca559c18ac Mon Sep 17 00:00:00 2001 From: Katrina Pierre Date: Sat, 30 Apr 2022 18:32:26 -0400 Subject: [PATCH 45/45] triggering build --- .../src/test/scala/AstToWorkflowDefinitionElementSpec.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala b/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala index b84c2e2502f..99731de6aad 100644 --- a/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala +++ b/wdl/transforms/draft3/src/test/scala/AstToWorkflowDefinitionElementSpec.scala @@ -94,4 +94,4 @@ class AstToWorkflowDefinitionElementSpec extends AnyFlatSpec{ } } -} +} \ No newline at end of file