[WX-1468] Implement returnCodes runtime attribute

.github/workflows/integration_tests.yml

@@ -104,6 +104,14 @@ jobs:
         set -e
         echo Running test.sh
         ./src/ci/bin/test.sh
+     # If a build step fails, activate SSH and idle for 30 minutes.
+    - name: Setup tmate session
+      if: ${{ failure() }}
+      uses: mxschmitt/action-tmate@v3
+      timeout-minutes: 90
+      with:
+       limit-access-to-actor: true
+       detached: true
     # always() is some github magic that forces the following step to run, even when the previous fails.
     # Without it, the if statement won't be evaluated on a test failure.
     - uses: ravsamhq/notify-slack-action@v2

backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala

@@ -11,7 +11,7 @@
 import cromwell.backend.BackendLifecycleActor._
 import cromwell.backend.BackendWorkflowInitializationActor._
 import cromwell.backend.async.{RuntimeAttributeValidationFailure, RuntimeAttributeValidationFailures}
-import cromwell.backend.validation.ContinueOnReturnCodeValidation
+import cromwell.backend.validation.{ContinueOnReturnCodeValidation, ReturnCodesValidation}
 import cromwell.core._
 import cromwell.services.metadata.{MetadataEvent, MetadataKey, MetadataValue}
 import cromwell.services.metadata.MetadataService.PutMetadataAction
@@ -134,6 +134,16 @@
       .default(configurationDescriptor.backendRuntimeAttributesConfig)
       .validateOptionalWomValue(womExpressionMaybe)
 
+  /**
+   * This predicate is only appropriate for validation during workflow initialization.  The logic does not differentiate
+   * between evaluation failures due to missing call inputs or evaluation failures due to malformed expressions, and will
+   * return `true` in both cases.
+   */
+  protected def returnCodesPredicate(valueRequired: Boolean)(womExpressionMaybe: Option[WomValue]): Boolean =
+    ReturnCodesValidation
+      .default(configurationDescriptor.backendRuntimeAttributesConfig)
+      .validateOptionalWomValue(womExpressionMaybe)
+
   protected def runtimeAttributeValidators: Map[String, Option[WomExpression] => Boolean]
 
   // FIXME: If a workflow executes jobs using multiple backends,

backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala

@@ -752,9 +752,26 @@ trait StandardAsyncExecutionActor
     *
     * @return the behavior for continuing on the return code.
     */
-  lazy val continueOnReturnCode: ContinueOnReturnCode =
+  lazy val continueOnReturnCode: ReturnCode =
     RuntimeAttributesValidation.extract(ContinueOnReturnCodeValidation.instance, validatedRuntimeAttributes)
 
+  /**
+   * Returns the behavior for continuing on the return code, obtained by converting `returnCodeContents` to an Int.
+   *
+   * @return the behavior for continuing on the return code.
+   */
+  lazy val returnCodes: ReturnCode =
+    RuntimeAttributesValidation.extract(ReturnCodesValidation.instance, validatedRuntimeAttributes)
+
+  /**
+   * Returns the true behavior for continuing on the return code. If `returnCodes` runtime attribute is specified,
+   * then `continueOnReturnCode` attribute is ignored. If `returnCodes` is unspecified, then the value in
+   * `continueOnReturnCode` will be used.
+   */
+  lazy val returnCode: ReturnCode =
+    if (returnCodes.isInstanceOf[ReturnCodeSet] && returnCodes.equals(ReturnCodeSet(Set(0)))) continueOnReturnCode
+    else returnCodes
+
   /**
     * Returns the max number of times that a failed job should be retried, obtained by converting `maxRetries` to an Int.
     */
@@ -1388,7 +1405,7 @@ trait StandardAsyncExecutionActor
               )
             )
             retryElseFail(executionHandle)
-          case Success(returnCodeAsInt) if continueOnReturnCode.continueFor(returnCodeAsInt) =>
+          case Success(returnCodeAsInt) if returnCode.continueFor(returnCodeAsInt) =>
             handleExecutionSuccess(status, oldHandle, returnCodeAsInt)
           // It's important that we check retryWithMoreMemory case before isAbort. RC could be 137 in either case;
           // if it was caused by OOM killer, want to handle as OOM and not job abort.
@@ -1422,7 +1439,7 @@ trait StandardAsyncExecutionActor
       } else {
         tryReturnCodeAsInt match {
           case Success(returnCodeAsInt)
-              if outOfMemoryDetected && memoryRetryRequested && !continueOnReturnCode.continueFor(returnCodeAsInt) =>
+              if outOfMemoryDetected && memoryRetryRequested && !returnCode.continueFor(returnCodeAsInt) =>
             val executionHandle = Future.successful(
               FailedNonRetryableExecutionHandle(
                 RetryWithMoreMemory(jobDescriptor.key.tag, stderrAsOption, memoryRetryErrorKeys, log),

backend/src/main/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilder.scala

@@ -32,6 +32,7 @@ object StandardValidatedRuntimeAttributesBuilder {
   def default(backendRuntimeConfig: Option[Config]): StandardValidatedRuntimeAttributesBuilder = {
     val required = Seq(
       ContinueOnReturnCodeValidation.default(backendRuntimeConfig),
+      ReturnCodesValidation.default(backendRuntimeConfig),
       FailOnStderrValidation.default(backendRuntimeConfig),
       MaxRetriesValidation.default(backendRuntimeConfig)
     )

backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCode.scala

@@ -9,18 +9,18 @@
 /**
   * Decides if a call/job continues upon a specific return code.
   */
-sealed trait ContinueOnReturnCode {
+sealed trait ContinueOnReturnCode extends ReturnCode {
 
   /**
     * Returns true if the call is a success based on the return code.
     *
     * @param returnCode Return code from the process / script.
     * @return True if the call is a success.
     */
-  final def continueFor(returnCode: Int): Boolean =
+  final override def continueFor(returnCode: Int): Boolean =
     this match {
       case ContinueOnReturnCodeFlag(continue) => continue || returnCode == 0
-      case ContinueOnReturnCodeSet(returnCodes) => returnCodes.contains(returnCode)
+      case _ => super.continueFor(returnCode)
     }
 }
 
@@ -31,14 +31,3 @@
 case class ContinueOnReturnCodeFlag(continue: Boolean) extends ContinueOnReturnCode {
   override def toString = continue.toString
 }
-
-/**
-  * Continues only if the call/job return code is found in returnCodes.
-  * @param returnCodes Inclusive set of return codes that specify a job success.
-  */
-case class ContinueOnReturnCodeSet(returnCodes: Set[Int]) extends ContinueOnReturnCode {
-  override def toString = returnCodes match {
-    case single if single.size == 1 => returnCodes.head.toString
-    case multiple => s"[${multiple.mkString(",")}]"
-  }
-}

backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala

@@ -1,9 +1,7 @@
 package cromwell.backend.validation
 
-import cats.data.Validated.{Invalid, Valid}
 import cats.implicits._
 import com.typesafe.config.Config
-import cromwell.backend.validation.RuntimeAttributesValidation._
 import common.validation.ErrorOr._
 import wom.RuntimeAttributesKeys
 import wom.types._
@@ -23,42 +21,29 @@ import scala.util.Try
   * `default` a validation with the default value specified by the reference.conf file.
   */
 object ContinueOnReturnCodeValidation {
-  lazy val instance: RuntimeAttributesValidation[ContinueOnReturnCode] = new ContinueOnReturnCodeValidation
-  def default(runtimeConfig: Option[Config]): RuntimeAttributesValidation[ContinueOnReturnCode] =
+  lazy val instance: RuntimeAttributesValidation[ReturnCode] = new ContinueOnReturnCodeValidation
+  def default(runtimeConfig: Option[Config]): RuntimeAttributesValidation[ReturnCode] =
     instance.withDefault(configDefaultWdlValue(runtimeConfig) getOrElse WomInteger(0))
   def configDefaultWdlValue(runtimeConfig: Option[Config]): Option[WomValue] =
     instance.configDefaultWomValue(runtimeConfig)
 }
 
-class ContinueOnReturnCodeValidation extends RuntimeAttributesValidation[ContinueOnReturnCode] {
+class ContinueOnReturnCodeValidation extends ReturnCodeValidation {
 
   override def key: String = RuntimeAttributesKeys.ContinueOnReturnCodeKey
 
   override def coercion: Set[WomType] = ContinueOnReturnCode.validWdlTypes
 
-  override def validateValue: PartialFunction[WomValue, ErrorOr[ContinueOnReturnCode]] = {
+  override def validateValue: PartialFunction[WomValue, ErrorOr[ReturnCode]] = {
     case WomBoolean(value) => ContinueOnReturnCodeFlag(value).validNel
     case WomString(value) if Try(value.toBoolean).isSuccess => ContinueOnReturnCodeFlag(value.toBoolean).validNel
-    case WomString(value) if Try(value.toInt).isSuccess => ContinueOnReturnCodeSet(Set(value.toInt)).validNel
-    case WomInteger(value) => ContinueOnReturnCodeSet(Set(value)).validNel
-    case value @ WomArray(_, seq) =>
-      val errorOrInts: ErrorOr[List[Int]] = (seq.toList map validateInt).sequence[ErrorOr, Int]
-      errorOrInts match {
-        case Valid(ints) => ContinueOnReturnCodeSet(ints.toSet).validNel
-        case Invalid(_) => invalidValueFailure(value)
-      }
+    case value => super.validateValue(value)
   }
 
   override def validateExpression: PartialFunction[WomValue, Boolean] = {
     case WomBoolean(_) => true
-    case WomString(value) if Try(value.toInt).isSuccess => true
     case WomString(value) if Try(value.toBoolean).isSuccess => true
-    case WomInteger(_) => true
-    case WomArray(WomArrayType(WomStringType), elements) =>
-      elements forall { value =>
-        Try(value.valueString.toInt).isSuccess
-      }
-    case WomArray(WomArrayType(WomIntegerType), _) => true
+    case value => super.validateExpression(value)
   }
 
   override protected def missingValueMessage: String = s"Expecting $key" +

backend/src/main/scala/cromwell/backend/validation/ReturnCode.scala

@@ -0,0 +1,34 @@
+package cromwell.backend.validation
+
+trait ReturnCode {
+
+  /**
+   * Returns true if the call is a success based on the return code.
+   *
+   * @param returnCode Return code from the process / script.
+   * @return True if the call is a success.
+   */
+  def continueFor(returnCode: Int): Boolean =
+    this match {
+      case ReturnCodeSet(returnCodes) => returnCodes.contains(returnCode)
+    }
+}
+
+/**
+ * Continues only if the call/job return code is found in returnCodes.
+ * @param returnCodes Inclusive set of return codes that specify a job success.
+ */
+case class ReturnCodeSet(returnCodes: Set[Int]) extends ReturnCode {
+  override def toString = returnCodes match {
+    case single if single.size == 1 => returnCodes.head.toString
+    case multiple => s"[${multiple.mkString(",")}]"
+  }
+
+  /**
+   * Returns true if the call is a success based on the return code.
+   *
+   * @param returnCode Return code from the process / script.
+   * @return True if the call is a success.
+   */
+  override def continueFor(returnCode: Int): Boolean = super.continueFor(returnCode)
+}

backend/src/main/scala/cromwell/backend/validation/ReturnCodeValidation.scala

@@ -0,0 +1,43 @@
+package cromwell.backend.validation
+
+import cats.data.Validated.{Invalid, Valid}
+import cats.implicits.{catsSyntaxValidatedId, toTraverseOps}
+import common.validation.ErrorOr.ErrorOr
+import cromwell.backend.validation.RuntimeAttributesValidation.validateInt
+import wom.types.{WomArrayType, WomIntegerType, WomStringType, WomType}
+import wom.values.{WomArray, WomInteger, WomString, WomValue}
+
+import scala.util.Try
+
+/**
+ * Validates the `continueOnReturnCode` and `returnCodes` runtime attributes, since they share the functionality of
+ * being a Set of Integers. 
+ */
+trait ReturnCodeValidation extends RuntimeAttributesValidation[ReturnCode] {
+  override def key: String
+  override def coercion: Iterable[WomType]
+  override def validateValue: PartialFunction[WomValue, ErrorOr[ReturnCode]] = {
+    case WomString(value) if Try(value.toInt).isSuccess => ReturnCodeSet(Set(value.toInt)).validNel
+    case WomInteger(value) => ReturnCodeSet(Set(value)).validNel
+    case value @ WomArray(_, seq) =>
+      val errorOrInts: ErrorOr[List[Int]] = (seq.toList map validateInt).sequence[ErrorOr, Int]
+      errorOrInts match {
+        case Valid(ints) => ReturnCodeSet(ints.toSet).validNel
+        case Invalid(_) => invalidValueFailure(value)
+      }
+    case value => invalidValueFailure(value)
+  }
+
+  override def validateExpression: PartialFunction[WomValue, Boolean] = {
+    case WomString(value) if Try(value.toInt).isSuccess => true
+    case WomInteger(_) => true
+    case WomArray(WomArrayType(WomStringType), elements) =>
+      elements forall { value =>
+        Try(value.valueString.toInt).isSuccess
+      }
+    case WomArray(WomArrayType(WomIntegerType), _) => true
+    case _ => false
+  }
+
+  override def usedInCallCaching: Boolean = true
+}

backend/src/main/scala/cromwell/backend/validation/ReturnCodes.scala

@@ -0,0 +1,33 @@
+package cromwell.backend.validation
+
+import wom.types._
+
+object ReturnCodes {
+  val validWdlTypes = Set[WomType](WomArrayType(WomIntegerType), WomStringType, WomIntegerType)
+}
+
+/**
+ * Decides if a call/job continues upon a specific return code.
+ */
+sealed trait ReturnCodes extends ReturnCode {
+
+  /**
+   * Returns true if the call is a success based on the return code.
+   *
+   * @param returnCode Return code from the process / script.
+   * @return True if the call is a success.
+   */
+  final override def continueFor(returnCode: Int): Boolean =
+    this match {
+      case ReturnCodesString(continue) => continue.equals("*") || returnCode == 0
+      case _ => super.continueFor(returnCode)
+    }
+}
+
+/**
+ * Continues based on a string, if "*" all return codes continue.
+ * @param returnCode If "*", all return codes are valid for continuing.
+ */
+case class ReturnCodesString(returnCode: String) extends ReturnCodes {
+  override def toString = returnCode
+}

backend/src/main/scala/cromwell/backend/validation/ReturnCodesValidation.scala

@@ -0,0 +1,51 @@
+package cromwell.backend.validation
+
+import cats.implicits.catsSyntaxValidatedId
+import com.typesafe.config.Config
+import common.validation.ErrorOr.ErrorOr
+import wom.RuntimeAttributesKeys
+import wom.types.WomType
+import wom.values.{WomInteger, WomString, WomValue}
+
+/**
+ * Validates the "returnCodes" runtime attribute as an Integer, returning the value as an `Int`, or as
+ *  an array of Integers, returning the value as Array[Int], or "*", returning the value as a String.
+ *
+ * `default` a hardcoded default WomValue for returnCodes.
+ *
+ * `configDefaultWdlValue` returns the value of the attribute as specified by the
+ * reference.conf file, coerced into a WomValue.
+ */
+object ReturnCodesValidation {
+  lazy val instance: RuntimeAttributesValidation[ReturnCode] = new ReturnCodesValidation
+  lazy val optional: OptionalRuntimeAttributesValidation[ReturnCode] = instance.optional
+  def default(runtimeConfig: Option[Config]): RuntimeAttributesValidation[ReturnCode] =
+    instance.withDefault(configDefaultWdlValue(runtimeConfig) getOrElse WomInteger(0))
+
+  def configDefaultWdlValue(runtimeConfig: Option[Config]): Option[WomValue] =
+    instance.configDefaultWomValue(runtimeConfig)
+  def configDefaultWomValue(config: Option[Config]): Option[WomValue] = instance.configDefaultWomValue(config)
+}
+
+class ReturnCodesValidation extends ReturnCodeValidation {
+
+  override def key: String = RuntimeAttributesKeys.ReturnCodesKey
+
+  override def coercion: Set[WomType] = ReturnCodes.validWdlTypes
+
+  override def validateValue: PartialFunction[WomValue, ErrorOr[ReturnCode]] = {
+    case WomString(value) if value.equals("*") =>
+      ReturnCodesString(value).validNel
+    case value => super.validateValue(value)
+  }
+
+  override def validateExpression: PartialFunction[WomValue, Boolean] = {
+    case WomString(value) if value.equals("*") => true
+    case value => super.validateExpression(value)
+  }
+
+  override protected def missingValueMessage: String = s"Expecting $key" +
+    " runtime attribute to be either a String '*' or an Array[Int]"
+
+  override def usedInCallCaching: Boolean = true
+}

backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala

@@ -34,9 +34,12 @@
 
   def validateContinueOnReturnCode(value: Option[WomValue],
                                    onMissingKey: => ErrorOr[ContinueOnReturnCode]
-  ): ErrorOr[ContinueOnReturnCode] =
+  ): ErrorOr[ReturnCode] =
     validateWithValidation(value, ContinueOnReturnCodeValidation.instance, onMissingKey)
 
+  def validateReturnCodes(value: Option[WomValue], onMissingKey: => ErrorOr[ReturnCode]): ErrorOr[ReturnCode] =
+    validateWithValidation(value, ReturnCodesValidation.instance, onMissingKey)
+
   def validateMemory(value: Option[WomValue], onMissingKey: => ErrorOr[MemorySize]): ErrorOr[MemorySize] =
     validateWithValidation(value, MemoryValidation.instance(), onMissingKey)
 
@@ -372,7 +375,8 @@
           case Success(womValue) => validateExpression.applyOrElse(womValue, (_: Any) => false)
           case Failure(_) => true // If we can't evaluate it, we'll let it pass for now...
         }
-      case Some(womValue) => validateExpression.applyOrElse(womValue, (_: Any) => false)
+      case Some(womValue) =>
+        validateExpression.applyOrElse(womValue, (_: Any) => false)
     }
 
   def validateOptionalWomExpression(womExpressionMaybe: Option[WomExpression]): Boolean =