Better QoB error propagation (patch from Tim Poterba) (#242)

populationgenomics · Oct 16, 2022 · 3c66763 · 3c66763
1 parent 0c057d6
commit 3c66763
Show file tree

Hide file tree

Showing 3 changed files with 12 additions and 4 deletions.
diff --git a/hail/src/main/scala/is/hail/backend/service/ServiceBackend.scala b/hail/src/main/scala/is/hail/backend/service/ServiceBackend.scala
@@ -214,9 +214,6 @@ class ServiceBackend(
     stageCount += 1
     implicit val formats: Formats = DefaultFormats
     val batchState = (batch \ "state").extract[String]
-    if (batchState == "failed") {
-      throw new HailBatchFailure(s"Update $updateId for batch $batchId failed")
-    }
 
     log.info(s"parallelizeAndComputeWithIndex: $token: reading results")
 
@@ -237,7 +234,12 @@ class ServiceBackend(
       availableGCSConnections.acquire()
       try {
         val bytes = retryTransientErrors {
-          using(open(s"$root/result.$i")) { is =>
+          val is = try {
+            open(s"$root/result.$i")
+          } catch {
+            case e: Throwable => throw new HailWorkerFailure(s"no result for failing job ${i}!", e)
+          }
+          using(is) { is =>
             resultOrHailException(new DataInputStream(is))
           }
         }
@@ -248,6 +250,8 @@ class ServiceBackend(
       }
     }
 
+    assert(batchState != "failed")  // a failure can't have all the correct outputs with no exceptions!
+
     log.info(s"all results complete")
     results.toArray[Array[Byte]]
   }

diff --git a/hail/src/main/scala/is/hail/backend/service/Worker.scala b/hail/src/main/scala/is/hail/backend/service/Worker.scala
@@ -157,6 +157,8 @@ object Worker {
         writeString(dos, shortMessage)
         writeString(dos, expandedMessage)
         dos.writeInt(errorId)
+        log.info(s"job $i/$n failed with user exception: $shortMessage (error id $errorId)\n  $expandedMessage")
+        throw userError
       }
     }
     timer.end("writeOutputs")

diff --git a/hail/src/main/scala/is/hail/utils/ErrorHandling.scala b/hail/src/main/scala/is/hail/utils/ErrorHandling.scala
@@ -7,6 +7,8 @@ class HailException(val msg: String, val logMsg: Option[String], cause: Throwabl
   def this(msg: String, errorId: Int) = this(msg, None, null, errorId)
 }
 
+class HailWorkerFailure(msg: String, cause: Throwable) extends RuntimeException(msg)
+
 class HailWorkerException(
   val shortMessage: String,
   val expandedMessage: String,