Merge pull request #1385 from NASA-AMMOS/feat/sim-error-ids

Return the Executing Directive Id in Simulation Error Response

e2e-tests/src/test/java/gov/nasa/jpl/aerie/e2e/SimulationTests.java

@@ -14,6 +14,7 @@
 import org.junit.jupiter.api.TestInstance;
 
 import javax.json.Json;
+import javax.json.JsonValue;
 import java.io.IOException;
 import java.util.Comparator;
 
@@ -365,4 +366,196 @@ void resimOnlyUpdatesConfigRevision() throws IOException {
       assertEquals(simConfig.simulationEndTime(), newSimConfig.simulationEndTime());
     }
   }
+
+  @Nested
+  class SimulationExceptionResponse {
+    private int fooId;
+    private int fooPlan;
+
+    @BeforeEach
+    void beforeEach() throws IOException, InterruptedException {
+      // Insert the Mission Model
+      // Foo contains ways to trigger exceptions on-demand
+      try (final var gateway = new GatewayRequests(playwright)) {
+        fooId = hasura.createMissionModel(
+            gateway.uploadFooJar(),
+            "Foo (e2e tests)",
+            "aerie_e2e_tests",
+            "Simulation Tests");
+      }
+      // Insert the Plan
+      fooPlan = hasura.createPlan(
+          fooId,
+          "Foo Plan - Error Simulation Tests",
+          "02:00:00",
+          planStartTimestamp);
+    }
+
+    @AfterEach
+    void afterEach() throws IOException {
+      // Remove Model and Plan
+      hasura.deletePlan(fooPlan);
+      hasura.deleteMissionModel(fooId);
+    }
+
+    /**
+     * If a directive throws an exception, the error response will include the executing directive id.
+     */
+    @Test
+    void directiveIdIncluded() throws IOException {
+      // Setup: Insert a directive that will throw
+      int dirId = hasura.insertActivity(
+          fooPlan,
+          "DaemonCheckerActivity",
+          "01:00:00",
+          Json.createObjectBuilder().add("minutesElapsed", 1).build());
+
+      final var simDatasetId = hasura.awaitFailingSimulation(fooPlan).simDatasetId();
+      final var simDataset = hasura.getSimulationDataset(simDatasetId);
+      assertEquals(SimulationDataset.SimulationStatus.failed, simDataset.status());
+
+      // The correct type of error was returned
+      assertTrue(simDataset.reason().isPresent());
+      final var reason = simDataset.reason().get();
+      assertEquals("SIMULATION_EXCEPTION", reason.type());
+
+      final var exception = reason.data();
+
+      // The error includes the directive id that was executing
+      assertTrue(exception.containsKey("executingDirectiveId"));
+      assertEquals(dirId, exception.getInt("executingDirectiveId"));
+
+      // The error message is correct
+      assertEquals("Minutes elapsed is incorrect. TimeTrackerDaemon may have stopped.\n\tExpected: 1 Actual: 59",
+                   reason.message());
+
+      // The error was thrown at the correct time
+      assertTrue(exception.containsKey("utcTimeDoy"));
+      assertEquals("2023-001T01:00:00", exception.getString("utcTimeDoy"));
+
+      // The trace starts at the original exception and doesn't include the intermediary SpanException and SimulationException
+      final var expectedStart = """
+          java.lang.RuntimeException: Minutes elapsed is incorrect. TimeTrackerDaemon may have stopped.
+          \tExpected: 1 Actual: 59
+          \tat gov.nasa.jpl.aerie.foomissionmodel.activities.DaemonCheckerActivity.run(DaemonCheckerActivity.java""";
+      assertTrue(reason.trace().startsWith(expectedStart));
+    }
+
+    /**
+     * If a child span throws an exception, the error response will include the parent directive's id.
+     */
+    @Test
+    void directiveIdOnDescendant() throws IOException {
+      // Setup: Insert a directive that will throw
+      int dirId = hasura.insertActivity(
+          fooPlan,
+          "DaemonCheckerSpawner",
+          "01:00:00",
+          Json.createObjectBuilder().add("minutesElapsed", 1).add("spawnDelay", 1).build());
+
+      final var simDatasetId = hasura.awaitFailingSimulation(fooPlan).simDatasetId();
+      final var simDataset = hasura.getSimulationDataset(simDatasetId);
+      assertEquals(SimulationDataset.SimulationStatus.failed, simDataset.status());
+
+      // The correct type of error was returned
+      assertTrue(simDataset.reason().isPresent());
+      final var reason = simDataset.reason().get();
+      assertEquals("SIMULATION_EXCEPTION", reason.type());
+
+      final var exception = reason.data();
+
+      // The error includes the directive id that was executing
+      assertTrue(exception.containsKey("executingDirectiveId"));
+      assertEquals(dirId, exception.getInt("executingDirectiveId"));
+
+      // The error message is correct
+      assertEquals("Minutes elapsed is incorrect. TimeTrackerDaemon may have stopped.\n\tExpected: 1 Actual: 60",
+                   reason.message());
+
+      // The error was thrown at the correct time
+      assertTrue(exception.containsKey("utcTimeDoy"));
+      assertEquals("2023-001T01:01:00", exception.getString("utcTimeDoy"));
+
+      // The trace starts at the original exception and doesn't include the intermediary SpanException and SimulationException
+      final var expectedStart = """
+          java.lang.RuntimeException: Minutes elapsed is incorrect. TimeTrackerDaemon may have stopped.
+          \tExpected: 1 Actual: 60
+          \tat gov.nasa.jpl.aerie.foomissionmodel.activities.DaemonCheckerActivity.run(DaemonCheckerActivity.java""";
+      assertTrue(reason.trace().startsWith(expectedStart));
+    }
+
+    /**
+     * If a daemon task throws an exception, the error response will not include any id
+     * but will include the exception message.
+     */
+    @Test
+    void noDirectiveIdOnDaemon() throws IOException {
+      // Set up: Update the config to force an exception 1hr into the plan
+      hasura.updateSimArguments(fooPlan, Json.createObjectBuilder().add("raiseException", JsonValue.TRUE).build());
+
+      final var simDatasetId = hasura.awaitFailingSimulation(fooPlan).simDatasetId();
+      final var simDataset = hasura.getSimulationDataset(simDatasetId);
+      assertEquals(SimulationDataset.SimulationStatus.failed, simDataset.status());
+
+      // The correct type of error was returned
+      assertTrue(simDataset.reason().isPresent());
+      final var reason = simDataset.reason().get();
+      assertEquals("SIMULATION_EXCEPTION", reason.type());
+
+      final var exception = reason.data();
+
+      // The error does not include any directive ids
+      assertFalse(exception.containsKey("executingDirectiveId"));
+
+      // The error message is correct
+      assertEquals("Daemon task exception raised.", reason.message());
+
+      // The error was thrown at the correct time
+      assertTrue(exception.containsKey("utcTimeDoy"));
+      assertEquals("2023-001T01:00:00", exception.getString("utcTimeDoy"));
+
+      // The trace starts at the original exception and doesn't include the intermediary SimulationException
+      final var expectedStart = """
+          java.lang.RuntimeException: Daemon task exception raised.
+          \tat gov.nasa.jpl.aerie.foomissionmodel.Mission.lambda$new$1(Mission.java""";
+      assertTrue(reason.trace().startsWith(expectedStart));
+    }
+
+    /**
+     * If a daemon task throws an exception while an activity directive is executing,
+     * the error response won't include the directive's id
+     */
+    @Test
+    void noDirectiveIdDaemonMidDirective() throws IOException {
+      // Set up: Update the config to force an exception 1hr into the plan
+      // and add an activity that will be executing at that time
+      hasura.updateSimArguments(fooPlan, Json.createObjectBuilder().add("raiseException", JsonValue.TRUE).build());
+      hasura.insertActivity(
+          fooPlan,
+          "DaemonCheckerSpawner",
+          "00:59:00",
+          Json.createObjectBuilder().add("minutesElapsed", 1).add("spawnDelay", 5).build());
+
+      final var simDatasetId = hasura.awaitFailingSimulation(fooPlan).simDatasetId();
+      final var simDataset = hasura.getSimulationDataset(simDatasetId);
+      assertEquals(SimulationDataset.SimulationStatus.failed, simDataset.status());
+
+      // The correct type of error was returned
+      assertTrue(simDataset.reason().isPresent());
+      final var reason = simDataset.reason().get();
+      assertEquals("SIMULATION_EXCEPTION", reason.type());
+
+      final var exception = reason.data();
+
+      // The error does not include any directive ids
+      assertFalse(exception.containsKey("executingDirectiveId"));
+
+      // The error message is correct
+      assertEquals("Daemon task exception raised.", reason.message());
+
+      // The error was thrown at the correct time
+      assertTrue(exception.containsKey("utcTimeDoy"));
+      assertEquals("2023-001T01:00:00", exception.getString("utcTimeDoy"));
+    }
+  }
 }

e2e-tests/src/test/java/gov/nasa/jpl/aerie/e2e/utils/GQL.java

@@ -590,6 +590,13 @@ mutation updateSchedulingSpec($planId: Int!, $planRev: Int!) {
         affected_rows
       }
     }"""),
+  UPDATE_SIMULATION_ARGUMENTS("""
+    mutation updateSimulationArguments($plan_id: Int!, $arguments: jsonb!) {
+      update_simulation(where: {plan_id: {_eq: $plan_id}},
+      _set: { arguments: $arguments }) {
+        affected_rows
+       }
+    }"""),
   UPDATE_SIMULATION_BOUNDS("""
     mutation updateSimulationBounds($plan_id: Int!, $simulation_start_time: timestamptz!, $simulation_end_time: timestamptz!) {
       update_simulation(where: {plan_id: {_eq: $plan_id}},

e2e-tests/src/test/java/gov/nasa/jpl/aerie/e2e/utils/HasuraRequests.java

@@ -397,6 +397,41 @@ public SimulationDataset cancelingSimulation(int planId, int timeout) throws IOE
     throw new TimeoutError("Simulation timed out after " + timeout + " seconds");
   }
 
+    /**
+   * Simulate the specified plan with a timeout of 30 seconds.
+   * Used when the simulation is expected to fail.
+   */
+  public SimulationResponse awaitFailingSimulation(int planId) throws IOException {
+    return awaitFailingSimulation(planId, 30);
+  }
+
+  /**
+   * Simulate the specified plan with a set timeout.
+   * Used when the simulation is expected to fail.
+   *
+   * @param planId the plan to simulate
+   * @param timeout the length of the timeout, in seconds
+   */
+  public SimulationResponse awaitFailingSimulation(int planId, int timeout) throws IOException {
+    for(int i = 0; i < timeout; ++i){
+      final var response = simulate(planId);
+        switch (response.status()) {
+          case "pending", "incomplete" -> {
+            try {
+              Thread.sleep(1000); // 1s
+            } catch (InterruptedException e) {
+              throw new RuntimeException(e);
+            }
+          }
+          case "failed" -> {
+            return response;
+          }
+          default -> fail("Simulation returned bad status " + response.status() + " with reason " +response.reason());
+        }
+    }
+    throw new TimeoutError("Simulation timed out after " + timeout + " seconds");
+  }
+
   public int getSimulationId(int planId) throws IOException {
     final var variables = Json.createObjectBuilder().add("plan_id", planId).build();
     return makeRequest(GQL.GET_SIMULATION_ID, variables).getJsonArray("simulation").getJsonObject(0).getInt("id");
@@ -436,13 +471,21 @@ public void deleteSimTemplate(int templateId) throws IOException {
     makeRequest(GQL.DELETE_SIMULATION_PRESET, variables);
   }
 
+  public void updateSimArguments(int planId, JsonObject arguments) throws IOException {
+    final var variables = Json.createObjectBuilder()
+                              .add("plan_id", planId)
+                              .add("arguments", arguments)
+                              .build();
+    makeRequest(GQL.UPDATE_SIMULATION_ARGUMENTS, variables);
+  }
+
   public void updateSimBounds(int planId, String simStartTime, String simEndTime) throws IOException {
     final var variables = Json.createObjectBuilder()
                               .add("plan_id", planId)
                               .add("simulation_start_time", simStartTime)
                               .add("simulation_end_time", simEndTime)
                               .build();
-    makeRequest(GQL.UPDATE_SIMULATION_BOUNDS, variables).getJsonObject("update_simulation");
+    makeRequest(GQL.UPDATE_SIMULATION_BOUNDS, variables);
   }
   //endregion
 

examples/foo-missionmodel/src/main/java/gov/nasa/jpl/aerie/foomissionmodel/Configuration.java

@@ -7,11 +7,20 @@ public final class Configuration {
   @Export.Parameter
   public Double sinkRate;
 
-  public Configuration(final Double sinkRate) {
+  // If enabled, will raise an exception 30 min into a plan
+  @Export.Parameter
+  public Boolean raiseException;
+
+  public Configuration(final Double sinkRate, final Boolean raiseException) {
     this.sinkRate = sinkRate;
+    this.raiseException = raiseException;
+  }
+
+  public Configuration(final Double sinkRate) {
+    this(sinkRate, false);
   }
 
   public Configuration() {
-    this(0.5);
+    this(0.5, false);
   }
 }

examples/foo-missionmodel/src/main/java/gov/nasa/jpl/aerie/foomissionmodel/Mission.java

@@ -81,6 +81,13 @@ public Mission(final Registrar registrar, final Instant planStart, final Configu
         ModelActions.delay(Duration.SECOND);
       }
     });
+
+    if(config.raiseException) {
+      spawn(() -> {
+        delay(Duration.HOUR);
+        throw new RuntimeException("Daemon task exception raised.");
+      });
+    }
   }
 
   public void test() {

examples/foo-missionmodel/src/main/java/gov/nasa/jpl/aerie/foomissionmodel/activities/DaemonCheckerSpawner.java

@@ -0,0 +1,24 @@
+package gov.nasa.jpl.aerie.foomissionmodel.activities;
+
+import gov.nasa.jpl.aerie.foomissionmodel.Mission;
+import gov.nasa.jpl.aerie.merlin.framework.annotations.ActivityType;
+import gov.nasa.jpl.aerie.merlin.protocol.types.Duration;
+
+import static gov.nasa.jpl.aerie.foomissionmodel.generated.ActivityActions.call;
+import static gov.nasa.jpl.aerie.merlin.framework.ModelActions.delay;
+
+/**
+ * An activity that spawns a DaemonCheckerActivity after delaying.
+ * Useful for testing the behavior of exceptions thrown by child activities.
+ *
+ * @param minutesElapsed The expected number of minutes elapsed when the DaemonCheckerActivity begins
+ * @param spawnDelay The number of minutes to delay between the start of this activity and spawning the DaemonCheckerActivity
+ */
+@ActivityType("DaemonCheckerSpawner")
+public record DaemonCheckerSpawner(int minutesElapsed, int spawnDelay) {
+  @ActivityType.EffectModel
+  public void run(final Mission mission) {
+    delay(Duration.of(spawnDelay, Duration.MINUTE));
+    call(mission, new DaemonCheckerActivity(minutesElapsed));
+  }
+}

examples/foo-missionmodel/src/main/java/gov/nasa/jpl/aerie/foomissionmodel/package-info.java

@@ -15,6 +15,7 @@
 @WithActivityType(BasicFooActivity.class)
 @WithActivityType(ZeroDurationUncontrollableActivity.class)
 @WithActivityType(DaemonCheckerActivity.class)
+@WithActivityType(DaemonCheckerSpawner.class)
 
 @WithActivityType(DecompositionTestActivities.ParentActivity.class)
 @WithActivityType(DecompositionTestActivities.ChildActivity.class)
@@ -28,6 +29,7 @@
 import gov.nasa.jpl.aerie.foomissionmodel.activities.BasicFooActivity;
 import gov.nasa.jpl.aerie.foomissionmodel.activities.ControllableDurationActivity;
 import gov.nasa.jpl.aerie.foomissionmodel.activities.DaemonCheckerActivity;
+import gov.nasa.jpl.aerie.foomissionmodel.activities.DaemonCheckerSpawner;
 import gov.nasa.jpl.aerie.foomissionmodel.activities.DecompositionTestActivities;
 import gov.nasa.jpl.aerie.foomissionmodel.activities.FooActivity;
 import gov.nasa.jpl.aerie.foomissionmodel.activities.LateRiserActivity;