Merge pull request #1876 from HubSpot/task-lag

Better alerting for task lag
HubSpot · Nov 15, 2018 · 379c957 · 379c957
2 parents 3f1019d + 6e8827d
commit 379c957
Show file tree

Hide file tree

Showing 3 changed files with 23 additions and 8 deletions.
diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityScheduledTasksInfo.java b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityScheduledTasksInfo.java
@@ -1,23 +1,28 @@
 package com.hubspot.singularity;
 
+import java.util.ArrayList;
 import java.util.List;
 
 public class SingularityScheduledTasksInfo {
 
-  private final int numLateTasks;
   private final int numFutureTasks;
   private final long maxTaskLag;
   private final long timestamp;
+  private final List<SingularityPendingTaskId> lateTasks;
 
-  private SingularityScheduledTasksInfo(int numLateTasks, int numFutureTasks, long maxTaskLag, long timestamp) {
-    this.numLateTasks = numLateTasks;
+  private SingularityScheduledTasksInfo(List<SingularityPendingTaskId> lateTasks, int numFutureTasks, long maxTaskLag, long timestamp) {
+    this.lateTasks = lateTasks;
     this.numFutureTasks = numFutureTasks;
     this.maxTaskLag = maxTaskLag;
     this.timestamp = timestamp;
   }
 
+  public List<SingularityPendingTaskId> getLateTasks() {
+    return lateTasks;
+  }
+
   public int getNumLateTasks() {
-    return numLateTasks;
+    return getLateTasks().size();
   }
 
   public int getNumFutureTasks() {
@@ -35,15 +40,15 @@ public long getTimestamp() {
   public static SingularityScheduledTasksInfo getInfo(List<SingularityPendingTask> pendingTasks, long millisDeltaForLateTasks) {
     final long now = System.currentTimeMillis();
 
-    int numLateTasks = 0;
     int numFutureTasks = 0;
     long maxTaskLag = 0;
+    List<SingularityPendingTaskId> lateTasks = new ArrayList<>();
 
     for (SingularityPendingTask pendingTask : pendingTasks) {
       long delta = now - pendingTask.getPendingTaskId().getNextRunAt();
 
       if (delta > millisDeltaForLateTasks) {
-        numLateTasks++;
+        lateTasks.add(pendingTask.getPendingTaskId());
       } else {
         numFutureTasks++;
       }
@@ -53,6 +58,6 @@ public static SingularityScheduledTasksInfo getInfo(List<SingularityPendingTask>
       }
     }
 
-    return new SingularityScheduledTasksInfo(numLateTasks, numFutureTasks, maxTaskLag, now);
+    return new SingularityScheduledTasksInfo(lateTasks, numFutureTasks, maxTaskLag, now);
   }
 }
diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityState.java b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityState.java
@@ -1,5 +1,6 @@
 package com.hubspot.singularity;
 
+import java.util.Collections;
 import java.util.List;
 
 import com.fasterxml.jackson.annotation.JsonCreator;
@@ -18,6 +19,7 @@ public class SingularityState {
   private final int cooldownRequests;
   private final int scheduledTasks;
   private final int lateTasks;
+  private final List<SingularityPendingTaskId> listLateTasks;
   private final int futureTasks;
   private final int cleaningTasks;
   private final int lbCleanupTasks;
@@ -85,6 +87,7 @@ public SingularityState(@JsonProperty("activeTasks") int activeTasks,
                           @JsonProperty("oldestDeployStep") long oldestDeployStep,
                           @JsonProperty("activeDeploys") List<SingularityDeployMarker> activeDeploys,
                           @JsonProperty("lateTasks") int lateTasks,
+                          @JsonProperty("listLateTasks") List<SingularityPendingTaskId> listLateTasks,
                           @JsonProperty("futureTasks") int futureTasks,
                           @JsonProperty("maxTaskLag") long maxTaskLag,
                           @JsonProperty("generatedAt") long generatedAt,
@@ -119,6 +122,7 @@ public SingularityState(@JsonProperty("activeTasks") int activeTasks,
     this.cleaningTasks = cleaningTasks;
     this.hostStates = hostStates;
     this.lateTasks = lateTasks;
+    this.listLateTasks = listLateTasks == null ? Collections.emptyList() : listLateTasks;
     this.finishedRequests = finishedRequests;
     this.futureTasks = futureTasks;
     this.maxTaskLag = maxTaskLag;
@@ -276,6 +280,11 @@ public int getLateTasks() {
     return lateTasks;
   }
 
+  @Schema(description = "The list of all late tasks that have not been launched in time")
+  public List<SingularityPendingTaskId> getListLateTasks() {
+    return listLateTasks;
+  }
+
   @Schema(description = "The count of pending tasks that will be launched at a future time")
   public int getFutureTasks() {
     return futureTasks;
@@ -356,6 +365,7 @@ public String toString() {
         ", cooldownRequests=" + cooldownRequests +
         ", scheduledTasks=" + scheduledTasks +
         ", lateTasks=" + lateTasks +
+        ", listLateTasks=" + listLateTasks +
         ", futureTasks=" + futureTasks +
         ", cleaningTasks=" + cleaningTasks +
         ", lbCleanupTasks=" + lbCleanupTasks +

diff --git a/SingularityService/src/main/java/com/hubspot/singularity/data/StateManager.java b/SingularityService/src/main/java/com/hubspot/singularity/data/StateManager.java
@@ -293,7 +293,7 @@ public SingularityState generateState(boolean includeRequestIds) {
     final Optional<Double> minimumPriorityLevel = getMinimumPriorityLevel();
 
     return new SingularityState(activeTasks, launchingTasks, numActiveRequests, cooldownRequests, numPausedRequests, scheduledTasks, pendingRequests, lbCleanupTasks, lbCleanupRequests, cleaningRequests, activeSlaves,
-        deadSlaves, decommissioningSlaves, activeRacks, deadRacks, decommissioningRacks, cleaningTasks, states, oldestDeploy, numDeploys, oldestDeployStep, activeDeploys, scheduledTasksInfo.getNumLateTasks(),
+        deadSlaves, decommissioningSlaves, activeRacks, deadRacks, decommissioningRacks, cleaningTasks, states, oldestDeploy, numDeploys, oldestDeployStep, activeDeploys, scheduledTasksInfo.getNumLateTasks(), scheduledTasksInfo.getLateTasks(),
         scheduledTasksInfo.getNumFutureTasks(), scheduledTasksInfo.getMaxTaskLag(), System.currentTimeMillis(), includeRequestIds ? overProvisionedRequestIds : null,
         includeRequestIds ? underProvisionedRequestIds : null, overProvisionedRequestIds.size(), underProvisionedRequestIds.size(), numFinishedRequests, unknownRacks, unknownSlaves, authDatastoreHealthy, minimumPriorityLevel,
         statusUpdateDeltaAvg.get(), lastHeartbeatTime.get());