diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityScheduledTasksInfo.java b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityScheduledTasksInfo.java index fef4dc138e..e094cd0e38 100644 --- a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityScheduledTasksInfo.java +++ b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityScheduledTasksInfo.java @@ -1,23 +1,28 @@ package com.hubspot.singularity; +import java.util.ArrayList; import java.util.List; public class SingularityScheduledTasksInfo { - private final int numLateTasks; private final int numFutureTasks; private final long maxTaskLag; private final long timestamp; + private final List lateTasks; - private SingularityScheduledTasksInfo(int numLateTasks, int numFutureTasks, long maxTaskLag, long timestamp) { - this.numLateTasks = numLateTasks; + private SingularityScheduledTasksInfo(List lateTasks, int numFutureTasks, long maxTaskLag, long timestamp) { + this.lateTasks = lateTasks; this.numFutureTasks = numFutureTasks; this.maxTaskLag = maxTaskLag; this.timestamp = timestamp; } + public List getLateTasks() { + return lateTasks; + } + public int getNumLateTasks() { - return numLateTasks; + return getLateTasks().size(); } public int getNumFutureTasks() { @@ -35,15 +40,15 @@ public long getTimestamp() { public static SingularityScheduledTasksInfo getInfo(List pendingTasks, long millisDeltaForLateTasks) { final long now = System.currentTimeMillis(); - int numLateTasks = 0; int numFutureTasks = 0; long maxTaskLag = 0; + List lateTasks = new ArrayList<>(); for (SingularityPendingTask pendingTask : pendingTasks) { long delta = now - pendingTask.getPendingTaskId().getNextRunAt(); if (delta > millisDeltaForLateTasks) { - numLateTasks++; + lateTasks.add(pendingTask.getPendingTaskId()); } else { numFutureTasks++; } @@ -53,6 +58,6 @@ public static SingularityScheduledTasksInfo getInfo(List } } - return new SingularityScheduledTasksInfo(numLateTasks, numFutureTasks, maxTaskLag, now); + return new SingularityScheduledTasksInfo(lateTasks, numFutureTasks, maxTaskLag, now); } } diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityState.java b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityState.java index 82c4f5ea45..c5fd15cfb4 100644 --- a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityState.java +++ b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityState.java @@ -1,5 +1,6 @@ package com.hubspot.singularity; +import java.util.Collections; import java.util.List; import com.fasterxml.jackson.annotation.JsonCreator; @@ -18,6 +19,7 @@ public class SingularityState { private final int cooldownRequests; private final int scheduledTasks; private final int lateTasks; + private final List listLateTasks; private final int futureTasks; private final int cleaningTasks; private final int lbCleanupTasks; @@ -85,6 +87,7 @@ public SingularityState(@JsonProperty("activeTasks") int activeTasks, @JsonProperty("oldestDeployStep") long oldestDeployStep, @JsonProperty("activeDeploys") List activeDeploys, @JsonProperty("lateTasks") int lateTasks, + @JsonProperty("listLateTasks") List listLateTasks, @JsonProperty("futureTasks") int futureTasks, @JsonProperty("maxTaskLag") long maxTaskLag, @JsonProperty("generatedAt") long generatedAt, @@ -119,6 +122,7 @@ public SingularityState(@JsonProperty("activeTasks") int activeTasks, this.cleaningTasks = cleaningTasks; this.hostStates = hostStates; this.lateTasks = lateTasks; + this.listLateTasks = listLateTasks == null ? Collections.emptyList() : listLateTasks; this.finishedRequests = finishedRequests; this.futureTasks = futureTasks; this.maxTaskLag = maxTaskLag; @@ -276,6 +280,11 @@ public int getLateTasks() { return lateTasks; } + @Schema(description = "The list of all late tasks that have not been launched in time") + public List getListLateTasks() { + return listLateTasks; + } + @Schema(description = "The count of pending tasks that will be launched at a future time") public int getFutureTasks() { return futureTasks; @@ -356,6 +365,7 @@ public String toString() { ", cooldownRequests=" + cooldownRequests + ", scheduledTasks=" + scheduledTasks + ", lateTasks=" + lateTasks + + ", listLateTasks=" + listLateTasks + ", futureTasks=" + futureTasks + ", cleaningTasks=" + cleaningTasks + ", lbCleanupTasks=" + lbCleanupTasks + diff --git a/SingularityService/src/main/java/com/hubspot/singularity/data/StateManager.java b/SingularityService/src/main/java/com/hubspot/singularity/data/StateManager.java index bab8652714..735112d3a8 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/data/StateManager.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/data/StateManager.java @@ -293,7 +293,7 @@ public SingularityState generateState(boolean includeRequestIds) { final Optional minimumPriorityLevel = getMinimumPriorityLevel(); return new SingularityState(activeTasks, launchingTasks, numActiveRequests, cooldownRequests, numPausedRequests, scheduledTasks, pendingRequests, lbCleanupTasks, lbCleanupRequests, cleaningRequests, activeSlaves, - deadSlaves, decommissioningSlaves, activeRacks, deadRacks, decommissioningRacks, cleaningTasks, states, oldestDeploy, numDeploys, oldestDeployStep, activeDeploys, scheduledTasksInfo.getNumLateTasks(), + deadSlaves, decommissioningSlaves, activeRacks, deadRacks, decommissioningRacks, cleaningTasks, states, oldestDeploy, numDeploys, oldestDeployStep, activeDeploys, scheduledTasksInfo.getNumLateTasks(), scheduledTasksInfo.getLateTasks(), scheduledTasksInfo.getNumFutureTasks(), scheduledTasksInfo.getMaxTaskLag(), System.currentTimeMillis(), includeRequestIds ? overProvisionedRequestIds : null, includeRequestIds ? underProvisionedRequestIds : null, overProvisionedRequestIds.size(), underProvisionedRequestIds.size(), numFinishedRequests, unknownRacks, unknownSlaves, authDatastoreHealthy, minimumPriorityLevel, statusUpdateDeltaAvg.get(), lastHeartbeatTime.get());