diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/RequestUtilization.java b/SingularityBase/src/main/java/com/hubspot/singularity/RequestUtilization.java index d71ccad5ced..a283e7ec046 100644 --- a/SingularityBase/src/main/java/com/hubspot/singularity/RequestUtilization.java +++ b/SingularityBase/src/main/java/com/hubspot/singularity/RequestUtilization.java @@ -7,8 +7,10 @@ public class RequestUtilization { private final String requestId; private final String deployId; - private long memBytesTotal = 0; - private double cpuTotal = 0; + private long memBytesUsed = 0; + private long memBytesReserved = 0; + private double cpuUsed = 0; + private double cpuReserved = 0; private int numTasks = 0; @JsonCreator @@ -18,13 +20,23 @@ public RequestUtilization(@JsonProperty("requestId") String requestId, this.deployId = deployId; } - public RequestUtilization addMemBytes(long memBytes) { - this.memBytesTotal += memBytes; + public RequestUtilization addMemBytesUsed(long memBytes) { + this.memBytesUsed += memBytes; return this; } - public RequestUtilization addCpu(double cpu) { - this.cpuTotal += cpu; + public RequestUtilization addMemBytesReserved(long memBytes) { + this.memBytesReserved += memBytes; + return this; + } + + public RequestUtilization addCpuUsed(double cpu) { + this.cpuUsed += cpu; + return this; + } + + public RequestUtilization addCpuReserved(double cpu) { + this.cpuReserved += cpu; return this; } @@ -33,12 +45,20 @@ public RequestUtilization incrementTaskCount() { return this; } - public long getMemBytesTotal() { - return memBytesTotal; + public long getMemBytesUsed() { + return memBytesUsed; + } + + public long getMemBytesReserved() { + return memBytesReserved; + } + + public double getCpuUsed() { + return cpuUsed; } - public double getCpuTotal() { - return cpuTotal; + public double getCpuReserved() { + return cpuReserved; } public int getNumTasks() { @@ -47,12 +67,12 @@ public int getNumTasks() { @JsonIgnore public double getAvgMemBytesUsed() { - return memBytesTotal / (double) numTasks; + return memBytesUsed / (double) numTasks; } @JsonIgnore public double getAvgCpuUsed() { - return cpuTotal / (double) numTasks; + return cpuUsed / (double) numTasks; } public String getDeployId() { @@ -68,8 +88,10 @@ public String toString() { return "RequestUtilization{" + "requestId=" + requestId + ", deployId=" + deployId + - ", memBytesTotal=" + memBytesTotal + - ", cpuTotal=" + cpuTotal + + ", memBytesUsed=" + memBytesUsed + + ", memBytesReserved=" + memBytesReserved + + ", cpuUsed=" + cpuUsed + + ", cpuReserved=" + cpuReserved + ", numTasks=" + numTasks + '}'; } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java index 53097635952..20cac2a9aad 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java @@ -84,10 +84,10 @@ public void runActionOnPoll() { longRunningTasksUsage.put(ResourceUsageType.CPU_USED, 0); Optional memoryMbTotal = Optional.absent(); Optional cpusTotal = Optional.absent(); - long memoryMbReserved = 0; - long cpuReserved = 0; - long memoryBytesUsed = 0; - double cpusUsed = 0; + long memoryMbReservedOnSlave = 0; + long cpuReservedOnSlave = 0; + long memoryBytesUsedOnSlave = 0; + double cpusUsedOnSlave = 0; try { List allTaskUsage = mesosClient.getSlaveResourceUsage(slave.getHost()); @@ -105,17 +105,19 @@ public void runActionOnPoll() { SingularityTaskUsage latestUsage = getUsage(taskUsage); List pastTaskUsages = usageManager.getTaskUsage(taskId); - updateRequestUtilization(utilizationPerRequestId, pastTaskUsages, latestUsage, task); clearOldUsage(pastTaskUsages, taskId); usageManager.saveSpecificTaskUsage(taskId, latestUsage); Optional maybeTask = taskManager.getTask(task); if (maybeTask.isPresent() && maybeTask.get().getTaskRequest().getDeploy().getResources().isPresent()) { - memoryMbReserved += maybeTask.get().getTaskRequest().getDeploy().getResources().get().getMemoryMb(); - cpuReserved += maybeTask.get().getTaskRequest().getDeploy().getResources().get().getCpus(); + double memoryMbReservedForTask = maybeTask.get().getTaskRequest().getDeploy().getResources().get().getMemoryMb(); + double cpuReservedForTask = maybeTask.get().getTaskRequest().getDeploy().getResources().get().getCpus(); + memoryMbReservedOnSlave += memoryMbReservedForTask; + cpuReservedOnSlave += cpuReservedForTask; + updateRequestUtilization(utilizationPerRequestId, pastTaskUsages, latestUsage, task, memoryMbReservedForTask, cpuReservedForTask); } - memoryBytesUsed += latestUsage.getMemoryTotalBytes(); + memoryBytesUsedOnSlave += latestUsage.getMemoryTotalBytes(); if (!pastTaskUsages.isEmpty()) { SingularityTaskUsage lastUsage = pastTaskUsages.get(pastTaskUsages.size() - 1); @@ -129,7 +131,7 @@ public void runActionOnPoll() { usageManager.saveCurrentTaskUsage(taskId, currentUsage); - cpusUsed += taskCpusUsed; + cpusUsedOnSlave += taskCpusUsed; } } @@ -142,7 +144,7 @@ public void runActionOnPoll() { cpusTotal = Optional.of(slave.getResources().get().getNumCpus().get().doubleValue()); } - SingularitySlaveUsage slaveUsage = new SingularitySlaveUsage(memoryBytesUsed, memoryMbReserved, now, cpusUsed, cpuReserved, allTaskUsage.size(), memoryMbTotal, cpusTotal, longRunningTasksUsage); + SingularitySlaveUsage slaveUsage = new SingularitySlaveUsage(memoryBytesUsedOnSlave, memoryMbReservedOnSlave, now, cpusUsedOnSlave, cpuReservedOnSlave, allTaskUsage.size(), memoryMbTotal, cpusTotal, longRunningTasksUsage); List slaveTimestamps = usageManager.getSlaveUsageTimestamps(slave.getId()); if (slaveTimestamps.size() + 1 > configuration.getNumUsageToKeep()) { usageManager.deleteSpecificSlaveUsage(slave.getId(), slaveTimestamps.get(0)); @@ -195,7 +197,7 @@ private void updateLongRunningTasksUsage(Map longRunn longRunningTasksUsage.compute(ResourceUsageType.CPU_USED, (k, v) -> (v == null) ? cpuUsed : v.doubleValue() + cpuUsed); } - private void updateRequestUtilization(Map utilizationPerRequestId, List pastTaskUsages, SingularityTaskUsage latestUsage, SingularityTaskId task) { + private void updateRequestUtilization(Map utilizationPerRequestId, List pastTaskUsages, SingularityTaskUsage latestUsage, SingularityTaskId task, double memoryMbReservedForTask, double cpuReservedForTask) { List pastTaskUsagesCopy = new ArrayList<>(); pastTaskUsagesCopy.add(new SingularityTaskUsage(0, task.getStartedAt(), 0)); // to calculate oldest cpu usage pastTaskUsagesCopy.addAll(pastTaskUsages); @@ -208,11 +210,16 @@ private void updateRequestUtilization(Map utilizatio SingularityTaskUsage newerUsage = pastTaskUsagesCopy.get(i + 1); double cpusUsed = (newerUsage.getCpuSeconds() - olderUsage.getCpuSeconds()) / (newerUsage.getTimestamp() - olderUsage.getTimestamp()); - requestUtilization.addCpu(cpusUsed); - requestUtilization.addMemBytes(newerUsage.getMemoryTotalBytes()); - requestUtilization.incrementTaskCount(); + requestUtilization + .addCpuUsed(cpusUsed) + .addMemBytesUsed(newerUsage.getMemoryTotalBytes()) + .incrementTaskCount(); } + requestUtilization + .addMemBytesReserved((long) (memoryMbReservedForTask * SingularitySlaveUsage.BYTES_PER_MEGABYTE * requestUtilization.getNumTasks())) + .addCpuReserved(cpuReservedForTask * requestUtilization.getNumTasks()); + utilizationPerRequestId.put(task.getRequestId(), requestUtilization); } diff --git a/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularityUsageTest.java b/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularityUsageTest.java index b70ad70cd8d..e2854e7386b 100644 --- a/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularityUsageTest.java +++ b/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularityUsageTest.java @@ -15,6 +15,7 @@ import com.hubspot.mesos.json.MesosTaskStatisticsObject; import com.hubspot.singularity.MachineState; import com.hubspot.singularity.SingularityClusterUtilization; +import com.hubspot.singularity.SingularitySlaveUsage; import com.hubspot.singularity.SingularityTask; import com.hubspot.singularity.SingularityTaskCurrentUsageWithId; import com.hubspot.singularity.SingularityTaskId; @@ -207,7 +208,9 @@ public void testUsagePollerComplex() throws InterruptedException { @Test public void itTracksClusterUtilizationSimple() { initRequest(); - initFirstDeployWithResources(10, .001); + double cpuReserved = 10; + double memMbReserved = .001; + initFirstDeployWithResources(cpuReserved, memMbReserved); saveAndSchedule(request.toBuilder().setInstances(Optional.of(1))); resourceOffers(1); @@ -229,7 +232,12 @@ public void itTracksClusterUtilizationSimple() { SingularityClusterUtilization utilization = usageManager.getClusterUtilization().get(); - Assert.assertEquals(2, usageManager.getTaskUsage(t1).size()); + int taskUsages = usageManager.getTaskUsage(t1).size(); + Assert.assertEquals(2, taskUsages); + + Assert.assertEquals(1, utilization.getRequestUtilizations().size()); + Assert.assertEquals(cpuReserved * taskUsages, utilization.getRequestUtilizations().get(0).getCpuReserved(), 0); + Assert.assertEquals(memMbReserved * SingularitySlaveUsage.BYTES_PER_MEGABYTE * taskUsages, utilization.getRequestUtilizations().get(0).getMemBytesReserved(), 0); Assert.assertEquals(0, utilization.getNumRequestsWithOverUtilizedCpu()); Assert.assertEquals(1, utilization.getNumRequestsWithUnderUtilizedCpu()); @@ -253,7 +261,9 @@ public void itTracksClusterUtilizationSimple() { @Test public void itDoesntIncludePerfectlyUtilizedRequestsInClusterUtilization() { initRequest(); - initFirstDeployWithResources(2, .001); + double cpuReserved = 2; + double memMbReserved = .001; + initFirstDeployWithResources(cpuReserved, memMbReserved); saveAndSchedule(request.toBuilder().setInstances(Optional.of(1))); resourceOffers(1); @@ -275,7 +285,12 @@ public void itDoesntIncludePerfectlyUtilizedRequestsInClusterUtilization() { SingularityClusterUtilization utilization = usageManager.getClusterUtilization().get(); - Assert.assertEquals(2, usageManager.getTaskUsage(t1).size()); + int taskUsages = usageManager.getTaskUsage(t1).size(); + Assert.assertEquals(2, taskUsages); + + Assert.assertEquals(1, utilization.getRequestUtilizations().size()); + Assert.assertEquals(cpuReserved * taskUsages, utilization.getRequestUtilizations().get(0).getCpuReserved(), 0); + Assert.assertEquals(memMbReserved * SingularitySlaveUsage.BYTES_PER_MEGABYTE * taskUsages, utilization.getRequestUtilizations().get(0).getMemBytesReserved(), 0); Assert.assertEquals(0, utilization.getNumRequestsWithOverUtilizedCpu()); Assert.assertEquals(0, utilization.getNumRequestsWithUnderUtilizedCpu()); @@ -298,7 +313,9 @@ public void itDoesntIncludePerfectlyUtilizedRequestsInClusterUtilization() { @Test public void itTracksOverusedCpuInClusterUtilization() { initRequest(); - initFirstDeployWithResources(2, .001); + double cpuReserved = 2; + double memMbReserved = .001; + initFirstDeployWithResources(cpuReserved, memMbReserved); saveAndSchedule(request.toBuilder().setInstances(Optional.of(1))); resourceOffers(1); @@ -320,7 +337,12 @@ public void itTracksOverusedCpuInClusterUtilization() { SingularityClusterUtilization utilization = usageManager.getClusterUtilization().get(); - Assert.assertEquals(2, usageManager.getTaskUsage(t1).size()); + int taskUsages = usageManager.getTaskUsage(t1).size(); + Assert.assertEquals(2, taskUsages); + + Assert.assertEquals(1, utilization.getRequestUtilizations().size()); + Assert.assertEquals(cpuReserved * taskUsages, utilization.getRequestUtilizations().get(0).getCpuReserved(), 0); + Assert.assertEquals(memMbReserved * SingularitySlaveUsage.BYTES_PER_MEGABYTE * taskUsages, utilization.getRequestUtilizations().get(0).getMemBytesReserved(), 0); Assert.assertEquals(1, utilization.getNumRequestsWithOverUtilizedCpu()); Assert.assertEquals(0, utilization.getNumRequestsWithUnderUtilizedCpu());