diff --git a/SingularityBase/src/main/java/com/hubspot/mesos/json/MesosTaskStatisticsObject.java b/SingularityBase/src/main/java/com/hubspot/mesos/json/MesosTaskStatisticsObject.java index 34f29b21bb..f85f5c1a7e 100644 --- a/SingularityBase/src/main/java/com/hubspot/mesos/json/MesosTaskStatisticsObject.java +++ b/SingularityBase/src/main/java/com/hubspot/mesos/json/MesosTaskStatisticsObject.java @@ -15,7 +15,8 @@ public class MesosTaskStatisticsObject { private final long memLimitBytes; private final long memMappedFileBytes; private final long memRssBytes; - private final double timestamp; + private final long memTotalBytes; + private final double timestampSeconds; @JsonCreator public MesosTaskStatisticsObject(@JsonProperty("cpus_limit") int cpusLimit, @@ -29,7 +30,8 @@ public MesosTaskStatisticsObject(@JsonProperty("cpus_limit") int cpusLimit, @JsonProperty("mem_limit_bytes") long memLimitBytes, @JsonProperty("mem_mapped_file_bytes") long memMappedFileBytes, @JsonProperty("mem_rss_bytes") long memRssBytes, - @JsonProperty("timestamp") double timestamp) { + @JsonProperty("mem_total_bytes") long memTotalBytes, + @JsonProperty("timestamp") double timestampSeconds) { this.cpusLimit = cpusLimit; this.cpusNrPeriods = cpusNrPeriods; this.cpusNrThrottled = cpusNrThrottled; @@ -41,7 +43,8 @@ public MesosTaskStatisticsObject(@JsonProperty("cpus_limit") int cpusLimit, this.memLimitBytes = memLimitBytes; this.memMappedFileBytes = memMappedFileBytes; this.memRssBytes = memRssBytes; - this.timestamp = timestamp; + this.memTotalBytes = memTotalBytes; + this.timestampSeconds = timestampSeconds; } public int getCpusLimit() { @@ -88,8 +91,12 @@ public long getMemRssBytes() { return memRssBytes; } - public double getTimestamp() { - return timestamp; + public long getMemTotalBytes() { + return memTotalBytes; + } + + public double getTimestampSeconds() { + return timestampSeconds; } @Override @@ -106,7 +113,8 @@ public String toString() { ", memLimitBytes=" + memLimitBytes + ", memMappedFileBytes=" + memMappedFileBytes + ", memRssBytes=" + memRssBytes + - ", timestamp=" + timestamp + + ", memTotalBytes=" + memTotalBytes + + ", timestampSeconds=" + timestampSeconds + '}'; } } diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/RequestUtilization.java b/SingularityBase/src/main/java/com/hubspot/singularity/RequestUtilization.java new file mode 100644 index 0000000000..a4103c19de --- /dev/null +++ b/SingularityBase/src/main/java/com/hubspot/singularity/RequestUtilization.java @@ -0,0 +1,140 @@ +package com.hubspot.singularity; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; + +public class RequestUtilization { + private final String requestId; + private final String deployId; + + private long memBytesUsed = 0; + private long memBytesReserved = 0; + private double cpuUsed = 0; + private double cpuReserved = 0; + private int numTasks = 0; + + private long maxMemBytesUsed = 0; + private long minMemBytesUsed = Long.MAX_VALUE; + private double maxCpuUsed = 0; + private double minCpuUsed = Double.MAX_VALUE; + + @JsonCreator + public RequestUtilization(@JsonProperty("requestId") String requestId, + @JsonProperty("deployId") String deployId) { + this.requestId = requestId; + this.deployId = deployId; + } + + public RequestUtilization addMemBytesUsed(long memBytes) { + this.memBytesUsed += memBytes; + return this; + } + + public RequestUtilization addMemBytesReserved(long memBytes) { + this.memBytesReserved += memBytes; + return this; + } + + public RequestUtilization addCpuUsed(double cpu) { + this.cpuUsed += cpu; + return this; + } + + public RequestUtilization addCpuReserved(double cpu) { + this.cpuReserved += cpu; + return this; + } + + public RequestUtilization incrementTaskCount() { + this.numTasks++; + return this; + } + + public long getMemBytesUsed() { + return memBytesUsed; + } + + public long getMemBytesReserved() { + return memBytesReserved; + } + + public double getCpuUsed() { + return cpuUsed; + } + + public double getCpuReserved() { + return cpuReserved; + } + + public int getNumTasks() { + return numTasks; + } + + @JsonIgnore + public double getAvgMemBytesUsed() { + return memBytesUsed / (double) numTasks; + } + + @JsonIgnore + public double getAvgCpuUsed() { + return cpuUsed / (double) numTasks; + } + + public String getDeployId() { + return deployId; + } + + public String getRequestId() { + return requestId; + } + + public long getMaxMemBytesUsed() { + return maxMemBytesUsed; + } + + public RequestUtilization setMaxMemBytesUsed(long maxMemBytesUsed) { + this.maxMemBytesUsed = maxMemBytesUsed; + return this; + } + + public double getMaxCpuUsed() { + return maxCpuUsed; + } + + public RequestUtilization setMaxCpuUsed(double maxCpuUsed) { + this.maxCpuUsed = maxCpuUsed; + return this; + } + + public long getMinMemBytesUsed() { + return minMemBytesUsed; + } + + public RequestUtilization setMinMemBytesUsed(long minMemBytesUsed) { + this.minMemBytesUsed = minMemBytesUsed; + return this; + } + + public double getMinCpuUsed() { + return minCpuUsed; + } + + public RequestUtilization setMinCpuUsed(double minCpuUsed) { + this.minCpuUsed = minCpuUsed; + return this; + } + + @Override + public String toString() { + return "RequestUtilization{" + + "requestId=" + requestId + + ", deployId=" + deployId + + ", memBytesUsed=" + memBytesUsed + + ", memBytesReserved=" + memBytesReserved + + ", cpuUsed=" + cpuUsed + + ", cpuReserved=" + cpuReserved + + ", numTasks=" + numTasks + + '}'; + } +} diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityClusterUtilization.java b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityClusterUtilization.java index de85928d6c..ea47ade535 100644 --- a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityClusterUtilization.java +++ b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityClusterUtilization.java @@ -1,21 +1,89 @@ package com.hubspot.singularity; +import java.util.List; + import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; public class SingularityClusterUtilization { - private long totalMemBytesUsed; - private long totalMemBytesAvailable; - private double totalCpuUsed; - private double totalCpuAvailable; + private final List requestUtilizations; + + private final int numRequestsWithUnderUtilizedCpu; + private final int numRequestsWithOverUtilizedCpu; + private final int numRequestsWithUnderUtilizedMemBytes; + + private final double totalUnderUtilizedCpu; + private final double totalOverUtilizedCpu; + private final long totalUnderUtilizedMemBytes; + + private final double avgUnderUtilizedCpu; + private final double avgOverUtilizedCpu; + private final long avgUnderUtilizedMemBytes; + + private final double maxUnderUtilizedCpu; + private final double maxOverUtilizedCpu; + private final long maxUnderUtilizedMemBytes; + + private final String maxUnderUtilizedCpuRequestId; + private final String maxOverUtilizedCpuRequestId; + private final String maxUnderUtilizedMemBytesRequestId; + + private final double minUnderUtilizedCpu; + private final double minOverUtilizedCpu; + private final long minUnderUtilizedMemBytes; + + private final long totalMemBytesUsed; + private final long totalMemBytesAvailable; + + private final double totalCpuUsed; + private final double totalCpuAvailable; + private final long timestamp; @JsonCreator - public SingularityClusterUtilization(@JsonProperty("totalMemBytesUsed") long totalMemBytesUsed, + public SingularityClusterUtilization(@JsonProperty("requestUtilizations") List requestUtilizations, + @JsonProperty("numRequestsWithUnderUtilizedCpu") int numRequestsWithUnderUtilizedCpu, + @JsonProperty("numRequestsWithOverUtilizedCpu") int numRequestsWithOverUtilizedCpu, + @JsonProperty("numRequestsWithUnderUtilizedMemBytes") int numRequestsWithUnderUtilizedMemBytes, + @JsonProperty("totalUnderUtilizedCpu") double totalUnderUtilizedCpu, + @JsonProperty("totalOverUtilizedCpu") double totalOverUtilizedCpu, + @JsonProperty("totalUnderUtilizedMemBytes") long totalUnderUtilizedMemBytes, + @JsonProperty("avgUnderUtilizedCpu") double avgUnderUtilizedCpu, + @JsonProperty("avgOverUtilizedCpu") double avgOverUtilizedCpu, + @JsonProperty("avgUnderUtilizedMemBytes") long avgUnderUtilizedMemBytes, + @JsonProperty("maxUnderUtilizedCpu") double maxUnderUtilizedCpu, + @JsonProperty("maxOverUtilizedCpu") double maxOverUtilizedCpu, + @JsonProperty("maxUnderUtilizedMemBytes") long maxUnderUtilizedMemBytes, + @JsonProperty("maxUnderUtilizedCpuRequestId") String maxUnderUtilizedCpuRequestId, + @JsonProperty("maxOverUtilizedCpuRequestId") String maxOverUtilizedCpuRequestId, + @JsonProperty("maxUnderUtilizedMemBytesRequestId") String maxUnderUtilizedMemBytesRequestId, + @JsonProperty("minUnderUtilizedCpu") double minUnderUtilizedCpu, + @JsonProperty("minOverUtilizedCpu") double minOverUtilizedCpu, + @JsonProperty("minUnderUtilizedMemBytes") long minUnderUtilizedMemBytes, + @JsonProperty("totalMemBytesUsed") long totalMemBytesUsed, @JsonProperty("totalMemBytesAvailable") long totalMemBytesAvailable, @JsonProperty("totalCpuUsed") double totalCpuUsed, @JsonProperty("totalCpuAvailable") double totalCpuAvailable, @JsonProperty("timestamp") long timestamp) { + this.requestUtilizations = requestUtilizations; + this.numRequestsWithUnderUtilizedCpu = numRequestsWithUnderUtilizedCpu; + this.numRequestsWithOverUtilizedCpu = numRequestsWithOverUtilizedCpu; + this.numRequestsWithUnderUtilizedMemBytes = numRequestsWithUnderUtilizedMemBytes; + this.totalUnderUtilizedCpu = totalUnderUtilizedCpu; + this.totalOverUtilizedCpu = totalOverUtilizedCpu; + this.totalUnderUtilizedMemBytes = totalUnderUtilizedMemBytes; + this.avgUnderUtilizedCpu = avgUnderUtilizedCpu; + this.avgOverUtilizedCpu = avgOverUtilizedCpu; + this.avgUnderUtilizedMemBytes = avgUnderUtilizedMemBytes; + this.maxUnderUtilizedCpu = maxUnderUtilizedCpu; + this.maxOverUtilizedCpu = maxOverUtilizedCpu; + this.maxUnderUtilizedMemBytes = maxUnderUtilizedMemBytes; + this.maxUnderUtilizedCpuRequestId = maxUnderUtilizedCpuRequestId; + this.maxOverUtilizedCpuRequestId = maxOverUtilizedCpuRequestId; + this.maxUnderUtilizedMemBytesRequestId = maxUnderUtilizedMemBytesRequestId; + this.minUnderUtilizedCpu = minUnderUtilizedCpu; + this.minOverUtilizedCpu = minOverUtilizedCpu; + this.minUnderUtilizedMemBytes = minUnderUtilizedMemBytes; this.totalMemBytesUsed = totalMemBytesUsed; this.totalMemBytesAvailable = totalMemBytesAvailable; this.totalCpuUsed = totalCpuUsed; @@ -23,6 +91,82 @@ public SingularityClusterUtilization(@JsonProperty("totalMemBytesUsed") long tot this.timestamp = timestamp; } + public List getRequestUtilizations() { + return requestUtilizations; + } + + public int getNumRequestsWithUnderUtilizedCpu() { + return numRequestsWithUnderUtilizedCpu; + } + + public int getNumRequestsWithOverUtilizedCpu() { + return numRequestsWithOverUtilizedCpu; + } + + public int getNumRequestsWithUnderUtilizedMemBytes() { + return numRequestsWithUnderUtilizedMemBytes; + } + + public double getTotalUnderUtilizedCpu() { + return totalUnderUtilizedCpu; + } + + public double getTotalOverUtilizedCpu() { + return totalOverUtilizedCpu; + } + + public long getTotalUnderUtilizedMemBytes() { + return totalUnderUtilizedMemBytes; + } + + public double getAvgUnderUtilizedCpu() { + return avgUnderUtilizedCpu; + } + + public double getAvgOverUtilizedCpu() { + return avgOverUtilizedCpu; + } + + public long getAvgUnderUtilizedMemBytes() { + return avgUnderUtilizedMemBytes; + } + + public double getMaxUnderUtilizedCpu() { + return maxUnderUtilizedCpu; + } + + public double getMaxOverUtilizedCpu() { + return maxOverUtilizedCpu; + } + + public long getMaxUnderUtilizedMemBytes() { + return maxUnderUtilizedMemBytes; + } + + public String getMaxUnderUtilizedCpuRequestId() { + return maxUnderUtilizedCpuRequestId; + } + + public String getMaxOverUtilizedCpuRequestId() { + return maxOverUtilizedCpuRequestId; + } + + public String getMaxUnderUtilizedMemBytesRequestId() { + return maxUnderUtilizedMemBytesRequestId; + } + + public double getMinUnderUtilizedCpu() { + return minUnderUtilizedCpu; + } + + public double getMinOverUtilizedCpu() { + return minOverUtilizedCpu; + } + + public long getMinUnderUtilizedMemBytes() { + return minUnderUtilizedMemBytes; + } + public long getTotalMemBytesUsed() { return totalMemBytesUsed; } @@ -46,12 +190,30 @@ public long getTimestamp() { @Override public String toString() { return "SingularityClusterUtilization [" + - "totalMemBytesUsed=" + totalMemBytesUsed + + ", requestUtilizations=" + requestUtilizations + + ", numRequestsWithUnderUtilizedCpu=" + numRequestsWithUnderUtilizedCpu + + ", numRequestsWithOverUtilizedCpu=" + numRequestsWithOverUtilizedCpu + + ", numRequestsWithUnderUtilizedMemBytes=" + numRequestsWithUnderUtilizedMemBytes + + ", totalUnderUtilizedCpu=" + totalUnderUtilizedCpu + + ", totalOverUtilizedCpu=" + totalOverUtilizedCpu + + ", totalUnderUtilizedMemBytes=" + totalUnderUtilizedMemBytes + + ", avgUnderUtilizedCpu=" + avgUnderUtilizedCpu + + ", avgOverUtilizedCpu=" + avgOverUtilizedCpu + + ", avgUnderUtilizedMemBytes=" + avgUnderUtilizedMemBytes + + ", maxUnderUtilizedCpu=" + maxUnderUtilizedCpu + + ", maxOverUtilizedCpu=" + maxOverUtilizedCpu + + ", maxUnderUtilizedMemBytes=" + maxUnderUtilizedMemBytes + + ", maxUnderUtilizedCpuRequestId=" + maxUnderUtilizedCpuRequestId + + ", maxOverUtilizedCpuRequestId=" + maxOverUtilizedCpuRequestId + + ", maxUnderUtilizedMemBytesRequestId=" + maxUnderUtilizedMemBytesRequestId + + ", minUnderUtilizedCpu=" + minUnderUtilizedCpu + + ", minOverUtilizedCpu=" + minOverUtilizedCpu + + ", minUnderUtilizedMemBytes=" + minUnderUtilizedMemBytes + + ", totalMemBytesUsed=" + totalMemBytesUsed + ", totalMemBytesAvailable=" + totalMemBytesAvailable + ", totalCpuUsed=" + totalCpuUsed + ", totalCpuAvailable=" + totalCpuAvailable + ", timestamp=" + timestamp + "]"; - } } diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/SingularitySlaveUsage.java b/SingularityBase/src/main/java/com/hubspot/singularity/SingularitySlaveUsage.java index e2168ec9e9..b6c6cfd5b9 100644 --- a/SingularityBase/src/main/java/com/hubspot/singularity/SingularitySlaveUsage.java +++ b/SingularityBase/src/main/java/com/hubspot/singularity/SingularitySlaveUsage.java @@ -12,7 +12,7 @@ public enum ResourceUsageType { CPU_USED, MEMORY_BYTES_USED, CPU_FREE, MEMORY_BYTES_FREE } - public static final long BYTES_PER_MEGABYTE = 1024L * 1024L; + public static final long BYTES_PER_MEGABYTE = 1000L * 1000L; private final long memoryBytesUsed; private final long memoryMbReserved; diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityTaskCurrentUsage.java b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityTaskCurrentUsage.java index b49fe707fb..4c778bc4a2 100644 --- a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityTaskCurrentUsage.java +++ b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityTaskCurrentUsage.java @@ -5,19 +5,19 @@ public class SingularityTaskCurrentUsage { - private final long memoryRssBytes; + private final long memoryTotalBytes; private final long timestamp; private final double cpusUsed; @JsonCreator - public SingularityTaskCurrentUsage(@JsonProperty("memoryRssBytes") long memoryRssBytes, @JsonProperty("long") long timestamp, @JsonProperty("cpusUsed") double cpusUsed) { - this.memoryRssBytes = memoryRssBytes; + public SingularityTaskCurrentUsage(@JsonProperty("memoryTotalBytes") long memoryTotalBytes, @JsonProperty("long") long timestamp, @JsonProperty("cpusUsed") double cpusUsed) { + this.memoryTotalBytes = memoryTotalBytes; this.timestamp = timestamp; this.cpusUsed = cpusUsed; } - public long getMemoryRssBytes() { - return memoryRssBytes; + public long getMemoryTotalBytes() { + return memoryTotalBytes; } public long getTimestamp() { @@ -30,7 +30,7 @@ public double getCpusUsed() { @Override public String toString() { - return "SingularityTaskCurrentUsage [memoryRssBytes=" + memoryRssBytes + ", timestamp=" + timestamp + ", cpusUsed=" + cpusUsed + "]"; + return "SingularityTaskCurrentUsage [memoryTotalBytes=" + memoryTotalBytes + ", timestamp=" + timestamp + ", cpusUsed=" + cpusUsed + "]"; } } diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityTaskCurrentUsageWithId.java b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityTaskCurrentUsageWithId.java index b07d8db927..21db2af008 100644 --- a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityTaskCurrentUsageWithId.java +++ b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityTaskCurrentUsageWithId.java @@ -5,7 +5,7 @@ public class SingularityTaskCurrentUsageWithId extends SingularityTaskCurrentUsa private final SingularityTaskId taskId; public SingularityTaskCurrentUsageWithId(SingularityTaskId taskId, SingularityTaskCurrentUsage taskCurrentUsage) { - super(taskCurrentUsage.getMemoryRssBytes(), taskCurrentUsage.getTimestamp(), taskCurrentUsage.getCpusUsed()); + super(taskCurrentUsage.getMemoryTotalBytes(), taskCurrentUsage.getTimestamp(), taskCurrentUsage.getCpusUsed()); this.taskId = taskId; } diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityTaskUsage.java b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityTaskUsage.java index 16951d0edb..82313b8132 100644 --- a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityTaskUsage.java +++ b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityTaskUsage.java @@ -5,19 +5,19 @@ public class SingularityTaskUsage { - private final long memoryRssBytes; - private final double timestamp; + private final long memoryTotalBytes; + private final double timestamp; // seconds private final double cpuSeconds; @JsonCreator - public SingularityTaskUsage(@JsonProperty("memoryRssBytes") long memoryRssBytes, @JsonProperty("timestamp") double timestamp, @JsonProperty("cpuSeconds") double cpuSeconds) { - this.memoryRssBytes = memoryRssBytes; + public SingularityTaskUsage(@JsonProperty("memoryTotalBytes") long memoryTotalBytes, @JsonProperty("timestamp") double timestamp, @JsonProperty("cpuSeconds") double cpuSeconds) { + this.memoryTotalBytes = memoryTotalBytes; this.timestamp = timestamp; this.cpuSeconds = cpuSeconds; } - public long getMemoryRssBytes() { - return memoryRssBytes; + public long getMemoryTotalBytes() { + return memoryTotalBytes; } public double getTimestamp() { @@ -30,7 +30,7 @@ public double getCpuSeconds() { @Override public String toString() { - return "SingularityTaskUsage [memoryRssBytes=" + memoryRssBytes + ", timestamp=" + timestamp + ", cpuSeconds=" + cpuSeconds + "]"; + return "SingularityTaskUsage [memoryTotalBytes=" + memoryTotalBytes + ", timestamp=" + timestamp + ", cpuSeconds=" + cpuSeconds + "]"; } } diff --git a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java index cf511b9f52..d671a01c77 100644 --- a/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java +++ b/SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java @@ -44,6 +44,7 @@ import com.hubspot.singularity.SingularityAction; import com.hubspot.singularity.SingularityAuthorizationScope; import com.hubspot.singularity.SingularityClientCredentials; +import com.hubspot.singularity.SingularityClusterUtilization; import com.hubspot.singularity.SingularityCreateResult; import com.hubspot.singularity.SingularityDeleteResult; import com.hubspot.singularity.SingularityDeploy; @@ -103,6 +104,9 @@ public class SingularityClient { private static final String STATE_FORMAT = "%s/state"; private static final String TASK_RECONCILIATION_FORMAT = STATE_FORMAT + "/task-reconciliation"; + private static final String USAGE_FORMAT = "%s/usage"; + private static final String CLUSTER_UTILIZATION_FORMAT = USAGE_FORMAT + "/cluster/utilization"; + private static final String RACKS_FORMAT = "%s/racks"; private static final String RACKS_DECOMISSION_FORMAT = RACKS_FORMAT + "/rack/%s/decommission"; private static final String RACKS_FREEZE_FORMAT = RACKS_FORMAT + "/rack/%s/freeze"; @@ -535,6 +539,12 @@ public Optional getTaskReconciliationSt return Optional.of(response.getAs(SingularityTaskReconciliationStatistics.class)); } + public Optional getClusterUtilization() { + final Function uri = (host) -> String.format(CLUSTER_UTILIZATION_FORMAT, getApiBase(host)); + + return getSingle(uri, "clusterUtilization", "", SingularityClusterUtilization.class); + } + // // ACTIONS ON A SINGLE SINGULARITY REQUEST // diff --git a/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java b/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java index b9b1494710..1bd201a686 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java @@ -73,9 +73,11 @@ public class SingularityConfiguration extends Configuration { private long checkUsageEveryMillis = TimeUnit.MINUTES.toMillis(1); + private int usageIntervalSeconds = 5760; // 15 saved each 5760 seconds (96 min) apart is 1 day of usage + private long cleanUsageEveryMillis = TimeUnit.MINUTES.toMillis(5); - private int numUsageToKeep = 5; + private int numUsageToKeep = 15; private long cleanupEverySeconds = 5; @@ -1419,6 +1421,15 @@ public void setCheckUsageEveryMillis(long checkUsageEveryMillis) { this.checkUsageEveryMillis = checkUsageEveryMillis; } + public int getUsageIntervalSeconds() { + return usageIntervalSeconds; + } + + public SingularityConfiguration setUsageIntervalSeconds(int usageIntervalSeconds) { + this.usageIntervalSeconds = usageIntervalSeconds; + return this; + } + public long getCleanUsageEveryMillis() { return cleanUsageEveryMillis; } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/data/UsageManager.java b/SingularityService/src/main/java/com/hubspot/singularity/data/UsageManager.java index dc18620458..fb6c5a77d7 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/data/UsageManager.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/data/UsageManager.java @@ -175,6 +175,14 @@ public List getTaskUsage(String taskId) { return children; } + private static final Comparator TASK_USAGE_PATH_COMPARATOR_TIMESTAMP_ASC = Comparator.comparingDouble(Double::parseDouble); + + public List getTaskUsagePaths(String taskId) { + List children = getChildren(getTaskUsageHistoryPath(taskId)); + children.sort(TASK_USAGE_PATH_COMPARATOR_TIMESTAMP_ASC); + return children; + } + public Optional getClusterUtilization() { return getData(USAGE_SUMMARY_PATH, clusterUtilizationTranscoder); } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/resources/UsageResource.java b/SingularityService/src/main/java/com/hubspot/singularity/resources/UsageResource.java index 3ec1aa45d8..fd9fba6a0e 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/resources/UsageResource.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/resources/UsageResource.java @@ -11,6 +11,7 @@ import com.google.common.base.Optional; import com.google.inject.Inject; +import com.hubspot.singularity.SingularityClusterUtilization; import com.hubspot.singularity.SingularityService; import com.hubspot.singularity.SingularitySlave; import com.hubspot.singularity.SingularitySlaveUsage; @@ -78,4 +79,11 @@ public List getTaskUsageHistory(@PathParam("taskId") Strin return usageManager.getTaskUsage(taskId); } + @GET + @Path("/cluster/utilization") + public SingularityClusterUtilization getClusterUtilization() { + WebExceptions.checkNotFound(usageManager.getClusterUtilization().isPresent(), "No cluster utilization has been saved yet"); + + return usageManager.getClusterUtilization().get(); + } } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsageCleanerPoller.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsageCleanerPoller.java index 35dc96c9a8..ad79666b0d 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsageCleanerPoller.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsageCleanerPoller.java @@ -65,5 +65,4 @@ private void deleteObsoleteTaskUsage() { LOG.debug("Deleted obsolete task usage {} - {}", taskIdWithUsage, result); } } - } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java index a86763f7c3..4d250efb99 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java @@ -1,19 +1,25 @@ package com.hubspot.singularity.scheduler; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Optional; import com.google.inject.Inject; import com.hubspot.mesos.client.MesosClient; import com.hubspot.mesos.json.MesosTaskMonitorObject; import com.hubspot.singularity.InvalidSingularityTaskIdException; +import com.hubspot.singularity.RequestUtilization; import com.hubspot.singularity.SingularityClusterUtilization; +import com.hubspot.singularity.SingularityDeleteResult; +import com.hubspot.singularity.SingularityDeploy; import com.hubspot.singularity.SingularityDeployStatistics; import com.hubspot.singularity.SingularityRequestWithState; import com.hubspot.singularity.SingularitySlave; @@ -66,7 +72,9 @@ public class SingularityUsagePoller extends SingularityLeaderOnlyPoller { @Override public void runActionOnPoll() { + Map utilizationPerRequestId = new HashMap<>(); final long now = System.currentTimeMillis(); + long totalMemBytesUsed = 0; long totalMemBytesAvailable = 0; double totalCpuUsed = 0.00; @@ -78,10 +86,10 @@ public void runActionOnPoll() { longRunningTasksUsage.put(ResourceUsageType.CPU_USED, 0); Optional memoryMbTotal = Optional.absent(); Optional cpusTotal = Optional.absent(); - long memoryMbReserved = 0; - long cpuReserved = 0; - long memoryBytesUsed = 0; - double cpusUsed = 0; + long memoryMbReservedOnSlave = 0; + double cpuReservedOnSlave = 0; + long memoryBytesUsedOnSlave = 0; + double cpusUsedOnSlave = 0; try { List allTaskUsage = mesosClient.getSlaveResourceUsage(slave.getHost()); @@ -96,35 +104,36 @@ public void runActionOnPoll() { continue; } - SingularityTaskUsage usage = getUsage(taskUsage); + SingularityTaskUsage latestUsage = getUsage(taskUsage); + List pastTaskUsages = usageManager.getTaskUsage(taskId); - List taskUsages = usageManager.getTaskUsage(taskId); - if (taskUsages.size() + 1 > configuration.getNumUsageToKeep()) { - usageManager.deleteSpecificTaskUsage(taskId, taskUsages.get(0).getTimestamp()); - } - usageManager.saveSpecificTaskUsage(taskId, usage); + clearOldUsage(taskId); + usageManager.saveSpecificTaskUsage(taskId, latestUsage); Optional maybeTask = taskManager.getTask(task); if (maybeTask.isPresent() && maybeTask.get().getTaskRequest().getDeploy().getResources().isPresent()) { - memoryMbReserved += maybeTask.get().getTaskRequest().getDeploy().getResources().get().getMemoryMb(); - cpuReserved += maybeTask.get().getTaskRequest().getDeploy().getResources().get().getCpus(); + double memoryMbReservedForTask = maybeTask.get().getTaskRequest().getDeploy().getResources().get().getMemoryMb(); + double cpuReservedForTask = maybeTask.get().getTaskRequest().getDeploy().getResources().get().getCpus(); + memoryMbReservedOnSlave += memoryMbReservedForTask; + cpuReservedOnSlave += cpuReservedForTask; + updateRequestUtilization(utilizationPerRequestId, pastTaskUsages, latestUsage, task, memoryMbReservedForTask, cpuReservedForTask); } - memoryBytesUsed += usage.getMemoryRssBytes(); + memoryBytesUsedOnSlave += latestUsage.getMemoryTotalBytes(); - if (!taskUsages.isEmpty()) { - SingularityTaskUsage lastUsage = taskUsages.get(taskUsages.size() - 1); + if (!pastTaskUsages.isEmpty()) { + SingularityTaskUsage lastUsage = pastTaskUsages.get(pastTaskUsages.size() - 1); - double taskCpusUsed = ((usage.getCpuSeconds() - lastUsage.getCpuSeconds()) / (usage.getTimestamp() - lastUsage.getTimestamp())); + double taskCpusUsed = ((latestUsage.getCpuSeconds() - lastUsage.getCpuSeconds()) / (latestUsage.getTimestamp() - lastUsage.getTimestamp())); if (isLongRunning(task) || isConsideredLongRunning(task)) { - updateLongRunningTasksUsage(longRunningTasksUsage, usage.getMemoryRssBytes(), taskCpusUsed); + updateLongRunningTasksUsage(longRunningTasksUsage, latestUsage.getMemoryTotalBytes(), taskCpusUsed); } - SingularityTaskCurrentUsage currentUsage = new SingularityTaskCurrentUsage(usage.getMemoryRssBytes(), now, taskCpusUsed); + SingularityTaskCurrentUsage currentUsage = new SingularityTaskCurrentUsage(latestUsage.getMemoryTotalBytes(), now, taskCpusUsed); usageManager.saveCurrentTaskUsage(taskId, currentUsage); - cpusUsed += taskCpusUsed; + cpusUsedOnSlave += taskCpusUsed; } } @@ -137,7 +146,7 @@ public void runActionOnPoll() { cpusTotal = Optional.of(slave.getResources().get().getNumCpus().get().doubleValue()); } - SingularitySlaveUsage slaveUsage = new SingularitySlaveUsage(memoryBytesUsed, memoryMbReserved, now, cpusUsed, cpuReserved, allTaskUsage.size(), memoryMbTotal, cpusTotal, longRunningTasksUsage); + SingularitySlaveUsage slaveUsage = new SingularitySlaveUsage(memoryBytesUsedOnSlave, memoryMbReservedOnSlave, now, cpusUsedOnSlave, cpuReservedOnSlave, allTaskUsage.size(), memoryMbTotal, cpusTotal, longRunningTasksUsage); List slaveTimestamps = usageManager.getSlaveUsageTimestamps(slave.getId()); if (slaveTimestamps.size() + 1 > configuration.getNumUsageToKeep()) { usageManager.deleteSpecificSlaveUsage(slave.getId(), slaveTimestamps.get(0)); @@ -158,15 +167,14 @@ public void runActionOnPoll() { LOG.error(message, e); exceptionNotifier.notify(message, e); } - - usageManager.saveClusterUtilization(new SingularityClusterUtilization(totalMemBytesUsed, totalMemBytesAvailable, totalCpuUsed, totalCpuAvailable, now)); } + usageManager.saveClusterUtilization(getClusterUtilization(utilizationPerRequestId, totalMemBytesUsed, totalMemBytesAvailable, totalCpuUsed, totalCpuAvailable, now)); } private SingularityTaskUsage getUsage(MesosTaskMonitorObject taskUsage) { double cpuSeconds = taskUsage.getStatistics().getCpusSystemTimeSecs() + taskUsage.getStatistics().getCpusUserTimeSecs(); - return new SingularityTaskUsage(taskUsage.getStatistics().getMemRssBytes(), taskUsage.getStatistics().getTimestamp(), cpuSeconds); + return new SingularityTaskUsage(taskUsage.getStatistics().getMemTotalBytes(), taskUsage.getStatistics().getTimestampSeconds(), cpuSeconds); } private boolean isLongRunning(SingularityTaskId task) { @@ -190,4 +198,172 @@ private void updateLongRunningTasksUsage(Map longRunn longRunningTasksUsage.compute(ResourceUsageType.MEMORY_BYTES_USED, (k, v) -> (v == null) ? memBytesUsed : v.longValue() + memBytesUsed); longRunningTasksUsage.compute(ResourceUsageType.CPU_USED, (k, v) -> (v == null) ? cpuUsed : v.doubleValue() + cpuUsed); } + + private void updateRequestUtilization(Map utilizationPerRequestId, List pastTaskUsages, SingularityTaskUsage latestUsage, SingularityTaskId task, double memoryMbReservedForTask, double cpuReservedForTask) { + String requestId = task.getRequestId(); + RequestUtilization requestUtilization = utilizationPerRequestId.getOrDefault(requestId, new RequestUtilization(requestId, task.getDeployId())); + long curMaxMemBytesUsed = 0; + long curMinMemBytesUsed = Long.MAX_VALUE; + double curMaxCpuUsed = 0; + double curMinCpuUsed = Double.MAX_VALUE; + + if (utilizationPerRequestId.containsKey(requestId)) { + curMaxMemBytesUsed = requestUtilization.getMaxMemBytesUsed(); + curMinMemBytesUsed = requestUtilization.getMinMemBytesUsed(); + curMaxCpuUsed = requestUtilization.getMaxCpuUsed(); + curMinCpuUsed = requestUtilization.getMinCpuUsed(); + } + + List pastTaskUsagesCopy = copyUsages(pastTaskUsages, latestUsage, task); + int numTasks = pastTaskUsagesCopy.size() - 1; + + for (int i = 0; i < numTasks; i++) { + SingularityTaskUsage olderUsage = pastTaskUsagesCopy.get(i); + SingularityTaskUsage newerUsage = pastTaskUsagesCopy.get(i + 1); + double cpusUsed = (newerUsage.getCpuSeconds() - olderUsage.getCpuSeconds()) / (newerUsage.getTimestamp() - olderUsage.getTimestamp()); + + curMaxCpuUsed = Math.max(cpusUsed, curMaxCpuUsed); + curMinCpuUsed = Math.min(cpusUsed, curMinCpuUsed); + curMaxMemBytesUsed = Math.max(newerUsage.getMemoryTotalBytes(), curMaxMemBytesUsed); + curMinMemBytesUsed = Math.min(newerUsage.getMemoryTotalBytes(), curMinMemBytesUsed); + + requestUtilization + .addCpuUsed(cpusUsed) + .addMemBytesUsed(newerUsage.getMemoryTotalBytes()) + .incrementTaskCount(); + } + + requestUtilization + .addMemBytesReserved((long) (memoryMbReservedForTask * SingularitySlaveUsage.BYTES_PER_MEGABYTE * numTasks)) + .addCpuReserved(cpuReservedForTask * numTasks) + .setMaxCpuUsed(curMaxCpuUsed) + .setMinCpuUsed(curMinCpuUsed) + .setMaxMemBytesUsed(curMaxMemBytesUsed) + .setMinMemBytesUsed(curMinMemBytesUsed); + + utilizationPerRequestId.put(requestId, requestUtilization); + } + + private List copyUsages(List pastTaskUsages, SingularityTaskUsage latestUsage, SingularityTaskId task) { + List pastTaskUsagesCopy = new ArrayList<>(); + pastTaskUsagesCopy.add(new SingularityTaskUsage(0, TimeUnit.MILLISECONDS.toSeconds(task.getStartedAt()), 0)); // to calculate oldest cpu usage + pastTaskUsagesCopy.addAll(pastTaskUsages); + pastTaskUsagesCopy.add(latestUsage); + + return pastTaskUsagesCopy; + } + + private SingularityClusterUtilization getClusterUtilization(Map utilizationPerRequestId, long totalMemBytesUsed, long totalMemBytesAvailable, double totalCpuUsed, double totalCpuAvailable, long now) { + int numRequestsWithUnderUtilizedCpu = 0; + int numRequestsWithOverUtilizedCpu = 0; + int numRequestsWithUnderUtilizedMemBytes = 0; + + double totalUnderUtilizedCpu = 0; + double totalOverUtilizedCpu = 0; + long totalUnderUtilizedMemBytes = 0; + + double maxUnderUtilizedCpu = 0; + double maxOverUtilizedCpu = 0; + long maxUnderUtilizedMemBytes = 0; + + String maxUnderUtilizedCpuRequestId = null; + String maxOverUtilizedCpuRequestId = null; + String maxUnderUtilizedMemBytesRequestId = null; + + double minUnderUtilizedCpu = Double.MAX_VALUE; + double minOverUtilizedCpu = Double.MAX_VALUE; + long minUnderUtilizedMemBytes = Long.MAX_VALUE; + + + for (RequestUtilization utilization : utilizationPerRequestId.values()) { + Optional maybeDeploy = deployManager.getDeploy(utilization.getRequestId(), utilization.getDeployId()); + + if (maybeDeploy.isPresent() && maybeDeploy.get().getResources().isPresent()) { + String requestId = utilization.getRequestId(); + long memoryBytesReserved = (long) (maybeDeploy.get().getResources().get().getMemoryMb() * SingularitySlaveUsage.BYTES_PER_MEGABYTE); + double cpuReserved = maybeDeploy.get().getResources().get().getCpus(); + + double unusedCpu = cpuReserved - utilization.getAvgCpuUsed(); + long unusedMemBytes = (long) (memoryBytesReserved - utilization.getAvgMemBytesUsed()); + + if (unusedCpu > 0) { + numRequestsWithUnderUtilizedCpu++; + totalUnderUtilizedCpu += unusedCpu; + if (unusedCpu > maxUnderUtilizedCpu) { + maxUnderUtilizedCpu = unusedCpu; + maxUnderUtilizedCpuRequestId = requestId; + } + minUnderUtilizedCpu = Math.min(unusedCpu, minUnderUtilizedCpu); + } else if (unusedCpu < 0) { + double overusedCpu = Math.abs(unusedCpu); + + numRequestsWithOverUtilizedCpu++; + totalOverUtilizedCpu += overusedCpu; + if (overusedCpu > maxOverUtilizedCpu) { + maxOverUtilizedCpu = overusedCpu; + maxOverUtilizedCpuRequestId = requestId; + } + minOverUtilizedCpu = Math.min(overusedCpu, minOverUtilizedCpu); + } + + if (unusedMemBytes > 0) { + numRequestsWithUnderUtilizedMemBytes++; + totalUnderUtilizedMemBytes += unusedMemBytes; + if (unusedMemBytes > maxUnderUtilizedMemBytes) { + maxUnderUtilizedMemBytes = unusedMemBytes; + maxUnderUtilizedMemBytesRequestId = requestId; + } + minUnderUtilizedMemBytes = Math.min(unusedMemBytes, minUnderUtilizedMemBytes); + } + } + } + + double avgUnderUtilizedCpu = numRequestsWithUnderUtilizedCpu != 0 ? totalUnderUtilizedCpu / numRequestsWithUnderUtilizedCpu : 0; + double avgOverUtilizedCpu = numRequestsWithOverUtilizedCpu != 0? totalOverUtilizedCpu / numRequestsWithOverUtilizedCpu : 0; + long avgUnderUtilizedMemBytes = numRequestsWithUnderUtilizedMemBytes != 0 ? totalUnderUtilizedMemBytes / numRequestsWithUnderUtilizedMemBytes : 0; + + return new SingularityClusterUtilization(new ArrayList<>(utilizationPerRequestId.values()), numRequestsWithUnderUtilizedCpu, numRequestsWithOverUtilizedCpu, + numRequestsWithUnderUtilizedMemBytes, totalUnderUtilizedCpu, totalOverUtilizedCpu, totalUnderUtilizedMemBytes, avgUnderUtilizedCpu, avgOverUtilizedCpu, + avgUnderUtilizedMemBytes, maxUnderUtilizedCpu, maxOverUtilizedCpu, maxUnderUtilizedMemBytes, maxUnderUtilizedCpuRequestId, maxOverUtilizedCpuRequestId, + maxUnderUtilizedMemBytesRequestId, getMin(minUnderUtilizedCpu), getMin(minOverUtilizedCpu), getMin(minUnderUtilizedMemBytes), totalMemBytesUsed, + totalMemBytesAvailable, totalCpuUsed, totalCpuAvailable, now); + } + + private double getMin(double value) { + return value == Double.MAX_VALUE ? 0 : value; + } + + private long getMin(long value) { + return value == Long.MAX_VALUE ? 0 : value; + } + + @VisibleForTesting + void clearOldUsage(String taskId) { + List pastTaskUsagePaths = usageManager.getTaskUsagePaths(taskId).stream().map(Double::parseDouble).collect(Collectors.toList()); + + while (pastTaskUsagePaths.size() + 1 > configuration.getNumUsageToKeep()) { + long minSecondsApart = configuration.getUsageIntervalSeconds(); + boolean deleted = false; + + for (int i = 0; i < pastTaskUsagePaths.size() - 1; i++) { + if (pastTaskUsagePaths.get(i + 1) - pastTaskUsagePaths.get(i) < minSecondsApart) { + SingularityDeleteResult result = usageManager.deleteSpecificTaskUsage(taskId, pastTaskUsagePaths.get(i + 1)); + + if (result.equals(SingularityDeleteResult.DIDNT_EXIST)) { + LOG.warn("Didn't delete taskUsage {} for taskId {}", pastTaskUsagePaths.get(i + 1).toString(), taskId); + } + + deleted = true; + pastTaskUsagePaths.remove(pastTaskUsagePaths.get(i + 1)); + break; + } + } + + if (!deleted) { + usageManager.deleteSpecificTaskUsage(taskId, pastTaskUsagePaths.get(0)); + pastTaskUsagePaths.remove(pastTaskUsagePaths.get(0)); + } + } + + } } diff --git a/SingularityService/src/test/java/com/hubspot/singularity/mesos/SingularityMesosOfferSchedulerTest.java b/SingularityService/src/test/java/com/hubspot/singularity/mesos/SingularityMesosOfferSchedulerTest.java index 35227e5989..e4c11a02be 100644 --- a/SingularityService/src/test/java/com/hubspot/singularity/mesos/SingularityMesosOfferSchedulerTest.java +++ b/SingularityService/src/test/java/com/hubspot/singularity/mesos/SingularityMesosOfferSchedulerTest.java @@ -289,7 +289,7 @@ private void assertValueIs(double expectedValue, double actualValue) { } private long mbToBytes(long memMb) { - return memMb * 1024L * 1024L; + return memMb * 1000L * 1000L; } private SingularitySlaveUsageWithId getUsage(long memMbReserved, long memMbTotal, double cpusReserved, double cpusTotal, Map longRunningTasksUsage) { diff --git a/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularityHealthchecksTest.java b/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularityHealthchecksTest.java index a0681b9bb1..2b68636fc4 100644 --- a/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularityHealthchecksTest.java +++ b/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularityHealthchecksTest.java @@ -223,7 +223,7 @@ public void testNewTaskCheckerRespectsDeployHealthcheckRetries() { HealthcheckOptions options = new HealthcheckOptionsBuilder("http://uri").setMaxRetries(Optional.of(1)).build(); SingularityDeployBuilder db = new SingularityDeployBuilder(requestId, deployId).setHealthcheck(Optional.of(options)); - SingularityDeploy deploy = initAndFinishDeploy(request, db); + SingularityDeploy deploy = initAndFinishDeploy(request, db, Optional.absent()); SingularityTask task = launchTask(request, deploy, System.currentTimeMillis(), 1, TaskState.TASK_RUNNING); @@ -369,9 +369,8 @@ public void testPortIndices() { setConfigurationForNoDelay(); initRequest(); HealthcheckOptions options = new HealthcheckOptionsBuilder("http://uri").setPortIndex(Optional.of(1)).build(); - firstDeploy = initAndFinishDeploy(request, new SingularityDeployBuilder(request.getId(), firstDeployId) - .setCommand(Optional.of("sleep 100")).setResources(Optional.of(new Resources(1, 64, 3, 0))) - .setHealthcheck(Optional.of(options))); + firstDeploy = initAndFinishDeploy(request, new SingularityDeployBuilder(request.getId(), firstDeployId).setCommand(Optional.of("sleep 100")) + .setHealthcheck(Optional.of(options)), Optional.of(new Resources(1, 64, 3, 0))); requestResource.postRequest(request.toBuilder().setInstances(Optional.of(2)).build()); scheduler.drainPendingQueue(stateCacheProvider.get()); @@ -404,7 +403,7 @@ public void testPortNumber() { HealthcheckOptions options = new HealthcheckOptionsBuilder("http://uri").setPortNumber(Optional.of(81L)).build(); firstDeploy = initAndFinishDeploy(request, new SingularityDeployBuilder(request.getId(), firstDeployId) .setCommand(Optional.of("sleep 100")).setResources(Optional.of(new Resources(1, 64, 3, 0))) - .setHealthcheck(Optional.of(options))); + .setHealthcheck(Optional.of(options)), Optional.absent()); requestResource.postRequest(request.toBuilder().setInstances(Optional.of(2)).build()); scheduler.drainPendingQueue(stateCacheProvider.get()); diff --git a/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySchedulerTest.java b/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySchedulerTest.java index b74faef24c..30f3c9a158 100644 --- a/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySchedulerTest.java +++ b/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySchedulerTest.java @@ -208,7 +208,7 @@ public void testLeftoverCachedOffersAreReturnedToCache() throws Exception { initRequest(); firstDeploy = initAndFinishDeploy(request, new SingularityDeployBuilder(request.getId(), firstDeployId) - .setCommand(Optional.of("sleep 100")).setResources(Optional.of(new Resources(1, 128, 2, 0))) + .setCommand(Optional.of("sleep 100")), Optional.of(new Resources(1, 128, 2, 0)) ); requestManager.addToPendingQueue( @@ -1862,10 +1862,10 @@ public void testMaxTasksPerOffer() { @Test public void testRequestedPorts() { - final SingularityDeployBuilder deployBuilder = dockerDeployWithPorts(3); + final SingularityDeployBuilder deployBuilder = dockerDeployWithPorts(); initRequest(); - initAndFinishDeploy(request, deployBuilder); + initAndFinishDeploy(request, deployBuilder, Optional.of(new Resources(1, 64, 3, 0))); requestResource.postRequest(request.toBuilder().setInstances(Optional.of(2)).build()); scheduler.drainPendingQueue(stateCacheProvider.get()); @@ -1886,7 +1886,7 @@ public void testRequestedPorts() { Assert.assertEquals(1, taskManager.getActiveTaskIds().size()); } - private SingularityDeployBuilder dockerDeployWithPorts(int numPorts) { + private SingularityDeployBuilder dockerDeployWithPorts() { final SingularityDockerPortMapping literalMapping = new SingularityDockerPortMapping(Optional.absent(), 80, Optional.of(SingularityPortMappingType.LITERAL), 8080, Optional.absent()); final SingularityDockerPortMapping offerMapping = new SingularityDockerPortMapping(Optional.absent(), 81, Optional.of(SingularityPortMappingType.FROM_OFFER), 0, Optional.of("udp")); final SingularityContainerInfo containerInfo = new SingularityContainerInfo( @@ -1901,7 +1901,7 @@ private SingularityDeployBuilder dockerDeployWithPorts(int numPorts) { Optional.>of(ImmutableMap.of("env", "var=value"))) )); final SingularityDeployBuilder deployBuilder = new SingularityDeployBuilder(requestId, "test-docker-ports-deploy"); - deployBuilder.setContainerInfo(Optional.of(containerInfo)).setResources(Optional.of(new Resources(1, 64, numPorts, 0))); + deployBuilder.setContainerInfo(Optional.of(containerInfo)); return deployBuilder; } diff --git a/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySchedulerTestBase.java b/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySchedulerTestBase.java index 5938477c9e..90232685f5 100644 --- a/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySchedulerTestBase.java +++ b/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySchedulerTestBase.java @@ -524,8 +524,12 @@ protected void initFirstDeploy() { firstDeploy = initAndFinishDeploy(request, firstDeployId); } + protected void initFirstDeployWithResources(double cpus, double memoryMb) { + firstDeploy = initAndFinishDeployWithResources(request, firstDeployId, cpus, memoryMb); + } + protected void initHCDeploy() { - firstDeploy = initAndFinishDeploy(request, new SingularityDeployBuilder(request.getId(), firstDeployId).setCommand(Optional.of("sleep 100")).setHealthcheck(Optional.of(new HealthcheckOptionsBuilder("http://uri").build()))); + firstDeploy = initAndFinishDeploy(request, new SingularityDeployBuilder(request.getId(), firstDeployId).setCommand(Optional.of("sleep 100")).setHealthcheck(Optional.of(new HealthcheckOptionsBuilder("http://uri").build())), Optional.absent()); } protected void initLoadBalancedDeploy() { @@ -533,15 +537,21 @@ protected void initLoadBalancedDeploy() { .setCommand(Optional.of("sleep 100")) .setServiceBasePath(Optional.of("/basepath")) .setLoadBalancerGroups(Optional.of(Collections.singleton("test"))); - firstDeploy = initAndFinishDeploy(request, builder); + firstDeploy = initAndFinishDeploy(request, builder, Optional.absent()); } protected SingularityDeploy initAndFinishDeploy(SingularityRequest request, String deployId) { - return initAndFinishDeploy(request, new SingularityDeployBuilder(request.getId(), deployId).setCommand(Optional.of("sleep 100"))); + return initAndFinishDeploy(request, new SingularityDeployBuilder(request.getId(), deployId).setCommand(Optional.of("sleep 100")), Optional.absent()); } - protected SingularityDeploy initAndFinishDeploy(SingularityRequest request, SingularityDeployBuilder builder) { - SingularityDeploy deploy = builder.build(); + protected SingularityDeploy initAndFinishDeployWithResources(SingularityRequest request, String deployId, double cpus, double memoryMb) { + Resources r = new Resources(cpus, memoryMb, 0); + + return initAndFinishDeploy(request, new SingularityDeployBuilder(request.getId(), deployId).setCommand(Optional.of("sleep 100")), Optional.of(r)); + } + + protected SingularityDeploy initAndFinishDeploy(SingularityRequest request, SingularityDeployBuilder builder, Optional maybeResources) { + SingularityDeploy deploy = builder.setResources(maybeResources).build(); SingularityDeployMarker marker = new SingularityDeployMarker(deploy.getRequestId(), deploy.getId(), System.currentTimeMillis(), Optional. absent(), Optional. absent()); diff --git a/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularityUsageTest.java b/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularityUsageTest.java index e6d2c34827..1f8fd4edcf 100644 --- a/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularityUsageTest.java +++ b/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularityUsageTest.java @@ -3,6 +3,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.concurrent.TimeUnit; import org.apache.mesos.Protos.TaskState; import org.junit.Assert; @@ -13,6 +14,8 @@ import com.hubspot.mesos.json.MesosTaskMonitorObject; import com.hubspot.mesos.json.MesosTaskStatisticsObject; import com.hubspot.singularity.MachineState; +import com.hubspot.singularity.SingularityClusterUtilization; +import com.hubspot.singularity.SingularitySlaveUsage; import com.hubspot.singularity.SingularityTask; import com.hubspot.singularity.SingularityTaskCurrentUsageWithId; import com.hubspot.singularity.SingularityTaskId; @@ -49,12 +52,9 @@ public void testUsagePollerSimple() { resourceOffers(1); SingularityTask firstTask = taskManager.getActiveTasks().get(0); - String hostname = firstTask.getHostname(); - MesosTaskMonitorObject usage = new MesosTaskMonitorObject(null, null, null, firstTask.getTaskId().getId(), getStatistics(2, 5, 100)); - + MesosTaskMonitorObject usage = getTaskMonitor(firstTask.getTaskId().getId(), 2, 5, 100); mesosClient.setSlaveResourceUsage(hostname, Collections.singletonList(usage)); - usagePoller.runActionOnPoll(); String slaveId = firstTask.getSlaveId().getValue(); @@ -70,7 +70,7 @@ public void testUsagePollerSimple() { SingularityTaskUsage first = usageManager.getTaskUsage(firstTask.getTaskId().getId()).get(0); Assert.assertEquals(2, first.getCpuSeconds(), 0); - Assert.assertEquals(100, first.getMemoryRssBytes(), 0); + Assert.assertEquals(100, first.getMemoryTotalBytes(), 0); Assert.assertEquals(5, first.getTimestamp(), 0); } @@ -89,8 +89,8 @@ public void testUsageCleaner() { String slaveId = slaveManager.getObjectIds().get(0); String host = slaveManager.getObjects().get(0).getHost(); - MesosTaskMonitorObject t1u1 = new MesosTaskMonitorObject(null, null, null, t1, getStatistics(2, 5, 100)); - MesosTaskMonitorObject t2u1 = new MesosTaskMonitorObject(null, null, null, t2, getStatistics(10, 5, 1000)); + MesosTaskMonitorObject t1u1 = getTaskMonitor(t1, 2, 5, 100); + MesosTaskMonitorObject t2u1 = getTaskMonitor(t2, 10, 5, 1000); mesosClient.setSlaveResourceUsage(host, Arrays.asList(t1u1, t2u1)); @@ -111,7 +111,7 @@ public void testUsageCleaner() { Assert.assertEquals(1, usageManager.getTasksWithUsage().size()); Assert.assertEquals(1, usageManager.getSlavesWithUsage().size()); - slaveManager.changeState(slaveId, MachineState.DEAD, Optional. absent(), Optional. absent()); + slaveManager.changeState(slaveId, MachineState.DEAD, Optional.absent(), Optional.absent()); cleaner.runActionOnPoll(); @@ -127,6 +127,8 @@ public void testUsagePollerComplex() throws InterruptedException { resourceOffers(1); configuration.setNumUsageToKeep(2); + configuration.setUsageIntervalSeconds(1); + configuration.setCheckUsageEveryMillis(1); List taskIds = taskManager.getActiveTaskIds(); @@ -136,8 +138,8 @@ public void testUsagePollerComplex() throws InterruptedException { String slaveId = slaveManager.getObjectIds().get(0); String host = slaveManager.getObjects().get(0).getHost(); - MesosTaskMonitorObject t1u1 = new MesosTaskMonitorObject(null, null, null, t1, getStatistics(2, 5, 100)); - MesosTaskMonitorObject t2u1 = new MesosTaskMonitorObject(null, null, null, t2, getStatistics(10, 5, 1000)); + MesosTaskMonitorObject t1u1 = getTaskMonitor(t1, 2, 5, 100); + MesosTaskMonitorObject t2u1 = getTaskMonitor(t2, 10, 5, 1000); mesosClient.setSlaveResourceUsage(host, Arrays.asList(t1u1, t2u1)); @@ -148,8 +150,8 @@ public void testUsagePollerComplex() throws InterruptedException { // 5 seconds have elapsed, t1 has used 1 CPU the whole time = 5 + 2 // t2 has used 2.5 CPUs the whole time = - MesosTaskMonitorObject t1u2 = new MesosTaskMonitorObject(null, null, null, t1, getStatistics(7, 10, 125)); - MesosTaskMonitorObject t2u2 = new MesosTaskMonitorObject(null, null, null, t2, getStatistics(22.5, 10, 750)); + MesosTaskMonitorObject t1u2 = getTaskMonitor(t1, 7, 10, 125); + MesosTaskMonitorObject t2u2 = getTaskMonitor(t2, 22.5, 10, 750); mesosClient.setSlaveResourceUsage(host, Arrays.asList(t1u2, t2u2)); @@ -167,8 +169,8 @@ public void testUsagePollerComplex() throws InterruptedException { Thread.sleep(2); - MesosTaskMonitorObject t1u3 = new MesosTaskMonitorObject(null, null, null, t1, getStatistics(8, 11, 125)); - MesosTaskMonitorObject t2u3 = new MesosTaskMonitorObject(null, null, null, t2, getStatistics(23.5, 11, 1000)); + MesosTaskMonitorObject t1u3 = getTaskMonitor(t1, 8, 11, 125); + MesosTaskMonitorObject t2u3 = getTaskMonitor(t2, 23.5, 11, 1000); mesosClient.setSlaveResourceUsage(host, Arrays.asList(t1u3, t2u3)); @@ -203,8 +205,365 @@ public void testUsagePollerComplex() throws InterruptedException { Assert.assertTrue(activeTaskIds.isEmpty()); } + @Test + public void itTracksClusterUtilizationSimple() { + initRequest(); + double cpuReserved = 10; + double memMbReserved = .001; + initFirstDeployWithResources(cpuReserved, memMbReserved); + saveAndSchedule(request.toBuilder().setInstances(Optional.of(1))); + resourceOffers(1); + + SingularityTaskId taskId = taskManager.getActiveTaskIds().get(0); + String t1 = taskId.getId(); + String host = slaveManager.getObjects().get(0).getHost(); + + // used 8 cpu + MesosTaskMonitorObject t1u1 = getTaskMonitor(t1, 40, getTimestampSeconds(taskId, 5), 800); + mesosClient.setSlaveResourceUsage(host, Collections.singletonList(t1u1)); + usagePoller.runActionOnPoll(); + + // used 8 cpu + MesosTaskMonitorObject t1u2 = getTaskMonitor(t1, 80, getTimestampSeconds(taskId, 10), 850); + mesosClient.setSlaveResourceUsage(host, Collections.singletonList(t1u2)); + usagePoller.runActionOnPoll(); + + Assert.assertTrue("Couldn't find cluster utilization", usageManager.getClusterUtilization().isPresent()); + + SingularityClusterUtilization utilization = usageManager.getClusterUtilization().get(); + + int taskUsages = usageManager.getTaskUsage(t1).size(); + testUtilization(utilization, 2, taskUsages, cpuReserved, memMbReserved, + 0, 1, 1, + 0, 2, 175, + 0, 2, 175, + 0, 2, 175); + + Assert.assertEquals(requestId, utilization.getMaxUnderUtilizedCpuRequestId()); + Assert.assertEquals(requestId, utilization.getMaxUnderUtilizedMemBytesRequestId()); + } + + @Test + public void itDoesntIncludePerfectlyUtilizedRequestsInClusterUtilization() { + initRequest(); + double cpuReserved = 2; + double memMbReserved = .001; + initFirstDeployWithResources(cpuReserved, memMbReserved); + saveAndSchedule(request.toBuilder().setInstances(Optional.of(1))); + resourceOffers(1); + + SingularityTaskId taskId = taskManager.getActiveTaskIds().get(0); + String t1 = taskId.getId(); + String host = slaveManager.getObjects().get(0).getHost(); + + // 2 cpus used + MesosTaskMonitorObject t1u1 = getTaskMonitor(t1, 10, getTimestampSeconds(taskId, 5), 1000); + mesosClient.setSlaveResourceUsage(host, Collections.singletonList(t1u1)); + usagePoller.runActionOnPoll(); + + // 2 cpus used + MesosTaskMonitorObject t1u2 = getTaskMonitor(t1, 20, getTimestampSeconds(taskId, 10), 900); + mesosClient.setSlaveResourceUsage(host, Collections.singletonList(t1u2)); + usagePoller.runActionOnPoll(); + + Assert.assertTrue("Couldn't find cluster utilization", usageManager.getClusterUtilization().isPresent()); + + SingularityClusterUtilization utilization = usageManager.getClusterUtilization().get(); + + int taskUsages = usageManager.getTaskUsage(t1).size(); + testUtilization(utilization, 2, taskUsages, cpuReserved, memMbReserved, + 0, 0, 1, + 0, 0, 50, + 0, 0, 50, + 0, 0, 50); + + Assert.assertEquals(requestId, utilization.getMaxUnderUtilizedMemBytesRequestId()); + } + + @Test + public void itTracksOverusedCpuInClusterUtilization() { + initRequest(); + double cpuReserved = 2; + double memMbReserved = .001; + initFirstDeployWithResources(cpuReserved, memMbReserved); + saveAndSchedule(request.toBuilder().setInstances(Optional.of(1))); + resourceOffers(1); + + SingularityTaskId taskId = taskManager.getActiveTaskIds().get(0); + String t1 = taskId.getId(); + String host = slaveManager.getObjects().get(0).getHost(); + + // 4 cpus used + MesosTaskMonitorObject t1u1 = getTaskMonitor(t1, 20, getTimestampSeconds(taskId, 5), 1000); + mesosClient.setSlaveResourceUsage(host, Collections.singletonList(t1u1)); + usagePoller.runActionOnPoll(); + + // 4 cpus used + MesosTaskMonitorObject t1u2 = getTaskMonitor(t1, 40, getTimestampSeconds(taskId, 10), 1000); + mesosClient.setSlaveResourceUsage(host, Collections.singletonList(t1u2)); + usagePoller.runActionOnPoll(); + + Assert.assertTrue("Couldn't find cluster utilization", usageManager.getClusterUtilization().isPresent()); + + SingularityClusterUtilization utilization = usageManager.getClusterUtilization().get(); + + int taskUsages = usageManager.getTaskUsage(t1).size(); + testUtilization(utilization, 2, taskUsages, cpuReserved, memMbReserved, + 1, 0, 0, + 2, 0, 0, + 2, 0, 0, + 2, 0, 0); + + Assert.assertEquals(requestId, utilization.getMaxOverUtilizedCpuRequestId()); + } + + @Test + public void itCorrectlyDeletesOldUsage() { + configuration.setNumUsageToKeep(3); + configuration.setUsageIntervalSeconds(180); + configuration.setCheckUsageEveryMillis(TimeUnit.MINUTES.toMillis(1)); + long now = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis()); + + // no usages exist, none are deleted + String taskId = "newTask"; + clearUsages(taskId); + + // 1 usage exists, none are deleted + taskId = "singleUsage"; + saveTaskUsage(taskId, now); + clearUsages(taskId); + Assert.assertEquals(1, usageManager.getTaskUsage(taskId).size()); + + // 2 usages exist 1 min apart, none are deleted + taskId = "twoUsages"; + saveTaskUsage(taskId, now, now + TimeUnit.MINUTES.toSeconds(1)); + clearUsages(taskId); + Assert.assertEquals(2, usageManager.getTaskUsage(taskId).size()); + + // x1 (3 min apart) x2 (1 min apart) x3 + // x3 is deleted + taskId = "threeUsages"; + saveTaskUsage(taskId, now, now + TimeUnit.MINUTES.toSeconds(3), now + TimeUnit.MINUTES.toSeconds(4)); + clearUsages(taskId); + Assert.assertEquals(2, usageManager.getTaskUsage(taskId).size()); + Assert.assertEquals(now, (long) usageManager.getTaskUsage(taskId).get(0).getTimestamp()); + Assert.assertEquals(now + TimeUnit.MINUTES.toSeconds(3), (long) usageManager.getTaskUsage(taskId).get(1).getTimestamp()); + + // x1 (1 min apart) x2 (1 min apart) x3 + // x2 is deleted + taskId = "threeUsages2"; + saveTaskUsage(taskId, now, now + TimeUnit.MINUTES.toSeconds(1), now + TimeUnit.MINUTES.toSeconds(2)); + clearUsages(taskId); + Assert.assertEquals(2, usageManager.getTaskUsage(taskId).size()); + Assert.assertEquals(now, (long) usageManager.getTaskUsage(taskId).get(0).getTimestamp()); + Assert.assertEquals(now + TimeUnit.MINUTES.toSeconds(2), (long) usageManager.getTaskUsage(taskId).get(1).getTimestamp()); + + // x1 (3 min apart) x2 (3 min apart) x3 + // x1 is deleted + taskId = "threeUsages3"; + saveTaskUsage(taskId, now, now + TimeUnit.MINUTES.toSeconds(3), now + TimeUnit.MINUTES.toSeconds(6)); + clearUsages(taskId); + Assert.assertEquals(2, usageManager.getTaskUsage(taskId).size()); + Assert.assertEquals(now + TimeUnit.MINUTES.toSeconds(3), (long) usageManager.getTaskUsage(taskId).get(0).getTimestamp()); + Assert.assertEquals(now + TimeUnit.MINUTES.toSeconds(6), (long) usageManager.getTaskUsage(taskId).get(1).getTimestamp()); + } + + @Test + public void itCorrectlyDeterminesResourcesReservedForRequestsWithMultipleTasks() { + initRequest(); + double cpuReserved = 10; + double memMbReserved = .001; + initFirstDeployWithResources(cpuReserved, memMbReserved); + saveAndSchedule(request.toBuilder().setInstances(Optional.of(2))); + resourceOffers(1); + + List taskIds = taskManager.getActiveTaskIds(); + SingularityTaskId t1 = taskIds.get(0); + SingularityTaskId t2 = taskIds.get(1); + String host = slaveManager.getObjects().get(0).getHost(); + + // used 6 cpu + MesosTaskMonitorObject t1u1 = getTaskMonitor(t1.getId(), 30, getTimestampSeconds(t1, 5), 800); + // used 6 cpu + MesosTaskMonitorObject t2u1 = getTaskMonitor(t2.getId(), 30, getTimestampSeconds(t2, 5), 800); + mesosClient.setSlaveResourceUsage(host, Arrays.asList(t1u1, t2u1)); + usagePoller.runActionOnPoll(); + + // used 8 cpu + MesosTaskMonitorObject t1u2 = getTaskMonitor(t1.getId(), 70, getTimestampSeconds(t1, 10), 850); + // used 8 cpu + MesosTaskMonitorObject t2u2 = getTaskMonitor(t2.getId(), 70, getTimestampSeconds(t2, 10), 850); + mesosClient.setSlaveResourceUsage(host, Arrays.asList(t1u2, t2u2)); + usagePoller.runActionOnPoll(); + + Assert.assertTrue("Couldn't find cluster utilization", usageManager.getClusterUtilization().isPresent()); + + SingularityClusterUtilization utilization = usageManager.getClusterUtilization().get(); + + int t1TaskUsages = usageManager.getTaskUsage(t1.getId()).size(); + int t2TaskUsages = usageManager.getTaskUsage(t2.getId()).size(); + Assert.assertEquals(2, t1TaskUsages); + Assert.assertEquals(2, t2TaskUsages); + + Assert.assertEquals(1, utilization.getRequestUtilizations().size()); + Assert.assertEquals(cpuReserved * (t1TaskUsages + t2TaskUsages), utilization.getRequestUtilizations().get(0).getCpuReserved(), 0); + Assert.assertEquals(memMbReserved * SingularitySlaveUsage.BYTES_PER_MEGABYTE * (t1TaskUsages + t2TaskUsages), utilization.getRequestUtilizations().get(0).getMemBytesReserved(), 0); + } + + @Test + public void itCorrectlyTracksMaxAndMinUtilizedPerRequest() { + initRequest(); + double cpuReserved = 10; + double memMbReserved = .001; + initFirstDeployWithResources(cpuReserved, memMbReserved); + saveAndSchedule(request.toBuilder().setInstances(Optional.of(2))); + resourceOffers(1); + + List taskIds = taskManager.getActiveTaskIds(); + SingularityTaskId t1 = taskIds.get(0); + SingularityTaskId t2 = taskIds.get(1); + String host = slaveManager.getObjects().get(0).getHost(); + + // used 10 cpu + MesosTaskMonitorObject t1u1 = getTaskMonitor(t1.getId(), 50, getTimestampSeconds(t1, 5), 800); + // used 8 cpu + MesosTaskMonitorObject t2u1 = getTaskMonitor(t2.getId(), 40, getTimestampSeconds(t2, 5), 700); + mesosClient.setSlaveResourceUsage(host, Arrays.asList(t1u1, t2u1)); + usagePoller.runActionOnPoll(); + + Assert.assertTrue("Couldn't find cluster utilization", usageManager.getClusterUtilization().isPresent()); + SingularityClusterUtilization utilization = usageManager.getClusterUtilization().get(); + + int t1TaskUsages = usageManager.getTaskUsage(t1.getId()).size(); + int t2TaskUsages = usageManager.getTaskUsage(t2.getId()).size(); + Assert.assertEquals(1, t1TaskUsages); + Assert.assertEquals(1, t2TaskUsages); + Assert.assertEquals(1, utilization.getRequestUtilizations().size()); + + double maxCpu = utilization.getRequestUtilizations().get(0).getMaxCpuUsed(); + double minCpu = utilization.getRequestUtilizations().get(0).getMinCpuUsed(); + long maxMemBytes = utilization.getRequestUtilizations().get(0).getMaxMemBytesUsed(); + long minMemBytes = utilization.getRequestUtilizations().get(0).getMinMemBytesUsed(); + Assert.assertEquals(10, maxCpu, 0); + Assert.assertEquals(8, minCpu, 0); + Assert.assertEquals(800, maxMemBytes); + Assert.assertEquals(700, minMemBytes); + + // new max and min after 2nd run + + // used 12 cpu + MesosTaskMonitorObject t1u2 = getTaskMonitor(t1.getId(), 110, getTimestampSeconds(t1, 10), 850); + // used 7 cpu + MesosTaskMonitorObject t2u2 = getTaskMonitor(t2.getId(), 75, getTimestampSeconds(t2, 10), 600); + mesosClient.setSlaveResourceUsage(host, Arrays.asList(t1u2, t2u2)); + usagePoller.runActionOnPoll(); + + Assert.assertTrue("Couldn't find cluster utilization", usageManager.getClusterUtilization().isPresent()); + utilization = usageManager.getClusterUtilization().get(); + + t1TaskUsages = usageManager.getTaskUsage(t1.getId()).size(); + t2TaskUsages = usageManager.getTaskUsage(t2.getId()).size(); + Assert.assertEquals(2, t1TaskUsages); + Assert.assertEquals(2, t2TaskUsages); + Assert.assertEquals(1, utilization.getRequestUtilizations().size()); + + maxCpu = utilization.getRequestUtilizations().get(0).getMaxCpuUsed(); + minCpu = utilization.getRequestUtilizations().get(0).getMinCpuUsed(); + maxMemBytes = utilization.getRequestUtilizations().get(0).getMaxMemBytesUsed(); + minMemBytes = utilization.getRequestUtilizations().get(0).getMinMemBytesUsed(); + Assert.assertEquals(12, maxCpu, 0); + Assert.assertEquals(7, minCpu, 0); + Assert.assertEquals(850, maxMemBytes); + Assert.assertEquals(600, minMemBytes); + + // same max and min after 3rd run + + // used 8 cpu + MesosTaskMonitorObject t1u3 = getTaskMonitor(t1.getId(), 150, getTimestampSeconds(t1, 15), 750); + // used 8 cpu + MesosTaskMonitorObject t2u3 = getTaskMonitor(t2.getId(), 120, getTimestampSeconds(t2, 15), 700); + mesosClient.setSlaveResourceUsage(host, Arrays.asList(t1u3, t2u3)); + usagePoller.runActionOnPoll(); + + Assert.assertTrue("Couldn't find cluster utilization", usageManager.getClusterUtilization().isPresent()); + utilization = usageManager.getClusterUtilization().get(); + + t1TaskUsages = usageManager.getTaskUsage(t1.getId()).size(); + t2TaskUsages = usageManager.getTaskUsage(t2.getId()).size(); + Assert.assertEquals(3, t1TaskUsages); + Assert.assertEquals(3, t2TaskUsages); + Assert.assertEquals(1, utilization.getRequestUtilizations().size()); + + maxCpu = utilization.getRequestUtilizations().get(0).getMaxCpuUsed(); + minCpu = utilization.getRequestUtilizations().get(0).getMinCpuUsed(); + maxMemBytes = utilization.getRequestUtilizations().get(0).getMaxMemBytesUsed(); + minMemBytes = utilization.getRequestUtilizations().get(0).getMinMemBytesUsed(); + Assert.assertEquals(12, maxCpu, 0); + Assert.assertEquals(7, minCpu, 0); + Assert.assertEquals(850, maxMemBytes); + Assert.assertEquals(600, minMemBytes); + } + private MesosTaskStatisticsObject getStatistics(double cpuSecs, double timestamp, long memBytes) { - return new MesosTaskStatisticsObject(1, 0L, 0L, 0, 0, cpuSecs, 0L, 0L, 0L, 0L, memBytes, timestamp); + return new MesosTaskStatisticsObject(1, 0L, 0L, 0, 0, cpuSecs, 0L, 0L, 0L, 0L, 0L, memBytes, timestamp); + } + + private long getTimestampSeconds(SingularityTaskId taskId, long seconds) { + return TimeUnit.MILLISECONDS.toSeconds(taskId.getStartedAt()) + seconds; + } + + private MesosTaskMonitorObject getTaskMonitor(String id, double cpuSecs, long timestampSeconds, int memBytes) { + return new MesosTaskMonitorObject(null, null, null, id, getStatistics(cpuSecs, timestampSeconds, memBytes)); } + private void saveTaskUsage(String taskId, long... times) { + for (long time : times) { + usageManager.saveSpecificTaskUsage(taskId, new SingularityTaskUsage(0, time, 0)); + } + } + + private void clearUsages(String taskId) { + usagePoller.clearOldUsage(taskId); + } + + private void testUtilization(SingularityClusterUtilization utilization, + int expectedTaskUsages, + int actualTaskUsages, + double cpuReserved, + double memMbReserved, + int expectedRequestsWithOverUtilizedCpu, + int expectedRequestsWithUnderUtilizedCpu, + int expectedRequestsWithUnderUtilizedMemBytes, + double expectedAvgOverUtilizedCpu, + double expectedAvgUnderUtilizedCpu, + double expectedAvgUnderUtilizedMemBytes, + double expectedMaxOverUtilizedCpu, + double expectedMaxUnderUtilizedCpu, + long expectedMaxUnderUtilizedMemBytes, + double expectedMinOverUtilizedCpu, + double expectedMinUnderUtilizedCpu, + long expectedMinUnderUtilizedMemBytes) { + + Assert.assertEquals(expectedTaskUsages, actualTaskUsages); + + Assert.assertEquals("Expected 1 request", 1, utilization.getRequestUtilizations().size()); + Assert.assertEquals("Incorrect cpu reserved", cpuReserved * actualTaskUsages, utilization.getRequestUtilizations().get(0).getCpuReserved(), 0); + Assert.assertEquals("Incorrect mem reserved", memMbReserved * SingularitySlaveUsage.BYTES_PER_MEGABYTE * actualTaskUsages, utilization.getRequestUtilizations().get(0).getMemBytesReserved(), 0); + + Assert.assertEquals(expectedRequestsWithOverUtilizedCpu, utilization.getNumRequestsWithOverUtilizedCpu()); + Assert.assertEquals(expectedRequestsWithUnderUtilizedCpu, utilization.getNumRequestsWithUnderUtilizedCpu()); + Assert.assertEquals(expectedRequestsWithUnderUtilizedMemBytes, utilization.getNumRequestsWithUnderUtilizedMemBytes()); + + Assert.assertEquals(expectedAvgOverUtilizedCpu, utilization.getAvgOverUtilizedCpu(), 0); + Assert.assertEquals(expectedAvgUnderUtilizedCpu, utilization.getAvgUnderUtilizedCpu(), 0); + Assert.assertEquals(expectedAvgUnderUtilizedMemBytes, utilization.getAvgUnderUtilizedMemBytes(), 0); + + Assert.assertEquals(expectedMaxOverUtilizedCpu, utilization.getMaxOverUtilizedCpu(), 0); + Assert.assertEquals(expectedMaxUnderUtilizedCpu, utilization.getMaxUnderUtilizedCpu(), 0); + Assert.assertEquals(expectedMaxUnderUtilizedMemBytes, utilization.getMaxUnderUtilizedMemBytes()); + + Assert.assertEquals(expectedMinOverUtilizedCpu, utilization.getMinOverUtilizedCpu(), 0); + Assert.assertEquals(expectedMinUnderUtilizedCpu, utilization.getMinUnderUtilizedCpu(), 0); + Assert.assertEquals(expectedMinUnderUtilizedMemBytes, utilization.getMinUnderUtilizedMemBytes()); + } }