From f41e82487b1c526e8f96c9a65844b7a6ca483779 Mon Sep 17 00:00:00 2001 From: Gowtam Lal Date: Wed, 8 May 2019 19:29:48 -0400 Subject: [PATCH 01/21] Add new cleanup type. --- .../src/main/java/com/hubspot/singularity/TaskCleanupType.java | 1 + 1 file changed, 1 insertion(+) diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/TaskCleanupType.java b/SingularityBase/src/main/java/com/hubspot/singularity/TaskCleanupType.java index 7805802234..4a03b56d60 100644 --- a/SingularityBase/src/main/java/com/hubspot/singularity/TaskCleanupType.java +++ b/SingularityBase/src/main/java/com/hubspot/singularity/TaskCleanupType.java @@ -20,6 +20,7 @@ public enum TaskCleanupType { PAUSING(false, false), PRIORITY_KILL(true, true), REBALANCE_CPU_USAGE(false, false), + REBALANCE_MEMORY_USAGE(false, false), REBALANCE_RACKS(false, false), REBALANCE_SLAVE_ATTRIBUTES(false, false), REQUEST_DELETING(true, true), From cb9fc14ac4a2455baef9b79c712cc4feb7ef5f84 Mon Sep 17 00:00:00 2001 From: Gowtam Lal Date: Wed, 8 May 2019 19:30:01 -0400 Subject: [PATCH 02/21] Add configs for memory shuffling. --- .../config/SingularityConfiguration.java | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java b/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java index 227f4cb100..dd49b269e8 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java @@ -84,6 +84,10 @@ public class SingularityConfiguration extends Configuration { private boolean shuffleTasksForOverloadedSlaves = false; // recommended 'true' when oversubscribing cpu for larger clusters + private boolean shuffleTasksForSlavesWithHighMemoryUsage = false; // recommended 'true' when oversubscribing memory for larger clusters + + private double shuffleTasksWhenSlaveMemoryUtilizationPercentageExceeds = 0.90; + private int maxTasksToShuffleTotal = 6; // Do not allow more than this many shuffle cleanups at once cluster-wide private int maxTasksToShufflePerHost = 2; @@ -1497,6 +1501,22 @@ public void setShuffleTasksForOverloadedSlaves(boolean shuffleTasksForOverloaded this.shuffleTasksForOverloadedSlaves = shuffleTasksForOverloadedSlaves; } + public boolean isShuffleTasksForSlavesWithHighMemoryUsage() { + return shuffleTasksForSlavesWithHighMemoryUsage; + } + + public void setShuffleTasksForSlavesWithHighMemoryUsage(boolean shuffleTasksForSlavesWithHighMemoryUsage) { + this.shuffleTasksForSlavesWithHighMemoryUsage = shuffleTasksForSlavesWithHighMemoryUsage; + } + + public double getShuffleTasksWhenSlaveMemoryUtilizationPercentageExceeds() { + return shuffleTasksWhenSlaveMemoryUtilizationPercentageExceeds; + } + + public void setShuffleTasksWhenSlaveMemoryUtilizationPercentageExceeds(double shuffleTasksWhenSlaveMemoryUtilizationPercentageExceeds) { + this.shuffleTasksWhenSlaveMemoryUtilizationPercentageExceeds = shuffleTasksWhenSlaveMemoryUtilizationPercentageExceeds; + } + public int getMaxTasksToShuffleTotal() { return maxTasksToShuffleTotal; } From 82644f762ce052a9d0f288a273c400f66fe7a076 Mon Sep 17 00:00:00 2001 From: Gowtam Lal Date: Wed, 8 May 2019 19:33:17 -0400 Subject: [PATCH 03/21] Also collect a list of mem-overloaded slaves. --- .../singularity/scheduler/SingularityUsageHelper.java | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsageHelper.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsageHelper.java index 71dfc7c99c..5f85a09932 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsageHelper.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsageHelper.java @@ -101,6 +101,7 @@ public void collectSlaveUsage( Map utilizationPerRequestId, Map previousUtilizations, Map> overLoadedHosts, + Map> highMemUsageHosts, AtomicLong totalMemBytesUsed, AtomicLong totalMemBytesAvailable, AtomicDouble totalCpuUsed, @@ -157,6 +158,7 @@ public void collectSlaveUsage( } boolean slaveOverloaded = systemCpusTotal > 0 && systemLoad / systemCpusTotal > 1.0; + boolean slaveExperiencingHighMemUsage = ((systemMemTotalBytes - systemMemFreeBytes) / systemMemTotalBytes) > configuration.getShuffleTasksWhenSlaveMemoryUtilizationPercentageExceeds(); List possibleTasksToShuffle = new ArrayList<>(); for (MesosTaskMonitorObject taskUsage : allTaskUsage) { @@ -216,11 +218,11 @@ public void collectSlaveUsage( cpusUsedOnSlave += taskCpusUsed; } - if (configuration.isShuffleTasksForOverloadedSlaves() && currentUsage != null && currentUsage.getCpusUsed() > 0) { + if (currentUsage != null && currentUsage.getCpusUsed() > 0) { if (isEligibleForShuffle(task)) { Optional maybeCleanupUpdate = taskManager.getTaskHistoryUpdate(task, ExtendedTaskState.TASK_CLEANING); if (maybeCleanupUpdate.isPresent() && isTaskAlreadyCleanedUpForShuffle(maybeCleanupUpdate.get())) { - LOG.trace("Task {} already being cleaned up to spread cpu usage, skipping", taskId); + LOG.trace("Task {} already being cleaned up to spread cpu or mem usage, skipping", taskId); } else { if (maybeResources.isPresent()) { possibleTasksToShuffle.add(new TaskIdWithUsage(task, maybeResources.get(), currentUsage)); @@ -248,6 +250,10 @@ public void collectSlaveUsage( overLoadedHosts.put(slaveUsage, possibleTasksToShuffle); } + if (slaveExperiencingHighMemUsage) { + highMemUsageHosts.put(slaveUsage, possibleTasksToShuffle); + } + if (slaveUsage.getMemoryBytesTotal().isPresent() && slaveUsage.getCpusTotal().isPresent()) { totalMemBytesUsed.getAndAdd((long) slaveUsage.getMemoryBytesUsed()); totalCpuUsed.getAndAdd(slaveUsage.getCpusUsed()); From 3f6cd24014dda1301f95e654fb40eb38bc424684 Mon Sep 17 00:00:00 2001 From: Gowtam Lal Date: Wed, 8 May 2019 19:33:41 -0400 Subject: [PATCH 04/21] Account for the new cleanup type here. --- .../singularity/scheduler/SingularityUsageHelper.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsageHelper.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsageHelper.java index 5f85a09932..00f2155714 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsageHelper.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsageHelper.java @@ -320,11 +320,13 @@ private boolean isLongRunning(SingularityTaskId task) { } private boolean isTaskAlreadyCleanedUpForShuffle(SingularityTaskHistoryUpdate taskHistoryUpdate) { - if (taskHistoryUpdate.getStatusMessage().or("").contains(TaskCleanupType.REBALANCE_CPU_USAGE.name())) { + String statusMessage = taskHistoryUpdate.getStatusMessage().or(""); + if (statusMessage.contains(TaskCleanupType.REBALANCE_CPU_USAGE.name()) || statusMessage.contains(TaskCleanupType.REBALANCE_MEMORY_USAGE.name())) { return true; } for (SingularityTaskHistoryUpdate previous : taskHistoryUpdate.getPrevious()) { - if (previous.getStatusMessage().or("").contains(TaskCleanupType.REBALANCE_CPU_USAGE.name())) { + statusMessage = previous.getStatusMessage().or(""); + if (statusMessage.contains(TaskCleanupType.REBALANCE_CPU_USAGE.name()) || statusMessage.contains(TaskCleanupType.REBALANCE_MEMORY_USAGE.name())) { return true; } } From e8be955a8a5f2fbca91206a164ecf09c242e316b Mon Sep 17 00:00:00 2001 From: Gowtam Lal Date: Wed, 8 May 2019 19:37:12 -0400 Subject: [PATCH 05/21] Handle the case where the same hosts are overloaded on both CPU and memory. --- .../scheduler/SingularityUsagePoller.java | 122 +++++++++++++----- 1 file changed, 92 insertions(+), 30 deletions(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java index 44a3c5cade..248e6ac803 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java @@ -1,6 +1,7 @@ package com.hubspot.singularity.scheduler; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -36,6 +37,8 @@ import com.hubspot.singularity.data.TaskManager; import com.hubspot.singularity.data.usage.UsageManager; +import io.dropwizard.util.SizeUnit; + public class SingularityUsagePoller extends SingularityLeaderOnlyPoller { private static final Logger LOG = LoggerFactory.getLogger(SingularityUsagePoller.class); @@ -86,13 +89,14 @@ public void runActionOnPoll() { AtomicLong totalDiskBytesAvailable = new AtomicLong(0); Map> overLoadedHosts = new ConcurrentHashMap<>(); + Map> highMemUsageHosts = new ConcurrentHashMap<>(); List> usageFutures = new ArrayList<>(); usageHelper.getSlavesToTrackUsageFor().forEach((slave) -> { usageFutures.add(usageCollectionSemaphore.call(() -> CompletableFuture.runAsync(() -> { - usageHelper.collectSlaveUsage(slave, now, utilizationPerRequestId, previousUtilizations, overLoadedHosts, totalMemBytesUsed, totalMemBytesAvailable, + usageHelper.collectSlaveUsage(slave, now, utilizationPerRequestId, previousUtilizations, overLoadedHosts, highMemUsageHosts, totalMemBytesUsed, totalMemBytesAvailable, totalCpuUsed, totalCpuAvailable, totalDiskBytesUsed, totalDiskBytesAvailable, false); }, usageExecutor) )); @@ -107,14 +111,14 @@ public void runActionOnPoll() { utilizationPerRequestId.values().forEach(usageManager::saveRequestUtilization); if (configuration.isShuffleTasksForOverloadedSlaves()) { - shuffleTasksOnOverloadedHosts(overLoadedHosts); + shuffleTasksOnOverloadedHosts(overLoadedHosts, highMemUsageHosts); } } - private void shuffleTasksOnOverloadedHosts(Map> overLoadedHosts) { + private void shuffleTasksOnOverloadedHosts(Map> overLoadedHosts, Map> highMemUsageHosts) { List shuffleCleanups = taskManager.getCleanupTasks() .stream() - .filter((taskCleanup) -> taskCleanup.getCleanupType() == TaskCleanupType.REBALANCE_CPU_USAGE) + .filter((taskCleanup) -> taskCleanup.getCleanupType() == TaskCleanupType.REBALANCE_CPU_USAGE || taskCleanup.getCleanupType() == TaskCleanupType.REBALANCE_MEMORY_USAGE) .collect(Collectors.toList()); long currentShuffleCleanupsTotal = shuffleCleanups.size(); Set requestsWithShuffledTasks = shuffleCleanups @@ -128,51 +132,109 @@ private void shuffleTasksOnOverloadedHosts(Map highMemUsedSlavesByUsage = highMemUsageHosts.keySet().stream() + .sorted((usage1, usage2) -> Double.compare( + usage2.getMemoryBytesUsed(), // TODO: memoryBytesUsed vs memoryBytesReserved here? + usage1.getMemoryBytesUsed() // TODO: absolute values vs percentages here? + )) + .collect(Collectors.toList()); + + // First handle the case where the same slaves are overloaded for both CPU & memory, and try and pick the resource that's worse off + Set overloadedForMemAndCpu = new HashSet<>(overLoadedHosts.keySet()); + overloadedForMemAndCpu.retainAll(highMemUsageHosts.keySet()); + + for (SingularitySlaveUsage overloadedSlave : overloadedForMemAndCpu) { if (currentShuffleCleanupsTotal >= configuration.getMaxTasksToShuffleTotal()) { LOG.debug("Not shuffling any more tasks (totalShuffleCleanups: {})", currentShuffleCleanupsTotal); break; } int shuffledTasksOnSlave = 0; - List possibleTasksToShuffle = overLoadedHosts.get(overloadedSlave); - possibleTasksToShuffle.sort((u1, u2) -> - Double.compare( - u2.getUsage().getCpusUsed() / u2.getRequestedResources().getCpus(), - u1.getUsage().getCpusUsed() / u1.getRequestedResources().getCpus() - )); double systemLoad = getSystemLoadForShuffle(overloadedSlave); double cpuOverage = systemLoad - overloadedSlave.getSystemCpusTotal(); + double cpuOverusage = cpuOverage / overloadedSlave.getSystemCpusTotal(); + + double memOverageBytes = overloadedSlave.getMemoryBytesUsed() - (configuration.getShuffleTasksWhenSlaveMemoryUtilizationPercentageExceeds() * overloadedSlave.getMemoryBytesTotal().get()); + double memOverusage = memOverageBytes / overloadedSlave.getMemoryBytesTotal().get(); + + List possibleTasksToShuffle; + boolean shufflingForCpu; + if (cpuOverusage > memOverusage) { + shufflingForCpu = true; + possibleTasksToShuffle = overLoadedHosts.get(overloadedSlave); + possibleTasksToShuffle.sort((u1, u2) -> + Double.compare( + u2.getUsage().getCpusUsed() / u2.getRequestedResources().getCpus(), + u1.getUsage().getCpusUsed() / u1.getRequestedResources().getCpus() + )); + } else { + shufflingForCpu = false; + possibleTasksToShuffle = highMemUsageHosts.get(overloadedSlave); + possibleTasksToShuffle.sort((u1, u2) -> + Double.compare( + u2.getUsage().getMemoryTotalBytes() / u2.getRequestedResources().getMemoryMb(), + u1.getUsage().getMemoryTotalBytes() / u1.getRequestedResources().getMemoryMb() + )); + } + for (TaskIdWithUsage taskIdWithUsage : possibleTasksToShuffle) { if (requestsWithShuffledTasks.contains(taskIdWithUsage.getTaskId().getRequestId())) { LOG.debug("Request {} already has a shuffling task, skipping", taskIdWithUsage.getTaskId().getRequestId()); continue; } - if (cpuOverage <= 0 || shuffledTasksOnSlave > configuration.getMaxTasksToShufflePerHost() || currentShuffleCleanupsTotal >= configuration.getMaxTasksToShuffleTotal()) { - LOG.debug("Not shuffling any more tasks (overage: {}, shuffledOnHost: {}, totalShuffleCleanups: {})", cpuOverage, shuffledTasksOnSlave, currentShuffleCleanupsTotal); + if ((shufflingForCpu && cpuOverage <= 0) || (!shufflingForCpu && memOverageBytes <= 0) || shuffledTasksOnSlave > configuration.getMaxTasksToShufflePerHost() || currentShuffleCleanupsTotal >= configuration.getMaxTasksToShuffleTotal()) { + LOG.debug("Not shuffling any more tasks (cpu overage: {}, mem overage: {}MiB, shuffledOnHost: {}, totalShuffleCleanups: {})", cpuOverage, SizeUnit.BYTES.toMegabytes(((long) memOverageBytes)), shuffledTasksOnSlave, currentShuffleCleanupsTotal); break; } - LOG.debug("Cleaning up task {} to free up cpu on overloaded host (remaining cpu overage: {})", taskIdWithUsage.getTaskId(), cpuOverage); - Optional message = Optional.of(String.format( - "Load on slave is %s / %s, shuffling task using %s / %s to less busy host", - systemLoad, - overloadedSlave.getSystemCpusTotal(), - taskIdWithUsage.getUsage().getCpusUsed(), - taskIdWithUsage.getRequestedResources().getCpus())); - taskManager.createTaskCleanup( - new SingularityTaskCleanup( - Optional.absent(), - TaskCleanupType.REBALANCE_CPU_USAGE, - System.currentTimeMillis(), - taskIdWithUsage.getTaskId(), - message, - Optional.of(UUID.randomUUID().toString()), - Optional.absent(), Optional.absent())); + + Optional message; + + if (shufflingForCpu) { + LOG.debug("Cleaning up task {} to free up cpu on overloaded host (remaining cpu overage: {})", taskIdWithUsage.getTaskId(), cpuOverage); + message = Optional.of(String.format( + "Load on slave is %s / %s, shuffling task using %s / %s to less busy host", + systemLoad, + overloadedSlave.getSystemCpusTotal(), + taskIdWithUsage.getUsage().getCpusUsed(), + taskIdWithUsage.getRequestedResources().getCpus())); + taskManager.createTaskCleanup( + new SingularityTaskCleanup( + Optional.absent(), + TaskCleanupType.REBALANCE_CPU_USAGE, + System.currentTimeMillis(), + taskIdWithUsage.getTaskId(), + message, + Optional.of(UUID.randomUUID().toString()), + Optional.absent(), Optional.absent())); + + cpuOverage -= taskIdWithUsage.getUsage().getCpusUsed(); + } else { + LOG.debug("Cleaning up task {} to free up mem on overloaded host (remaining mem overage: {}MiB)", taskIdWithUsage.getTaskId(), SizeUnit.BYTES.toMegabytes(((long) memOverageBytes))); + message = Optional.of(String.format( + "Mem usage on slave is %sMiB / %sMiB, shuffling task using %sMiB / %sMiB to less busy host", + SizeUnit.BYTES.toMegabytes(((long) overloadedSlave.getMemoryBytesUsed())), + SizeUnit.BYTES.toMegabytes(overloadedSlave.getMemoryBytesTotal().get()), + SizeUnit.BYTES.toMegabytes(taskIdWithUsage.getUsage().getMemoryTotalBytes()), + taskIdWithUsage.getRequestedResources().getMemoryMb())); + taskManager.createTaskCleanup( + new SingularityTaskCleanup( + Optional.absent(), + TaskCleanupType.REBALANCE_MEMORY_USAGE, + System.currentTimeMillis(), + taskIdWithUsage.getTaskId(), + message, + Optional.of(UUID.randomUUID().toString()), + Optional.absent(), Optional.absent())); + + memOverageBytes -= taskIdWithUsage.getUsage().getMemoryTotalBytes(); + } + requestManager.addToPendingQueue(new SingularityPendingRequest(taskIdWithUsage.getTaskId().getRequestId(), taskIdWithUsage.getTaskId() .getDeployId(), System.currentTimeMillis(), Optional.absent(), PendingType.TASK_BOUNCE, Optional.absent(), Optional.absent(), Optional.absent(), message, Optional.of(UUID.randomUUID().toString()))); - cpuOverage -= taskIdWithUsage.getUsage().getCpusUsed(); + shuffledTasksOnSlave++; currentShuffleCleanupsTotal++; requestsWithShuffledTasks.add(taskIdWithUsage.getTaskId().getRequestId()); From d86a44fca550d5bceffacb34aed96485385c800c Mon Sep 17 00:00:00 2001 From: Gowtam Lal Date: Fri, 17 May 2019 13:56:23 -0400 Subject: [PATCH 06/21] Let's not do a separate config for this. --- .../singularity/config/SingularityConfiguration.java | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java b/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java index dd49b269e8..93d5805c63 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java @@ -84,8 +84,6 @@ public class SingularityConfiguration extends Configuration { private boolean shuffleTasksForOverloadedSlaves = false; // recommended 'true' when oversubscribing cpu for larger clusters - private boolean shuffleTasksForSlavesWithHighMemoryUsage = false; // recommended 'true' when oversubscribing memory for larger clusters - private double shuffleTasksWhenSlaveMemoryUtilizationPercentageExceeds = 0.90; private int maxTasksToShuffleTotal = 6; // Do not allow more than this many shuffle cleanups at once cluster-wide @@ -1501,14 +1499,6 @@ public void setShuffleTasksForOverloadedSlaves(boolean shuffleTasksForOverloaded this.shuffleTasksForOverloadedSlaves = shuffleTasksForOverloadedSlaves; } - public boolean isShuffleTasksForSlavesWithHighMemoryUsage() { - return shuffleTasksForSlavesWithHighMemoryUsage; - } - - public void setShuffleTasksForSlavesWithHighMemoryUsage(boolean shuffleTasksForSlavesWithHighMemoryUsage) { - this.shuffleTasksForSlavesWithHighMemoryUsage = shuffleTasksForSlavesWithHighMemoryUsage; - } - public double getShuffleTasksWhenSlaveMemoryUtilizationPercentageExceeds() { return shuffleTasksWhenSlaveMemoryUtilizationPercentageExceeds; } From d21b7376b266ecc96b7da5e6e1bfba47b2c6a6b0 Mon Sep 17 00:00:00 2001 From: Gowtam Lal Date: Fri, 17 May 2019 14:02:04 -0400 Subject: [PATCH 07/21] Minor cleanup. --- .../singularity/scheduler/SingularityUsageHelper.java | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsageHelper.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsageHelper.java index 00f2155714..a611602d84 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsageHelper.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsageHelper.java @@ -101,7 +101,6 @@ public void collectSlaveUsage( Map utilizationPerRequestId, Map previousUtilizations, Map> overLoadedHosts, - Map> highMemUsageHosts, AtomicLong totalMemBytesUsed, AtomicLong totalMemBytesAvailable, AtomicDouble totalCpuUsed, @@ -157,7 +156,7 @@ public void collectSlaveUsage( break; } - boolean slaveOverloaded = systemCpusTotal > 0 && systemLoad / systemCpusTotal > 1.0; + boolean slaveOverloadedForCpu = systemCpusTotal > 0 && systemLoad / systemCpusTotal > 1.0; boolean slaveExperiencingHighMemUsage = ((systemMemTotalBytes - systemMemFreeBytes) / systemMemTotalBytes) > configuration.getShuffleTasksWhenSlaveMemoryUtilizationPercentageExceeds(); List possibleTasksToShuffle = new ArrayList<>(); @@ -246,14 +245,10 @@ public void collectSlaveUsage( memoryMbTotal, diskMbUsedOnSlave, diskMbReservedOnSlave, diskMbTotal, allTaskUsage.size(), now, systemMemTotalBytes, systemMemFreeBytes, systemCpusTotal, systemLoad1Min, systemLoad5Min, systemLoad15Min, slaveDiskUsed, slaveDiskTotal); - if (slaveOverloaded) { + if (slaveOverloadedForCpu || slaveExperiencingHighMemUsage) { overLoadedHosts.put(slaveUsage, possibleTasksToShuffle); } - if (slaveExperiencingHighMemUsage) { - highMemUsageHosts.put(slaveUsage, possibleTasksToShuffle); - } - if (slaveUsage.getMemoryBytesTotal().isPresent() && slaveUsage.getCpusTotal().isPresent()) { totalMemBytesUsed.getAndAdd((long) slaveUsage.getMemoryBytesUsed()); totalCpuUsed.getAndAdd(slaveUsage.getCpusUsed()); From 589ff01dc77948400c5df4b5d51e012851e5186e Mon Sep 17 00:00:00 2001 From: Gowtam Lal Date: Fri, 17 May 2019 14:02:31 -0400 Subject: [PATCH 08/21] Compute overusage of CPU & mem, and shuffle based on whichever is the worst off. --- .../scheduler/SingularityUsagePoller.java | 97 +++++++++++-------- 1 file changed, 58 insertions(+), 39 deletions(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java index 248e6ac803..261b7ff139 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java @@ -1,7 +1,6 @@ package com.hubspot.singularity.scheduler; import java.util.ArrayList; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -36,6 +35,7 @@ import com.hubspot.singularity.data.RequestManager; import com.hubspot.singularity.data.TaskManager; import com.hubspot.singularity.data.usage.UsageManager; +import com.hubspot.singularity.scheduler.SingularityUsagePoller.OverusedResource.Type; import io.dropwizard.util.SizeUnit; @@ -89,14 +89,13 @@ public void runActionOnPoll() { AtomicLong totalDiskBytesAvailable = new AtomicLong(0); Map> overLoadedHosts = new ConcurrentHashMap<>(); - Map> highMemUsageHosts = new ConcurrentHashMap<>(); List> usageFutures = new ArrayList<>(); usageHelper.getSlavesToTrackUsageFor().forEach((slave) -> { usageFutures.add(usageCollectionSemaphore.call(() -> CompletableFuture.runAsync(() -> { - usageHelper.collectSlaveUsage(slave, now, utilizationPerRequestId, previousUtilizations, overLoadedHosts, highMemUsageHosts, totalMemBytesUsed, totalMemBytesAvailable, + usageHelper.collectSlaveUsage(slave, now, utilizationPerRequestId, previousUtilizations, overLoadedHosts, totalMemBytesUsed, totalMemBytesAvailable, totalCpuUsed, totalCpuAvailable, totalDiskBytesUsed, totalDiskBytesAvailable, false); }, usageExecutor) )); @@ -111,11 +110,42 @@ public void runActionOnPoll() { utilizationPerRequestId.values().forEach(usageManager::saveRequestUtilization); if (configuration.isShuffleTasksForOverloadedSlaves()) { - shuffleTasksOnOverloadedHosts(overLoadedHosts, highMemUsageHosts); + shuffleTasksOnOverloadedHosts(overLoadedHosts); } } - private void shuffleTasksOnOverloadedHosts(Map> overLoadedHosts, Map> highMemUsageHosts) { + static class OverusedResource { + enum Type { MEMORY, CPU }; + + double usage; + double overusage; + Type resourceType; + + OverusedResource(double usage, double overusage, Type resourceType) { + this.usage = usage; + this.overusage = overusage; + this.resourceType = resourceType; + } + } + + private OverusedResource getMostOverusedResource(SingularitySlaveUsage overloadedSlave, double currentCpuLoad, double currentMemUsageBytes) { + double cpuOverage = currentCpuLoad - overloadedSlave.getSystemCpusTotal(); + + double cpuOverusage = cpuOverage / overloadedSlave.getSystemCpusTotal(); + + // TODO: memoryBytesUsed vs memoryBytesReserved here? + double targetMemUsageBytes = (configuration.getShuffleTasksWhenSlaveMemoryUtilizationPercentageExceeds() * overloadedSlave.getSystemMemTotalBytes()); + double memOverageBytes = currentMemUsageBytes - targetMemUsageBytes; + double memOverusage = memOverageBytes / targetMemUsageBytes; + + if (cpuOverusage > memOverusage) { + return new OverusedResource(currentCpuLoad, cpuOverusage, Type.CPU); + } else { + return new OverusedResource(overloadedSlave.getMemoryBytesUsed(), memOverusage, Type.MEMORY); + } + } + + private void shuffleTasksOnOverloadedHosts(Map> overLoadedHosts) { List shuffleCleanups = taskManager.getCleanupTasks() .stream() .filter((taskCleanup) -> taskCleanup.getCleanupType() == TaskCleanupType.REBALANCE_CPU_USAGE || taskCleanup.getCleanupType() == TaskCleanupType.REBALANCE_MEMORY_USAGE) @@ -126,42 +156,30 @@ private void shuffleTasksOnOverloadedHosts(Map taskCleanup.getTaskId().getRequestId()) .collect(Collectors.toSet()); - List overLoadedSlavesByUsage = overLoadedHosts.keySet().stream() - .sorted((usage1, usage2) -> Double.compare( - getSystemLoadForShuffle(usage2), - getSystemLoadForShuffle(usage1) - )) - .collect(Collectors.toList()); + List overloadedSlavesByOverusage = overLoadedHosts.keySet().stream() + .sorted((usage1, usage2) -> { + OverusedResource mostOverusedResource1 = getMostOverusedResource(usage1, getSystemLoadForShuffle(usage1), usage1.getMemoryBytesUsed()); + OverusedResource mostOverusedResource2 = getMostOverusedResource(usage2, getSystemLoadForShuffle(usage2), usage2.getMemoryBytesUsed()); - List highMemUsedSlavesByUsage = highMemUsageHosts.keySet().stream() - .sorted((usage1, usage2) -> Double.compare( - usage2.getMemoryBytesUsed(), // TODO: memoryBytesUsed vs memoryBytesReserved here? - usage1.getMemoryBytesUsed() // TODO: absolute values vs percentages here? - )) + return Double.compare(mostOverusedResource2.overusage, mostOverusedResource1.overusage); + }) .collect(Collectors.toList()); - // First handle the case where the same slaves are overloaded for both CPU & memory, and try and pick the resource that's worse off - Set overloadedForMemAndCpu = new HashSet<>(overLoadedHosts.keySet()); - overloadedForMemAndCpu.retainAll(highMemUsageHosts.keySet()); - - for (SingularitySlaveUsage overloadedSlave : overloadedForMemAndCpu) { + for (SingularitySlaveUsage overloadedSlave : overloadedSlavesByOverusage) { if (currentShuffleCleanupsTotal >= configuration.getMaxTasksToShuffleTotal()) { LOG.debug("Not shuffling any more tasks (totalShuffleCleanups: {})", currentShuffleCleanupsTotal); break; } int shuffledTasksOnSlave = 0; - double systemLoad = getSystemLoadForShuffle(overloadedSlave); - double cpuOverage = systemLoad - overloadedSlave.getSystemCpusTotal(); + double currentCpuLoad = getSystemLoadForShuffle(overloadedSlave); + double currentMemUsageBytes = overloadedSlave.getMemoryBytesUsed(); - double cpuOverusage = cpuOverage / overloadedSlave.getSystemCpusTotal(); - - double memOverageBytes = overloadedSlave.getMemoryBytesUsed() - (configuration.getShuffleTasksWhenSlaveMemoryUtilizationPercentageExceeds() * overloadedSlave.getMemoryBytesTotal().get()); - double memOverusage = memOverageBytes / overloadedSlave.getMemoryBytesTotal().get(); + OverusedResource mostOverusedResource = getMostOverusedResource(overloadedSlave, currentCpuLoad, currentMemUsageBytes); List possibleTasksToShuffle; boolean shufflingForCpu; - if (cpuOverusage > memOverusage) { + if (mostOverusedResource.resourceType == Type.CPU) { shufflingForCpu = true; possibleTasksToShuffle = overLoadedHosts.get(overloadedSlave); possibleTasksToShuffle.sort((u1, u2) -> @@ -171,7 +189,7 @@ private void shuffleTasksOnOverloadedHosts(Map Double.compare( u2.getUsage().getMemoryTotalBytes() / u2.getRequestedResources().getMemoryMb(), @@ -184,18 +202,19 @@ private void shuffleTasksOnOverloadedHosts(Map configuration.getMaxTasksToShufflePerHost() || currentShuffleCleanupsTotal >= configuration.getMaxTasksToShuffleTotal()) { - LOG.debug("Not shuffling any more tasks (cpu overage: {}, mem overage: {}MiB, shuffledOnHost: {}, totalShuffleCleanups: {})", cpuOverage, SizeUnit.BYTES.toMegabytes(((long) memOverageBytes)), shuffledTasksOnSlave, currentShuffleCleanupsTotal); + if ((mostOverusedResource.overusage <= 0) || shuffledTasksOnSlave > configuration.getMaxTasksToShufflePerHost() || currentShuffleCleanupsTotal >= configuration.getMaxTasksToShuffleTotal()) { + LOG.debug("Not shuffling any more tasks ({} overage : {}%, shuffledOnHost: {}, totalShuffleCleanups: {})", mostOverusedResource.resourceType, mostOverusedResource.overusage * 100, shuffledTasksOnSlave, currentShuffleCleanupsTotal); break; } Optional message; if (shufflingForCpu) { - LOG.debug("Cleaning up task {} to free up cpu on overloaded host (remaining cpu overage: {})", taskIdWithUsage.getTaskId(), cpuOverage); + currentCpuLoad -= taskIdWithUsage.getUsage().getCpusUsed(); + LOG.debug("Cleaning up task {} to free up cpu on overloaded host (remaining cpu overage: {})", taskIdWithUsage.getTaskId(), currentCpuLoad - overloadedSlave.getSystemCpusTotal()); message = Optional.of(String.format( "Load on slave is %s / %s, shuffling task using %s / %s to less busy host", - systemLoad, + mostOverusedResource.usage, overloadedSlave.getSystemCpusTotal(), taskIdWithUsage.getUsage().getCpusUsed(), taskIdWithUsage.getRequestedResources().getCpus())); @@ -209,13 +228,15 @@ private void shuffleTasksOnOverloadedHosts(Map Date: Fri, 17 May 2019 14:02:45 -0400 Subject: [PATCH 09/21] Add test. --- .../scheduler/SingularityUsageTest.java | 54 +++++++++++++++++-- 1 file changed, 50 insertions(+), 4 deletions(-) diff --git a/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularityUsageTest.java b/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularityUsageTest.java index 775d82cdab..801f237f37 100644 --- a/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularityUsageTest.java +++ b/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularityUsageTest.java @@ -492,16 +492,62 @@ public void itDelaysTaskShuffles() { } @Test - public void itCreatesTaskCleanupsWhenAMachineIsOverloaded() { + public void itCreatesTaskCleanupsWhenAMachineIsOverloadedOnMemory() { try { configuration.setShuffleTasksForOverloadedSlaves(true); configuration.setMinutesBeforeNewTaskEligibleForShuffle(0); + configuration.setShuffleTasksWhenSlaveMemoryUtilizationPercentageExceeds(0.90); initRequest(); initFirstDeployWithResources(configuration.getMesosConfiguration().getDefaultCpus(), configuration.getMesosConfiguration().getDefaultMemory()); saveAndSchedule(requestManager.getRequest(requestId).get().getRequest().toBuilder().setInstances(Optional.of(3))); resourceOffers(1); - SingularitySlaveUsage highUsage = new SingularitySlaveUsage(15, 10, Optional.of(10.0), 1, 1, Optional.of(30L), 1, 1, Optional.of(1024L), 1, System.currentTimeMillis(), 1, 30000, 10, 15, 15, 15, 0, 107374182); + SingularitySlaveUsage highUsage = new SingularitySlaveUsage(10, 10, Optional.of(10.0), 1, 1, Optional.of(30L), 1, 1, Optional.of(1024L), 1, System.currentTimeMillis(), 200000, 10000, 10, 10, 10, 10, 0, 107374182); + usageManager.saveCurrentSlaveUsage(new SingularitySlaveUsageWithId(highUsage, "host1")); + + SingularityTaskId taskId1 = taskManager.getActiveTaskIds().get(0); + String t1 = taskId1.getId(); + SingularityTaskId taskId2 = taskManager.getActiveTaskIds().get(1); + String t2 = taskId2.getId(); + SingularityTaskId taskId3 = taskManager.getActiveTaskIds().get(2); + String t3 = taskId3.getId(); + statusUpdate(taskManager.getTask(taskId1).get(), TaskState.TASK_STARTING, Optional.of(taskId1.getStartedAt())); + statusUpdate(taskManager.getTask(taskId2).get(), TaskState.TASK_STARTING, Optional.of(taskId2.getStartedAt())); + statusUpdate(taskManager.getTask(taskId3).get(), TaskState.TASK_STARTING, Optional.of(taskId3.getStartedAt())); + // task 1 using 3G mem + MesosTaskMonitorObject t1u1 = getTaskMonitor(t1, 2, TimeUnit.MILLISECONDS.toSeconds(taskId1.getStartedAt()) + 5, 95000); + // task 2 using 2G mem + MesosTaskMonitorObject t2u1 = getTaskMonitor(t2, 5, TimeUnit.MILLISECONDS.toSeconds(taskId2.getStartedAt()) + 5, 63333); + // task 3 using 1G mem + MesosTaskMonitorObject t3u1 = getTaskMonitor(t3, 5, TimeUnit.MILLISECONDS.toSeconds(taskId3.getStartedAt()) + 5, 31667); + mesosClient.setSlaveResourceUsage("host1", Arrays.asList(t1u1, t2u1, t3u1)); + mesosClient.setSlaveMetricsSnapshot( + "host1", + new MesosSlaveMetricsSnapshotObject(0, 0, 0, 10.0, 0, 0, 0, 0, 0, 0, 0, 0, 10.0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 0, 0, 10, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 10, 200000, 0, 10000, 0, 0, 0, 10, 0, 0, 0, 0) + ); + + usagePoller.runActionOnPoll(); + + // First task is cleaned up + Assert.assertEquals(TaskCleanupType.REBALANCE_MEMORY_USAGE, taskManager.getTaskCleanup(taskId1.getId()).get().getCleanupType()); + // Second task is not cleaned up because it is from the same request as task 1 + Assert.assertFalse(taskManager.getTaskCleanup(taskId2.getId()).isPresent()); + } finally { + configuration.setShuffleTasksForOverloadedSlaves(false); + } + } + + @Test + public void itCreatesTaskCleanupsWhenAMachineIsOverloadedOnCpu() { + try { + configuration.setShuffleTasksForOverloadedSlaves(true); + configuration.setMinutesBeforeNewTaskEligibleForShuffle(0); + + initRequest(); + initFirstDeployWithResources(configuration.getMesosConfiguration().getDefaultCpus(), configuration.getMesosConfiguration().getDefaultMemory()); + saveAndSchedule(requestManager.getRequest(requestId).get().getRequest().toBuilder().setInstances(Optional.of(3))); + resourceOffers(1); + SingularitySlaveUsage highUsage = new SingularitySlaveUsage(15, 10, Optional.of(10.0), 1, 1, Optional.of(30L), 1, 1, Optional.of(1024L), 1, System.currentTimeMillis(), 200000, 30000, 10, 15, 15, 15, 0, 107374182); usageManager.saveCurrentSlaveUsage(new SingularitySlaveUsageWithId(highUsage, "host1")); SingularityTaskId taskId1 = taskManager.getActiveTaskIds().get(0); @@ -522,13 +568,13 @@ public void itCreatesTaskCleanupsWhenAMachineIsOverloaded() { mesosClient.setSlaveResourceUsage("host1", Arrays.asList(t1u1, t2u1, t3u1)); mesosClient.setSlaveMetricsSnapshot( "host1", - new MesosSlaveMetricsSnapshotObject(0, 0, 0, 10.0, 0, 0, 0, 0, 0, 0, 0, 0, 10.0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0) + new MesosSlaveMetricsSnapshotObject(0, 0, 0, 10.0, 0, 0, 0, 0, 0, 0, 0, 0, 10.0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 200000, 0, 30000, 0, 0, 0, 15, 0, 0, 0, 0) ); usagePoller.runActionOnPoll(); // First task is cleaned up - Assert.assertEquals(taskManager.getTaskCleanup(taskId1.getId()).get().getCleanupType(), TaskCleanupType.REBALANCE_CPU_USAGE); + Assert.assertEquals(TaskCleanupType.REBALANCE_CPU_USAGE, taskManager.getTaskCleanup(taskId1.getId()).get().getCleanupType()); // Second task is not cleaned up because it is from the same request as task 1 Assert.assertFalse(taskManager.getTaskCleanup(taskId2.getId()).isPresent()); } finally { From 4110caff7a81c0c38cc9e1768d4edf5a875b21f5 Mon Sep 17 00:00:00 2001 From: Gowtam Lal Date: Fri, 17 May 2019 14:12:02 -0400 Subject: [PATCH 10/21] Adjust fixtures for other tests. --- .../singularity/scheduler/SingularityUsageTest.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularityUsageTest.java b/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularityUsageTest.java index 801f237f37..6659f3d0ee 100644 --- a/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularityUsageTest.java +++ b/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularityUsageTest.java @@ -450,7 +450,7 @@ public void itDelaysTaskShuffles() { initFirstDeployWithResources(configuration.getMesosConfiguration().getDefaultCpus(), configuration.getMesosConfiguration().getDefaultMemory()); saveAndSchedule(requestManager.getRequest(requestId).get().getRequest().toBuilder().setInstances(Optional.of(3))); resourceOffers(1); - SingularitySlaveUsage highUsage = new SingularitySlaveUsage(15, 10, Optional.of(10.0), 1, 1, Optional.of(30L), 1, 1, Optional.of(1024L), 1, System.currentTimeMillis(), 1, 30000, 10, 15, 15, 15, 0, 107374182); + SingularitySlaveUsage highUsage = new SingularitySlaveUsage(15, 10, Optional.of(10.0), 1, 1, Optional.of(30L), 1, 1, Optional.of(1024L), 1, System.currentTimeMillis(), 200000, 30000, 10, 15, 15, 15, 0, 107374182); usageManager.saveCurrentSlaveUsage(new SingularitySlaveUsageWithId(highUsage, "host1")); SingularityTaskId taskId1 = taskManager.getActiveTaskIds().get(0); @@ -474,7 +474,7 @@ public void itDelaysTaskShuffles() { mesosClient.setSlaveResourceUsage("host1", Arrays.asList(t1u1, t2u1, t3u1)); mesosClient.setSlaveMetricsSnapshot( "host1", - new MesosSlaveMetricsSnapshotObject(0, 0, 0, 10.0, 0, 0, 0, 0, 0, 0, 0, 0, 10.0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0) + new MesosSlaveMetricsSnapshotObject(0, 0, 0, 10.0, 0, 0, 0, 0, 0, 0, 0, 0, 10.0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 200000, 0, 30000, 0, 0, 0, 15, 0, 0, 0, 0) ); usagePoller.runActionOnPoll(); @@ -484,7 +484,7 @@ public void itDelaysTaskShuffles() { Assert.assertFalse(taskManager.getTaskCleanup(taskId3.getId()).isPresent()); // Even though it's not the worst offender, task 2 is cleaned up because it's been running long enough. - Assert.assertEquals(taskManager.getTaskCleanup(taskId2.getId()).get().getCleanupType(), TaskCleanupType.REBALANCE_CPU_USAGE); + Assert.assertEquals(TaskCleanupType.REBALANCE_CPU_USAGE, taskManager.getTaskCleanup(taskId2.getId()).get().getCleanupType()); } finally { configuration.setShuffleTasksForOverloadedSlaves(false); } @@ -593,7 +593,7 @@ public void itLimitsTheNumberOfTaskCleanupsToCreate() { initFirstDeployWithResources(configuration.getMesosConfiguration().getDefaultCpus(), configuration.getMesosConfiguration().getDefaultMemory()); saveAndSchedule(requestManager.getRequest(requestId).get().getRequest().toBuilder().setInstances(Optional.of(3))); resourceOffers(1); - SingularitySlaveUsage highUsage = new SingularitySlaveUsage(15, 10, Optional.of(10.0), 1, 1, Optional.of(30L), 1, 1, Optional.of(1024L), 1, System.currentTimeMillis(), 1, 30000, 10, 15, 15, 15, 0, 107374182); + SingularitySlaveUsage highUsage = new SingularitySlaveUsage(15, 10, Optional.of(10.0), 1, 1, Optional.of(30L), 1, 1, Optional.of(1024L), 1, System.currentTimeMillis(), 200000, 30000, 10, 15, 15, 15, 0, 107374182); usageManager.saveCurrentSlaveUsage(new SingularitySlaveUsageWithId(highUsage, "host1")); SingularityTaskId taskId1 = taskManager.getActiveTaskIds().get(0); @@ -609,13 +609,13 @@ public void itLimitsTheNumberOfTaskCleanupsToCreate() { mesosClient.setSlaveResourceUsage("host1", Arrays.asList(t1u1, t2u1)); mesosClient.setSlaveMetricsSnapshot( "host1", - new MesosSlaveMetricsSnapshotObject(0, 0, 0, 10.0, 0, 0, 0, 0, 0, 0, 0, 0, 10.0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0) + new MesosSlaveMetricsSnapshotObject(0, 0, 0, 10.0, 0, 0, 0, 0, 0, 0, 0, 0, 10.0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 15, 200000, 0, 30000, 0, 0, 0, 15, 0, 0, 0, 0) ); usagePoller.runActionOnPoll(); // First task is cleaned up - Assert.assertEquals(taskManager.getTaskCleanup(taskId1.getId()).get().getCleanupType(), TaskCleanupType.REBALANCE_CPU_USAGE); + Assert.assertEquals(TaskCleanupType.REBALANCE_CPU_USAGE, taskManager.getTaskCleanup(taskId1.getId()).get().getCleanupType()); // Second task doesn't get cleaned up dur to cluster wide limit Assert.assertFalse(taskManager.getTaskCleanup(taskId2.getId()).isPresent()); } finally { From 9aa722bfadbb6a8b7a775979ed83bde6cd21d409 Mon Sep 17 00:00:00 2001 From: Gowtam Lal Date: Fri, 17 May 2019 14:42:26 -0400 Subject: [PATCH 11/21] Update comment. --- .../hubspot/singularity/config/SingularityConfiguration.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java b/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java index 93d5805c63..e1d76cb23d 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java @@ -82,7 +82,7 @@ public class SingularityConfiguration extends Configuration { private int maxConcurrentUsageCollections = 15; - private boolean shuffleTasksForOverloadedSlaves = false; // recommended 'true' when oversubscribing cpu for larger clusters + private boolean shuffleTasksForOverloadedSlaves = false; // recommended 'true' when oversubscribing resources for larger clusters private double shuffleTasksWhenSlaveMemoryUtilizationPercentageExceeds = 0.90; From aacb1935b24761d9f10a3bde3c1075a9e58669c2 Mon Sep 17 00:00:00 2001 From: Gowtam Lal Date: Fri, 17 May 2019 14:52:38 -0400 Subject: [PATCH 12/21] Add dep. --- SingularityService/pom.xml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/SingularityService/pom.xml b/SingularityService/pom.xml index 12fca5c1c6..8719324e2f 100644 --- a/SingularityService/pom.xml +++ b/SingularityService/pom.xml @@ -121,6 +121,11 @@ dropwizard-guicier + + io.dropwizard + dropwizard-util + + com.hubspot.jackson jackson-datatype-protobuf From 6e25b84b26128f2a8c767ae58bed46dc91225c6e Mon Sep 17 00:00:00 2001 From: Gowtam Lal Date: Fri, 17 May 2019 14:58:42 -0400 Subject: [PATCH 13/21] Reduce the default target mem utilization. --- .../hubspot/singularity/config/SingularityConfiguration.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java b/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java index e1d76cb23d..335db94f33 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java @@ -84,7 +84,7 @@ public class SingularityConfiguration extends Configuration { private boolean shuffleTasksForOverloadedSlaves = false; // recommended 'true' when oversubscribing resources for larger clusters - private double shuffleTasksWhenSlaveMemoryUtilizationPercentageExceeds = 0.90; + private double shuffleTasksWhenSlaveMemoryUtilizationPercentageExceeds = 0.82; private int maxTasksToShuffleTotal = 6; // Do not allow more than this many shuffle cleanups at once cluster-wide From f915de8c680a74d140ff1b7cd8b871b36f6afac5 Mon Sep 17 00:00:00 2001 From: Gowtam Lal Date: Tue, 21 May 2019 12:33:24 -0400 Subject: [PATCH 14/21] Bail out when we're no longer in the red. --- .../hubspot/singularity/scheduler/SingularityUsagePoller.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java index 261b7ff139..ece611c82d 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java @@ -202,7 +202,9 @@ private void shuffleTasksOnOverloadedHosts(Map configuration.getMaxTasksToShufflePerHost() || currentShuffleCleanupsTotal >= configuration.getMaxTasksToShuffleTotal()) { + if (((shufflingForCpu && currentCpuLoad <= overloadedSlave.getSystemCpusTotal()) || + (!shufflingForCpu && currentMemUsageBytes <= configuration.getShuffleTasksWhenSlaveMemoryUtilizationPercentageExceeds() * overloadedSlave.getSystemMemTotalBytes())) + || shuffledTasksOnSlave > configuration.getMaxTasksToShufflePerHost() || currentShuffleCleanupsTotal >= configuration.getMaxTasksToShuffleTotal()) { LOG.debug("Not shuffling any more tasks ({} overage : {}%, shuffledOnHost: {}, totalShuffleCleanups: {})", mostOverusedResource.resourceType, mostOverusedResource.overusage * 100, shuffledTasksOnSlave, currentShuffleCleanupsTotal); break; } From c68b2b0e89b9fbb8355a78d952243cc9c56bdfe8 Mon Sep 17 00:00:00 2001 From: Gowtam Lal Date: Tue, 21 May 2019 15:24:25 -0400 Subject: [PATCH 15/21] Use consistent units. --- .../hubspot/singularity/scheduler/SingularityUsagePoller.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java index ece611c82d..2444d9f77d 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java @@ -173,7 +173,7 @@ private void shuffleTasksOnOverloadedHosts(Map Date: Tue, 21 May 2019 15:26:15 -0400 Subject: [PATCH 16/21] Logging. --- .../hubspot/singularity/scheduler/SingularityUsagePoller.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java index 2444d9f77d..eb3e9a57c6 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java @@ -205,7 +205,7 @@ private void shuffleTasksOnOverloadedHosts(Map configuration.getMaxTasksToShufflePerHost() || currentShuffleCleanupsTotal >= configuration.getMaxTasksToShuffleTotal()) { - LOG.debug("Not shuffling any more tasks ({} overage : {}%, shuffledOnHost: {}, totalShuffleCleanups: {})", mostOverusedResource.resourceType, mostOverusedResource.overusage * 100, shuffledTasksOnSlave, currentShuffleCleanupsTotal); + LOG.debug("Not shuffling any more tasks on slave {} ({} overage : {}%, shuffledOnHost: {}, totalShuffleCleanups: {})", taskIdWithUsage.getTaskId().getSanitizedHost(), mostOverusedResource.resourceType, mostOverusedResource.overusage * 100, shuffledTasksOnSlave, currentShuffleCleanupsTotal); break; } From 2534c71519f2a2b4bf37cef49624e680ba0126cf Mon Sep 17 00:00:00 2001 From: Gowtam Lal Date: Tue, 21 May 2019 15:55:38 -0400 Subject: [PATCH 17/21] Use the cumulative values here. --- .../scheduler/SingularityUsagePoller.java | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java index eb3e9a57c6..1186f8db1c 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java @@ -212,14 +212,16 @@ private void shuffleTasksOnOverloadedHosts(Map message; if (shufflingForCpu) { - currentCpuLoad -= taskIdWithUsage.getUsage().getCpusUsed(); - LOG.debug("Cleaning up task {} to free up cpu on overloaded host (remaining cpu overage: {})", taskIdWithUsage.getTaskId(), currentCpuLoad - overloadedSlave.getSystemCpusTotal()); message = Optional.of(String.format( "Load on slave is %s / %s, shuffling task using %s / %s to less busy host", - mostOverusedResource.usage, + currentCpuLoad, overloadedSlave.getSystemCpusTotal(), taskIdWithUsage.getUsage().getCpusUsed(), taskIdWithUsage.getRequestedResources().getCpus())); + + currentCpuLoad -= taskIdWithUsage.getUsage().getCpusUsed(); + LOG.debug("Cleaning up task {} to free up cpu on overloaded host (remaining cpu overage: {})", taskIdWithUsage.getTaskId(), currentCpuLoad - overloadedSlave.getSystemCpusTotal()); + taskManager.createTaskCleanup( new SingularityTaskCleanup( Optional.absent(), @@ -231,16 +233,18 @@ private void shuffleTasksOnOverloadedHosts(Map Date: Tue, 21 May 2019 16:04:42 -0400 Subject: [PATCH 18/21] Don't need this. --- .../singularity/scheduler/SingularityUsagePoller.java | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java index 1186f8db1c..1d26fb17a6 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java @@ -117,12 +117,10 @@ public void runActionOnPoll() { static class OverusedResource { enum Type { MEMORY, CPU }; - double usage; double overusage; Type resourceType; - OverusedResource(double usage, double overusage, Type resourceType) { - this.usage = usage; + OverusedResource(double overusage, Type resourceType) { this.overusage = overusage; this.resourceType = resourceType; } @@ -139,9 +137,9 @@ private OverusedResource getMostOverusedResource(SingularitySlaveUsage overloade double memOverusage = memOverageBytes / targetMemUsageBytes; if (cpuOverusage > memOverusage) { - return new OverusedResource(currentCpuLoad, cpuOverusage, Type.CPU); + return new OverusedResource(cpuOverusage, Type.CPU); } else { - return new OverusedResource(overloadedSlave.getMemoryBytesUsed(), memOverusage, Type.MEMORY); + return new OverusedResource(memOverusage, Type.MEMORY); } } From 56ab734eb54e5510cf71d4521561caf87e9a78eb Mon Sep 17 00:00:00 2001 From: Gowtam Lal Date: Tue, 21 May 2019 18:00:16 -0400 Subject: [PATCH 19/21] Cast this to a long as well to get a consistent log statement. --- .../hubspot/singularity/scheduler/SingularityUsagePoller.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java index 1d26fb17a6..58ce772cdf 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java @@ -236,7 +236,7 @@ private void shuffleTasksOnOverloadedHosts(Map Date: Wed, 22 May 2019 11:58:02 -0400 Subject: [PATCH 20/21] Minor cleanup. --- .../scheduler/SingularityUsagePoller.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java index 58ce772cdf..eaa36b0f5d 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java @@ -126,13 +126,16 @@ enum Type { MEMORY, CPU }; } } + private double getTargetMemoryUtilizationForHost(SingularitySlaveUsage usage) { + return configuration.getShuffleTasksWhenSlaveMemoryUtilizationPercentageExceeds() * usage.getSystemMemTotalBytes(); + } + private OverusedResource getMostOverusedResource(SingularitySlaveUsage overloadedSlave, double currentCpuLoad, double currentMemUsageBytes) { double cpuOverage = currentCpuLoad - overloadedSlave.getSystemCpusTotal(); double cpuOverusage = cpuOverage / overloadedSlave.getSystemCpusTotal(); - // TODO: memoryBytesUsed vs memoryBytesReserved here? - double targetMemUsageBytes = (configuration.getShuffleTasksWhenSlaveMemoryUtilizationPercentageExceeds() * overloadedSlave.getSystemMemTotalBytes()); + double targetMemUsageBytes = getTargetMemoryUtilizationForHost(overloadedSlave); double memOverageBytes = currentMemUsageBytes - targetMemUsageBytes; double memOverusage = memOverageBytes / targetMemUsageBytes; @@ -201,8 +204,8 @@ private void shuffleTasksOnOverloadedHosts(Map configuration.getMaxTasksToShufflePerHost() || currentShuffleCleanupsTotal >= configuration.getMaxTasksToShuffleTotal()) { + (!shufflingForCpu && currentMemUsageBytes <= getTargetMemoryUtilizationForHost(overloadedSlave)) + || shuffledTasksOnSlave > configuration.getMaxTasksToShufflePerHost() || currentShuffleCleanupsTotal >= configuration.getMaxTasksToShuffleTotal())) { LOG.debug("Not shuffling any more tasks on slave {} ({} overage : {}%, shuffledOnHost: {}, totalShuffleCleanups: {})", taskIdWithUsage.getTaskId().getSanitizedHost(), mostOverusedResource.resourceType, mostOverusedResource.overusage * 100, shuffledTasksOnSlave, currentShuffleCleanupsTotal); break; } @@ -229,7 +232,6 @@ private void shuffleTasksOnOverloadedHosts(Map Date: Wed, 22 May 2019 12:07:29 -0400 Subject: [PATCH 21/21] More clarity. --- .../singularity/scheduler/SingularityUsagePoller.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java index eaa36b0f5d..38ceaf5c77 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityUsagePoller.java @@ -203,9 +203,11 @@ private void shuffleTasksOnOverloadedHosts(Map configuration.getMaxTasksToShufflePerHost() || currentShuffleCleanupsTotal >= configuration.getMaxTasksToShuffleTotal())) { + + boolean resourceNoLongerOverutilized = (shufflingForCpu && currentCpuLoad <= overloadedSlave.getSystemCpusTotal()) || (!shufflingForCpu && currentMemUsageBytes <= getTargetMemoryUtilizationForHost(overloadedSlave)); + boolean shufflingTooManyTasks = shuffledTasksOnSlave > configuration.getMaxTasksToShufflePerHost() || currentShuffleCleanupsTotal >= configuration.getMaxTasksToShuffleTotal(); + + if (resourceNoLongerOverutilized || shufflingTooManyTasks) { LOG.debug("Not shuffling any more tasks on slave {} ({} overage : {}%, shuffledOnHost: {}, totalShuffleCleanups: {})", taskIdWithUsage.getTaskId().getSanitizedHost(), mostOverusedResource.resourceType, mostOverusedResource.overusage * 100, shuffledTasksOnSlave, currentShuffleCleanupsTotal); break; }