diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityPendingTask.java b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityPendingTask.java index 940156ab5f..1ad1da5e42 100644 --- a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityPendingTask.java +++ b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityPendingTask.java @@ -20,6 +20,7 @@ public class SingularityPendingTask { private final Optional skipHealthchecks; private final Optional message; private final Optional resources; + private final Optional actionId; public static Predicate matchingRequest(final String requestId) { return new Predicate() { @@ -46,7 +47,7 @@ public boolean apply(@Nonnull SingularityPendingTask input) { @JsonCreator public SingularityPendingTask(@JsonProperty("pendingTaskId") SingularityPendingTaskId pendingTaskId, @JsonProperty("cmdLineArgsList") Optional> cmdLineArgsList, @JsonProperty("user") Optional user, @JsonProperty("runId") Optional runId, @JsonProperty("skipHealthchecks") Optional skipHealthchecks, - @JsonProperty("message") Optional message, @JsonProperty("resources") Optional resources) { + @JsonProperty("message") Optional message, @JsonProperty("resources") Optional resources, @JsonProperty("actionId") Optional actionId) { this.pendingTaskId = pendingTaskId; this.user = user; this.message = message; @@ -54,6 +55,7 @@ public SingularityPendingTask(@JsonProperty("pendingTaskId") SingularityPendingT this.runId = runId; this.skipHealthchecks = skipHealthchecks; this.resources = resources; + this.actionId = actionId; } @Override @@ -104,10 +106,14 @@ public Optional getResources() { return resources; } + public Optional getActionId() { + return actionId; + } + @Override public String toString() { return "SingularityPendingTask [pendingTaskId=" + pendingTaskId + ", cmdLineArgsList=" + cmdLineArgsList + ", user=" + user + ", runId=" + runId + ", skipHealthchecks=" + skipHealthchecks - + ", message=" + message + ", resources=" + resources + "]"; + + ", message=" + message + ", resources=" + resources + ", actionId=" + actionId + "]"; } } diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityRequest.java b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityRequest.java index d9d06edcc4..5563168ece 100644 --- a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityRequest.java +++ b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityRequest.java @@ -58,6 +58,8 @@ public class SingularityRequest { private final Optional taskPriorityLevel; + private final Optional allowBounceToSameHost; + @JsonCreator public SingularityRequest(@JsonProperty("id") String id, @JsonProperty("requestType") RequestType requestType, @JsonProperty("owners") Optional> owners, @JsonProperty("numRetriesOnFailure") Optional numRetriesOnFailure, @JsonProperty("schedule") Optional schedule, @JsonProperty("instances") Optional instances, @@ -73,7 +75,7 @@ public SingularityRequest(@JsonProperty("id") String id, @JsonProperty("requestT @JsonProperty("emailConfigurationOverrides") Optional>> emailConfigurationOverrides, @JsonProperty("daemon") @Deprecated Optional daemon, @JsonProperty("hideEvenNumberAcrossRacks") Optional hideEvenNumberAcrossRacksHint, @JsonProperty("taskLogErrorRegex") Optional taskLogErrorRegex, @JsonProperty("taskLogErrorRegexCaseSensitive") Optional taskLogErrorRegexCaseSensitive, - @JsonProperty("taskPriorityLevel") Optional taskPriorityLevel) { + @JsonProperty("taskPriorityLevel") Optional taskPriorityLevel, @JsonProperty("allowBounceToSameHost") Optional allowBounceToSameHost) { this.id = checkNotNull(id, "id cannot be null"); this.owners = owners; this.numRetriesOnFailure = numRetriesOnFailure; @@ -102,6 +104,7 @@ public SingularityRequest(@JsonProperty("id") String id, @JsonProperty("requestT this.taskLogErrorRegex = taskLogErrorRegex; this.taskLogErrorRegexCaseSensitive = taskLogErrorRegexCaseSensitive; this.taskPriorityLevel = taskPriorityLevel; + this.allowBounceToSameHost = allowBounceToSameHost; if (requestType == null) { this.requestType = RequestType.fromDaemonAndScheduleAndLoadBalanced(schedule, daemon, loadBalanced); } else { @@ -137,7 +140,8 @@ public SingularityRequestBuilder toBuilder() { .setHideEvenNumberAcrossRacksHint(hideEvenNumberAcrossRacksHint) .setTaskLogErrorRegex(taskLogErrorRegex) .setTaskLogErrorRegexCaseSensitive(taskLogErrorRegexCaseSensitive) - .setTaskPriorityLevel(taskPriorityLevel); + .setTaskPriorityLevel(taskPriorityLevel) + .setAllowBounceToSameHost(allowBounceToSameHost); } @ApiModelProperty(required=true, value="A unique id for the request") @@ -230,6 +234,11 @@ public Optional> getAllowedSlaveAttributes() { return allowedSlaveAttributes; } + @ApiModelProperty(required=false, value="If set to true, allow tasks to be scheduled on the same host as an existing active task when bouncing") + public Optional getAllowBounceToSameHost() { + return allowBounceToSameHost; + } + @JsonIgnore public int getInstancesSafe() { return getInstances().or(1); @@ -364,6 +373,7 @@ public String toString() { .add("taskLogErrorRegex", taskLogErrorRegex) .add("taskLogErrorRegexCaseSensitive", taskLogErrorRegexCaseSensitive) .add("taskPriorityLevel", taskPriorityLevel) + .add("allowBounceToSameHost", allowBounceToSameHost) .toString(); } @@ -403,11 +413,12 @@ public boolean equals(Object o) { Objects.equals(hideEvenNumberAcrossRacksHint, request.hideEvenNumberAcrossRacksHint) && Objects.equals(taskLogErrorRegex, request.taskLogErrorRegex) && Objects.equals(taskLogErrorRegexCaseSensitive, request.taskLogErrorRegexCaseSensitive) && - Objects.equals(taskPriorityLevel, request.taskPriorityLevel); + Objects.equals(taskPriorityLevel, request.taskPriorityLevel) && + Objects.equals(allowBounceToSameHost, request.allowBounceToSameHost); } @Override public int hashCode() { - return Objects.hash(id, requestType, owners, numRetriesOnFailure, schedule, quartzSchedule, scheduleTimeZone, scheduleType, killOldNonLongRunningTasksAfterMillis, taskExecutionTimeLimitMillis, scheduledExpectedRuntimeMillis, waitAtLeastMillisAfterTaskFinishesForReschedule, instances, rackSensitive, rackAffinity, slavePlacement, requiredSlaveAttributes, allowedSlaveAttributes, loadBalanced, group, readWriteGroups, readOnlyGroups, bounceAfterScale, emailConfigurationOverrides, hideEvenNumberAcrossRacksHint, taskLogErrorRegex, taskLogErrorRegexCaseSensitive, taskPriorityLevel); + return Objects.hash(id, requestType, owners, numRetriesOnFailure, schedule, quartzSchedule, scheduleTimeZone, scheduleType, killOldNonLongRunningTasksAfterMillis, taskExecutionTimeLimitMillis, scheduledExpectedRuntimeMillis, waitAtLeastMillisAfterTaskFinishesForReschedule, instances, rackSensitive, rackAffinity, slavePlacement, requiredSlaveAttributes, allowedSlaveAttributes, loadBalanced, group, readWriteGroups, readOnlyGroups, bounceAfterScale, emailConfigurationOverrides, hideEvenNumberAcrossRacksHint, taskLogErrorRegex, taskLogErrorRegexCaseSensitive, taskPriorityLevel, allowBounceToSameHost); } } diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityRequestBuilder.java b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityRequestBuilder.java index 398d9d394f..f557697416 100644 --- a/SingularityBase/src/main/java/com/hubspot/singularity/SingularityRequestBuilder.java +++ b/SingularityBase/src/main/java/com/hubspot/singularity/SingularityRequestBuilder.java @@ -48,6 +48,7 @@ public class SingularityRequestBuilder { private Optional taskLogErrorRegex; private Optional taskLogErrorRegexCaseSensitive; private Optional taskPriorityLevel; + private Optional allowBounceToSameHost; public SingularityRequestBuilder(String id, RequestType requestType) { this.id = checkNotNull(id, "id cannot be null"); @@ -79,12 +80,13 @@ public SingularityRequestBuilder(String id, RequestType requestType) { this.taskLogErrorRegex = Optional.absent(); this.taskLogErrorRegexCaseSensitive = Optional.absent(); this.taskPriorityLevel = Optional.absent(); + this.allowBounceToSameHost = Optional.absent(); } public SingularityRequest build() { return new SingularityRequest(id, requestType, owners, numRetriesOnFailure, schedule, instances, rackSensitive, loadBalanced, killOldNonLongRunningTasksAfterMillis, taskExecutionTimeLimitMillis, scheduleType, quartzSchedule, scheduleTimeZone, rackAffinity, slavePlacement, requiredSlaveAttributes, allowedSlaveAttributes, scheduledExpectedRuntimeMillis, waitAtLeastMillisAfterTaskFinishesForReschedule, group, readWriteGroups, readOnlyGroups, - bounceAfterScale, skipHealthchecks, emailConfigurationOverrides, Optional.absent(), hideEvenNumberAcrossRacksHint, taskLogErrorRegex, taskLogErrorRegexCaseSensitive, taskPriorityLevel); + bounceAfterScale, skipHealthchecks, emailConfigurationOverrides, Optional.absent(), hideEvenNumberAcrossRacksHint, taskLogErrorRegex, taskLogErrorRegexCaseSensitive, taskPriorityLevel, allowBounceToSameHost); } public Optional getSkipHealthchecks() { @@ -325,6 +327,15 @@ public SingularityRequestBuilder setTaskPriorityLevel(Optional taskPrior return this; } + public Optional getAllowBounceToSameHost() { + return allowBounceToSameHost; + } + + public SingularityRequestBuilder setAllowBounceToSameHost(Optional allowBounceToSameHost) { + this.allowBounceToSameHost = allowBounceToSameHost; + return this; + } + @Override public String toString() { return "SingularityRequestBuilder[" + @@ -357,6 +368,7 @@ public String toString() { ", taskLogErrorRegex=" + taskLogErrorRegex + ", taskLogErrorRegexCaseSensitive=" + taskLogErrorRegexCaseSensitive + ", taskPriorityLevel=" + taskPriorityLevel + + ", allowBounceToSameHost=" + allowBounceToSameHost + ']'; } @@ -397,7 +409,8 @@ public boolean equals(Object o) { Objects.equals(hideEvenNumberAcrossRacksHint, that.hideEvenNumberAcrossRacksHint) && Objects.equals(taskLogErrorRegex, that.taskLogErrorRegex) && Objects.equals(taskLogErrorRegexCaseSensitive, that.taskLogErrorRegexCaseSensitive) && - Objects.equals(taskPriorityLevel, that.taskPriorityLevel); + Objects.equals(taskPriorityLevel, that.taskPriorityLevel) && + Objects.equals(allowBounceToSameHost, that.allowBounceToSameHost); } @Override @@ -405,7 +418,7 @@ public int hashCode() { return Objects.hash(id, requestType, owners, numRetriesOnFailure, schedule, quartzSchedule, scheduleTimeZone, scheduleType, killOldNonLongRunningTasksAfterMillis, taskExecutionTimeLimitMillis, scheduledExpectedRuntimeMillis, waitAtLeastMillisAfterTaskFinishesForReschedule, instances, rackSensitive, rackAffinity, slavePlacement, requiredSlaveAttributes, allowedSlaveAttributes, loadBalanced, group, readOnlyGroups, readWriteGroups, bounceAfterScale, skipHealthchecks, emailConfigurationOverrides, - hideEvenNumberAcrossRacksHint, taskLogErrorRegex, taskLogErrorRegexCaseSensitive, taskPriorityLevel); + hideEvenNumberAcrossRacksHint, taskLogErrorRegex, taskLogErrorRegexCaseSensitive, taskPriorityLevel, allowBounceToSameHost); } } diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/api/SingularityScaleRequest.java b/SingularityBase/src/main/java/com/hubspot/singularity/api/SingularityScaleRequest.java index 5d72c517bd..de4e1a8579 100644 --- a/SingularityBase/src/main/java/com/hubspot/singularity/api/SingularityScaleRequest.java +++ b/SingularityBase/src/main/java/com/hubspot/singularity/api/SingularityScaleRequest.java @@ -9,13 +9,18 @@ public class SingularityScaleRequest extends SingularityExpiringRequestParent { private final Optional instances; private final Optional skipHealthchecks; + private final Optional bounce; + private final Optional incremental; @JsonCreator public SingularityScaleRequest(@JsonProperty("instances") Optional instances, @JsonProperty("durationMillis") Optional durationMillis, - @JsonProperty("skipHealthchecks") Optional skipHealthchecks, @JsonProperty("actionId") Optional actionId, @JsonProperty("message") Optional message) { + @JsonProperty("skipHealthchecks") Optional skipHealthchecks, @JsonProperty("actionId") Optional actionId, @JsonProperty("message") Optional message, + @JsonProperty("bounce") Optional bounce, @JsonProperty("incremental") Optional incremental) { super(durationMillis, actionId, message); this.instances = instances; this.skipHealthchecks = skipHealthchecks; + this.bounce = bounce; + this.incremental = incremental; } @ApiModelProperty(required=false, value="If set to true, healthchecks will be skipped while scaling this request (only)") @@ -28,9 +33,19 @@ public Optional getInstances() { return instances; } + @ApiModelProperty(required=false, value="Bounce the request to get to the new scale") + public Optional getBounce() { + return bounce; + } + + @ApiModelProperty(required=false, value="If present and set to true, old tasks will be killed as soon as replacement tasks are available, instead of waiting for all replacement tasks to be healthy") + public Optional getIncremental() { + return incremental; + } + @Override public String toString() { - return "SingularityScaleRequest [instances=" + instances + ", skipHealthchecks=" + skipHealthchecks + ", toString()=" + super.toString() + "]"; + return "SingularityScaleRequest [instances=" + instances + ", skipHealthchecks=" + skipHealthchecks + ", bounce=" + bounce + ", incremental=" + incremental + ", toString()=" + super.toString() + "]"; } } diff --git a/SingularityBase/src/main/java/com/hubspot/singularity/expiring/SingularityExpiringScale.java b/SingularityBase/src/main/java/com/hubspot/singularity/expiring/SingularityExpiringScale.java index be5097cc06..fdfb4ea38c 100644 --- a/SingularityBase/src/main/java/com/hubspot/singularity/expiring/SingularityExpiringScale.java +++ b/SingularityBase/src/main/java/com/hubspot/singularity/expiring/SingularityExpiringScale.java @@ -7,22 +7,28 @@ public class SingularityExpiringScale extends SingularityExpiringRequestActionParent { private final Optional revertToInstances; + private final Optional bounce; public SingularityExpiringScale(@JsonProperty("requestId") String requestId, @JsonProperty("user") Optional user, @JsonProperty("startMillis") long startMillis, @JsonProperty("expiringAPIRequestObject") SingularityScaleRequest scaleRequest, @JsonProperty("revertToInstances") Optional revertToInstances, - @JsonProperty("actionId") String actionId) { + @JsonProperty("actionId") String actionId, @JsonProperty("bounce") Optional bounce) { super(scaleRequest, user, startMillis, actionId, requestId); this.revertToInstances = revertToInstances; + this.bounce = bounce; } public Optional getRevertToInstances() { return revertToInstances; } + public Optional getBounce() { + return bounce; + } + @Override public String toString() { - return "SingularityExpiringScale [revertToInstances=" + revertToInstances + "]"; + return "SingularityExpiringScale [revertToInstances=" + revertToInstances + ", bounce=" + bounce + "]"; } } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java b/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java index af16a44a98..a57d547a2d 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java @@ -302,6 +302,8 @@ public class SingularityConfiguration extends Configuration { private boolean rebalanceRacksOnScaleDown = false; + private boolean allowBounceToSameHost = false; + public long getAskDriverToKillTasksAgainAfterMillis() { return askDriverToKillTasksAgainAfterMillis; } @@ -1207,4 +1209,13 @@ public boolean isRebalanceRacksOnScaleDown() { public void setRebalanceRacksOnScaleDown(boolean rebalanceRacksOnScaleDown) { this.rebalanceRacksOnScaleDown = rebalanceRacksOnScaleDown; } + + public boolean isAllowBounceToSameHost() { + return allowBounceToSameHost; + } + + public SingularityConfiguration setAllowBounceToSameHost(boolean allowBounceToSameHost) { + this.allowBounceToSameHost = allowBounceToSameHost; + return this; + } } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/data/RequestManager.java b/SingularityService/src/main/java/com/hubspot/singularity/data/RequestManager.java index 6f0eb108bb..b427c21cbd 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/data/RequestManager.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/data/RequestManager.java @@ -176,6 +176,10 @@ public boolean cleanupRequestExists(String requestId) { return false; } + public boolean cleanupRequestExists(String requestId, RequestCleanupType type) { + return checkExists(getCleanupPath(requestId, type)).isPresent(); + } + public void deleteCleanRequest(String requestId, RequestCleanupType type) { delete(getCleanupPath(requestId, type)); } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/data/SingularityValidator.java b/SingularityService/src/main/java/com/hubspot/singularity/data/SingularityValidator.java index 65766775e4..29f0bfd414 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/data/SingularityValidator.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/data/SingularityValidator.java @@ -92,6 +92,7 @@ public class SingularityValidator { private final boolean allowRequestsWithoutOwners; private final boolean createDeployIds; private final int deployIdLength; + private final boolean allowBounceToSameHost; private final UIConfiguration uiConfiguration; private final SlavePlacement defaultSlavePlacement; private final DeployHistoryHelper deployHistoryHelper; @@ -125,6 +126,8 @@ public SingularityValidator(SingularityConfiguration configuration, DeployHistor this.maxMemoryMbPerRequest = configuration.getMesosConfiguration().getMaxMemoryMbPerRequest(); this.maxInstancesPerRequest = configuration.getMesosConfiguration().getMaxNumInstancesPerRequest(); + this.allowBounceToSameHost = configuration.isAllowBounceToSameHost(); + this.maxTotalHealthcheckTimeoutSeconds = configuration.getHealthcheckMaxTotalTimeoutSeconds(); this.defaultHealthcheckIntervalSeconds = configuration.getHealthcheckIntervalSeconds(); this.defaultHealthcheckStartupTimeooutSeconds = configuration.getStartupTimeoutSeconds(); @@ -524,7 +527,8 @@ private String getNewDayOfWeekValue(String schedule, int dayOfWeekValue) { public void checkResourcesForBounce(SingularityRequest request, boolean isIncremental) { SlavePlacement placement = request.getSlavePlacement().or(defaultSlavePlacement); - if (placement != SlavePlacement.GREEDY && placement != SlavePlacement.OPTIMISTIC) { + if ((isAllowBounceToSameHost(request) && placement == SlavePlacement.SEPARATE_BY_REQUEST) + || (!isAllowBounceToSameHost(request) && placement != SlavePlacement.GREEDY && placement != SlavePlacement.OPTIMISTIC)) { int currentActiveSlaveCount = slaveManager.getNumObjectsAtState(MachineState.ACTIVE); int requiredSlaveCount = isIncremental ? request.getInstancesSafe() + 1 : request.getInstancesSafe() * 2; @@ -532,6 +536,14 @@ public void checkResourcesForBounce(SingularityRequest request, boolean isIncrem } } + private boolean isAllowBounceToSameHost(SingularityRequest request) { + if (request.getAllowBounceToSameHost().isPresent()) { + return request.getAllowBounceToSameHost().get(); + } else { + return allowBounceToSameHost; + } + } + public void checkScale(SingularityRequest request, Optional previousScale) { SlavePlacement placement = request.getSlavePlacement().or(defaultSlavePlacement); diff --git a/SingularityService/src/main/java/com/hubspot/singularity/data/zkmigrations/SingularityCmdLineArgsMigration.java b/SingularityService/src/main/java/com/hubspot/singularity/data/zkmigrations/SingularityCmdLineArgsMigration.java index 36fb1c3f7c..f21401fa54 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/data/zkmigrations/SingularityCmdLineArgsMigration.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/data/zkmigrations/SingularityCmdLineArgsMigration.java @@ -143,7 +143,7 @@ private void checkPendingTasks() { Optional cmdLineArgs = getCmdLineArgs(pendingTaskId); SingularityCreateResult result = taskManager.savePendingTask(new SingularityPendingTask(pendingTaskId, getCmdLineArgs(cmdLineArgs), Optional. absent(), - Optional. absent(), Optional. absent(), Optional. absent(), Optional.absent())); + Optional. absent(), Optional. absent(), Optional. absent(), Optional.absent(), Optional.absent())); LOG.info("Saving {} ({}) {}", pendingTaskId, cmdLineArgs, result); } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/data/zkmigrations/SingularityPendingTaskIdMigration.java b/SingularityService/src/main/java/com/hubspot/singularity/data/zkmigrations/SingularityPendingTaskIdMigration.java index c9eaab6e74..e4b8d06195 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/data/zkmigrations/SingularityPendingTaskIdMigration.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/data/zkmigrations/SingularityPendingTaskIdMigration.java @@ -60,7 +60,7 @@ public void applyMigration() { Optional cmdLineArgs = getCmdLineArgs(pendingTaskId); taskManager.savePendingTask(new SingularityPendingTask(newPendingTaskId, cmdLineArgs.isPresent() ? Optional.of(Collections.singletonList(cmdLineArgs.get())) : - Optional.> absent(), Optional. absent(), Optional. absent(), Optional. absent(), Optional. absent(), Optional.absent())); + Optional.> absent(), Optional. absent(), Optional. absent(), Optional. absent(), Optional. absent(), Optional.absent(), Optional.absent())); curator.delete().forPath(ZKPaths.makePath(PENDING_TASKS_ROOT, pendingTaskId)); } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/helpers/RequestHelper.java b/SingularityService/src/main/java/com/hubspot/singularity/helpers/RequestHelper.java index 9db87c368a..f71403136a 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/helpers/RequestHelper.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/helpers/RequestHelper.java @@ -1,18 +1,25 @@ package com.hubspot.singularity.helpers; +import java.util.UUID; + import com.google.common.base.Optional; import com.google.inject.Inject; import com.google.inject.Singleton; +import com.hubspot.singularity.RequestCleanupType; import com.hubspot.singularity.RequestState; +import com.hubspot.singularity.SingularityCreateResult; import com.hubspot.singularity.SingularityDeploy; import com.hubspot.singularity.SingularityPendingRequest; import com.hubspot.singularity.SingularityPendingRequest.PendingType; import com.hubspot.singularity.SingularityRequest; +import com.hubspot.singularity.SingularityRequestCleanup; import com.hubspot.singularity.SingularityRequestDeployState; import com.hubspot.singularity.SingularityRequestHistory.RequestHistoryType; +import com.hubspot.singularity.api.SingularityBounceRequest; import com.hubspot.singularity.data.DeployManager; import com.hubspot.singularity.data.RequestManager; import com.hubspot.singularity.data.SingularityValidator; +import com.hubspot.singularity.expiring.SingularityExpiringBounce; import com.hubspot.singularity.smtp.SingularityMailer; @Singleton @@ -79,7 +86,7 @@ private boolean shouldReschedule(SingularityRequest newRequest, SingularityReque return false; } - private void checkReschedule(SingularityRequest newRequest, Optional maybeOldRequest, Optional user, long timestamp, Optional skipHealthchecks, Optional message) { + private void checkReschedule(SingularityRequest newRequest, Optional maybeOldRequest, Optional user, long timestamp, Optional skipHealthchecks, Optional message, Optional maybeBounceRequest) { if (!maybeOldRequest.isPresent()) { return; } @@ -88,14 +95,30 @@ private void checkReschedule(SingularityRequest newRequest, Optional maybeDeployId = deployManager.getInUseDeployId(newRequest.getId()); if (maybeDeployId.isPresent()) { - requestManager.addToPendingQueue(new SingularityPendingRequest(newRequest.getId(), maybeDeployId.get(), timestamp, user, PendingType.UPDATED_REQUEST, + if (maybeBounceRequest.isPresent()) { + Optional actionId = maybeBounceRequest.get().getActionId().or(Optional.of(UUID.randomUUID().toString())); + SingularityCreateResult createResult = requestManager.createCleanupRequest( + new SingularityRequestCleanup(user, maybeBounceRequest.get().getIncremental().or(true) ? RequestCleanupType.INCREMENTAL_BOUNCE : RequestCleanupType.BOUNCE, + System.currentTimeMillis(), Optional. absent(), newRequest.getId(), Optional.of(maybeDeployId.get()), skipHealthchecks, message, actionId, maybeBounceRequest.get().getRunShellCommandBeforeKill())); + + if (createResult != SingularityCreateResult.EXISTED) { + requestManager.bounce(newRequest, System.currentTimeMillis(), user, Optional.of(String.format("Bouncing due to bounce after scale by %s", user))); + final SingularityBounceRequest validatedBounceRequest = validator.checkBounceRequest(maybeBounceRequest.get()); + requestManager.saveExpiringObject(new SingularityExpiringBounce(newRequest.getId(), maybeDeployId.get(), user, System.currentTimeMillis(), validatedBounceRequest, actionId.get())); + } else { + requestManager.addToPendingQueue(new SingularityPendingRequest(newRequest.getId(), maybeDeployId.get(), timestamp, user, PendingType.UPDATED_REQUEST, + skipHealthchecks, message)); + } + } else { + requestManager.addToPendingQueue(new SingularityPendingRequest(newRequest.getId(), maybeDeployId.get(), timestamp, user, PendingType.UPDATED_REQUEST, skipHealthchecks, message)); + } } } } public void updateRequest(SingularityRequest request, Optional maybeOldRequest, RequestState requestState, Optional historyType, - Optional user, Optional skipHealthchecks, Optional message) { + Optional user, Optional skipHealthchecks, Optional message, Optional maybeBounceRequest) { SingularityRequestDeployHolder deployHolder = getDeployHolder(request.getId()); SingularityRequest newRequest = validator.checkSingularityRequest(request, maybeOldRequest, deployHolder.getActiveDeploy(), deployHolder.getPendingDeploy()); @@ -118,7 +141,7 @@ public void updateRequest(SingularityRequest request, Optional countPerRack = HashMultiset.create(stateCache.getNumActiveRacks()); double numOnSlave = 0; double numCleaningOnSlave = 0; + double numFromSameBounceOnSlave = 0; double numOtherDeploysOnSlave = 0; + boolean taskLaunchedFromBounceWithActionId = taskRequest.getPendingTask().getPendingTaskId().getPendingType() == PendingType.BOUNCE && taskRequest.getPendingTask().getActionId().isPresent(); final String sanitizedHost = JavaUtils.getReplaceHyphensWithUnderscores(host); final String sanitizedRackId = JavaUtils.getReplaceHyphensWithUnderscores(rackId); @@ -151,6 +157,29 @@ public SlaveMatchState doesOfferMatch(Protos.Offer offer, SingularityTaskRequest } else { numOnSlave++; } + if (taskLaunchedFromBounceWithActionId) { + Optional maybeTask = taskManager.getTask(taskId); + boolean errorInTaskData = false; + if (maybeTask.isPresent()) { + SingularityPendingTask pendingTask = maybeTask.get().getTaskRequest().getPendingTask(); + if (pendingTask.getPendingTaskId().getPendingType() == PendingType.BOUNCE) { + if (pendingTask.getActionId().isPresent()) { + if (pendingTask.getActionId().get().equals(taskRequest.getPendingTask().getActionId().get())) { + numFromSameBounceOnSlave++; + } + } else { + // No actionId present on bounce, fall back to more restrictive placement strategy + errorInTaskData = true; + } + } + } else { + // Could not find appropriate task data, fall back to more restrictive placement strategy + errorInTaskData = true; + } + if (errorInTaskData) { + allowBounceToSameHost = false; + } + } } else { numOtherDeploysOnSlave++; } @@ -171,14 +200,21 @@ public SlaveMatchState doesOfferMatch(Protos.Offer offer, SingularityTaskRequest switch (slavePlacement) { case SEPARATE: case SEPARATE_BY_DEPLOY: - if (numOnSlave > 0 || numCleaningOnSlave > 0) { - LOG.trace("Rejecting SEPARATE task {} from slave {} ({}) due to numOnSlave {} numCleaningOnSlave {}", taskRequest.getRequest().getId(), slaveId, host, numOnSlave, numCleaningOnSlave); - return SlaveMatchState.SLAVE_SATURATED; + if (allowBounceToSameHost && taskLaunchedFromBounceWithActionId) { + if (numFromSameBounceOnSlave > 0) { + LOG.trace("Rejecting SEPARATE task {} from slave {} ({}) due to numFromSameBounceOnSlave {}", taskRequest.getRequest().getId(), slaveId, host, numFromSameBounceOnSlave); + return SlaveMatchState.SLAVE_SATURATED; + } + } else { + if (numOnSlave > 0 || numCleaningOnSlave > 0) { + LOG.trace("Rejecting SEPARATE task {} from slave {} ({}) due to numOnSlave {} numCleaningOnSlave {}", taskRequest.getRequest().getId(), slaveId, host, numOnSlave, numCleaningOnSlave); + return SlaveMatchState.SLAVE_SATURATED; + } } break; case SEPARATE_BY_REQUEST: if (numOnSlave > 0 || numCleaningOnSlave > 0 || numOtherDeploysOnSlave > 0) { - LOG.trace("Rejecting SEPARATE task {} from slave {} ({}) due to numOnSlave {} numCleaningOnSlave {} numOtherDeploysOnSlave {}", taskRequest.getRequest().getId(), slaveId, host, numOnSlave, numCleaningOnSlave, numOtherDeploysOnSlave); + LOG.trace("Rejecting SEPARATE_BY_REQUEST task {} from slave {} ({}) due to numOnSlave {} numCleaningOnSlave {} numOtherDeploysOnSlave {}", taskRequest.getRequest().getId(), slaveId, host, numOnSlave, numCleaningOnSlave, numOtherDeploysOnSlave); return SlaveMatchState.SLAVE_SATURATED; } break; @@ -198,6 +234,14 @@ public SlaveMatchState doesOfferMatch(Protos.Offer offer, SingularityTaskRequest return SlaveMatchState.OK; } + private boolean isAllowBounceToSameHost(SingularityRequest request) { + if (request.getAllowBounceToSameHost().isPresent()) { + return request.getAllowBounceToSameHost().get(); + } else { + return configuration.isAllowBounceToSameHost(); + } + } + private boolean isRackOk(Multiset countPerRack, String sanitizedRackId, int numDesiredInstances, String requestId, String slaveId, String host, double numCleaningOnSlave, SingularitySchedulerStateCache stateCache) { int racksAccountedFor = countPerRack.elementSet().size(); double numPerRack = numDesiredInstances / (double) stateCache.getNumActiveRacks(); diff --git a/SingularityService/src/main/java/com/hubspot/singularity/resources/RequestResource.java b/SingularityService/src/main/java/com/hubspot/singularity/resources/RequestResource.java index 3f6acef8ef..943df6d2e8 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/resources/RequestResource.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/resources/RequestResource.java @@ -96,7 +96,7 @@ public RequestResource(SingularityValidator validator, DeployManager deployManag } private void submitRequest(SingularityRequest request, Optional oldRequestWithState, Optional historyType, - Optional skipHealthchecks, Optional message) { + Optional skipHealthchecks, Optional message, Optional maybeBounceRequest) { checkNotNullBadRequest(request.getId(), "Request must have an id"); checkConflict(!requestManager.cleanupRequestExists(request.getId()), "Request %s is currently cleaning. Try again after a few moments", request.getId()); @@ -125,7 +125,7 @@ private void submitRequest(SingularityRequest request, Optional absent(), Optional. absent(), Optional. absent()); + submitRequest(request, requestManager.getRequest(request.getId()), Optional. absent(), Optional. absent(), Optional. absent(), Optional.absent()); return fillEntireRequest(fetchRequestWithState(request.getId())); } @@ -568,16 +568,32 @@ public SingularityRequestParent scale(@ApiParam("The Request ID to scale") @Path validator.checkActionEnabled(SingularityAction.SCALE_REQUEST); SingularityRequest newRequest = oldRequest.toBuilder().setInstances(scaleRequest.getInstances()).build(); - validator.checkScale(newRequest, Optional.absent()); checkBadRequest(oldRequest.getInstancesSafe() != newRequest.getInstancesSafe(), "Scale request has no affect on the # of instances (%s)", newRequest.getInstancesSafe()); - submitRequest(newRequest, Optional.of(oldRequestWithState), Optional.of(RequestHistoryType.SCALED), scaleRequest.getSkipHealthchecks(), scaleRequest.getMessage()); + if (newRequest.getBounceAfterScale().or(scaleRequest.getBounce().or(false))) { + validator.checkActionEnabled(SingularityAction.BOUNCE_REQUEST); + + checkBadRequest(newRequest.isLongRunning(), "Can not bounce a %s request (%s)", newRequest.getRequestType(), newRequest); + checkConflict(oldRequestWithState.getState() != RequestState.PAUSED, "Request %s is paused. Unable to bounce (it must be manually unpaused first)", newRequest.getId()); + checkConflict(!requestManager.cleanupRequestExists(newRequest.getId(), RequestCleanupType.BOUNCE), "Request %s is already bouncing cannot bounce again", newRequest.getId()); + + final boolean isIncrementalBounce = scaleRequest.getIncremental().or(true); + + validator.checkResourcesForBounce(newRequest, isIncrementalBounce); + validator.checkRequestForPriorityFreeze(newRequest); + + SingularityBounceRequest bounceRequest = new SingularityBounceRequest(Optional.of(isIncrementalBounce), scaleRequest.getSkipHealthchecks(), Optional.absent(), Optional.of(UUID.randomUUID().toString()), Optional.absent(), Optional.absent()); + + submitRequest(newRequest, Optional.of(oldRequestWithState), Optional.of(RequestHistoryType.SCALED), scaleRequest.getSkipHealthchecks(), scaleRequest.getMessage(), Optional.of(bounceRequest)); + } else { + submitRequest(newRequest, Optional.of(oldRequestWithState), Optional.of(RequestHistoryType.SCALED), scaleRequest.getSkipHealthchecks(), scaleRequest.getMessage(), Optional.absent()); + } if (scaleRequest.getDurationMillis().isPresent()) { requestManager.saveExpiringObject(new SingularityExpiringScale(requestId, JavaUtils.getUserEmail(user), - System.currentTimeMillis(), scaleRequest, oldRequest.getInstances(), scaleRequest.getActionId().or(UUID.randomUUID().toString()))); + System.currentTimeMillis(), scaleRequest, oldRequest.getInstances(), scaleRequest.getActionId().or(UUID.randomUUID().toString()), scaleRequest.getBounce())); } mailer.sendRequestScaledMail(newRequest, Optional.of(scaleRequest), oldRequest.getInstances(), JavaUtils.getUserEmail(user)); @@ -673,7 +689,7 @@ public SingularityRequestParent skipHealthchecks(@ApiParam("The Request ID to sc SingularityRequest oldRequest = oldRequestWithState.getRequest(); SingularityRequest newRequest = oldRequest.toBuilder().setSkipHealthchecks(skipHealthchecksRequest.getSkipHealthchecks()).build(); - submitRequest(newRequest, Optional.of(oldRequestWithState), Optional. absent(), Optional. absent(), skipHealthchecksRequest.getMessage()); + submitRequest(newRequest, Optional.of(oldRequestWithState), Optional. absent(), Optional. absent(), skipHealthchecksRequest.getMessage(), Optional.absent()); if (skipHealthchecksRequest.getDurationMillis().isPresent()) { requestManager.saveExpiringObject(new SingularityExpiringSkipHealthchecks(requestId, JavaUtils.getUserEmail(user), diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityExpiringUserActionPoller.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityExpiringUserActionPoller.java index 7d54ea2568..8ac79e690f 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityExpiringUserActionPoller.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityExpiringUserActionPoller.java @@ -29,14 +29,15 @@ import com.hubspot.singularity.SingularityRequestHistory.RequestHistoryType; import com.hubspot.singularity.SingularityRequestWithState; import com.hubspot.singularity.SingularitySlave; -import com.hubspot.singularity.SingularityTask; import com.hubspot.singularity.SingularityTaskCleanup; import com.hubspot.singularity.SingularityTaskHistoryUpdate; import com.hubspot.singularity.SingularityTaskId; import com.hubspot.singularity.SingularityTaskShellCommandRequestId; import com.hubspot.singularity.TaskCleanupType; +import com.hubspot.singularity.api.SingularityBounceRequest; import com.hubspot.singularity.api.SingularityScaleRequest; import com.hubspot.singularity.config.SingularityConfiguration; +import com.hubspot.singularity.data.DeployManager; import com.hubspot.singularity.data.RackManager; import com.hubspot.singularity.data.RequestManager; import com.hubspot.singularity.data.SlaveManager; @@ -58,6 +59,7 @@ public class SingularityExpiringUserActionPoller extends SingularityLeaderOnlyPo private static final Logger LOG = LoggerFactory.getLogger(SingularityExpiringUserActionPoller.class); private final RequestManager requestManager; + private final DeployManager deployManager; private final TaskManager taskManager; private final SingularityMailer mailer; private final RequestHelper requestHelper; @@ -67,10 +69,11 @@ public class SingularityExpiringUserActionPoller extends SingularityLeaderOnlyPo private final SingularityConfiguration configuration; @Inject - SingularityExpiringUserActionPoller(SingularityConfiguration configuration, RequestManager requestManager, TaskManager taskManager, SlaveManager slaveManager, RackManager rackManager, + SingularityExpiringUserActionPoller(SingularityConfiguration configuration, RequestManager requestManager, DeployManager deployManager, TaskManager taskManager, SlaveManager slaveManager, RackManager rackManager, @Named(SingularityMesosModule.SCHEDULER_LOCK_NAME) final Lock lock, RequestHelper requestHelper, SingularityMailer mailer) { super(configuration.getCheckExpiringUserActionEveryMillis(), TimeUnit.MILLISECONDS, lock); + this.deployManager = deployManager; this.requestManager = requestManager; this.requestHelper = requestHelper; this.mailer = mailer; @@ -252,8 +255,20 @@ protected void handleExpiringObject(SingularityExpiringScale expiringObject, Sin final SingularityRequest newRequest = oldRequest.toBuilder().setInstances(expiringObject.getRevertToInstances()).build(); try { + Optional maybeBounceRequest = Optional.absent(); + + if (expiringObject.getBounce().or(false) || newRequest.getBounceAfterScale().or(false)) { + LOG.info("Attempting to bounce request {} after expiring scale", newRequest.getId()); + Optional maybeActiveDeployId = deployManager.getInUseDeployId(newRequest.getId()); + if (maybeActiveDeployId.isPresent()) { + maybeBounceRequest = Optional.of(SingularityBounceRequest.defaultRequest()); + } else { + LOG.debug("No active deploy id present for request {}, not bouncing after expiring scale", newRequest.getId()); + } + } + requestHelper.updateRequest(newRequest, Optional.of(oldRequest), requestWithState.getState(), Optional.of(RequestHistoryType.SCALE_REVERTED), expiringObject.getUser(), - Optional. absent(), Optional.of(message)); + Optional. absent(), Optional.of(message), maybeBounceRequest); mailer.sendRequestScaledMail(newRequest, Optional. absent(), oldRequest.getInstances(), expiringObject.getUser()); } catch (WebApplicationException wae) { @@ -281,7 +296,7 @@ protected void handleExpiringObject(SingularityExpiringSkipHealthchecks expiring try { requestHelper.updateRequest(newRequest, Optional.of(oldRequest), requestWithState.getState(), Optional. absent(), expiringObject.getUser(), - Optional. absent(), Optional.of(message)); + Optional. absent(), Optional.of(message), Optional.absent()); } catch (WebApplicationException wae) { LOG.error("While trying to apply {} for {}", expiringObject, expiringObject.getRequestId(), wae); } diff --git a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityScheduler.java b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityScheduler.java index e034b3fe4b..0e1c08f768 100644 --- a/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityScheduler.java +++ b/SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityScheduler.java @@ -709,7 +709,7 @@ private List getScheduledTaskIds(int numMissingInstances newTasks .add(new SingularityPendingTask(new SingularityPendingTaskId(request.getId(), deployId, nextRunAt.get(), nextInstanceNumber, pendingRequest.getPendingType(), pendingRequest.getTimestamp()), - pendingRequest.getCmdLineArgsList(), pendingRequest.getUser(), pendingRequest.getRunId(), pendingRequest.getSkipHealthchecks(), pendingRequest.getMessage(), pendingRequest.getResources())); + pendingRequest.getCmdLineArgsList(), pendingRequest.getUser(), pendingRequest.getRunId(), pendingRequest.getSkipHealthchecks(), pendingRequest.getMessage(), pendingRequest.getResources(), pendingRequest.getActionId())); nextInstanceNumber++; } diff --git a/SingularityService/src/test/java/com/hubspot/singularity/SingularityHistoryTest.java b/SingularityService/src/test/java/com/hubspot/singularity/SingularityHistoryTest.java index 0633980d31..060a76fb8c 100644 --- a/SingularityService/src/test/java/com/hubspot/singularity/SingularityHistoryTest.java +++ b/SingularityService/src/test/java/com/hubspot/singularity/SingularityHistoryTest.java @@ -488,7 +488,7 @@ public void testMessage() { msg = msg + i; } - requestResource.scale(requestId, new SingularityScaleRequest(Optional.of(2), Optional. absent(), Optional. absent(), Optional. absent(), Optional.of(msg))); + requestResource.scale(requestId, new SingularityScaleRequest(Optional.of(2), Optional. absent(), Optional. absent(), Optional. absent(), Optional.of(msg), Optional.absent(), Optional.absent())); requestResource.deleteRequest(requestId, Optional.of(new SingularityDeleteRequestRequest(Optional.of("a msg"), Optional. absent()))); cleaner.drainCleanupQueue(); diff --git a/SingularityService/src/test/java/com/hubspot/singularity/mesos/SingularityMesosTaskBuilderTest.java b/SingularityService/src/test/java/com/hubspot/singularity/mesos/SingularityMesosTaskBuilderTest.java index 24ffbe91a7..f557b0a8c8 100644 --- a/SingularityService/src/test/java/com/hubspot/singularity/mesos/SingularityMesosTaskBuilderTest.java +++ b/SingularityService/src/test/java/com/hubspot/singularity/mesos/SingularityMesosTaskBuilderTest.java @@ -64,7 +64,7 @@ public class SingularityMesosTaskBuilderTest { @Before public void createMocks() { pendingTask = new SingularityPendingTask(new SingularityPendingTaskId("test", "1", 0, 1, PendingType.IMMEDIATE, 0), Optional.> absent(), - Optional. absent(), Optional. absent(), Optional. absent(), Optional. absent(), Optional.absent()); + Optional. absent(), Optional. absent(), Optional. absent(), Optional. absent(), Optional.absent(), Optional.absent()); final SingularitySlaveAndRackHelper slaveAndRackHelper = mock(SingularitySlaveAndRackHelper.class); final ExecutorIdGenerator idGenerator = mock(ExecutorIdGenerator.class); diff --git a/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularityDeploysTest.java b/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularityDeploysTest.java index 0759abb1bb..4f922d3d29 100644 --- a/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularityDeploysTest.java +++ b/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularityDeploysTest.java @@ -74,16 +74,16 @@ public void testDeployClearsObsoleteScheduledTasks() { initSecondDeploy(); SingularityPendingTaskId taskIdOne = new SingularityPendingTaskId(requestId, firstDeployId, System.currentTimeMillis() + TimeUnit.DAYS.toMillis(3), 1, PendingType.IMMEDIATE, System.currentTimeMillis()); - SingularityPendingTask taskOne = new SingularityPendingTask(taskIdOne, Optional.> absent(), Optional. absent(), Optional. absent(), Optional. absent(), Optional. absent(), Optional.absent()); + SingularityPendingTask taskOne = new SingularityPendingTask(taskIdOne, Optional.> absent(), Optional. absent(), Optional. absent(), Optional. absent(), Optional. absent(), Optional.absent(), Optional.absent()); SingularityPendingTaskId taskIdTwo = new SingularityPendingTaskId(requestId, firstDeployId, System.currentTimeMillis() + TimeUnit.DAYS.toMillis(1), 2, PendingType.IMMEDIATE, System.currentTimeMillis()); - SingularityPendingTask taskTwo = new SingularityPendingTask(taskIdTwo, Optional.> absent(), Optional. absent(), Optional. absent(), Optional. absent(), Optional. absent(), Optional.absent()); + SingularityPendingTask taskTwo = new SingularityPendingTask(taskIdTwo, Optional.> absent(), Optional. absent(), Optional. absent(), Optional. absent(), Optional. absent(), Optional.absent(), Optional.absent()); SingularityPendingTaskId taskIdThree = new SingularityPendingTaskId(requestId, secondDeployId, System.currentTimeMillis() + TimeUnit.DAYS.toMillis(3), 1, PendingType.IMMEDIATE, System.currentTimeMillis()); - SingularityPendingTask taskThree = new SingularityPendingTask(taskIdThree, Optional.> absent(), Optional. absent(), Optional. absent(), Optional. absent(), Optional. absent(), Optional.absent()); + SingularityPendingTask taskThree = new SingularityPendingTask(taskIdThree, Optional.> absent(), Optional. absent(), Optional. absent(), Optional. absent(), Optional. absent(), Optional.absent(), Optional.absent()); SingularityPendingTaskId taskIdFour = new SingularityPendingTaskId(requestId + "hi", firstDeployId, System.currentTimeMillis() + TimeUnit.DAYS.toMillis(3), 5, PendingType.IMMEDIATE, System.currentTimeMillis()); - SingularityPendingTask taskFour = new SingularityPendingTask(taskIdFour,Optional.> absent(), Optional. absent(), Optional. absent(), Optional. absent(), Optional. absent(), Optional.absent()); + SingularityPendingTask taskFour = new SingularityPendingTask(taskIdFour,Optional.> absent(), Optional. absent(), Optional. absent(), Optional. absent(), Optional. absent(), Optional.absent(), Optional.absent()); taskManager.savePendingTask(taskOne); taskManager.savePendingTask(taskTwo); diff --git a/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularityExpiringActionsTest.java b/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularityExpiringActionsTest.java index 36af7356d0..13e5b07570 100644 --- a/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularityExpiringActionsTest.java +++ b/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularityExpiringActionsTest.java @@ -4,8 +4,11 @@ import org.junit.Assert; import org.junit.Test; +import com.fasterxml.jackson.annotation.JsonTypeInfo.As; import com.google.common.base.Optional; import com.hubspot.singularity.RequestState; +import com.hubspot.singularity.SingularityRequestHistory; +import com.hubspot.singularity.SingularityRequestHistory.RequestHistoryType; import com.hubspot.singularity.SingularityShellCommand; import com.hubspot.singularity.SingularityTask; import com.hubspot.singularity.api.SingularityBounceRequest; @@ -128,7 +131,7 @@ public void testExpiringNonIncrementalBounce() { public void testExpiringIncrementalBounce() { initRequest(); - requestResource.scale(requestId, new SingularityScaleRequest(Optional.of(3), Optional. absent(), Optional. absent(), Optional. absent(), Optional.absent())); + requestResource.scale(requestId, new SingularityScaleRequest(Optional.of(3), Optional. absent(), Optional. absent(), Optional. absent(), Optional.absent(), Optional.absent(), Optional.absent())); initFirstDeploy(); @@ -186,7 +189,7 @@ public void testExpiringScale() { initRequest(); initFirstDeploy(); - requestResource.scale(requestId, new SingularityScaleRequest(Optional.of(5), Optional.of(1L), Optional. absent(), Optional. absent(), Optional.absent())); + requestResource.scale(requestId, new SingularityScaleRequest(Optional.of(5), Optional.of(1L), Optional. absent(), Optional. absent(), Optional.absent(), Optional.absent(), Optional.absent())); try { Thread.sleep(2); @@ -229,4 +232,41 @@ public void testExpiringSkipHealthchecks() { Assert.assertTrue(healthchecker.cancelHealthcheck(thirdTask.getTaskId().getId())); } + + @Test + public void testExpiringScaleWithBounce() { + initRequest(); + initFirstDeploy(); + + requestResource.postRequest(request.toBuilder().setBounceAfterScale(Optional.of(true)).build()); + + requestResource.scale(requestId, new SingularityScaleRequest(Optional.of(5), Optional.of(1L), Optional. absent(), Optional. absent(), Optional.absent(), Optional.absent(), Optional.absent())); + + Assert.assertEquals(1, requestManager.getCleanupRequests().size()); + cleaner.drainCleanupQueue(); + resourceOffers(); + resourceOffers(); + resourceOffers(); + resourceOffers(); + cleaner.drainCleanupQueue(); + killKilledTasks(); + Assert.assertEquals(5, taskManager.getNumActiveTasks()); + + + try { + Thread.sleep(2); + } catch (InterruptedException e) { + + } + + expiringUserActionPoller.runActionOnPoll(); + Assert.assertEquals(1, requestManager.getCleanupRequests().size()); + cleaner.drainCleanupQueue(); + Assert.assertEquals(4, taskManager.getKilledTaskIdRecords().size()); + + launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); + + cleaner.drainCleanupQueue(); + Assert.assertEquals(5, taskManager.getKilledTaskIdRecords().size()); + } } diff --git a/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySchedulerTest.java b/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySchedulerTest.java index addc34c57e..eb6d9da1ad 100644 --- a/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySchedulerTest.java +++ b/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySchedulerTest.java @@ -96,7 +96,7 @@ public SingularitySchedulerTest() { private SingularityPendingTask pendingTask(String requestId, String deployId, PendingType pendingType) { return new SingularityPendingTask(new SingularityPendingTaskId(requestId, deployId, System.currentTimeMillis(), 1, pendingType, System.currentTimeMillis()), - Optional.> absent(), Optional. absent(), Optional. absent(), Optional. absent(), Optional. absent(), Optional.absent()); + Optional.> absent(), Optional. absent(), Optional. absent(), Optional. absent(), Optional. absent(), Optional.absent(), Optional.absent()); } @Test @@ -963,7 +963,7 @@ public void testPause() { public void testBounce() { initRequest(); - requestResource.scale(requestId, new SingularityScaleRequest(Optional.of(3), Optional. absent(), Optional. absent(), Optional. absent(), Optional.absent())); + requestResource.scale(requestId, new SingularityScaleRequest(Optional.of(3), Optional. absent(), Optional. absent(), Optional. absent(), Optional.absent(), Optional.absent(), Optional.absent())); initFirstDeploy(); @@ -1009,7 +1009,7 @@ public void testBounce() { public void testIncrementalBounceShutsDownOldTasksPerNewHealthyTask() { initRequest(); - requestResource.scale(requestId, new SingularityScaleRequest(Optional.of(3), Optional. absent(), Optional. absent(), Optional. absent(), Optional.absent())); + requestResource.scale(requestId, new SingularityScaleRequest(Optional.of(3), Optional. absent(), Optional. absent(), Optional. absent(), Optional.absent(), Optional.absent(), Optional.absent())); initFirstDeploy(); @@ -1287,7 +1287,7 @@ public void testScaleDownTakesHighestInstances() { Assert.assertEquals(5, taskManager.getActiveTaskIds().size()); requestResource.scale(requestId, new SingularityScaleRequest(Optional.of(2), Optional. absent(), Optional. absent(), - Optional. absent(), Optional.absent())); + Optional. absent(), Optional.absent(), Optional.absent(), Optional.absent())); resourceOffers(); cleaner.drainCleanupQueue(); @@ -1716,4 +1716,20 @@ public void testCleanerFindsTasksWithSkippedHealthchecks() { taskManager.saveHealthcheckResult(new SingularityTaskHealthcheckResult(Optional.of(200), Optional.of(1000L), now + 6000, Optional. absent(), Optional. absent(), unhealthyTaskThree.getTaskId(), Optional.absent())); Assert.assertEquals(DeployHealth.HEALTHY, deployHealthHelper.getDeployHealth(updatedRequest, Optional.of(firstDeploy), activeTaskIds, false)); } + + @Test + public void testScaleWithBounceDoesNotLaunchExtraInstances() { + initRequest(); + initFirstDeploy(); + launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING); + + requestResource.scale(requestId, new SingularityScaleRequest(Optional.of(5), Optional.of(1L), Optional. absent(), Optional. absent(), Optional.absent(), Optional.of(true), Optional.absent())); + + Assert.assertEquals(1, requestManager.getCleanupRequests().size()); + cleaner.drainCleanupQueue(); + Assert.assertEquals(1, taskManager.getNumCleanupTasks()); + + scheduler.drainPendingQueue(stateCacheProvider.get()); + Assert.assertEquals(5, taskManager.getPendingTaskIds().size()); + } } diff --git a/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySchedulerTestBase.java b/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySchedulerTestBase.java index 2d1d762848..f3bf081d0b 100644 --- a/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySchedulerTestBase.java +++ b/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySchedulerTestBase.java @@ -261,7 +261,7 @@ protected SingularityTask launchTask(SingularityRequest request, SingularityDepl protected SingularityPendingTask buildPendingTask(SingularityRequest request, SingularityDeploy deploy, long launchTime, int instanceNo, Optional runId) { SingularityPendingTaskId pendingTaskId = new SingularityPendingTaskId(request.getId(), deploy.getId(), launchTime, instanceNo, PendingType.IMMEDIATE, launchTime); - SingularityPendingTask pendingTask = new SingularityPendingTask(pendingTaskId, Optional.> absent(), Optional. absent(), runId, Optional. absent(), Optional. absent(), Optional.absent()); + SingularityPendingTask pendingTask = new SingularityPendingTask(pendingTaskId, Optional.> absent(), Optional. absent(), runId, Optional. absent(), Optional. absent(), Optional.absent(), Optional.absent()); return pendingTask; } @@ -442,7 +442,7 @@ protected void initRequest() { protected void initWithTasks(int num) { initRequest(); - requestResource.scale(requestId, new SingularityScaleRequest(Optional.of(num), Optional. absent(), Optional. absent(), Optional. absent(), Optional.absent())); + requestResource.scale(requestId, new SingularityScaleRequest(Optional.of(num), Optional. absent(), Optional. absent(), Optional. absent(), Optional.absent(), Optional.absent(), Optional.absent())); initFirstDeploy(); @@ -583,7 +583,7 @@ protected SingularityPendingTask createAndSchedulePendingTask(String deployId) { SingularityPendingTaskId pendingTaskId = new SingularityPendingTaskId(requestId, deployId, System.currentTimeMillis() + TimeUnit.DAYS.toMillis(random.nextInt(3)), random.nextInt(10), PendingType.IMMEDIATE, System.currentTimeMillis()); - SingularityPendingTask pendingTask = new SingularityPendingTask(pendingTaskId, Optional.> absent(), Optional. absent(), Optional. absent(), Optional. absent(), Optional. absent(), Optional.absent()); + SingularityPendingTask pendingTask = new SingularityPendingTask(pendingTaskId, Optional.> absent(), Optional. absent(), Optional. absent(), Optional. absent(), Optional. absent(), Optional.absent(), Optional.absent()); taskManager.savePendingTask(pendingTask); diff --git a/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySlavePlacementTest.java b/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySlavePlacementTest.java index b2b88dd9af..f8787157e6 100644 --- a/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySlavePlacementTest.java +++ b/SingularityService/src/test/java/com/hubspot/singularity/scheduler/SingularitySlavePlacementTest.java @@ -11,6 +11,7 @@ import com.google.common.base.Optional; import com.google.common.collect.ImmutableMap; import com.hubspot.singularity.ExtendedTaskState; +import com.hubspot.singularity.SingularityRequest; import com.hubspot.singularity.SingularityTaskCleanup; import com.hubspot.singularity.SingularityTaskHistoryUpdate; import com.hubspot.singularity.SlavePlacement; @@ -283,4 +284,46 @@ public void testRackPlacementOnScaleDown() { configuration.setRebalanceRacksOnScaleDown(false); } } + + @Test + public void testPlacementOfBounceTasks() { + // Set up 1 active rack + sms.resourceOffers(driver, Arrays.asList(createOffer(1, 128, "slave1", "host1", Optional.of("rack1")))); + + initRequest(); + initFirstDeploy(); + SingularityRequest newRequest = request.toBuilder() + .setInstances(Optional.of(2)) + .setRackSensitive(Optional.of(true)) + .setSlavePlacement(Optional.of(SlavePlacement.SEPARATE)) + .setAllowBounceToSameHost(Optional.of(true)) + .build(); + saveAndSchedule(newRequest.toBuilder()); + scheduler.drainPendingQueue(stateCacheProvider.get()); + + sms.resourceOffers(driver, Arrays.asList(createOffer(1, 128, "slave1", "host1", Optional.of("rack1")))); + sms.resourceOffers(driver, Arrays.asList(createOffer(1, 128, "slave2", "host2", Optional.of("rack1")))); + Assert.assertEquals(2, taskManager.getActiveTaskIds().size()); + + requestResource.bounce(requestId); + cleaner.drainCleanupQueue(); + scheduler.drainPendingQueue(stateCacheProvider.get()); + + Assert.assertEquals(2, taskManager.getNumCleanupTasks()); + Assert.assertEquals(2, taskManager.getPendingTaskIds().size()); + Assert.assertEquals(taskManager.getCleanupTasks().get(0).getActionId().get(), taskManager.getPendingTasks().get(0).getActionId().get()); + + // BOUNCE should allow a task to launch on the same host + sms.resourceOffers(driver, Arrays.asList(createOffer(1, 128, "slave1", "host1", Optional.of("rack1")))); + Assert.assertEquals(3, taskManager.getActiveTaskIds().size()); + + // But not a second one from the same bounce + sms.resourceOffers(driver, Arrays.asList(createOffer(1, 128, "slave1", "host1", Optional.of("rack1")))); + Assert.assertEquals(3, taskManager.getActiveTaskIds().size()); + + // Other pending type should not allow tasks on same host + saveAndSchedule(newRequest.toBuilder().setInstances(Optional.of(2))); + sms.resourceOffers(driver, Arrays.asList(createOffer(1, 128, "slave1", "host1", Optional.of("rack1")))); + Assert.assertEquals(3, taskManager.getActiveTaskIds().size()); + } } diff --git a/SingularityUI/app/actions/api/requests.es6 b/SingularityUI/app/actions/api/requests.es6 index 95de0b7131..dbf6bed74a 100644 --- a/SingularityUI/app/actions/api/requests.es6 +++ b/SingularityUI/app/actions/api/requests.es6 @@ -119,9 +119,9 @@ export const PersistSkipRequestHealthchecks = buildJsonApiAction( export const ScaleRequest = buildJsonApiAction( 'SCALE_REQUEST', 'PUT', - (requestId, {instances, skipHealthchecks, durationMillis, message, actionId}) => ({ + (requestId, {instances, skipHealthchecks, durationMillis, message, actionId, bounce, incremental }) => ({ url: `/requests/request/${requestId}/scale`, - body: { instances, skipHealthchecks, durationMillis, message, actionId } + body: { instances, skipHealthchecks, durationMillis, message, actionId, bounce, incremental } }) ); diff --git a/SingularityUI/app/components/common/modalButtons/ScaleButton.jsx b/SingularityUI/app/components/common/modalButtons/ScaleButton.jsx index e6fef93930..abc3f51150 100644 --- a/SingularityUI/app/components/common/modalButtons/ScaleButton.jsx +++ b/SingularityUI/app/components/common/modalButtons/ScaleButton.jsx @@ -17,6 +17,7 @@ const scaleTooltip = ( export default class ScaleButton extends Component { static propTypes = { requestId: PropTypes.string.isRequired, + bounceAfterScaleDefault: PropTypes.bool.isRequired, currentInstances: PropTypes.number, children: PropTypes.node, then: PropTypes.func @@ -41,6 +42,7 @@ export default class ScaleButton extends Component { requestId={this.props.requestId} currentInstances={this.props.currentInstances} then={this.props.then} + bounceAfterScaleDefault={this.props.bounceAfterScaleDefault} /> ); diff --git a/SingularityUI/app/components/common/modalButtons/ScaleModal.jsx b/SingularityUI/app/components/common/modalButtons/ScaleModal.jsx index 034c86ed42..88455b5752 100644 --- a/SingularityUI/app/components/common/modalButtons/ScaleModal.jsx +++ b/SingularityUI/app/components/common/modalButtons/ScaleModal.jsx @@ -11,8 +11,8 @@ import FormModal from '../modal/FormModal'; class ScaleModal extends Component { static propTypes = { requestId: PropTypes.string.isRequired, + bounceAfterScaleDefault: PropTypes.bool.isRequired, scaleRequest: PropTypes.func.isRequired, - bounceRequest: PropTypes.func.isRequired, currentInstances: PropTypes.number, then: PropTypes.func }; @@ -24,31 +24,26 @@ class ScaleModal extends Component { static INCREMENTAL_BOUNCE_VALUE = { INCREMENTAL: { label: 'Kill old tasks as new tasks become healthy', - value: true + value: 'incremental' }, ALL: { label: 'Kill old tasks once ALL new tasks are healthy', - value: false + value: 'non-incremental' } }; handleScale(data) { - const { instances, durationMillis, message } = data; + const { instances, durationMillis, message, bounce, incremental } = data; + const isIncremental = incremental === 'incremental'; this.props.scaleRequest( { instances, durationMillis, - message + message, + bounce, + incremental: isIncremental } - ).then(() => { - if (data.bounce) { - this.props.bounceRequest( - { - incremental: !!data.incremental - } - ); - } - }); + ); } show() { @@ -76,7 +71,7 @@ class ScaleModal extends Component { name: 'bounce', type: FormModal.INPUT_TYPES.BOOLEAN, label: 'Bounce after scaling', - defaultValue: false + defaultValue: this.props.bounceAfterScaleDefault }, { name: 'incremental', @@ -105,7 +100,6 @@ class ScaleModal extends Component { const mapDispatchToProps = (dispatch, ownProps) => ({ scaleRequest: (data) => dispatch(ScaleRequest.trigger(ownProps.requestId, data)).then((response) => ownProps.then && ownProps.then(response)), - bounceRequest: (data) => dispatch(BounceRequest.trigger(ownProps.requestId, data)) }); export default connect( diff --git a/SingularityUI/app/components/requestDetail/header/RequestActionButtons.jsx b/SingularityUI/app/components/requestDetail/header/RequestActionButtons.jsx index 654d04aea0..c310d2c9c6 100644 --- a/SingularityUI/app/components/requestDetail/header/RequestActionButtons.jsx +++ b/SingularityUI/app/components/requestDetail/header/RequestActionButtons.jsx @@ -82,7 +82,12 @@ const RequestActionButtons = ({requestParent, fetchRequest, fetchRequestHistory, let maybeScaleButton; if (Utils.request.canBeScaled(requestParent)) { maybeScaleButton = ( - + diff --git a/SingularityUI/app/components/requestForm/RequestForm.jsx b/SingularityUI/app/components/requestForm/RequestForm.jsx index 4446ae2246..870fe48ed4 100644 --- a/SingularityUI/app/components/requestForm/RequestForm.jsx +++ b/SingularityUI/app/components/requestForm/RequestForm.jsx @@ -348,6 +348,15 @@ const RequestForm = (props) => { /> ); + const allowBounceToSameHost = ( + updateField('allowBounceToSameHost', newValue)} + /> + ); + const waitAtLeastMillisAfterTaskFinishesForReschedule = ( { { shouldRenderField('rackSensitive') && rackSensitive } { shouldRenderField('hideEvenNumberAcrossRacksHint') && hideEvenNumberAcrossRacksHint } { shouldRenderField('loadBalanced') && loadBalanced } + { shouldRenderField('allowBounceToSameHost') && allowBounceToSameHost } { shouldRenderField('waitAtLeastMillisAfterTaskFinishesForReschedule') && waitAtLeastMillisAfterTaskFinishesForReschedule } { shouldRenderField('rackAffinity') && rackAffinity } { shouldRenderField('scheduleType') && scheduleTypeField } diff --git a/SingularityUI/app/components/requestForm/fields.es6 b/SingularityUI/app/components/requestForm/fields.es6 index 16b987c0e0..aa5bf6c154 100644 --- a/SingularityUI/app/components/requestForm/fields.es6 +++ b/SingularityUI/app/components/requestForm/fields.es6 @@ -84,6 +84,7 @@ export const FIELDS_BY_REQUEST_TYPE = { RACK_SENSITIVE_FIELD, HIDE_EVEN_NUMBERS_ACROSS_RACKS_HINT_FIELD, {id: 'loadBalanced', type: 'bool'}, + {id: 'allowBounceToSameHost', type: 'bool'}, RACK_AFFINITY_FIELD, BOUNCE_AFTER_SCALE_FIELD ], @@ -92,6 +93,7 @@ export const FIELDS_BY_REQUEST_TYPE = { RACK_SENSITIVE_FIELD, HIDE_EVEN_NUMBERS_ACROSS_RACKS_HINT_FIELD, {id: 'waitAtLeastMillisAfterTaskFinishesForReschedule', type: 'number'}, + {id: 'allowBounceToSameHost', type: 'bool'}, RACK_AFFINITY_FIELD, BOUNCE_AFTER_SCALE_FIELD ], diff --git a/SingularityUI/app/components/requests/Columns.jsx b/SingularityUI/app/components/requests/Columns.jsx index 89af2979aa..0148c1b876 100644 --- a/SingularityUI/app/components/requests/Columns.jsx +++ b/SingularityUI/app/components/requests/Columns.jsx @@ -209,6 +209,7 @@ export const Actions = ( );