Bounce updates for placement and scaling

SingularityBase/src/main/java/com/hubspot/singularity/SingularityPendingTask.java

@@ -20,6 +20,7 @@ public class SingularityPendingTask {
   private final Optional<Boolean> skipHealthchecks;
   private final Optional<String> message;
   private final Optional<Resources> resources;
+  private final Optional<String> actionId;
 
   public static Predicate<SingularityPendingTask> matchingRequest(final String requestId) {
     return new Predicate<SingularityPendingTask>() {
@@ -46,14 +47,15 @@ public boolean apply(@Nonnull SingularityPendingTask input) {
   @JsonCreator
   public SingularityPendingTask(@JsonProperty("pendingTaskId") SingularityPendingTaskId pendingTaskId, @JsonProperty("cmdLineArgsList") Optional<List<String>> cmdLineArgsList,
       @JsonProperty("user") Optional<String> user, @JsonProperty("runId") Optional<String> runId, @JsonProperty("skipHealthchecks") Optional<Boolean> skipHealthchecks,
-      @JsonProperty("message") Optional<String> message, @JsonProperty("resources") Optional<Resources> resources) {
+      @JsonProperty("message") Optional<String> message, @JsonProperty("resources") Optional<Resources> resources, @JsonProperty("actionId") Optional<String> actionId) {
     this.pendingTaskId = pendingTaskId;
     this.user = user;
     this.message = message;
     this.cmdLineArgsList = cmdLineArgsList;
     this.runId = runId;
     this.skipHealthchecks = skipHealthchecks;
     this.resources = resources;
+    this.actionId = actionId;
   }
 
   @Override
@@ -104,10 +106,14 @@ public Optional<Resources> getResources() {
     return resources;
   }
 
+  public Optional<String> getActionId() {
+    return actionId;
+  }
+
   @Override
   public String toString() {
     return "SingularityPendingTask [pendingTaskId=" + pendingTaskId + ", cmdLineArgsList=" + cmdLineArgsList + ", user=" + user + ", runId=" + runId + ", skipHealthchecks=" + skipHealthchecks
-        + ", message=" + message + ", resources=" + resources + "]";
+        + ", message=" + message + ", resources=" + resources + ", actionId=" + actionId + "]";
   }
 
 }

SingularityBase/src/main/java/com/hubspot/singularity/SingularityRequest.java

@@ -58,6 +58,8 @@ public class SingularityRequest {
 
   private final Optional<Double> taskPriorityLevel;
 
+  private final Optional<Boolean> allowBounceToSameHost;
+
   @JsonCreator
   public SingularityRequest(@JsonProperty("id") String id, @JsonProperty("requestType") RequestType requestType, @JsonProperty("owners") Optional<List<String>> owners,
       @JsonProperty("numRetriesOnFailure") Optional<Integer> numRetriesOnFailure, @JsonProperty("schedule") Optional<String> schedule, @JsonProperty("instances") Optional<Integer> instances,
@@ -73,7 +75,7 @@ public SingularityRequest(@JsonProperty("id") String id, @JsonProperty("requestT
       @JsonProperty("emailConfigurationOverrides") Optional<Map<SingularityEmailType, List<SingularityEmailDestination>>> emailConfigurationOverrides,
       @JsonProperty("daemon") @Deprecated Optional<Boolean> daemon, @JsonProperty("hideEvenNumberAcrossRacks") Optional<Boolean> hideEvenNumberAcrossRacksHint,
       @JsonProperty("taskLogErrorRegex") Optional<String> taskLogErrorRegex, @JsonProperty("taskLogErrorRegexCaseSensitive") Optional<Boolean> taskLogErrorRegexCaseSensitive,
-      @JsonProperty("taskPriorityLevel") Optional<Double> taskPriorityLevel) {
+      @JsonProperty("taskPriorityLevel") Optional<Double> taskPriorityLevel, @JsonProperty("allowBounceToSameHost") Optional<Boolean> allowBounceToSameHost) {
     this.id = checkNotNull(id, "id cannot be null");
     this.owners = owners;
     this.numRetriesOnFailure = numRetriesOnFailure;
@@ -102,6 +104,7 @@ public SingularityRequest(@JsonProperty("id") String id, @JsonProperty("requestT
     this.taskLogErrorRegex = taskLogErrorRegex;
     this.taskLogErrorRegexCaseSensitive = taskLogErrorRegexCaseSensitive;
     this.taskPriorityLevel = taskPriorityLevel;
+    this.allowBounceToSameHost = allowBounceToSameHost;
     if (requestType == null) {
       this.requestType = RequestType.fromDaemonAndScheduleAndLoadBalanced(schedule, daemon, loadBalanced);
     } else {
@@ -137,7 +140,8 @@ public SingularityRequestBuilder toBuilder() {
     .setHideEvenNumberAcrossRacksHint(hideEvenNumberAcrossRacksHint)
     .setTaskLogErrorRegex(taskLogErrorRegex)
     .setTaskLogErrorRegexCaseSensitive(taskLogErrorRegexCaseSensitive)
-    .setTaskPriorityLevel(taskPriorityLevel);
+    .setTaskPriorityLevel(taskPriorityLevel)
+    .setAllowBounceToSameHost(allowBounceToSameHost);
   }
 
   @ApiModelProperty(required=true, value="A unique id for the request")
@@ -230,6 +234,11 @@ public Optional<Map<String, String>> getAllowedSlaveAttributes() {
     return allowedSlaveAttributes;
   }
 
+  @ApiModelProperty(required=false, value="If set to true, allow tasks to be scheduled on the same host as an existing active task when bouncing")
+  public Optional<Boolean> getAllowBounceToSameHost() {
+    return allowBounceToSameHost;
+  }
+
   @JsonIgnore
   public int getInstancesSafe() {
     return getInstances().or(1);
@@ -364,6 +373,7 @@ public String toString() {
       .add("taskLogErrorRegex", taskLogErrorRegex)
       .add("taskLogErrorRegexCaseSensitive", taskLogErrorRegexCaseSensitive)
       .add("taskPriorityLevel", taskPriorityLevel)
+      .add("allowBounceToSameHost", allowBounceToSameHost)
       .toString();
   }
 
@@ -403,11 +413,12 @@ public boolean equals(Object o) {
             Objects.equals(hideEvenNumberAcrossRacksHint, request.hideEvenNumberAcrossRacksHint) &&
             Objects.equals(taskLogErrorRegex, request.taskLogErrorRegex) &&
             Objects.equals(taskLogErrorRegexCaseSensitive, request.taskLogErrorRegexCaseSensitive) &&
-            Objects.equals(taskPriorityLevel, request.taskPriorityLevel);
+            Objects.equals(taskPriorityLevel, request.taskPriorityLevel) &&
+            Objects.equals(allowBounceToSameHost, request.allowBounceToSameHost);
   }
 
   @Override
   public int hashCode() {
-    return Objects.hash(id, requestType, owners, numRetriesOnFailure, schedule, quartzSchedule, scheduleTimeZone, scheduleType, killOldNonLongRunningTasksAfterMillis, taskExecutionTimeLimitMillis, scheduledExpectedRuntimeMillis, waitAtLeastMillisAfterTaskFinishesForReschedule, instances, rackSensitive, rackAffinity, slavePlacement, requiredSlaveAttributes, allowedSlaveAttributes, loadBalanced, group, readWriteGroups, readOnlyGroups, bounceAfterScale, emailConfigurationOverrides, hideEvenNumberAcrossRacksHint, taskLogErrorRegex, taskLogErrorRegexCaseSensitive, taskPriorityLevel);
+    return Objects.hash(id, requestType, owners, numRetriesOnFailure, schedule, quartzSchedule, scheduleTimeZone, scheduleType, killOldNonLongRunningTasksAfterMillis, taskExecutionTimeLimitMillis, scheduledExpectedRuntimeMillis, waitAtLeastMillisAfterTaskFinishesForReschedule, instances, rackSensitive, rackAffinity, slavePlacement, requiredSlaveAttributes, allowedSlaveAttributes, loadBalanced, group, readWriteGroups, readOnlyGroups, bounceAfterScale, emailConfigurationOverrides, hideEvenNumberAcrossRacksHint, taskLogErrorRegex, taskLogErrorRegexCaseSensitive, taskPriorityLevel, allowBounceToSameHost);
   }
 }

SingularityBase/src/main/java/com/hubspot/singularity/SingularityRequestBuilder.java

@@ -48,6 +48,7 @@ public class SingularityRequestBuilder {
   private Optional<String> taskLogErrorRegex;
   private Optional<Boolean> taskLogErrorRegexCaseSensitive;
   private Optional<Double> taskPriorityLevel;
+  private Optional<Boolean> allowBounceToSameHost;
 
   public SingularityRequestBuilder(String id, RequestType requestType) {
     this.id = checkNotNull(id, "id cannot be null");
@@ -79,12 +80,13 @@ public SingularityRequestBuilder(String id, RequestType requestType) {
     this.taskLogErrorRegex = Optional.absent();
     this.taskLogErrorRegexCaseSensitive = Optional.absent();
     this.taskPriorityLevel = Optional.absent();
+    this.allowBounceToSameHost = Optional.absent();
   }
 
   public SingularityRequest build() {
     return new SingularityRequest(id, requestType, owners, numRetriesOnFailure, schedule, instances, rackSensitive, loadBalanced, killOldNonLongRunningTasksAfterMillis, taskExecutionTimeLimitMillis, scheduleType, quartzSchedule, scheduleTimeZone,
         rackAffinity, slavePlacement, requiredSlaveAttributes, allowedSlaveAttributes, scheduledExpectedRuntimeMillis, waitAtLeastMillisAfterTaskFinishesForReschedule, group, readWriteGroups, readOnlyGroups,
-        bounceAfterScale, skipHealthchecks, emailConfigurationOverrides, Optional.<Boolean>absent(), hideEvenNumberAcrossRacksHint, taskLogErrorRegex, taskLogErrorRegexCaseSensitive, taskPriorityLevel);
+        bounceAfterScale, skipHealthchecks, emailConfigurationOverrides, Optional.<Boolean>absent(), hideEvenNumberAcrossRacksHint, taskLogErrorRegex, taskLogErrorRegexCaseSensitive, taskPriorityLevel, allowBounceToSameHost);
   }
 
   public Optional<Boolean> getSkipHealthchecks() {
@@ -325,6 +327,15 @@ public SingularityRequestBuilder setTaskPriorityLevel(Optional<Double> taskPrior
     return this;
   }
 
+  public Optional<Boolean> getAllowBounceToSameHost() {
+    return allowBounceToSameHost;
+  }
+
+  public SingularityRequestBuilder setAllowBounceToSameHost(Optional<Boolean> allowBounceToSameHost) {
+    this.allowBounceToSameHost = allowBounceToSameHost;
+    return this;
+  }
+
   @Override
   public String toString() {
     return "SingularityRequestBuilder[" +
@@ -357,6 +368,7 @@ public String toString() {
             ", taskLogErrorRegex=" + taskLogErrorRegex +
             ", taskLogErrorRegexCaseSensitive=" + taskLogErrorRegexCaseSensitive +
             ", taskPriorityLevel=" + taskPriorityLevel +
+            ", allowBounceToSameHost=" + allowBounceToSameHost +
             ']';
   }
 
@@ -397,15 +409,16 @@ public boolean equals(Object o) {
             Objects.equals(hideEvenNumberAcrossRacksHint, that.hideEvenNumberAcrossRacksHint) &&
             Objects.equals(taskLogErrorRegex, that.taskLogErrorRegex) &&
             Objects.equals(taskLogErrorRegexCaseSensitive, that.taskLogErrorRegexCaseSensitive) &&
-            Objects.equals(taskPriorityLevel, that.taskPriorityLevel);
+            Objects.equals(taskPriorityLevel, that.taskPriorityLevel) &&
+            Objects.equals(allowBounceToSameHost, that.allowBounceToSameHost);
   }
 
   @Override
   public int hashCode() {
     return Objects.hash(id, requestType, owners, numRetriesOnFailure, schedule, quartzSchedule, scheduleTimeZone, scheduleType, killOldNonLongRunningTasksAfterMillis,
         taskExecutionTimeLimitMillis, scheduledExpectedRuntimeMillis, waitAtLeastMillisAfterTaskFinishesForReschedule, instances, rackSensitive, rackAffinity, slavePlacement,
         requiredSlaveAttributes, allowedSlaveAttributes, loadBalanced, group, readOnlyGroups, readWriteGroups, bounceAfterScale, skipHealthchecks, emailConfigurationOverrides,
-        hideEvenNumberAcrossRacksHint, taskLogErrorRegex, taskLogErrorRegexCaseSensitive, taskPriorityLevel);
+        hideEvenNumberAcrossRacksHint, taskLogErrorRegex, taskLogErrorRegexCaseSensitive, taskPriorityLevel, allowBounceToSameHost);
   }
 
 }

SingularityBase/src/main/java/com/hubspot/singularity/api/SingularityScaleRequest.java

@@ -9,13 +9,18 @@ public class SingularityScaleRequest extends SingularityExpiringRequestParent {
 
   private final Optional<Integer> instances;
   private final Optional<Boolean> skipHealthchecks;
+  private final Optional<Boolean> bounce;
+  private final Optional<Boolean> incremental;
 
   @JsonCreator
   public SingularityScaleRequest(@JsonProperty("instances") Optional<Integer> instances, @JsonProperty("durationMillis") Optional<Long> durationMillis,
-      @JsonProperty("skipHealthchecks") Optional<Boolean> skipHealthchecks, @JsonProperty("actionId") Optional<String> actionId, @JsonProperty("message") Optional<String> message) {
+      @JsonProperty("skipHealthchecks") Optional<Boolean> skipHealthchecks, @JsonProperty("actionId") Optional<String> actionId, @JsonProperty("message") Optional<String> message,
+      @JsonProperty("bounce") Optional<Boolean> bounce, @JsonProperty("incremental") Optional<Boolean> incremental) {
     super(durationMillis, actionId, message);
     this.instances = instances;
     this.skipHealthchecks = skipHealthchecks;
+    this.bounce = bounce;
+    this.incremental = incremental;
   }
 
   @ApiModelProperty(required=false, value="If set to true, healthchecks will be skipped while scaling this request (only)")
@@ -28,9 +33,19 @@ public Optional<Integer> getInstances() {
     return instances;
   }
 
+  @ApiModelProperty(required=false, value="Bounce the request to get to the new scale")
+  public Optional<Boolean> getBounce() {
+    return bounce;
+  }
+
+  @ApiModelProperty(required=false, value="If present and set to true, old tasks will be killed as soon as replacement tasks are available, instead of waiting for all replacement tasks to be healthy")
+  public Optional<Boolean> getIncremental() {
+    return incremental;
+  }
+
   @Override
   public String toString() {
-    return "SingularityScaleRequest [instances=" + instances + ", skipHealthchecks=" + skipHealthchecks + ", toString()=" + super.toString() + "]";
+    return "SingularityScaleRequest [instances=" + instances + ", skipHealthchecks=" + skipHealthchecks + ", bounce=" + bounce + ", incremental=" + incremental + ", toString()=" + super.toString() + "]";
   }
 
 }

SingularityBase/src/main/java/com/hubspot/singularity/expiring/SingularityExpiringScale.java

@@ -7,22 +7,28 @@
 public class SingularityExpiringScale extends SingularityExpiringRequestActionParent<SingularityScaleRequest> {
 
   private final Optional<Integer> revertToInstances;
+  private final Optional<Boolean> bounce;
 
   public SingularityExpiringScale(@JsonProperty("requestId") String requestId, @JsonProperty("user") Optional<String> user,
       @JsonProperty("startMillis") long startMillis, @JsonProperty("expiringAPIRequestObject") SingularityScaleRequest scaleRequest, @JsonProperty("revertToInstances") Optional<Integer> revertToInstances,
-      @JsonProperty("actionId") String actionId) {
+      @JsonProperty("actionId") String actionId, @JsonProperty("bounce") Optional<Boolean> bounce) {
     super(scaleRequest, user, startMillis, actionId, requestId);
 
     this.revertToInstances = revertToInstances;
+    this.bounce = bounce;
   }
 
   public Optional<Integer> getRevertToInstances() {
     return revertToInstances;
   }
 
+  public Optional<Boolean> getBounce() {
+    return bounce;
+  }
+
   @Override
   public String toString() {
-    return "SingularityExpiringScale [revertToInstances=" + revertToInstances + "]";
+    return "SingularityExpiringScale [revertToInstances=" + revertToInstances + ", bounce=" + bounce + "]";
   }
 
 }

SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java

@@ -302,6 +302,8 @@ public class SingularityConfiguration extends Configuration {
 
   private boolean rebalanceRacksOnScaleDown = false;
 
+  private boolean allowBounceToSameHost = false;
+
   public long getAskDriverToKillTasksAgainAfterMillis() {
     return askDriverToKillTasksAgainAfterMillis;
   }
@@ -1207,4 +1209,13 @@ public boolean isRebalanceRacksOnScaleDown() {
   public void setRebalanceRacksOnScaleDown(boolean rebalanceRacksOnScaleDown) {
     this.rebalanceRacksOnScaleDown = rebalanceRacksOnScaleDown;
   }
+
+  public boolean isAllowBounceToSameHost() {
+    return allowBounceToSameHost;
+  }
+
+  public SingularityConfiguration setAllowBounceToSameHost(boolean allowBounceToSameHost) {
+    this.allowBounceToSameHost = allowBounceToSameHost;
+    return this;
+  }
 }

SingularityService/src/main/java/com/hubspot/singularity/data/RequestManager.java

@@ -176,6 +176,10 @@ public boolean cleanupRequestExists(String requestId) {
     return false;
   }
 
+  public boolean cleanupRequestExists(String requestId, RequestCleanupType type) {
+    return checkExists(getCleanupPath(requestId, type)).isPresent();
+  }
+
   public void deleteCleanRequest(String requestId, RequestCleanupType type) {
     delete(getCleanupPath(requestId, type));
   }

SingularityService/src/main/java/com/hubspot/singularity/data/SingularityValidator.java

@@ -92,6 +92,7 @@ public class SingularityValidator {
   private final boolean allowRequestsWithoutOwners;
   private final boolean createDeployIds;
   private final int deployIdLength;
+  private final boolean allowBounceToSameHost;
   private final UIConfiguration uiConfiguration;
   private final SlavePlacement defaultSlavePlacement;
   private final DeployHistoryHelper deployHistoryHelper;
@@ -125,6 +126,8 @@ public SingularityValidator(SingularityConfiguration configuration, DeployHistor
     this.maxMemoryMbPerRequest = configuration.getMesosConfiguration().getMaxMemoryMbPerRequest();
     this.maxInstancesPerRequest = configuration.getMesosConfiguration().getMaxNumInstancesPerRequest();
 
+    this.allowBounceToSameHost = configuration.isAllowBounceToSameHost();
+
     this.maxTotalHealthcheckTimeoutSeconds = configuration.getHealthcheckMaxTotalTimeoutSeconds();
     this.defaultHealthcheckIntervalSeconds = configuration.getHealthcheckIntervalSeconds();
     this.defaultHealthcheckStartupTimeooutSeconds = configuration.getStartupTimeoutSeconds();
@@ -524,14 +527,23 @@ private String getNewDayOfWeekValue(String schedule, int dayOfWeekValue) {
   public void checkResourcesForBounce(SingularityRequest request, boolean isIncremental) {
     SlavePlacement placement = request.getSlavePlacement().or(defaultSlavePlacement);
 
-    if (placement != SlavePlacement.GREEDY && placement != SlavePlacement.OPTIMISTIC) {
+    if ((isAllowBounceToSameHost(request) && placement == SlavePlacement.SEPARATE_BY_REQUEST)
+      || (!isAllowBounceToSameHost(request) && placement != SlavePlacement.GREEDY && placement != SlavePlacement.OPTIMISTIC)) {
       int currentActiveSlaveCount = slaveManager.getNumObjectsAtState(MachineState.ACTIVE);
       int requiredSlaveCount = isIncremental ? request.getInstancesSafe() + 1 : request.getInstancesSafe() * 2;
 
       checkBadRequest(currentActiveSlaveCount >= requiredSlaveCount, "Not enough active slaves to successfully scale request %s to %s instances (minimum required: %s, current: %s).", request.getId(), request.getInstancesSafe(), requiredSlaveCount, currentActiveSlaveCount);
     }
   }
 
+  private boolean isAllowBounceToSameHost(SingularityRequest request) {
+    if (request.getAllowBounceToSameHost().isPresent()) {
+      return request.getAllowBounceToSameHost().get();
+    } else {
+      return allowBounceToSameHost;
+    }
+  }
+
   public void checkScale(SingularityRequest request, Optional<Integer> previousScale) {
     SlavePlacement placement = request.getSlavePlacement().or(defaultSlavePlacement);
 

SingularityService/src/main/java/com/hubspot/singularity/data/zkmigrations/SingularityCmdLineArgsMigration.java

@@ -143,7 +143,7 @@ private void checkPendingTasks() {
         Optional<String> cmdLineArgs = getCmdLineArgs(pendingTaskId);
 
         SingularityCreateResult result = taskManager.savePendingTask(new SingularityPendingTask(pendingTaskId, getCmdLineArgs(cmdLineArgs), Optional.<String> absent(),
-            Optional.<String> absent(), Optional.<Boolean> absent(), Optional.<String> absent(), Optional.<Resources>absent()));
+            Optional.<String> absent(), Optional.<Boolean> absent(), Optional.<String> absent(), Optional.<Resources>absent(), Optional.<String>absent()));
 
         LOG.info("Saving {} ({}) {}", pendingTaskId, cmdLineArgs, result);
       }

SingularityService/src/main/java/com/hubspot/singularity/data/zkmigrations/SingularityPendingTaskIdMigration.java

@@ -60,7 +60,7 @@ public void applyMigration() {
           Optional<String> cmdLineArgs = getCmdLineArgs(pendingTaskId);
 
           taskManager.savePendingTask(new SingularityPendingTask(newPendingTaskId, cmdLineArgs.isPresent() ? Optional.of(Collections.singletonList(cmdLineArgs.get())) :
-            Optional.<List<String>> absent(), Optional.<String> absent(), Optional.<String> absent(), Optional.<Boolean> absent(), Optional.<String> absent(), Optional.<Resources>absent()));
+            Optional.<List<String>> absent(), Optional.<String> absent(), Optional.<String> absent(), Optional.<Boolean> absent(), Optional.<String> absent(), Optional.<Resources>absent(), Optional.<String>absent()));
 
           curator.delete().forPath(ZKPaths.makePath(PENDING_TASKS_ROOT, pendingTaskId));
         }