Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Report currentActiveInstances on SingularityDeployProgress #1377

Merged
merged 6 commits into from
Feb 15, 2017
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -7,18 +7,24 @@

public class SingularityDeployProgress {
private final int targetActiveInstances;
private final int currentActiveInstances;
private final int deployInstanceCountPerStep;
private final long deployStepWaitTimeMs;
private final boolean stepComplete;
private final boolean autoAdvanceDeploySteps;
private final Set<SingularityTaskId> failedDeployTasks;
private final long timestamp;

@JsonCreator
public SingularityDeployProgress(@JsonProperty("targetActiveInstances") int targetActiveInstances, @JsonProperty("deployInstanceCountPerStep") int deployInstanceCountPerStep,
@JsonProperty("deployStepWaitTimeMs") long deployStepWaitTimeMs, @JsonProperty("stepComplete") boolean stepComplete, @JsonProperty("autoAdvanceDeploySteps") boolean autoAdvanceDeploySteps,
@JsonProperty("failedDeployTasks") Set<SingularityTaskId> failedDeployTasks, @JsonProperty("timestamp") long timestamp) {
public SingularityDeployProgress(@JsonProperty("targetActiveInstances") int targetActiveInstances,
@JsonProperty("currentActiveInstances") int currentActiveInstances,
@JsonProperty("deployInstanceCountPerStep") int deployInstanceCountPerStep,
@JsonProperty("deployStepWaitTimeMs") long deployStepWaitTimeMs,
@JsonProperty("stepComplete") boolean stepComplete,
@JsonProperty("autoAdvanceDeploySteps") boolean autoAdvanceDeploySteps,
@JsonProperty("failedDeployTasks") Set<SingularityTaskId> failedDeployTasks,
@JsonProperty("timestamp") long timestamp) {
this.targetActiveInstances = targetActiveInstances;
this.currentActiveInstances = currentActiveInstances;
this.deployInstanceCountPerStep = deployInstanceCountPerStep;
this.deployStepWaitTimeMs = deployStepWaitTimeMs;
this.stepComplete = stepComplete;
@@ -31,6 +37,10 @@ public int getTargetActiveInstances() {
return targetActiveInstances;
}

public int getCurrentActiveInstances() {
return currentActiveInstances;
}

public int getDeployInstanceCountPerStep() {
return deployInstanceCountPerStep;
}
@@ -55,22 +65,27 @@ public long getTimestamp() {
return timestamp;
}

public SingularityDeployProgress withNewInstances(int instances) {
return new SingularityDeployProgress(instances, deployInstanceCountPerStep, deployStepWaitTimeMs, false, autoAdvanceDeploySteps, failedDeployTasks, System.currentTimeMillis());
public SingularityDeployProgress withNewTargetInstances(int instances) {
return new SingularityDeployProgress(instances, currentActiveInstances, deployInstanceCountPerStep, deployStepWaitTimeMs, false, autoAdvanceDeploySteps, failedDeployTasks, System.currentTimeMillis());
}

public SingularityDeployProgress withNewActiveInstances(int instances) {
return new SingularityDeployProgress(targetActiveInstances, instances, deployInstanceCountPerStep, deployStepWaitTimeMs, false, autoAdvanceDeploySteps, failedDeployTasks, System.currentTimeMillis());
}

public SingularityDeployProgress withCompletedStep() {
return new SingularityDeployProgress(targetActiveInstances, deployInstanceCountPerStep, deployStepWaitTimeMs, true, autoAdvanceDeploySteps, failedDeployTasks, System.currentTimeMillis());
return new SingularityDeployProgress(targetActiveInstances, currentActiveInstances, deployInstanceCountPerStep, deployStepWaitTimeMs, true, autoAdvanceDeploySteps, failedDeployTasks, System.currentTimeMillis());
}

public SingularityDeployProgress withFailedTasks(Set<SingularityTaskId> failedTasks) {
return new SingularityDeployProgress(targetActiveInstances, deployInstanceCountPerStep, deployStepWaitTimeMs, false, autoAdvanceDeploySteps, failedTasks, System.currentTimeMillis());
return new SingularityDeployProgress(targetActiveInstances, currentActiveInstances, deployInstanceCountPerStep, deployStepWaitTimeMs, false, autoAdvanceDeploySteps, failedTasks, System.currentTimeMillis());
}

@Override
public String toString() {
return "SingularityIncrementalDeployProgress{" +
"targetActiveInstances=" + targetActiveInstances +
", currentActiveInstances=" + currentActiveInstances +
", deployInstanceCountPerStep=" + deployInstanceCountPerStep +
", deployStepWaitTimeMs=" + deployStepWaitTimeMs +
", stepComplete=" + stepComplete +
Original file line number Diff line number Diff line change
@@ -124,6 +124,7 @@ public SingularityRequestParent deploy(@ApiParam(required=true) SingularityDeplo
if (request.isLongRunning()) {
deployProgress = Optional.of(new SingularityDeployProgress(
Math.min(deploy.getDeployInstanceCountPerStep().or(request.getInstancesSafe()), request.getInstancesSafe()),
0,
deploy.getDeployInstanceCountPerStep().or(request.getInstancesSafe()),
deploy.getDeployStepWaitTimeMs().or(configuration.getDefaultDeployStepWaitTimeMs()),
false,
Original file line number Diff line number Diff line change
@@ -527,7 +527,7 @@ private SingularityDeployResult checkDeployProgress(final SingularityRequest req
return checkOverdue(request, deploy, pendingDeploy, deployActiveTasks, isDeployOverdue);
case HEALTHY:
if (!request.isLoadBalanced()) {
return markStepFinished(pendingDeploy, deploy, otherActiveTasks, request, updatePendingDeployRequest);
return markStepFinished(pendingDeploy, deploy, deployActiveTasks, otherActiveTasks, request, updatePendingDeployRequest);
}

if (updatePendingDeployRequest.isPresent() && updatePendingDeployRequest.get().getTargetActiveInstances() != deployProgress.getTargetActiveInstances()) {
@@ -560,7 +560,7 @@ private SingularityDeployResult checkCanMoveToNextDeployStep(SingularityRequest
Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest) {
SingularityDeployProgress deployProgress = pendingDeploy.getDeployProgress().get();
if (canMoveToNextStep(deployProgress) || updatePendingDeployRequest.isPresent()) {
SingularityDeployProgress newProgress = deployProgress.withNewInstances(getNewTargetInstances(deployProgress, request, updatePendingDeployRequest));
SingularityDeployProgress newProgress = deployProgress.withNewTargetInstances(getNewTargetInstances(deployProgress, request, updatePendingDeployRequest));
updatePendingDeploy(pendingDeploy, pendingDeploy.getLastLoadBalancerUpdate(), DeployState.WAITING, Optional.of(newProgress));
requestManager.addToPendingQueue(
new SingularityPendingRequest(request.getId(), pendingDeploy.getDeployMarker().getDeployId(), System.currentTimeMillis(), pendingDeploy.getDeployMarker().getUser(),
@@ -607,7 +607,7 @@ private SingularityDeployResult processLbState(SingularityRequest request, Optio
DeployState deployState = interpretLoadBalancerState(lbUpdate, pendingDeploy.getCurrentDeployState());
if (deployState == DeployState.SUCCEEDED) {
updatePendingDeploy(pendingDeploy, Optional.of(lbUpdate), DeployState.WAITING); // A step has completed, markStepFinished will determine SUCCEEDED/WAITING
return markStepFinished(pendingDeploy, deploy, otherActiveTasks, request, updatePendingDeployRequest);
return markStepFinished(pendingDeploy, deploy, deployActiveTasks, otherActiveTasks, request, updatePendingDeployRequest);
} else if (deployState == DeployState.WAITING) {
updatePendingDeploy(pendingDeploy, Optional.of(lbUpdate), deployState);
maybeUpdatePendingRequest(pendingDeploy, deploy, request, updatePendingDeployRequest, Optional.of(lbUpdate));
@@ -628,7 +628,7 @@ private void maybeUpdatePendingRequest(SingularityPendingDeploy pendingDeploy, O
Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest, Optional<SingularityLoadBalancerUpdate> lbUpdate) {
if (updatePendingDeployRequest.isPresent() && pendingDeploy.getDeployProgress().isPresent()) {
SingularityDeployProgress newProgress =
pendingDeploy.getDeployProgress().get().withNewInstances(Math.min(updatePendingDeployRequest.get().getTargetActiveInstances(), request.getInstancesSafe()));
pendingDeploy.getDeployProgress().get().withNewTargetInstances(Math.min(updatePendingDeployRequest.get().getTargetActiveInstances(), request.getInstancesSafe()));
updatePendingDeploy(pendingDeploy, lbUpdate.or(pendingDeploy.getLastLoadBalancerUpdate()), DeployState.WAITING, Optional.of(newProgress));
requestManager
.addToPendingQueue(new SingularityPendingRequest(request.getId(), pendingDeploy.getDeployMarker().getDeployId(), System.currentTimeMillis(), pendingDeploy.getDeployMarker().getUser(),
@@ -647,16 +647,17 @@ private boolean isLastStepFinished(SingularityDeployProgress deployProgress, Sin
return deployProgress.isStepComplete() && deployProgress.getTargetActiveInstances() >= request.getInstancesSafe();
}

private SingularityDeployResult markStepFinished(SingularityPendingDeploy pendingDeploy, Optional<SingularityDeploy> deploy, Collection<SingularityTaskId> otherActiveTasks,
SingularityRequest request, Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest) {
private SingularityDeployResult markStepFinished(SingularityPendingDeploy pendingDeploy, Optional<SingularityDeploy> deploy, Collection<SingularityTaskId> deployActiveTasks,
Collection<SingularityTaskId> otherActiveTasks, SingularityRequest request,
Optional<SingularityUpdatePendingDeployRequest> updatePendingDeployRequest) {
SingularityDeployProgress deployProgress = pendingDeploy.getDeployProgress().get();

if (updatePendingDeployRequest.isPresent() && getNewTargetInstances(deployProgress, request, updatePendingDeployRequest) != deployProgress.getTargetActiveInstances()) {
maybeUpdatePendingRequest(pendingDeploy, deploy, request, updatePendingDeployRequest);
return new SingularityDeployResult(DeployState.WAITING);
}

SingularityDeployProgress newProgress = deployProgress.withCompletedStep();
SingularityDeployProgress newProgress = deployProgress.withNewActiveInstances(deployActiveTasks.size()).withCompletedStep();
DeployState deployState = isLastStepFinished(newProgress, request) ? DeployState.SUCCEEDED : DeployState.WAITING;

String message = deployState == DeployState.SUCCEEDED ? "New deploy succeeded" : "New deploy is progressing, this task is being replaced";
Original file line number Diff line number Diff line change
@@ -847,4 +847,58 @@ public void testDeployTimesOut() {

Assert.assertEquals(DeployState.OVERDUE, deployManager.getDeployResult(requestId, deployId).get().getDeployState());
}

@Test
public void testIncrementalDeployInstanceCounter() {
initRequest();
SingularityRequest request = requestResource.getRequest(requestId).getRequest();
requestResource.postRequest(request.toBuilder().setInstances(Optional.of(4)).build());
initFirstDeploy();

SingularityTask firstTask = launchTask(request, firstDeploy, 1, TaskState.TASK_RUNNING);
SingularityTask secondTask = launchTask(request, firstDeploy, 2, TaskState.TASK_RUNNING);
SingularityTask thirdTask = launchTask(request, firstDeploy, 3, TaskState.TASK_RUNNING);

deploy(secondDeployId, Optional.<Boolean>absent(), Optional.of(1), Optional.<Boolean>absent(), false);
deployChecker.checkDeploys();
scheduler.drainPendingQueue(stateCacheProvider.get());
resourceOffers();
SingularityTaskId firstNewTaskId = taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId).get(0);
statusUpdate(taskManager.getTask(firstNewTaskId).get(), TaskState.TASK_RUNNING);
deployChecker.checkDeploys();

SingularityDeployProgress deployProgress = deployManager.getPendingDeploy(requestId).get().getDeployProgress().get();
Assert.assertEquals(1, deployProgress.getTargetActiveInstances());
Assert.assertEquals(1, deployProgress.getCurrentActiveInstances());

cleaner.drainCleanupQueue();
statusUpdate(firstTask, TaskState.TASK_KILLED);

deployChecker.checkDeploys();
scheduler.drainPendingQueue(stateCacheProvider.get());
resourceOffers();
for (SingularityTaskId taskId : taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId)) {
statusUpdate(taskManager.getTask(taskId).get(), TaskState.TASK_RUNNING);
}

deployChecker.checkDeploys();
deployProgress = deployManager.getPendingDeploy(requestId).get().getDeployProgress().get();
Assert.assertEquals(2, deployProgress.getTargetActiveInstances());
Assert.assertEquals(2, deployProgress.getCurrentActiveInstances());

cleaner.drainCleanupQueue();
statusUpdate(secondTask, TaskState.TASK_KILLED);

deployChecker.checkDeploys();
scheduler.drainPendingQueue(stateCacheProvider.get());
resourceOffers();
for (SingularityTaskId taskId : taskManager.getActiveTaskIdsForDeploy(requestId, secondDeployId)) {
statusUpdate(taskManager.getTask(taskId).get(), TaskState.TASK_RUNNING);
}

deployChecker.checkDeploys();
deployProgress = deployManager.getPendingDeploy(requestId).get().getDeployProgress().get();
Assert.assertEquals(3, deployProgress.getTargetActiveInstances());
Assert.assertEquals(3, deployProgress.getCurrentActiveInstances());
}
}
Original file line number Diff line number Diff line change
@@ -503,7 +503,7 @@ protected SingularityDeployMarker initSecondDeploy() {
}

protected void startDeploy(SingularityDeployMarker deployMarker, long timestamp) {
SingularityDeployProgress startingDeployProgress = new SingularityDeployProgress(1, 1, 10, false, true, Collections.<SingularityTaskId>emptySet(), timestamp);
SingularityDeployProgress startingDeployProgress = new SingularityDeployProgress(1, 0, 1, 10, false, true, Collections.<SingularityTaskId>emptySet(), timestamp);
deployManager.savePendingDeploy(new SingularityPendingDeploy(deployMarker, Optional.<SingularityLoadBalancerUpdate>absent(), DeployState.WAITING, Optional.of(startingDeployProgress), Optional.<SingularityRequest>absent()));
}