Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add retry logic to JenkinsQueueJobV2 #15211

Merged
merged 15 commits into from
Aug 26, 2021
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,10 @@
"loc.input.help.jobParameters": "Specify job parameters, one per line, in the form <b>`<parameterName>=<parameterValue>`</b><p>To set a parameter to an empty value (useful for overriding a default value), leave off the parameter value. For example, specify <b>`<parameterName>=`</b><p>Variables are supported. For example, to set a <b>`commitId`</b> parameter value to the Git commit ID of the build, use: <b>`commitId=$(Build.SourceVersion)`</b>. See the [documentation on variables](https://go.microsoft.com/fwlink/?linkid=875288) for more details.<p>Supported Jenkins parameter types are: <ul><li>`Boolean`</li><li>`Choice`</li><li>`Password`</li><li>`String`</li></ul>",
"loc.input.label.failOnUnstableResult": "Fail on unstable result",
"loc.input.help.failOnUnstableResult": "Specifies strictness of a success definition: whether to consider unstable as a failure or not. False for non-strict, and true for strict version.",
"loc.input.label.retryNumber": "Number of retries for failed connection",
"loc.input.help.retryNumber": "Specify number of retries on errors or failures. Default is 3 times",
"loc.input.label.retryTimer": "Time between retries",
"loc.input.help.retryTimer": "Specify time between retries. Default is 60 seconds",
"loc.messages.FailedToGenerateSummary": "Failed to generate build summary.",
"loc.messages.succeeded": "succeeded",
"loc.messages.unstable": "unstable",
Expand Down
7 changes: 7 additions & 0 deletions Tasks/JenkinsQueueJobV2/jenkinsqueuejobtask.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,10 @@ export class TaskOptions {

pollIntervalMillis: number;

//initialize retry count and timer
retryNumber: number;
retryTimer: number;

parameterizedJob: boolean;
// jobParameters are only possible if parameterizedJob is enabled
jobParameters: string[];
Expand Down Expand Up @@ -71,6 +75,9 @@ export class TaskOptions {

this.pollIntervalMillis = 5000; // five seconds is what the Jenkins Web UI uses

this.retryNumber = parseInt(tl.getInput('retryNumber', false));
this.retryTimer = parseInt(tl.getInput('retryTimer', false));

this.parameterizedJob = tl.getBoolInput('parameterizedJob', true);
// jobParameters are only possible if parameterizedJob is enabled
this.jobParameters = this.parameterizedJob ? tl.getDelimitedInput('jobParameters', '\n', false) : [];
Expand Down
32 changes: 26 additions & 6 deletions Tasks/JenkinsQueueJobV2/job.ts
Original file line number Diff line number Diff line change
Expand Up @@ -131,6 +131,14 @@ export class Job {
this.working = false;
}


private RetryConnection(): void {
const thisJob: Job = this;
kuleshovilya marked this conversation as resolved.
Show resolved Hide resolved
thisJob.queue.TaskOptions.retryNumber--;
kuleshovilya marked this conversation as resolved.
Show resolved Hide resolved
thisJob.consoleLog(`Connection error. Retrying again in ${thisJob.queue.TaskOptions.retryTimer} seconds`);
thisJob.stopWork(thisJob.queue.TaskOptions.retryTimer*1000, thisJob.State);
}

public IsActive(): boolean {
return this.State === JobState.New ||
this.State === JobState.Locating ||
Expand Down Expand Up @@ -404,9 +412,14 @@ export class Job {
request.get({ url: fullUrl, strictSSL: thisJob.queue.TaskOptions.strictSSL }, function requestCallback(err, httpResponse, body) {
tl.debug('streamConsole().requestCallback()');
if (err) {
Util.handleConnectionResetError(err); // something went bad
thisJob.stopWork(thisJob.queue.TaskOptions.pollIntervalMillis, thisJob.State);
return;
if (thisJob.queue.TaskOptions.retryNumber < 1) {
Util.handleConnectionResetError(err); // something went bad
thisJob.stopWork(thisJob.queue.TaskOptions.pollIntervalMillis, thisJob.State);
return;
}
else {
thisJob.RetryConnection();
}
} else if (httpResponse.statusCode === 404) {
// got here too fast, stream not yet available, try again in the future
thisJob.stopWork(thisJob.queue.TaskOptions.pollIntervalMillis, thisJob.State);
Expand All @@ -418,8 +431,13 @@ export class Job {
thisJob.queue.TaskOptions.failureMsg = 'Job progress tracking failed to read job progress';
thisJob.stopWork(0, JobState.Finishing);
} else if (httpResponse.statusCode !== 200) {
Util.failReturnCode(httpResponse, 'Job progress tracking failed to read job progress');
thisJob.stopWork(thisJob.queue.TaskOptions.pollIntervalMillis, thisJob.State);
if (thisJob.queue.TaskOptions.retryNumber < 1) {
Util.failReturnCode(httpResponse, 'Job progress tracking failed to read job progress');
thisJob.stopWork(thisJob.queue.TaskOptions.pollIntervalMillis, thisJob.State);
}
else {
thisJob.RetryConnection();
kuleshovilya marked this conversation as resolved.
Show resolved Hide resolved
}
} else {
thisJob.consoleLog(body); // redirect Jenkins console to task console
const xMoreData: string = httpResponse.headers['x-more-data'];
Expand All @@ -433,7 +451,9 @@ export class Job {
}
}).auth(thisJob.queue.TaskOptions.username, thisJob.queue.TaskOptions.password, true)
.on('error', (err) => {
throw err;
if (thisJob.queue.TaskOptions.retryNumber < 1) {
throw err;
}
});
}

Expand Down
20 changes: 19 additions & 1 deletion Tasks/JenkinsQueueJobV2/task.json
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
"version": {
"Major": 2,
"Minor": 192,
"Patch": 0
"Patch": 1
},
"groups": [
{
Expand Down Expand Up @@ -110,6 +110,24 @@
"required": false,
"helpMarkDown": "Specifies strictness of a success definition: whether to consider unstable as a failure or not. False for non-strict, and true for strict version.",
"groupName": "advanced"
},
{
"name": "retryNumber",
kuleshovilya marked this conversation as resolved.
Show resolved Hide resolved
"type": "string",
"label": "Number of retries for failed connection",
"defaultValue": "3",
"required": false,
"helpMarkDown": "Specify number of retries on errors or failures. Default is 3 times",
kuleshovilya marked this conversation as resolved.
Show resolved Hide resolved
"groupName": "advanced"
},
{
"name": "retryTimer",
"type": "string",
"label": "Time between retries",
"defaultValue": "60",
kuleshovilya marked this conversation as resolved.
Show resolved Hide resolved
"required": false,
"helpMarkDown": "Specify time between retries. Default is 60 seconds",
"groupName": "advanced"
}
],
"outputVariables": [
Expand Down
20 changes: 19 additions & 1 deletion Tasks/JenkinsQueueJobV2/task.loc.json
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
"version": {
"Major": 2,
"Minor": 192,
"Patch": 0
"Patch": 1
},
"groups": [
{
Expand Down Expand Up @@ -110,6 +110,24 @@
"required": false,
"helpMarkDown": "ms-resource:loc.input.help.failOnUnstableResult",
"groupName": "advanced"
},
{
"name": "retryNumber",
"type": "string",
"label": "ms-resource:loc.input.label.retryNumber",
"defaultValue": "3",
"required": false,
"helpMarkDown": "ms-resource:loc.input.help.retryNumber",
"groupName": "advanced"
},
{
"name": "retryTimer",
"type": "string",
"label": "ms-resource:loc.input.label.retryTimer",
"defaultValue": "60",
"required": false,
"helpMarkDown": "ms-resource:loc.input.help.retryTimer",
"groupName": "advanced"
}
],
"outputVariables": [
Expand Down