Skip to content

Commit

Permalink
[JenkinsQueueJobV2] Fix job hang out after unexpected response from J…
Browse files Browse the repository at this point in the history
…enkins (#13034)

Fix:
* Fixed situation when the job hang out after unexpected response from Jenkins
* Fixed "About this task" link

Code refactor:
* Resolved all tslint warnings
* JobState enum and checkStateTransitions function was exported into a separate module and refactored.
* Documentation for job states was rewritten on JSdocs

Unit tests:
* Added test with the longest test scenario of the state transitions
* Added test for checking that transition rules are defined for all job states
  • Loading branch information
alexander-smolyakov authored Jun 1, 2020
1 parent 07db48a commit b3a36e9
Show file tree
Hide file tree
Showing 9 changed files with 206 additions and 75 deletions.
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"loc.friendlyName": "Jenkins queue job",
"loc.helpMarkDown": "This task queues a job on a [Jenkins](https://jenkins.io/) server. Full integration capabilities require installation of the [Team Foundation Server Plugin](https://wiki.jenkins-ci.org/display/JENKINS/Team+Foundation+Server+Plugin) on Jenkins. [More Information](http://go.microsoft.com/fwlink/?LinkId=816956).",
"loc.helpMarkDown": "[Learn more about this task](http://go.microsoft.com/fwlink/?LinkId=816956). This task queues a job on a [Jenkins](https://jenkins.io/) server. Full integration capabilities require installation of the [Team Foundation Server Plugin](https://wiki.jenkins-ci.org/display/JENKINS/Team+Foundation+Server+Plugin) on Jenkins.",
"loc.description": "Queue a job on a Jenkins server",
"loc.instanceNameFormat": "Queue Jenkins job: $(jobName)",
"loc.group.displayName.advanced": "Advanced",
Expand Down
44 changes: 44 additions & 0 deletions Tasks/JenkinsQueueJobV2/Tests/L0.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import path = require('path');
import os = require('os');
import process = require('process');
import fs = require('fs');
import {JobState, checkStateTransitions} from '../states';

import * as ttm from 'azure-pipelines-task-lib/mock-test';

Expand Down Expand Up @@ -157,4 +158,47 @@ describe('JenkinsQueueJob L0 Suite', function () {
}
});

it('[Job state] Run the longest test scenario of the state transitions', (done) => {
let currentState: JobState = JobState.New;

// the longest scenario from possible
const expectedScenario: Array<JobState> = [
JobState.Locating,
JobState.Streaming,
JobState.Finishing,
JobState.Downloading,
JobState.Done
];

const executedScenario: Array<JobState> = [];

for (const newState of expectedScenario) {
const isValidTransition: boolean = checkStateTransitions(currentState, newState);
if (isValidTransition) {
executedScenario.push(newState);
currentState = newState;
} else {
console.log(`Invalid state transition from: ${JobState[currentState]} to: ${JobState[newState]}`);
break;
}
}

assert.deepEqual(expectedScenario, executedScenario);
done();
});

it('[Job state] Check that transition rules are defined for all states', (done) => {
try {
const stateList = Object.keys(JobState).filter((element) => isNaN(Number(element)));

for (const testedState of stateList) {
for (const state of stateList) {
checkStateTransitions(JobState[testedState], JobState[state]);
}
}
done();
} catch (error) {
done(error);
}
});
});
96 changes: 40 additions & 56 deletions Tasks/JenkinsQueueJobV2/job.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,30 +11,10 @@ import request = require('request');
import { JobSearch } from './jobsearch';
import { JobQueue } from './jobqueue';
import { unzip } from './unzip';
import {JobState, checkStateTransitions} from './states';

import * as Util from './util';

// Jobs transition between states as follows:
// ------------------------------------------
// BEGINNING STATE: New
// New → Locating, Streaming, Joined, Cut
// Locating → Streaming, Joined, Cut
// Streaming → Finishing
// Finishing → Downloading, Queued, Done
// Downloading → Done
// TERMINAL STATES: Done, Queued, Joined, Cut
export enum JobState {
New, // 0 - The job is yet to begin
Locating, // 1 - The job is being located
Streaming, // 2 - The job is running and its console output is streaming
Finishing, // 3 - The job has run and is "finishing"
Done, // 4 - The job has run and is done
Joined, // 5 - The job is considered complete because it has been joined to the execution of another matching job execution
Queued, // 6 - The job was queued and will not be tracked for completion (as specified by the "Capture..." task setting)
Cut, // 7 - The job was cut from execution by the pipeline
Downloading// 8 - The job has run and its results are being downloaded (occurs when the TFS Plugin for Jenkins is installed)
}

export class Job {
public Parent: Job; // if this job is a pipelined job, its parent that started it.
public Children: Job[] = []; // any pipelined jobs
Expand Down Expand Up @@ -92,28 +72,15 @@ export class Job {
* This defines all and validates all state transitions.
*/
private changeState(newState: JobState) {
const oldState: JobState = this.State;
this.State = newState;
if (oldState !== newState) {
this.debug('state changed from: ' + oldState);
let validStateChange: boolean = false;
if (oldState === JobState.New) {
validStateChange = (newState === JobState.Locating || newState === JobState.Streaming || newState === JobState.Joined || newState === JobState.Cut);
} else if (oldState === JobState.Locating) {
validStateChange = (newState === JobState.Streaming || newState === JobState.Joined || newState === JobState.Cut);
} else if (oldState === JobState.Streaming) {
validStateChange = (newState === JobState.Finishing);
} else if (oldState === JobState.Finishing) {
validStateChange = (newState === JobState.Downloading || newState === JobState.Queued || newState === JobState.Done);
} else if (oldState === JobState.Downloading) {
validStateChange = (newState === JobState.Done);
} else if (oldState === JobState.Done || oldState === JobState.Joined || oldState === JobState.Cut) {
validStateChange = false; // these are terminal states
}
if (!validStateChange) {
Util.fail('Invalid state change from: ' + oldState + ' ' + this);
}
const currentState: JobState = this.State;
this.debug(`state changed from ${JobState[currentState]} to ${JobState[newState]}`);

const validStateChange: boolean = checkStateTransitions(currentState, newState);
if (!validStateChange) {
Util.fail(`Invalid state change from: ${JobState[currentState]} to: ${JobState[newState]} ${this}`);
}

this.State = newState;
}

public DoWork() {
Expand All @@ -122,17 +89,32 @@ export class Job {
} else {
this.working = true;
setTimeout(() => {
if (this.State === JobState.New) {
this.initialize();
} else if (this.State === JobState.Streaming) {
this.streamConsole();
} else if (this.State === JobState.Downloading) {
this.downloadResults();
} else if (this.State === JobState.Finishing) {
this.finish();
} else {
// usually do not get here, but this can happen if another callback caused this job to be joined
this.stopWork(this.queue.TaskOptions.pollIntervalMillis, null);
switch (this.State) {
case (JobState.New): {
this.initialize();
break;
}

case (JobState.Streaming): {
this.streamConsole();
break;
}

case (JobState.Downloading): {
this.downloadResults();
break;
}

case (JobState.Finishing): {
this.finish();
break;
}

default: {
// usually do not get here, but this can happen if another callback caused this job to be joined
this.stopWork(this.queue.TaskOptions.pollIntervalMillis, null);
break;
}
}
}, this.workDelay);
}
Expand Down Expand Up @@ -313,8 +295,10 @@ export class Job {
Util.handleConnectionResetError(err); // something went bad
thisJob.stopWork(thisJob.queue.TaskOptions.pollIntervalMillis, thisJob.State);
return;
} else if (httpResponse.statusCode != 200) {
} else if (httpResponse.statusCode !== 200) {
Util.failReturnCode(httpResponse, 'Job progress tracking failed to read job result');
tl.error(`Job was killed because of an response with unexpected status code from Jenkins - ${httpResponse.statusCode}`);
thisJob.stopWork(0, JobState.Killed);
} else {
const parsedBody: {result: string, timestamp: number} = JSON.parse(body);
thisJob.debug(`parsedBody for: ${resultUrl} : ${JSON.stringify(parsedBody)}`);
Expand Down Expand Up @@ -448,8 +432,8 @@ export class Job {
}
}
}).auth(thisJob.queue.TaskOptions.username, thisJob.queue.TaskOptions.password, true)
.on('error', (err) => {
throw err;
.on('error', (err) => {
throw err;
});
}

Expand Down
3 changes: 2 additions & 1 deletion Tasks/JenkinsQueueJobV2/jobqueue.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,10 @@ import fs = require('fs');
import path = require('path');
import shell = require('shelljs');

import { Job, JobState } from './job';
import { Job } from './job';
import { JobSearch } from './jobsearch';
import { TaskOptions } from './jenkinsqueuejobtask';
import { JobState } from './states';

import util = require('./util');

Expand Down
13 changes: 7 additions & 6 deletions Tasks/JenkinsQueueJobV2/jobsearch.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,9 @@ import tl = require('azure-pipelines-task-lib/task');
import Q = require('q');
import request = require('request');

import { Job, JobState } from './job';
import { Job } from './job';
import { JobQueue } from './jobqueue';
import { JobState } from './states';

import util = require('./util');

Expand Down Expand Up @@ -301,13 +302,13 @@ export class JobSearch {
}

interface Project {
name: string,
url: string,
color: string
name: string;
url: string;
color: string;
}
interface ParsedTaskBody {
downstreamProjects?: Project[],
downstreamProjects?: Project[];
lastBuild?: {
number: number
}
};
}
103 changes: 103 additions & 0 deletions Tasks/JenkinsQueueJobV2/states.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
/**
* @enum {number} JobState
* @readonly
* @description Enum for job states
*/
export enum JobState {
/** The job is yet to begin */
New,

/** The job is being located */
Locating,

/** The job is running and its console output is streaming */
Streaming,

/** The job has run and is "finishing" */
Finishing,

/** The job has run and is done */
Done,

/** The job is considered complete because it has been joined to the execution of another matching job execution */
Joined,

/** The job was queued and will not be tracked for completion (as specified by the "Capture..." task setting) */
Queued,

/** The job was cut from execution by the pipeline */
Cut,

/** The job has run and its results are being downloaded (occurs when the TFS Plugin for Jenkins is installed) */
Downloading,

/** The job has run and while "finishing" Jenkins provided unexpected answer via an HTTP request */
Killed,
}

/**
* @function checkStateTransitions
* @description Check validation of transition between states
* @param {JobState} currentState - current job state
* @param {JobState} newState - future job state
* @throws {Error} When there was no transition rule for the current state
*
* @example
* Jobs transition between states as follows:
* BEGINNING STATE: New
* New → Locating, Streaming, Joined, Cut
* Locating → Streaming, Joined, Cut
* Streaming → Finishing
* Finishing → Downloading, Queued, Done, Killed
* Downloading → Done
* TERMINAL STATES: Done, Queued, Joined, Cut, Killed
*/
export function checkStateTransitions (currentState: JobState, newState: JobState): boolean {
let isValidTransition: boolean = false;
let possibleStates: Array<JobState> = [];

switch (currentState) {
case (JobState.New): {
possibleStates = [JobState.Locating, JobState.Streaming, JobState.Joined, JobState.Cut];
break;
}

case (JobState.Locating): {
possibleStates = [JobState.Streaming, JobState.Joined, JobState.Cut];
break;
}

case (JobState.Streaming): {
possibleStates = [JobState.Finishing];
break;
}

case (JobState.Finishing): {
possibleStates = [JobState.Downloading, JobState.Queued, JobState.Done, JobState.Killed];
break;
}

case (JobState.Downloading): {
possibleStates = [JobState.Done];
break;
}

case (JobState.Done):
case (JobState.Joined):
case (JobState.Queued):
case (JobState.Cut):
case (JobState.Killed): {
break;
}

default: {
throw new Error(`No transition rules defined for the ${currentState} state!`);
}
}

if (possibleStates.length > 0) {
isValidTransition = (possibleStates.indexOf(newState) !== -1);
}

return isValidTransition;
}
6 changes: 3 additions & 3 deletions Tasks/JenkinsQueueJobV2/task.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
"friendlyName": "Jenkins queue job",
"description": "Queue a job on a Jenkins server",
"helpUrl": "https://docs.microsoft.com/azure/devops/pipelines/tasks/build/jenkins-queue-job",
"helpMarkDown": "This task queues a job on a [Jenkins](https://jenkins.io/) server. Full integration capabilities require installation of the [Team Foundation Server Plugin](https://wiki.jenkins-ci.org/display/JENKINS/Team+Foundation+Server+Plugin) on Jenkins. [More Information](http://go.microsoft.com/fwlink/?LinkId=816956).",
"helpMarkDown": "[Learn more about this task](http://go.microsoft.com/fwlink/?LinkId=816956). This task queues a job on a [Jenkins](https://jenkins.io/) server. Full integration capabilities require installation of the [Team Foundation Server Plugin](https://wiki.jenkins-ci.org/display/JENKINS/Team+Foundation+Server+Plugin) on Jenkins.",
"category": "Build",
"visibility": [
"Build",
Expand All @@ -14,8 +14,8 @@
"demands": [],
"version": {
"Major": 2,
"Minor": 167,
"Patch": 2
"Minor": 171,
"Patch": 0
},
"groups": [
{
Expand Down
4 changes: 2 additions & 2 deletions Tasks/JenkinsQueueJobV2/task.loc.json
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,8 @@
"demands": [],
"version": {
"Major": 2,
"Minor": 167,
"Patch": 2
"Minor": 171,
"Patch": 0
},
"groups": [
{
Expand Down
Loading

0 comments on commit b3a36e9

Please sign in to comment.