Skip to content

Commit

Permalink
Publishing Automated Test Results using Command (#19580)
Browse files Browse the repository at this point in the history
* Initial Publish Function

* Generated Files

* Updating Calling place

* Updating Task Version

* Addressing Build Failures

* Removing extra lines

* Updating data structures for passing list of AutomatedTestPoints

* updating generated files

* Making nit changes

* Updating the files for publishing

* Nit Changes
  • Loading branch information
adityashahms authored Mar 6, 2024
1 parent 2c43475 commit c445e55
Show file tree
Hide file tree
Showing 25 changed files with 674 additions and 157 deletions.
2 changes: 1 addition & 1 deletion Tasks/AzureTestPlanV0/Invokers/pythonivoker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ export async function executepythontests(testsToBeExecuted: string[]) {
args.push(testcase);
}

args.push('--junitxml=junit.xml')
args.push('--junitxml=TEST-python-junit.xml')

tl.debug("Executing python tests with executable : " + executable);
tl.debug("Executing python tests with args :" + args);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,16 @@
"loc.input.help.testConfiguration": "Select Test Configuration.",
"loc.input.label.testLanguageInput": "Select Test framework language",
"loc.input.help.testLanguageInput": "Test Framework Language of automated tests in test plan",
"loc.input.label.mergeTestResults": "Merge test results",
"loc.input.help.mergeTestResults": "A test run is created for each results file. Check this option to merge results into a single test run. To optimize for better performance, results will be merged into a single run if there are more than 100 result files, irrespective of this option.",
"loc.input.label.publishRunAttachments": "Upload test results files",
"loc.input.help.publishRunAttachments": "Upload logs and other files containing diagnostic information collected when the tests were run.",
"loc.input.label.failTaskOnFailedTests": "Fail if there are test failures",
"loc.input.help.failTaskOnFailedTests": "Fail the task if there are any test failures. Check this option to fail the task if test failures are detected in the result files.",
"loc.input.label.failTaskOnFailureToPublishResults": "Fail if there is failure in publishing test results",
"loc.input.help.failTaskOnFailureToPublishResults": "Fail if there is failure in publishing test results. Check this option to fail the task if publishing test results is failed partially.",
"loc.input.label.failTaskOnMissingResultsFile": "Fail if no result files are found",
"loc.input.help.failTaskOnMissingResultsFile": "Fail the task if no result files are found.",
"loc.messages.testPlanInput": "Test plan Id : %s",
"loc.messages.testplanConfigInput": "Test plan configuration Id : %s",
"loc.messages.testSuiteSelected": "Test suite Id selected: %s",
Expand Down
3 changes: 2 additions & 1 deletion Tasks/AzureTestPlanV0/automatedTests.ts
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import tl = require('azure-pipelines-task-lib/task');
import { testInvoker } from './automatedTestInvoker'
import { TestPlanData } from './testPlanData'
import { TestCase } from 'azure-devops-node-api/interfaces/TestPlanInterfaces';
import { publishAutomatedTestResult } from './publishAutomatedTests'


export async function automatedTestsFlow(testPlanInfo: TestPlanData, testSelectorInput: string) {
Expand All @@ -13,6 +13,7 @@ export async function automatedTestsFlow(testPlanInfo: TestPlanData, testSelecto
if (listOfTestsToBeExecuted !== null && listOfTestsToBeExecuted !== undefined && listOfTestsToBeExecuted.length > 0) {
tl.debug("Invoking test execution for tests: " + listOfTestsToBeExecuted);
testInvoker(listOfTestsToBeExecuted);
publishAutomatedTestResult(JSON.stringify(testPlanInfo.listOfAutomatedTestPoints));
}
else {
console.log("No automated tests found for given test plan inputs ");
Expand Down
1 change: 1 addition & 0 deletions Tasks/AzureTestPlanV0/constants.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,3 +7,4 @@ export const AUTOMATED_TEST_NAME = "Microsoft.VSTS.TCM.AutomatedTestName";
export const AUTOMATED_TEST_STORAGE = "Microsoft.VSTS.TCM.AutomatedTestStorage";
export const AUTOMATION_STATUS = 'Microsoft.VSTS.TCM.AutomationStatus';
export const NOT_AUTOMATED = 'Not Automated'
export const MERGE_THRESHOLD = 100;
86 changes: 86 additions & 0 deletions Tasks/AzureTestPlanV0/publishAutomatedTests.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
import tl = require('azure-pipelines-task-lib/task');
import * as path from 'path';
import constants = require('./constants');

function publish(testRunner, resultFiles, mergeResults, failTaskOnFailedTests, platform, publishRunAttachments, testRunSystem , failTaskOnFailureToPublishResults, listOfAutomatedTestPoints) {
var properties = <{ [key: string]: string }>{};
properties['type'] = testRunner;
properties['mergeResults'] = mergeResults;
properties['platform'] = platform;
properties['publishRunAttachments'] = publishRunAttachments;
properties['resultFiles'] = resultFiles;
properties['failTaskOnFailedTests'] = failTaskOnFailedTests;
properties['failTaskOnFailureToPublishResults'] = failTaskOnFailureToPublishResults;
properties['testRunSystem'] = testRunSystem;
properties['listOfAutomatedTestPoints'] = listOfAutomatedTestPoints;
properties['testPlanId'] = tl.getVariable('TestPlanId');

tl.command('results.publish', properties, '');
}

export async function publishAutomatedTestResult(listOfAutomatedTestPoints: string) {
try{
const testRunner = "JUnit";
const testResultsFiles: string[] = ["**/TEST-*.xml"];
const mergeResults = tl.getInput('mergeTestResults');
const platform = "any cpu";
const publishRunAttachments = tl.getInput('publishRunAttachments');
const failTaskOnFailedTests = tl.getInput('failTaskOnFailedTests');
const failTaskOnMissingResultsFile: boolean = tl.getBoolInput('failTaskOnMissingResultsFile');
const failTaskOnFailureToPublishResults = tl.getInput('failTaskOnFailureToPublishResults');
const testRunSystem = "AzureTestPlan : " + tl.getInput("testLanguageInput");

let searchFolder = tl.getVariable('System.DefaultWorkingDirectory');

tl.debug('testRunner: ' + testRunner);
tl.debug('testResultsFiles: ' + testResultsFiles);
tl.debug('mergeResults: ' + mergeResults);
tl.debug('platform: ' + platform);
tl.debug('publishRunAttachments: ' + publishRunAttachments);
tl.debug('failTaskOnFailedTests: ' + failTaskOnFailedTests);
tl.debug('failTaskOnMissingResultsFile: ' + failTaskOnMissingResultsFile);
tl.debug('failTaskOnFailureToPublishResults: ' + failTaskOnFailureToPublishResults);

if(tl.getVariable('System.DefaultWorkingDirectory') && (!path.isAbsolute(searchFolder)))
{
searchFolder = path.join(tl.getVariable('System.DefaultWorkingDirectory'),searchFolder);
}

const findOptions = <tl.FindOptions>{
allowBrokenSymbolicLinks: true,
followSpecifiedSymbolicLink: true,
followSymbolicLinks: true
};

const matchingTestResultsFiles = tl.findMatch(searchFolder, testResultsFiles, findOptions);

const testResultsFilesCount = matchingTestResultsFiles ? matchingTestResultsFiles.length : 0;

tl.debug(`Detected ${testResultsFilesCount} test result files`);

const forceMerge = testResultsFilesCount > constants.MERGE_THRESHOLD;
if (forceMerge) {
tl.debug('Detected large number of test result files. Merged all of them into a single file and published a single test run to optimize for test result publish performance instead of publishing hundreds of test runs');
}

if (testResultsFilesCount === 0) {
if (failTaskOnMissingResultsFile) {
tl.setResult(tl.TaskResult.Failed, tl.loc('NoMatchingFilesFound', testResultsFiles));
} else {
tl.warning(tl.loc('NoMatchingFilesFound', testResultsFiles));
}
} else {
publish(testRunner, matchingTestResultsFiles,
forceMerge ? true.toString() : mergeResults,
failTaskOnFailedTests,
platform,
publishRunAttachments,
testRunSystem,
failTaskOnFailureToPublishResults,
listOfAutomatedTestPoints);
}
} catch (err) {
tl.setResult(tl.TaskResult.Failed, err);
}
}

43 changes: 42 additions & 1 deletion Tasks/AzureTestPlanV0/task.json
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
"version": {
"Major": 0,
"Minor": 236,
"Patch": 0
"Patch": 1
},
"preview": true,
"demands": [],
Expand Down Expand Up @@ -84,6 +84,47 @@
"properties": {
"MultiSelectFlatList": "True"
}
},
{
"name": "mergeTestResults",
"type": "boolean",
"label": "Merge test results",
"defaultValue": "false",
"required": false,
"helpMarkDown": "A test run is created for each results file. Check this option to merge results into a single test run. To optimize for better performance, results will be merged into a single run if there are more than 100 result files, irrespective of this option."
},
{
"name": "publishRunAttachments",
"type": "boolean",
"label": "Upload test results files",
"defaultValue": "true",
"required": false,
"helpMarkDown": "Upload logs and other files containing diagnostic information collected when the tests were run.",
"groupName": "advanced"
},
{
"name": "failTaskOnFailedTests",
"type": "boolean",
"label": "Fail if there are test failures",
"defaultValue": "false",
"required": false,
"helpMarkDown": "Fail the task if there are any test failures. Check this option to fail the task if test failures are detected in the result files."
},
{
"name": "failTaskOnFailureToPublishResults",
"type": "boolean",
"label": "Fail if there is failure in publishing test results",
"defaultValue": false,
"required": false,
"helpMarkDown": "Fail if there is failure in publishing test results. Check this option to fail the task if publishing test results is failed partially."
},
{
"name": "failTaskOnMissingResultsFile",
"type": "boolean",
"label": "Fail if no result files are found",
"defaultValue": false,
"required": false,
"helpMarkDown": "Fail the task if no result files are found."
}
],
"dataSourceBindings": [
Expand Down
47 changes: 46 additions & 1 deletion Tasks/AzureTestPlanV0/task.loc.json
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
"version": {
"Major": 0,
"Minor": 236,
"Patch": 0
"Patch": 1
},
"preview": true,
"demands": [],
Expand Down Expand Up @@ -84,6 +84,47 @@
"properties": {
"MultiSelectFlatList": "True"
}
},
{
"name": "mergeTestResults",
"type": "boolean",
"label": "ms-resource:loc.input.label.mergeTestResults",
"defaultValue": "false",
"required": false,
"helpMarkDown": "ms-resource:loc.input.help.mergeTestResults"
},
{
"name": "publishRunAttachments",
"type": "boolean",
"label": "ms-resource:loc.input.label.publishRunAttachments",
"defaultValue": "true",
"required": false,
"helpMarkDown": "ms-resource:loc.input.help.publishRunAttachments",
"groupName": "advanced"
},
{
"name": "failTaskOnFailedTests",
"type": "boolean",
"label": "ms-resource:loc.input.label.failTaskOnFailedTests",
"defaultValue": "false",
"required": false,
"helpMarkDown": "ms-resource:loc.input.help.failTaskOnFailedTests"
},
{
"name": "failTaskOnFailureToPublishResults",
"type": "boolean",
"label": "ms-resource:loc.input.label.failTaskOnFailureToPublishResults",
"defaultValue": false,
"required": false,
"helpMarkDown": "ms-resource:loc.input.help.failTaskOnFailureToPublishResults"
},
{
"name": "failTaskOnMissingResultsFile",
"type": "boolean",
"label": "ms-resource:loc.input.label.failTaskOnMissingResultsFile",
"defaultValue": false,
"required": false,
"helpMarkDown": "ms-resource:loc.input.help.failTaskOnMissingResultsFile"
}
],
"dataSourceBindings": [
Expand Down Expand Up @@ -133,5 +174,9 @@
"ErrorFailTaskOnExecutingTests": "ms-resource:loc.messages.ErrorFailTaskOnExecutingTests",
"ErrorFailTaskOnAPIFailure": "ms-resource:loc.messages.ErrorFailTaskOnAPIFailure",
"ErrorFailTaskOnCreateRunFailure": "ms-resource:loc.messages.ErrorFailTaskOnCreateRunFailure"
},
"_buildConfigMapping": {
"Default": "0.236.1",
"Node20-225": "0.236.2"
}
}
Loading

0 comments on commit c445e55

Please sign in to comment.