diff --git a/.travis.yml b/.travis.yml index a6a633711a66..a2b4b38975d7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,19 +7,11 @@ services: env: matrix: - MODE=branchStrategy - - MODE=syntax CHECK_NAME="Syntax Validator" - - MODE=semantic PR_ONLY=true CHECK_NAME="Semantic Validator" - - MODE=semantic PR_ONLY=false - - MODE=model PR_ONLY=true CHECK_NAME="Model Validator" - # - MODE=model PR_ONLY=false - MODE=BreakingChange PR_ONLY=true CHECK_NAME="Breaking Changes" - MODE=lintdiff PR_ONLY=true CHECK_NAME="Linter Diff" NODE_OPTIONS=--max-old-space-size=8192 matrix: fast_finish: true allow_failures: - - env: MODE=semantic PR_ONLY=false - - env: MODE=model PR_ONLY=false - - env: MODE=model PR_ONLY=true CHECK_NAME="Model Validator" - env: MODE=BreakingChange PR_ONLY=true CHECK_NAME="Breaking Changes" install: true script: @@ -28,23 +20,6 @@ script: # Check to ensure CI is not executing for a PR against the master branch in the private repository ! [[ $TRAVIS_PULL_REQUEST != 'false' && $TRAVIS_REPO_SLUG == 'Azure/azure-rest-api-specs-pr' && $TRAVIS_BRANCH == 'master' ]] fi - - >- - if [[ $MODE == 'syntax' ]]; then - npm install - npm test -- test/syntax.js - fi - - >- - if [[ $MODE == 'semantic' ]]; then - npm install - npm run tsc - node scripts/semanticValidation.js - fi - - >- - if [[ $MODE == 'model' ]]; then - npm install - npm run tsc - node scripts/modelValidation.js - fi - >- if [[ $MODE == 'BreakingChange' ]]; then scripts/install-dotnet.sh diff --git a/package.json b/package.json index f76623b5e517..c9e130ae6788 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "devDependencies": { "@azure/avocado": "^0.3.3", "@azure/oad": "^0.5.1", + "@azure/rest-api-specs-scripts": "^0.1.8", "@microsoft.azure/async-io": "^2.0.21", "@microsoft.azure/literate": "^1.0.25", "@microsoft.azure/polyfill": "^1.0.19", diff --git a/scripts/breaking-change.ts b/scripts/breaking-change.ts index 70825c3d42da..b79c27007b1a 100644 --- a/scripts/breaking-change.ts +++ b/scripts/breaking-change.ts @@ -1,245 +1,10 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. See License in the project root for license information. -import * as stringMap from '@ts-common/string-map' -import * as tsUtils from './ts-utils' -import * as utils from '../test/util/utils' -import * as path from 'path' -import * as fs from 'fs-extra' -import * as os from 'os' -import * as childProcess from 'child_process' -import * as oad from '@azure/oad' -import * as util from 'util' - -const exec = util.promisify(childProcess.exec) - -// This map is used to store the mapping between files resolved and stored location -var resolvedMapForNewSpecs: stringMap.MutableStringMap = {}; -let outputFolder = path.join(os.tmpdir(), "resolved"); -// Used to enable running script outside TravisCI for debugging -let isRunningInTravisCI = process.env.TRAVIS === 'true'; - -const headerText = ` -| | Rule | Location | Message | -|-|------|----------|---------| -`; - -function iconFor(type: unknown) { - if (type === 'Error') { - return ':x:'; - } else if (type === 'Warning') { - return ':warning:'; - } else if (type === 'Info') { - return ':speech_balloon:'; - } else { - return ''; - } -} - -function shortName(filePath: string) { - return `${path.basename(path.dirname(filePath))}/​${path.basename(filePath)}`; -} - -type Diff = { - readonly type: unknown - readonly id: string - readonly code: unknown - readonly message: unknown -} - -function tableLine(filePath: string, diff: Diff) { - return `|${iconFor(diff['type'])}|[${diff['type']} ${diff['id']} - ${diff['code']}](https://github.com/Azure/openapi-diff/blob/master/docs/rules/${diff['id']}.md)|[${shortName(filePath)}](${blobHref(filePath)} "${filePath}")|${diff['message']}|\n`; -} - -function blobHref(file: unknown) { - return `https://github.com/${process.env.TRAVIS_PULL_REQUEST_SLUG}/blob/${process.env.TRAVIS_PULL_REQUEST_SHA}/${file}`; -} - -/** - * Compares old and new specifications for breaking change detection. - * - * @param oldSpec Path to the old swagger specification file. - * - * @param newSpec Path to the new swagger specification file. - */ -async function runOad(oldSpec: string, newSpec: string) { - if (oldSpec === null || oldSpec === undefined || typeof oldSpec.valueOf() !== 'string' || !oldSpec.trim().length) { - throw new Error('oldSpec is a required parameter of type "string" and it cannot be an empty string.'); - } - - if (newSpec === null || newSpec === undefined || typeof newSpec.valueOf() !== 'string' || !newSpec.trim().length) { - throw new Error('newSpec is a required parameter of type "string" and it cannot be an empty string.'); - } - - console.log(`>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>`); - console.log(`Old Spec: "${oldSpec}"`); - console.log(`New Spec: "${newSpec}"`); - console.log(`>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>`); - - let result = await oad.compare(oldSpec, newSpec, { consoleLogLevel: 'warn' }); - console.log(result); - - if (!result) { - return; - } - - // fix up output from OAD, it does not output valid JSON - result = '[' + result.replace(/}\s+{/gi,"},{") + ']' - - return JSON.parse(result); -} - -/** - * Processes the given swagger and stores the resolved swagger on to disk - * - * @param swaggerPath Path to the swagger specification file. - */ -async function processViaAutoRest(swaggerPath: string) { - if (swaggerPath === null || swaggerPath === undefined || typeof swaggerPath.valueOf() !== 'string' || !swaggerPath.trim().length) { - throw new Error('swaggerPath is a required parameter of type "string" and it cannot be an empty string.'); - } - - const swaggerOutputFolder = path.join(outputFolder, path.dirname(swaggerPath)); - const swaggerOutputFileNameWithoutExt = path.basename(swaggerPath, '.json'); - const autoRestCmd = `autorest --input-file=${swaggerPath} --output-artifact=swagger-document.json --output-file=${swaggerOutputFileNameWithoutExt} --output-folder=${swaggerOutputFolder}`; - - console.log(`Executing : ${autoRestCmd}`); - - try { - await fs.ensureDir(swaggerOutputFolder); - await exec(`${autoRestCmd}`, { encoding: 'utf8', maxBuffer: 1024 * 1024 * 64 }); - resolvedMapForNewSpecs[swaggerPath] = path.join(swaggerOutputFolder, swaggerOutputFileNameWithoutExt + '.json'); - } catch (err) { - console.log(`Error processing via AutoRest: ${err}`); - } -} - -//main function -async function runScript() { - // See whether script is in Travis CI context - console.log(`isRunningInTravisCI: ${isRunningInTravisCI}`); - - let targetBranch = utils.getTargetBranch(); - let swaggersToProcess = utils.getFilesChangedInPR(); - - console.log('Processing swaggers:'); - console.log(swaggersToProcess); - - console.log('Finding new swaggers...') - let newSwaggers: unknown[] = []; - if (isRunningInTravisCI && swaggersToProcess.length > 0) { - newSwaggers = await utils.doOnBranch(utils.getTargetBranch(), async () => { - return swaggersToProcess.filter((s: string) => !fs.existsSync(s)) - }); - } - - console.log('Processing via AutoRest...'); - for (const swagger of swaggersToProcess) { - if (!newSwaggers.includes(swagger)) { - await processViaAutoRest(swagger); - } - } - - console.log(`Resolved map for the new specifications:`); - console.dir(resolvedMapForNewSpecs); - - let errors = 0, warnings = 0; - const diffFiles: stringMap.MutableStringMap = {}; - const newFiles = []; - - for (const swagger of swaggersToProcess) { - // If file does not exists in the previous commits then we ignore it as it's new file - if (newSwaggers.includes(swagger)) { - console.log(`File: "${swagger}" looks to be newly added in this PR.`); - newFiles.push(swagger); - continue; - } - - const resolved = resolvedMapForNewSpecs[swagger] - if (resolved) { - const diffs = await runOad(swagger, resolved); - if (diffs) { - diffFiles[swagger] = diffs; - for (const diff of diffs) { - if (diff['type'] === 'Error') { - if (errors === 0) { - console.log(`There are potential breaking changes in this PR. Please review before moving forward. Thanks!`); - process.exitCode = 1; - } - errors += 1; - } else if (diff['type'] === 'Warning') { - warnings += 1; - } - } - } - } - } - - if (isRunningInTravisCI) { - let summary = ''; - if (errors > 0) { - summary += '**There are potential breaking changes in this PR. Please review before moving forward. Thanks!**\n\n'; - } - summary += `Compared to the target branch (**${targetBranch}**), this pull request introduces:\n\n`; - summary += `   ${errors > 0 ? iconFor('Error') : ':white_check_mark:'}   **${errors}** new error${errors !== 1 ? 's' : ''}\n\n`; - summary += `   ${warnings > 0 ? iconFor('Warning') : ':white_check_mark:'}   **${warnings}** new warning${warnings !== 1 ? 's' : ''}\n\n`; - - let message = ''; - if (newFiles.length > 0) { - message += '### The following files look to be newly added in this PR:\n'; - newFiles.sort(); - for (const swagger of newFiles) { - message += `* [${swagger}](${blobHref(swagger)})\n`; - } - message += '

\n'; - } - - const diffFileNames = Object.keys(diffFiles); - if (diffFileNames.length > 0) { - message += '### OpenAPI diff results\n'; - message += headerText; - - diffFileNames.sort(); - for (const swagger of diffFileNames) { - const diffs = tsUtils.asNonUndefined(diffFiles[swagger]); - diffs.sort((a, b) => { - if (a.type === b.type) { - return a.id.localeCompare(b.id); - } else if (a.type === "Error") { - return 1; - } else if (b.type === "Error") { - return -1; - } else if (a.type === "Warning") { - return 1; - } else { - return -1; - } - }); - - for (const diff of diffs) { - message += tableLine(swagger, diff); - } - } - } else { - message += '**There were no files containing new errors or warnings.**\n'; - } - - message += '\n

\nThanks for using breaking change tool to review.\nIf you encounter any issue(s), please open issue(s) at https://github.com/Azure/openapi-diff/issues.'; - - const output = { - title: `${errors === 0 ? 'No' : errors} potential breaking change${errors !== 1 ? 's' : ''}`, - summary, - text: message - }; - - console.log('---output'); - console.log(JSON.stringify(output)); - console.log('---'); - } -} +import * as scripts from '@azure/rest-api-specs-scripts' // magic starts here -runScript().then(() => { +scripts.breakingChange().then(() => { console.log(`Thanks for using breaking change tool to review.`); console.log(`If you encounter any issue(s), please open issue(s) at https://github.com/Azure/openapi-diff/issues .`); }).catch(err => { diff --git a/scripts/momentOfTruth.ts b/scripts/momentOfTruth.ts index 70f89f16f2d3..100d750deae1 100644 --- a/scripts/momentOfTruth.ts +++ b/scripts/momentOfTruth.ts @@ -1,144 +1,11 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. See License.txt in the project root for license information. -import * as stringMap from '@ts-common/string-map' -import * as tsUtils from './ts-utils' -import { exec } from 'child_process' -import * as path from 'path' -import * as utils from '../test/util/utils' -import * as fs from 'fs' - -let configsToProcess = utils.getConfigFilesChangedInPR(); -let pullRequestNumber = utils.getPullRequestNumber(); -let linterCmd = `npx autorest --validation --azure-validator --message-format=json `; -var filename = `${pullRequestNumber}.json`; -var logFilepath = path.join(getLogDir(), filename); - -type FinalResult = { - readonly pullRequest: unknown, - readonly repositoryUrl: unknown, - readonly files: stringMap.MutableStringMap> -} - -var finalResult: FinalResult = { - pullRequest: pullRequestNumber, - repositoryUrl: utils.getRepoUrl(), - files: {} -} - -// Creates and returns path to the logging directory -function getLogDir() { - let logDir = path.join(__dirname, '../', 'output'); - if (!fs.existsSync(logDir)) { - try { - fs.mkdirSync(logDir); - } catch (e) { - if (e.code !== 'EEXIST') throw e; - } - } - return logDir; -} - -//creates the log file if it has not been created -function createLogFile() { - if (!fs.existsSync(logFilepath)) { - fs.writeFileSync(logFilepath, ''); - } -} - -//appends the content to the log file -function writeContent(content: unknown) { - fs.writeFileSync(logFilepath, content); -} - -// Executes linter on given swagger path and returns structured JSON of linter output -async function getLinterResult(swaggerPath: string|null|undefined) { - if (swaggerPath === null || swaggerPath === undefined || typeof swaggerPath.valueOf() !== 'string' || !swaggerPath.trim().length) { - throw new Error('swaggerPath is a required parameter of type "string" and it cannot be an empty string.'); - } - - let jsonResult = []; - if (!fs.existsSync(swaggerPath)) { - return []; - } - let cmd = "npx autorest --reset && " + linterCmd + swaggerPath; - console.log(`Executing: ${cmd}`); - const { err, stdout, stderr } = await new Promise(res => exec(cmd, { encoding: 'utf8', maxBuffer: 1024 * 1024 * 64 }, - (err: unknown, stdout: unknown, stderr: unknown) => res({ err: err, stdout: stdout, stderr: stderr }))); - - if (err && stderr.indexOf("Process() cancelled due to exception") !== -1) { - console.error(`AutoRest exited with code ${err.code}`); - console.error(stderr); - throw new Error("AutoRest failed"); - } - - let resultString = stdout + stderr; - if (resultString.indexOf('{') !== -1) { - resultString = resultString.replace(/Processing batch task - {.*} \.\n/g, ""); - resultString = "[" + resultString.substring(resultString.indexOf('{')).trim().replace(/\}\n\{/g, "},\n{") + "]"; - //console.log('>>>>>> Trimmed Result...'); - //console.log(resultString); - try { - jsonResult = JSON.parse(resultString); - //console.log('>>>>>> Parsed Result...'); - //console.dir(resultObject, {depth: null, colors: true}); - return jsonResult; - } catch (e) { - console.error(`An error occurred while executing JSON.parse() on the linter output for ${swaggerPath}:`); - console.dir(resultString); - console.dir(e, { depth: null, colors: true }); - process.exit(1) - } - } - return []; -}; - -// Run linter tool -async function runTools(swagger: string, beforeOrAfter: string) { - console.log(`Processing "${swagger}":`); - const linterErrors = await getLinterResult(swagger); - console.log(linterErrors); - await updateResult(swagger, linterErrors, beforeOrAfter); -}; - -// Updates final result json to be written to the output file -async function updateResult(spec: string, errors: unknown, beforeOrAfter: string) { - const files = finalResult['files'] - if (!files[spec]) { - files[spec] = {}; - } - const filesSpec = tsUtils.asNonUndefined(files[spec]) - if (!filesSpec[beforeOrAfter]) { - filesSpec[beforeOrAfter] = {}; - } - filesSpec[beforeOrAfter] = errors; -} - -//main function -async function runScript() { - console.log('Processing configs:'); - console.log(configsToProcess); - createLogFile(); - console.log(`The results will be logged here: "${logFilepath}".`) - - if (configsToProcess.length > 0) { - for (const configFile of configsToProcess) { - await runTools(configFile, 'after'); - } - - await utils.doOnBranch(utils.getTargetBranch(), async () => { - for (const configFile of configsToProcess) { - await runTools(configFile, 'before'); - } - }); - } - - writeContent(JSON.stringify(finalResult, null, 2)); -} +import * as scripts from '@azure/rest-api-specs-scripts' // magic starts here -runScript().then(_ => { - process.exit(0); +scripts.momentOfTruth().then(_ => { + process.exit(0); }).catch(_ => { - process.exit(1); -}) + process.exit(1); +}) \ No newline at end of file