From a2de81891501e700a94da3d05536808d8757e61d Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Mon, 31 Oct 2022 13:47:58 -0400 Subject: [PATCH] v4 new release (#253) --- README.md | 6 +- jest.config.js | 3 +- lib/index.js | 198 ++++++++++++++++++++++++++++---- src/run.ts | 6 +- src/types/errorable.ts | 48 ++++++++ src/types/privatekubectl.ts | 26 ++++- src/utilities/fileUtils.test.ts | 74 +++++++++--- src/utilities/fileUtils.ts | 107 ++++++++++++++++- 8 files changed, 419 insertions(+), 49 deletions(-) create mode 100644 src/types/errorable.ts diff --git a/README.md b/README.md index 0ba34f122..4fbdd2934 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ Following are the key capabilities of this action: manifests

(Required) - Path to the manifest files to be used for deployment. These can also be directories containing manifest files, in which case, all manifest files in the referenced directory at every depth will be deployed. Files not ending in .yml or .yaml will be ignored. + Path to the manifest files to be used for deployment. These can also be directories containing manifest files, in which case, all manifest files in the referenced directory at every depth will be deployed, or URLs to manifest files (like https://raw.githubusercontent.com/kubernetes/website/main/content/en/examples/controllers/nginx-deployment.yaml). Files and URLs not ending in .yml or .yaml will be ignored. strategy

(Required) @@ -471,3 +471,7 @@ provided by the bot. You will only need to do this once across all repos using o This project has adopted the [Microsoft Open Source Code of Conduct](https://opensource.microsoft.com/codeofconduct/). For more information see the [Code of Conduct FAQ](https://opensource.microsoft.com/codeofconduct/faq/) or contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additional questions or comments. + +## Support + +k8s-deploy is an open source project that is [**not** covered by the Microsoft Azure support policy](https://support.microsoft.com/en-us/help/2941892/support-for-linux-and-open-source-technology-in-azure). [Please search open issues here](https://github.com/Azure/k8s-deploy/issues), and if your issue isn't already represented please [open a new one](https://github.com/Azure/k8s-deploy/issues/new/choose). The project maintainers will respond to the best of their abilities. diff --git a/jest.config.js b/jest.config.js index 5b7734fc8..dfbf2d242 100644 --- a/jest.config.js +++ b/jest.config.js @@ -6,5 +6,6 @@ module.exports = { transform: { '^.+\\.ts$': 'ts-jest' }, - verbose: true + verbose: true, + testTimeout: 9000 } diff --git a/lib/index.js b/lib/index.js index db55942e7..89eeee6b9 100644 --- a/lib/index.js +++ b/lib/index.js @@ -20221,7 +20221,7 @@ function run() { .split(/[\n,;]+/) // split into each individual manifest .map((manifest) => manifest.trim()) // remove surrounding whitespace .filter((manifest) => manifest.length > 0); // remove any blanks - const fullManifestFilePaths = fileUtils_1.getFilesFromDirectories(manifestFilePaths); + const fullManifestFilePaths = yield fileUtils_1.getFilesFromDirectoriesAndURLs(manifestFilePaths); const kubectlPath = yield kubectl_1.getKubectlPath(); const namespace = core.getInput('namespace') || 'default'; const isPrivateCluster = core.getInput('private-cluster').toLowerCase() === 'true'; @@ -21867,6 +21867,53 @@ class DockerExec { exports.DockerExec = DockerExec; +/***/ }), + +/***/ 9939: +/***/ ((__unused_webpack_module, exports) => { + +"use strict"; + +Object.defineProperty(exports, "__esModule", ({ value: true })); +exports.getErrorMessage = exports.combine = exports.map = exports.failed = exports.succeeded = void 0; +function succeeded(e) { + return e.succeeded; +} +exports.succeeded = succeeded; +function failed(e) { + return !e.succeeded; +} +exports.failed = failed; +function map(e, fn) { + if (failed(e)) { + return { succeeded: false, error: e.error }; + } + return { succeeded: true, result: fn(e.result) }; +} +exports.map = map; +function combine(es) { + const failures = es.filter(failed); + if (failures.length > 0) { + return { + succeeded: false, + error: failures.map((f) => f.error).join('\n') + }; + } + return { + succeeded: true, + result: es.map((e) => e.result) + }; +} +exports.combine = combine; +function getErrorMessage(error) { + if (error instanceof Error) { + return error.message; + } + return String(error); +} +exports.getErrorMessage = getErrorMessage; + + /***/ }), /***/ 3067: @@ -22199,7 +22246,11 @@ class PrivateKubectl extends kubectl_1.Kubectl { args.unshift('kubectl'); let kubectlCmd = args.join(' '); let addFileFlag = false; - let eo = { silent }; + let eo = { + silent: true, + failOnStdErr: false, + ignoreReturnCode: true + }; if (this.containsFilenames(kubectlCmd)) { // For private clusters, files will referenced solely by their basename kubectlCmd = this.replaceFilnamesWithBasenames(kubectlCmd); @@ -22231,7 +22282,18 @@ class PrivateKubectl extends kubectl_1.Kubectl { } } core.debug(`private cluster Kubectl run with invoke command: ${kubectlCmd}`); - return yield exec_1.getExecOutput('az', privateClusterArgs, eo); + const runOutput = yield exec_1.getExecOutput('az', [...privateClusterArgs, '-o', 'json'], eo); + const runObj = JSON.parse(runOutput.stdout); + if (!silent) + core.info(runObj.logs); + if (runOutput.exitCode !== 0 && runObj.exitCode !== 0) { + throw Error(`failed private cluster Kubectl command: ${kubectlCmd}`); + } + return { + exitCode: runObj.exitCode, + stdout: runObj.logs, + stderr: '' + }; }); } replaceFilnamesWithBasenames(kubectlCmd) { @@ -22443,17 +22505,31 @@ exports.checkDockerPath = checkDockerPath; /***/ }), /***/ 7446: -/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { +/***/ (function(__unused_webpack_module, exports, __nccwpck_require__) { "use strict"; +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; Object.defineProperty(exports, "__esModule", ({ value: true })); -exports.getFilesFromDirectories = exports.writeManifestToFile = exports.writeObjectsToFile = exports.getTempDirectory = void 0; +exports.writeYamlFromURLToFile = exports.getFilesFromDirectoriesAndURLs = exports.writeManifestToFile = exports.writeObjectsToFile = exports.getTempDirectory = exports.urlFileKind = void 0; const fs = __nccwpck_require__(7147); +const https = __nccwpck_require__(5687); const path = __nccwpck_require__(1017); const core = __nccwpck_require__(6024); const os = __nccwpck_require__(2037); +const yaml = __nccwpck_require__(3607); +const errorable_1 = __nccwpck_require__(9939); const timeUtils_1 = __nccwpck_require__(4046); +const githubUtils_1 = __nccwpck_require__(9976); +exports.urlFileKind = 'urlfile'; function getTempDirectory() { return process.env['runner.tempDirectory'] || os.tmpdir(); } @@ -22499,30 +22575,104 @@ function getManifestFileName(kind, name) { const tempDirectory = getTempDirectory(); return path.join(tempDirectory, path.basename(filePath)); } -function getFilesFromDirectories(filePaths) { - const fullPathSet = new Set(); - filePaths.forEach((fileName) => { - try { - if (fs.lstatSync(fileName).isDirectory()) { - recurisveManifestGetter(fileName).forEach((file) => { - fullPathSet.add(file); - }); - } - else if (getFileExtension(fileName) === 'yml' || - getFileExtension(fileName) === 'yaml') { - fullPathSet.add(fileName); +function getFilesFromDirectoriesAndURLs(filePaths) { + return __awaiter(this, void 0, void 0, function* () { + const fullPathSet = new Set(); + let fileCounter = 0; + for (const fileName of filePaths) { + try { + if (githubUtils_1.isHttpUrl(fileName)) { + try { + const tempFilePath = yield writeYamlFromURLToFile(fileName, fileCounter++); + fullPathSet.add(tempFilePath); + } + catch (e) { + throw Error(`encountered error trying to pull YAML from URL ${fileName}: ${e}`); + } + } + else if (fs.lstatSync(fileName).isDirectory()) { + recurisveManifestGetter(fileName).forEach((file) => { + fullPathSet.add(file); + }); + } + else if (getFileExtension(fileName) === 'yml' || + getFileExtension(fileName) === 'yaml') { + fullPathSet.add(fileName); + } + else { + core.debug(`Detected non-manifest file, ${fileName}, continuing... `); + } } - else { - core.debug(`Detected non-manifest file, ${fileName}, continuing... `); + catch (ex) { + throw Error(`Exception occurred while reading the file ${fileName}: ${ex}`); } } - catch (ex) { - throw Error(`Exception occurred while reading the file ${fileName}: ${ex}`); - } + const arr = Array.from(fullPathSet); + return arr; + }); +} +exports.getFilesFromDirectoriesAndURLs = getFilesFromDirectoriesAndURLs; +function writeYamlFromURLToFile(url, fileNumber) { + return __awaiter(this, void 0, void 0, function* () { + return new Promise((resolve, reject) => { + https + .get(url, (response) => __awaiter(this, void 0, void 0, function* () { + var _a; + const code = (_a = response.statusCode) !== null && _a !== void 0 ? _a : 0; + if (code >= 400) { + reject(Error(`received response status ${response.statusMessage} from url ${url}`)); + } + const targetPath = getManifestFileName(exports.urlFileKind, fileNumber.toString()); + // save the file to disk + const fileWriter = fs + .createWriteStream(targetPath) + .on('finish', () => { + const verification = verifyYaml(targetPath, url); + if (errorable_1.succeeded(verification)) { + core.debug(`outputting YAML contents from ${url} to ${targetPath}: ${JSON.stringify(verification.result)}`); + resolve(targetPath); + } + else { + reject(verification.error); + } + }); + response.pipe(fileWriter); + })) + .on('error', (error) => { + reject(error); + }); + }); }); - return Array.from(fullPathSet); } -exports.getFilesFromDirectories = getFilesFromDirectories; +exports.writeYamlFromURLToFile = writeYamlFromURLToFile; +function verifyYaml(filepath, url) { + const fileContents = fs.readFileSync(filepath).toString(); + let inputObjects; + try { + inputObjects = yaml.safeLoadAll(fileContents); + } + catch (e) { + return { + succeeded: false, + error: `failed to parse manifest from url ${url}: ${e}` + }; + } + if (!inputObjects || inputObjects.length == 0) { + return { + succeeded: false, + error: `failed to parse manifest from url ${url}: no objects detected in manifest` + }; + } + for (const obj of inputObjects) { + if (!obj.kind || !obj.apiVersion || !obj.metadata) { + return { + succeeded: false, + error: `failed to parse manifest from ${url}: missing fields` + }; + } + } + return { succeeded: true, result: inputObjects }; +} function recurisveManifestGetter(dirName) { const toRet = []; fs.readdirSync(dirName).forEach((fileName) => { diff --git a/src/run.ts b/src/run.ts index 525390470..ca3012ff9 100644 --- a/src/run.ts +++ b/src/run.ts @@ -5,7 +5,7 @@ import {promote} from './actions/promote' import {reject} from './actions/reject' import {Action, parseAction} from './types/action' import {parseDeploymentStrategy} from './types/deploymentStrategy' -import {getFilesFromDirectories} from './utilities/fileUtils' +import {getFilesFromDirectoriesAndURLs} from './utilities/fileUtils' import {PrivateKubectl} from './types/privatekubectl' export async function run() { @@ -26,7 +26,9 @@ export async function run() { .map((manifest) => manifest.trim()) // remove surrounding whitespace .filter((manifest) => manifest.length > 0) // remove any blanks - const fullManifestFilePaths = getFilesFromDirectories(manifestFilePaths) + const fullManifestFilePaths = await getFilesFromDirectoriesAndURLs( + manifestFilePaths + ) const kubectlPath = await getKubectlPath() const namespace = core.getInput('namespace') || 'default' const isPrivateCluster = diff --git a/src/types/errorable.ts b/src/types/errorable.ts new file mode 100644 index 000000000..b507ebe8c --- /dev/null +++ b/src/types/errorable.ts @@ -0,0 +1,48 @@ +export interface Succeeded { + readonly succeeded: true + readonly result: T +} + +export interface Failed { + readonly succeeded: false + readonly error: string +} + +export type Errorable = Succeeded | Failed + +export function succeeded(e: Errorable): e is Succeeded { + return e.succeeded +} + +export function failed(e: Errorable): e is Failed { + return !e.succeeded +} + +export function map(e: Errorable, fn: (t: T) => U): Errorable { + if (failed(e)) { + return {succeeded: false, error: e.error} + } + return {succeeded: true, result: fn(e.result)} +} + +export function combine(es: Errorable[]): Errorable { + const failures = es.filter(failed) + if (failures.length > 0) { + return { + succeeded: false, + error: failures.map((f) => f.error).join('\n') + } + } + + return { + succeeded: true, + result: es.map((e) => (e as Succeeded).result) + } +} + +export function getErrorMessage(error: unknown) { + if (error instanceof Error) { + return error.message + } + return String(error) +} diff --git a/src/types/privatekubectl.ts b/src/types/privatekubectl.ts index f7e8c0478..390ae3e82 100644 --- a/src/types/privatekubectl.ts +++ b/src/types/privatekubectl.ts @@ -10,7 +10,11 @@ export class PrivateKubectl extends Kubectl { args.unshift('kubectl') let kubectlCmd = args.join(' ') let addFileFlag = false - let eo = {silent} + let eo = { + silent: true, + failOnStdErr: false, + ignoreReturnCode: true + } if (this.containsFilenames(kubectlCmd)) { // For private clusters, files will referenced solely by their basename @@ -52,7 +56,25 @@ export class PrivateKubectl extends Kubectl { core.debug( `private cluster Kubectl run with invoke command: ${kubectlCmd}` ) - return await getExecOutput('az', privateClusterArgs, eo) + + const runOutput = await getExecOutput( + 'az', + [...privateClusterArgs, '-o', 'json'], + eo + ) + const runObj: {logs: string; exitCode: number} = JSON.parse( + runOutput.stdout + ) + if (!silent) core.info(runObj.logs) + if (runOutput.exitCode !== 0 && runObj.exitCode !== 0) { + throw Error(`failed private cluster Kubectl command: ${kubectlCmd}`) + } + + return { + exitCode: runObj.exitCode, + stdout: runObj.logs, + stderr: '' + } as ExecOutput } private replaceFilnamesWithBasenames(kubectlCmd: string) { diff --git a/src/utilities/fileUtils.test.ts b/src/utilities/fileUtils.test.ts index 9b6d17482..546e3a1dd 100644 --- a/src/utilities/fileUtils.test.ts +++ b/src/utilities/fileUtils.test.ts @@ -1,11 +1,45 @@ -import {getFilesFromDirectories} from './fileUtils' +import { + getFilesFromDirectoriesAndURLs, + getTempDirectory, + urlFileKind, + writeYamlFromURLToFile +} from './fileUtils' +import * as yaml from 'js-yaml' +import * as fs from 'fs' import * as path from 'path' +import {succeeded} from '../types/errorable' +const sampleYamlUrl = + 'https://raw.githubusercontent.com/kubernetes/website/main/content/en/examples/controllers/nginx-deployment.yaml' describe('File utils', () => { - it('detects files in nested directories and ignores non-manifest files and empty dirs', () => { + test('correctly parses a yaml file from a URL', async () => { + const tempFile = await writeYamlFromURLToFile(sampleYamlUrl, 0) + const fileContents = fs.readFileSync(tempFile).toString() + const inputObjects = yaml.safeLoadAll(fileContents) + expect(inputObjects).toHaveLength(1) + + for (const obj of inputObjects) { + expect(obj.metadata.name).toBe('nginx-deployment') + expect(obj.kind).toBe('Deployment') + } + }) + + it('fails when a bad URL is given among other files', async () => { + const badUrl = 'https://www.github.com' + const testPath = path.join('test', 'unit', 'manifests') - const testSearch: string[] = getFilesFromDirectories([testPath]) + await expect( + getFilesFromDirectoriesAndURLs([testPath, badUrl]) + ).rejects.toThrow() + }) + + it('detects files in nested directories and ignores non-manifest files and empty dirs', async () => { + const testPath = path.join('test', 'unit', 'manifests') + const testSearch: string[] = await getFilesFromDirectoriesAndURLs([ + testPath, + sampleYamlUrl + ]) const expectedManifests = [ 'test/unit/manifests/manifest_test_dir/another_layer/deep-ingress.yaml', @@ -17,13 +51,18 @@ describe('File utils', () => { ] // is there a more efficient way to test equality w random order? - expect(testSearch).toHaveLength(7) + expect(testSearch).toHaveLength(8) expectedManifests.forEach((fileName) => { - expect(testSearch).toContain(fileName) + if (fileName.startsWith('test/unit')) { + expect(testSearch).toContain(fileName) + } else { + expect(fileName.includes(urlFileKind)).toBe(true) + expect(fileName.startsWith(getTempDirectory())) + } }) }) - it('crashes when an invalid file is provided', () => { + it('crashes when an invalid file is provided', async () => { const badPath = path.join('test', 'unit', 'manifests', 'nonexistent.yaml') const goodPath = path.join( 'test', @@ -32,12 +71,12 @@ describe('File utils', () => { 'manifest_test_dir' ) - expect(() => { - getFilesFromDirectories([badPath, goodPath]) - }).toThrowError() + expect( + getFilesFromDirectoriesAndURLs([badPath, goodPath]) + ).rejects.toThrowError() }) - it("doesn't duplicate files when nested dir included", () => { + it("doesn't duplicate files when nested dir included", async () => { const outerPath = path.join('test', 'unit', 'manifests') const fileAtOuter = path.join( 'test', @@ -53,11 +92,16 @@ describe('File utils', () => { ) expect( - getFilesFromDirectories([outerPath, fileAtOuter, innerPath]) + await getFilesFromDirectoriesAndURLs([ + outerPath, + fileAtOuter, + innerPath + ]) ).toHaveLength(7) }) -}) -// files that don't exist / nested files that don't exist / something else with non-manifest -// lots of combinations of pointing to a directory and non yaml/yaml file -// similarly named files in different folders + it('throws an error for an invalid URL', async () => { + const badUrl = 'https://www.github.com' + await expect(writeYamlFromURLToFile(badUrl, 0)).rejects.toBeTruthy() + }) +}) diff --git a/src/utilities/fileUtils.ts b/src/utilities/fileUtils.ts index d4bfcf5d5..03ebcbcdf 100644 --- a/src/utilities/fileUtils.ts +++ b/src/utilities/fileUtils.ts @@ -1,8 +1,15 @@ import * as fs from 'fs' +import * as https from 'https' import * as path from 'path' import * as core from '@actions/core' import * as os from 'os' +import * as yaml from 'js-yaml' +import {Errorable, succeeded, failed, Failed} from '../types/errorable' import {getCurrentTime} from './timeUtils' +import {isHttpUrl} from './githubUtils' +import {K8sObject} from '../types/k8sObject' + +export const urlFileKind = 'urlfile' export function getTempDirectory(): string { return process.env['runner.tempDirectory'] || os.tmpdir() @@ -62,12 +69,27 @@ function getManifestFileName(kind: string, name: string) { return path.join(tempDirectory, path.basename(filePath)) } -export function getFilesFromDirectories(filePaths: string[]): string[] { +export async function getFilesFromDirectoriesAndURLs( + filePaths: string[] +): Promise { const fullPathSet: Set = new Set() - filePaths.forEach((fileName) => { + let fileCounter = 0 + for (const fileName of filePaths) { try { - if (fs.lstatSync(fileName).isDirectory()) { + if (isHttpUrl(fileName)) { + try { + const tempFilePath: string = await writeYamlFromURLToFile( + fileName, + fileCounter++ + ) + fullPathSet.add(tempFilePath) + } catch (e) { + throw Error( + `encountered error trying to pull YAML from URL ${fileName}: ${e}` + ) + } + } else if (fs.lstatSync(fileName).isDirectory()) { recurisveManifestGetter(fileName).forEach((file) => { fullPathSet.add(file) }) @@ -86,9 +108,86 @@ export function getFilesFromDirectories(filePaths: string[]): string[] { `Exception occurred while reading the file ${fileName}: ${ex}` ) } + } + + const arr = Array.from(fullPathSet) + return arr +} + +export async function writeYamlFromURLToFile( + url: string, + fileNumber: number +): Promise { + return new Promise((resolve, reject) => { + https + .get(url, async (response) => { + const code = response.statusCode ?? 0 + if (code >= 400) { + reject( + Error( + `received response status ${response.statusMessage} from url ${url}` + ) + ) + } + + const targetPath = getManifestFileName( + urlFileKind, + fileNumber.toString() + ) + // save the file to disk + const fileWriter = fs + .createWriteStream(targetPath) + .on('finish', () => { + const verification = verifyYaml(targetPath, url) + if (succeeded(verification)) { + core.debug( + `outputting YAML contents from ${url} to ${targetPath}: ${JSON.stringify( + verification.result + )}` + ) + resolve(targetPath) + } else { + reject(verification.error) + } + }) + + response.pipe(fileWriter) + }) + .on('error', (error) => { + reject(error) + }) }) +} + +function verifyYaml(filepath: string, url: string): Errorable { + const fileContents = fs.readFileSync(filepath).toString() + let inputObjects + try { + inputObjects = yaml.safeLoadAll(fileContents) + } catch (e) { + return { + succeeded: false, + error: `failed to parse manifest from url ${url}: ${e}` + } + } + + if (!inputObjects || inputObjects.length == 0) { + return { + succeeded: false, + error: `failed to parse manifest from url ${url}: no objects detected in manifest` + } + } + + for (const obj of inputObjects) { + if (!obj.kind || !obj.apiVersion || !obj.metadata) { + return { + succeeded: false, + error: `failed to parse manifest from ${url}: missing fields` + } + } + } - return Array.from(fullPathSet) + return {succeeded: true, result: inputObjects} } function recurisveManifestGetter(dirName: string): string[] {