From 0609e295a508bd29c57fab4711a7fd5568d1d192 Mon Sep 17 00:00:00 2001 From: Paul Gschwendtner Date: Tue, 14 Mar 2017 19:31:15 +0100 Subject: [PATCH] build: create a util directory (#3558) * Introduces a new directory called `util` in the gulp setup. This should make the gulp setup more clear and maintainable. * The utility `task_helpers` no longer includes random utilities like for Firebase. Those have been moved to their own util file (similar as in `e2e/`) * Refactors the Github Status utility because it can be useful for other things as well (e.g upcoming coverage, payload) --- tools/gulp/tasks/aot.ts | 2 +- tools/gulp/tasks/clean.ts | 2 +- tools/gulp/tasks/components.ts | 2 +- tools/gulp/tasks/coverage.ts | 3 +- tools/gulp/tasks/development.ts | 2 +- tools/gulp/tasks/e2e.ts | 2 +- tools/gulp/tasks/lint.ts | 2 +- tools/gulp/tasks/payload.ts | 3 +- tools/gulp/tasks/release.ts | 2 +- tools/gulp/tasks/screenshots.ts | 16 ++++- tools/gulp/tasks/serve.ts | 2 +- tools/gulp/tasks/unit-test.ts | 2 +- tools/gulp/util/firebase.ts | 61 +++++++++++++++++ .../{util-functions.ts => util/github.ts} | 27 +++++--- tools/gulp/{ => util}/task_helpers.ts | 68 +------------------ tools/gulp/util/travis-ci.ts | 4 ++ 16 files changed, 111 insertions(+), 89 deletions(-) create mode 100644 tools/gulp/util/firebase.ts rename tools/gulp/{util-functions.ts => util/github.ts} (53%) rename tools/gulp/{ => util}/task_helpers.ts (66%) create mode 100644 tools/gulp/util/travis-ci.ts diff --git a/tools/gulp/tasks/aot.ts b/tools/gulp/tasks/aot.ts index 073535532435..5b1b1796d97f 100644 --- a/tools/gulp/tasks/aot.ts +++ b/tools/gulp/tasks/aot.ts @@ -1,7 +1,7 @@ import {task} from 'gulp'; import {join} from 'path'; import {DIST_ROOT} from '../constants'; -import {execNodeTask, sequenceTask} from '../task_helpers'; +import {execNodeTask, sequenceTask} from '../util/task_helpers'; /** Copies the source files of the demo-app to the dist folder. */ task('aot:copy', [':build:devapp:scss', ':build:devapp:assets']); diff --git a/tools/gulp/tasks/clean.ts b/tools/gulp/tasks/clean.ts index c64d99df5e5b..d3fabb9e47a9 100644 --- a/tools/gulp/tasks/clean.ts +++ b/tools/gulp/tasks/clean.ts @@ -1,6 +1,6 @@ import {task} from 'gulp'; import {DIST_ROOT} from '../constants'; -import {cleanTask} from '../task_helpers'; +import {cleanTask} from '../util/task_helpers'; task('clean', cleanTask(DIST_ROOT)); diff --git a/tools/gulp/tasks/components.ts b/tools/gulp/tasks/components.ts index 379e0b559256..282b83f2aada 100644 --- a/tools/gulp/tasks/components.ts +++ b/tools/gulp/tasks/components.ts @@ -7,7 +7,7 @@ import { import { sassBuildTask, tsBuildTask, execNodeTask, copyTask, sequenceTask, triggerLivereload -} from '../task_helpers'; +} from '../util/task_helpers'; // No typings for these. const inlineResources = require('../../../scripts/release/inline-resources'); diff --git a/tools/gulp/tasks/coverage.ts b/tools/gulp/tasks/coverage.ts index ea12470bf172..690ae631d506 100644 --- a/tools/gulp/tasks/coverage.ts +++ b/tools/gulp/tasks/coverage.ts @@ -2,7 +2,8 @@ import {task} from 'gulp'; import {existsSync} from 'fs-extra'; import {COVERAGE_RESULT_FILE} from '../constants'; import {spawnSync} from 'child_process'; -import {isTravisPushBuild, openFirebaseDashboardDatabase} from '../task_helpers'; +import {isTravisPushBuild} from '../util/travis-ci'; +import {openFirebaseDashboardDatabase} from '../util/firebase'; task('coverage:upload', () => { if (!existsSync(COVERAGE_RESULT_FILE)) { diff --git a/tools/gulp/tasks/development.ts b/tools/gulp/tasks/development.ts index 846d4eecd257..c8d318899b0d 100644 --- a/tools/gulp/tasks/development.ts +++ b/tools/gulp/tasks/development.ts @@ -5,7 +5,7 @@ import {DIST_ROOT, SOURCE_ROOT} from '../constants'; import { sassBuildTask, tsBuildTask, copyTask, buildAppTask, vendorTask, serverTask, sequenceTask, triggerLivereload -} from '../task_helpers'; +} from '../util/task_helpers'; const appDir = path.join(SOURCE_ROOT, 'demo-app'); diff --git a/tools/gulp/tasks/e2e.ts b/tools/gulp/tasks/e2e.ts index cf1249fcd9a1..009dc1bbe3a0 100644 --- a/tools/gulp/tasks/e2e.ts +++ b/tools/gulp/tasks/e2e.ts @@ -5,7 +5,7 @@ import {SOURCE_ROOT, DIST_ROOT, PROJECT_ROOT} from '../constants'; import { tsBuildTask, copyTask, buildAppTask, execNodeTask, vendorTask, sequenceTask, serverTask -} from '../task_helpers'; +} from '../util/task_helpers'; const gulpRunSequence = require('run-sequence'); const gulpConnect = require('gulp-connect'); diff --git a/tools/gulp/tasks/lint.ts b/tools/gulp/tasks/lint.ts index 3de5960740f1..fdbf62a16a05 100644 --- a/tools/gulp/tasks/lint.ts +++ b/tools/gulp/tasks/lint.ts @@ -1,5 +1,5 @@ import gulp = require('gulp'); -import {execNodeTask} from '../task_helpers'; +import {execNodeTask} from '../util/task_helpers'; gulp.task('lint', ['tslint', 'stylelint', 'madge']); diff --git a/tools/gulp/tasks/payload.ts b/tools/gulp/tasks/payload.ts index 45ec8bb25238..d7054da3621a 100644 --- a/tools/gulp/tasks/payload.ts +++ b/tools/gulp/tasks/payload.ts @@ -2,8 +2,9 @@ import {task} from 'gulp'; import {join} from 'path'; import {statSync, readFileSync} from 'fs'; import {DIST_COMPONENTS_ROOT} from '../constants'; -import {openFirebaseDashboardDatabase, isTravisPushBuild} from '../task_helpers'; import {spawnSync} from 'child_process'; +import {isTravisPushBuild} from '../util/travis-ci'; +import {openFirebaseDashboardDatabase} from '../util/firebase'; // Those imports lack types. const uglifyJs = require('uglify-js'); diff --git a/tools/gulp/tasks/release.ts b/tools/gulp/tasks/release.ts index 5e982c0ac6fa..e6bcce200de4 100644 --- a/tools/gulp/tasks/release.ts +++ b/tools/gulp/tasks/release.ts @@ -5,7 +5,7 @@ import gulpRunSequence = require('run-sequence'); import path = require('path'); import minimist = require('minimist'); -import {execTask, cleanTask} from '../task_helpers'; +import {execTask, cleanTask} from '../util/task_helpers'; import {DIST_COMPONENTS_ROOT} from '../constants'; const argv = minimist(process.argv.slice(3)); diff --git a/tools/gulp/tasks/screenshots.ts b/tools/gulp/tasks/screenshots.ts index f9cfc0ab0b0d..8ad9b9fbd91a 100644 --- a/tools/gulp/tasks/screenshots.ts +++ b/tools/gulp/tasks/screenshots.ts @@ -2,8 +2,8 @@ import {task} from 'gulp'; import {readdirSync, statSync, existsSync, mkdirp} from 'fs-extra'; import * as path from 'path'; import * as admin from 'firebase-admin'; -import {openScreenshotsBucket, openFirebaseScreenshotsDatabase} from '../task_helpers'; -import {updateGithubStatus} from '../util-functions'; +import {openScreenshotsBucket, openFirebaseScreenshotsDatabase} from '../util/firebase'; +import {setGithubStatus} from '../util/github'; const imageDiff = require('image-diff'); @@ -19,7 +19,7 @@ task('screenshots', () => { return getScreenshotFiles(database) .then((files: any[]) => downloadAllGoldsAndCompare(files, database, prNumber)) .then((results: boolean) => updateResult(database, prNumber, results)) - .then((result: boolean) => updateGithubStatus(result, prNumber)) + .then((result: boolean) => updateGithubStatus(prNumber, result)) .then(() => uploadScreenshots('diff', prNumber)) .then(() => uploadScreenshots('test', prNumber)) .then(() => updateTravis(database, prNumber)) @@ -159,3 +159,13 @@ function setScreenFilenames(database: admin.database.Database, database.ref(FIREBASE_FILELIST); return filelistDatabase.set(filenames); } + +/** Updates the Github Status of the given Pullrequest. */ +function updateGithubStatus(prNumber: number, result: boolean) { + setGithubStatus(process.env['TRAVIS_PULL_REQUEST_SHA'], { + result: result, + name: 'Screenshot Tests', + description: `Screenshot Tests ${result ? 'passed' : 'failed'})`, + url: `http://material2-screenshots.firebaseapp.com/${prNumber}` + }); +} diff --git a/tools/gulp/tasks/serve.ts b/tools/gulp/tasks/serve.ts index be7a1c20bef8..d33123dc7c0b 100644 --- a/tools/gulp/tasks/serve.ts +++ b/tools/gulp/tasks/serve.ts @@ -1,5 +1,5 @@ import {task} from 'gulp'; -import {serverTask} from '../task_helpers'; +import {serverTask} from '../util/task_helpers'; task('serve', serverTask()); diff --git a/tools/gulp/tasks/unit-test.ts b/tools/gulp/tasks/unit-test.ts index d1dfc649dce2..40908baf9301 100644 --- a/tools/gulp/tasks/unit-test.ts +++ b/tools/gulp/tasks/unit-test.ts @@ -3,7 +3,7 @@ import path = require('path'); import gulpMerge = require('merge2'); import {PROJECT_ROOT, COMPONENTS_DIR} from '../constants'; -import {sequenceTask} from '../task_helpers'; +import {sequenceTask} from '../util/task_helpers'; const karma = require('karma'); const runSequence = require('run-sequence'); diff --git a/tools/gulp/util/firebase.ts b/tools/gulp/util/firebase.ts new file mode 100644 index 000000000000..d3ccaabf836f --- /dev/null +++ b/tools/gulp/util/firebase.ts @@ -0,0 +1,61 @@ +const firebaseAdmin = require('firebase-admin'); +const gcloud = require('google-cloud'); + +/** Opens a connection to the firebase realtime database. */ +export function openFirebaseDashboardDatabase() { + // Initialize the Firebase application with admin credentials. + // Credentials need to be for a Service Account, which can be created in the Firebase console. + firebaseAdmin.initializeApp({ + credential: firebaseAdmin.credential.cert({ + project_id: 'material2-dashboard', + client_email: 'firebase-adminsdk-ch1ob@material2-dashboard.iam.gserviceaccount.com', + // In Travis CI the private key will be incorrect because the line-breaks are escaped. + // The line-breaks need to persist in the service account private key. + private_key: (process.env['MATERIAL2_FIREBASE_PRIVATE_KEY'] || '').replace(/\\n/g, '\n') + }), + databaseURL: 'https://material2-dashboard.firebaseio.com' + }); + + return firebaseAdmin.database(); +} + +/** + * Open Google Cloud Storage for screenshots. + * The files uploaded to google cloud are also available to firebase storage. + */ +export function openScreenshotsBucket() { + let gcs = gcloud.storage({ + projectId: 'material2-screenshots', + credentials: { + client_email: 'firebase-adminsdk-t4209@material2-screenshots.iam.gserviceaccount.com', + private_key: decode(process.env['MATERIAL2_SCREENSHOT_FIREBASE_KEY']) + }, + }); + + // Reference the existing appspot bucket. + return gcs.bucket('material2-screenshots.appspot.com'); +} + +/** Opens a connection to the firebase database for screenshots. */ +export function openFirebaseScreenshotsDatabase() { + // Initialize the Firebase application with admin credentials. + // Credentials need to be for a Service Account, which can be created in the Firebase console. + let screenshotApp = firebaseAdmin.initializeApp({ + credential: firebaseAdmin.credential.cert({ + project_id: 'material2-screenshots', + client_email: 'firebase-adminsdk-t4209@material2-screenshots.iam.gserviceaccount.com', + private_key: decode(process.env['MATERIAL2_SCREENSHOT_FIREBASE_KEY']) + }), + databaseURL: 'https://material2-screenshots.firebaseio.com' + }, 'material2-screenshots'); + + return screenshotApp.database(); +} + +/** Decodes a Travis CI variable that is public in favor for PRs. */ +export function decode(str: string): string { + // In Travis CI the private key will be incorrect because the line-breaks are escaped. + // The line-breaks need to persist in the service account private key. + return (str || '').split('\\n').reverse().join('\\n').replace(/\\n/g, '\n'); +} + diff --git a/tools/gulp/util-functions.ts b/tools/gulp/util/github.ts similarity index 53% rename from tools/gulp/util-functions.ts rename to tools/gulp/util/github.ts index 8acc5d4ca3b6..91aa1aa54758 100644 --- a/tools/gulp/util-functions.ts +++ b/tools/gulp/util/github.ts @@ -1,28 +1,37 @@ +import {MATERIAL_VERSION} from '../constants'; + const request = require('request'); -/** Update github pr status to success/failure */ -export function updateGithubStatus(result: boolean, prNumber: string) { - let state = result ? 'success' : 'failure'; - let sha = process.env['TRAVIS_PULL_REQUEST_SHA']; +/** Data that must be specified to set a Github PR status. */ +export type GithubStatusData = { + result: boolean; + name: string; + description: string; + url: string; +}; + +/** Function that sets a Github commit status */ +export function setGithubStatus(commitSHA: number, statusData: GithubStatusData) { + let state = statusData.result ? 'success' : 'failure'; let token = decode(process.env['MATERIAL2_GITHUB_STATUS_TOKEN']); let data = JSON.stringify({ state: state, - target_url: `http://material2-screenshots.firebaseapp.com/${prNumber}`, - context: 'screenshot-diff', - description: `Screenshot test ${state}` + target_url: statusData.url, + context: statusData.name, + description: statusData.description }); let headers = { 'Authorization': `token ${token}`, - 'User-Agent': 'ScreenshotDiff/1.0.0', + 'User-Agent': `${statusData.name}/${MATERIAL_VERSION}`, 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(data) }; return new Promise((resolve) => { request({ - url: `https://api.github.com/repos/angular/material2/statuses/${sha}`, + url: `https://api.github.com/repos/angular/material2/statuses/${commitSHA}`, method: 'POST', form: data, headers: headers diff --git a/tools/gulp/task_helpers.ts b/tools/gulp/util/task_helpers.ts similarity index 66% rename from tools/gulp/task_helpers.ts rename to tools/gulp/util/task_helpers.ts index 3f32afed07d8..effc066048bf 100644 --- a/tools/gulp/task_helpers.ts +++ b/tools/gulp/util/task_helpers.ts @@ -2,7 +2,7 @@ import * as child_process from 'child_process'; import * as fs from 'fs'; import * as gulp from 'gulp'; import * as path from 'path'; -import {NPM_VENDOR_FILES, PROJECT_ROOT, DIST_ROOT, SASS_AUTOPREFIXER_OPTIONS} from './constants'; +import {NPM_VENDOR_FILES, PROJECT_ROOT, DIST_ROOT, SASS_AUTOPREFIXER_OPTIONS} from '../constants'; /** Those imports lack typings. */ @@ -14,8 +14,7 @@ const gulpSourcemaps = require('gulp-sourcemaps'); const gulpAutoprefixer = require('gulp-autoprefixer'); const gulpConnect = require('gulp-connect'); const resolveBin = require('resolve-bin'); -const firebaseAdmin = require('firebase-admin'); -const gcloud = require('google-cloud'); + /** If the string passed in is a glob, returns it, otherwise append '**\/*' to it. */ @@ -185,66 +184,3 @@ export function sequenceTask(...args: any[]) { ); }; } - -/** Opens a connection to the firebase realtime database. */ -export function openFirebaseDashboardDatabase() { - // Initialize the Firebase application with admin credentials. - // Credentials need to be for a Service Account, which can be created in the Firebase console. - firebaseAdmin.initializeApp({ - credential: firebaseAdmin.credential.cert({ - project_id: 'material2-dashboard', - client_email: 'firebase-adminsdk-ch1ob@material2-dashboard.iam.gserviceaccount.com', - // In Travis CI the private key will be incorrect because the line-breaks are escaped. - // The line-breaks need to persist in the service account private key. - private_key: (process.env['MATERIAL2_FIREBASE_PRIVATE_KEY'] || '').replace(/\\n/g, '\n') - }), - databaseURL: 'https://material2-dashboard.firebaseio.com' - }); - - return firebaseAdmin.database(); -} - -/** Whether gulp currently runs inside of Travis as a push. */ -export function isTravisPushBuild() { - return process.env['TRAVIS_PULL_REQUEST'] === 'false'; -} - -/** - * Open Google Cloud Storage for screenshots. - * The files uploaded to google cloud are also available to firebase storage. - */ -export function openScreenshotsBucket() { - let gcs = gcloud.storage({ - projectId: 'material2-screenshots', - credentials: { - client_email: 'firebase-adminsdk-t4209@material2-screenshots.iam.gserviceaccount.com', - private_key: decode(process.env['MATERIAL2_SCREENSHOT_FIREBASE_KEY']) - }, - }); - - // Reference an existing bucket. - return gcs.bucket('material2-screenshots.appspot.com'); -} - -/** Opens a connection to the firebase realtime database for screenshots. */ -export function openFirebaseScreenshotsDatabase() { - // Initialize the Firebase application with admin credentials. - // Credentials need to be for a Service Account, which can be created in the Firebase console. - let screenshotApp = firebaseAdmin.initializeApp({ - credential: firebaseAdmin.credential.cert({ - project_id: 'material2-screenshots', - client_email: 'firebase-adminsdk-t4209@material2-screenshots.iam.gserviceaccount.com', - private_key: decode(process.env['MATERIAL2_SCREENSHOT_FIREBASE_KEY']) - }), - databaseURL: 'https://material2-screenshots.firebaseio.com' - }, 'material2-screenshots'); - - return screenshotApp.database(); -} - -/** Decode the token for Travis to use. */ -function decode(str: string): string { - // In Travis CI the private key will be incorrect because the line-breaks are escaped. - // The line-breaks need to persist in the service account private key. - return (str || '').split('\\n').reverse().join('\\n').replace(/\\n/g, '\n'); -} diff --git a/tools/gulp/util/travis-ci.ts b/tools/gulp/util/travis-ci.ts new file mode 100644 index 000000000000..3d6524ac05a7 --- /dev/null +++ b/tools/gulp/util/travis-ci.ts @@ -0,0 +1,4 @@ +/** Whether gulp currently runs inside of Travis as a push. */ +export function isTravisPushBuild() { + return process.env['TRAVIS_PULL_REQUEST'] === 'false'; +}