feat(screenshot): Add a gulp to compare screenshot diffs, upload results to gcs & firebase

package.json

@@ -41,6 +41,7 @@
     "@angular/platform-browser-dynamic": "^2.3.0",
     "@angular/platform-server": "^2.3.0",
     "@angular/router": "^3.3.0",
+    "@types/fs-extra": "0.0.37",
     "@types/glob": "^5.0.30",
     "@types/gulp": "^3.8.32",
     "@types/hammerjs": "^2.0.34",
@@ -59,6 +60,7 @@
     "firebase-tools": "^2.2.1",
     "fs-extra": "^2.0.0",
     "glob": "^7.1.1",
+    "google-cloud": "^0.45.1",
     "gulp": "^3.9.1",
     "gulp-autoprefixer": "^3.1.1",
     "gulp-better-rollup": "^1.0.2",
@@ -77,6 +79,7 @@
     "gulp-transform": "^1.1.0",
     "hammerjs": "^2.0.8",
     "highlight.js": "^9.9.0",
+    "image-diff": "^1.6.3",
     "jasmine-core": "^2.5.2",
     "karma": "^1.4.1",
     "karma-browserstack-launcher": "^1.2.0",

tools/gulp/gulpfile.ts

@@ -7,6 +7,7 @@ import './tasks/docs';
 import './tasks/e2e';
 import './tasks/lint';
 import './tasks/release';
+import './tasks/screenshots';
 import './tasks/serve';
 import './tasks/unit-test';
 import './tasks/docs';

tools/gulp/task_helpers.ts

@@ -2,7 +2,6 @@ import * as child_process from 'child_process';
 import * as fs from 'fs';
 import * as gulp from 'gulp';
 import * as path from 'path';
-
 import {NPM_VENDOR_FILES, PROJECT_ROOT, DIST_ROOT, SASS_AUTOPREFIXER_OPTIONS} from './constants';
 
 
@@ -16,6 +15,7 @@ const gulpAutoprefixer = require('gulp-autoprefixer');
 const gulpConnect = require('gulp-connect');
 const resolveBin = require('resolve-bin');
 const firebaseAdmin = require('firebase-admin');
+const gcloud = require('google-cloud');
 
 
 /** If the string passed in is a glob, returns it, otherwise append '**\/*' to it. */
@@ -186,7 +186,7 @@ export function sequenceTask(...args: any[]) {
 }
 
 /** Opens a connection to the firebase realtime database. */
-export function openFirebaseDatabase() {
+export function openFirebaseDashboardDatabase() {
   // Initialize the Firebase application with admin credentials.
   // Credentials need to be for a Service Account, which can be created in the Firebase console.
   firebaseAdmin.initializeApp({
@@ -207,3 +207,43 @@ export function openFirebaseDatabase() {
 export function isTravisPushBuild() {
   return process.env['TRAVIS_PULL_REQUEST'] === 'false';
 }
+
+/**
+ * Open Google Cloud Storage for screenshots.
+ * The files uploaded to google cloud are also available to firebase storage.
+ */
+export function openScreenshotsBucket() {
+  let gcs = gcloud.storage({
+    projectId: 'material2-screenshots',
+    credentials: {
+      client_email: 'firebase-adminsdk-t4209@material2-screenshots.iam.gserviceaccount.com',
+      private_key: decode(process.env['MATERIAL2_SCREENSHOT_FIREBASE_KEY'])
+    },
+  });
+
+  // Reference an existing bucket.
+  return gcs.bucket('material2-screenshots.appspot.com');
+}
+
+/** Opens a connection to the firebase realtime database for screenshots. */
+export function openFirebaseScreenshotsDatabase() {
+  // Initialize the Firebase application with admin credentials.
+  // Credentials need to be for a Service Account, which can be created in the Firebase console.
+  let screenshotApp = firebaseAdmin.initializeApp({
+    credential: firebaseAdmin.credential.cert({
+      project_id: 'material2-screenshots',
+      client_email: 'firebase-adminsdk-t4209@material2-screenshots.iam.gserviceaccount.com',
+      private_key: decode(process.env['MATERIAL2_SCREENSHOT_FIREBASE_KEY'])
+    }),
+    databaseURL: 'https://material2-screenshots.firebaseio.com'
+  }, 'material2-screenshots');
+
+  return screenshotApp.database();
+}
+
+/** Decode the token for Travis to use. */
+function decode(str: string): string {
+  // In Travis CI the private key will be incorrect because the line-breaks are escaped.
+  // The line-breaks need to persist in the service account private key.
+  return (str || '').split('\\n').reverse().join('\\n').replace(/\\n/g, '\n');
+}

tools/gulp/tasks/e2e.ts

@@ -64,6 +64,7 @@ task('e2e', (done: (err?: string) => void) => {
     'serve:e2eapp',
     ':test:protractor',
     ':serve:e2eapp:stop',
+    'screenshots',
     (err: any) => done(err)
   );
 });

tools/gulp/tasks/payload.ts

@@ -2,7 +2,7 @@ import {task} from 'gulp';
 import {join} from 'path';
 import {statSync, readFileSync} from 'fs';
 import {DIST_COMPONENTS_ROOT} from '../constants';
-import {openFirebaseDatabase, isTravisPushBuild} from '../task_helpers';
+import {openFirebaseDashboardDatabase, isTravisPushBuild} from '../task_helpers';
 import {spawnSync} from 'child_process';
 
 // Those imports lack types.
@@ -48,7 +48,7 @@ function getUglifiedSize(filePath: string) {
 /** Publishes the given results to the firebase database. */
 function publishResults(results: any) {
   let latestSha = spawnSync('git', ['rev-parse', 'HEAD']).stdout.toString().trim();
-  let database = openFirebaseDatabase();
+  let database = openFirebaseDashboardDatabase();
 
   // Write the results to the payloads object with the latest Git SHA as key.
   return database.ref('payloads').child(latestSha).set(results)

tools/gulp/tasks/screenshots.ts

@@ -0,0 +1,153 @@
+import {task} from 'gulp';
+import {readdirSync, statSync, existsSync, mkdirp} from 'fs-extra';
+import * as path from 'path';
+import * as admin from 'firebase-admin';
+import {openScreenshotsBucket, openFirebaseScreenshotsDatabase} from '../task_helpers';
+import {updateGithubStatus} from '../util-functions';
+
+const request = require('request');
+const imageDiff = require('image-diff');
+
+const SCREENSHOT_DIR = './screenshots';
+const FIREBASE_REPORT = 'screenshot/reports';
+const FIREBASE_FILELIST = 'screenshot/filenames';
+
+/** Task which upload screenshots generated from e2e test. */
+task('screenshots', () => {
+  let prNumber = process.env['TRAVIS_PULL_REQUEST'];
+  if (prNumber) {
+    let database = openFirebaseScreenshotsDatabase();
+    return getScreenshotFiles(database)
+      .then((files: any[]) => downloadAllGoldsAndCompare(files, database, prNumber))
+      .then((results: boolean) => updateResult(database, prNumber, results))
+      .then((result: boolean) => updateGithubStatus(result, prNumber))
+      .then(() => uploadScreenshots(prNumber, 'diff'))
+      .then(() => uploadScreenshots(prNumber, 'test'))
+      .then(() => updateTravis(database, prNumber))
+      .then(() => setScreenFilenames(database, prNumber))
+      .then(() => database.goOffline(), () => database.goOffline());
+  }
+});
+
+function updateFileResult(database: admin.database.Database, prNumber: string,
+                          filenameKey: string, result: boolean) {
+  return database.ref(FIREBASE_REPORT).child(prNumber).child('results').child(filenameKey).set(result);
+}
+
+function updateResult(database: admin.database.Database, prNumber: string,
+                      result: boolean) {
+  return database.ref(FIREBASE_REPORT).child(prNumber).child('result').set(result).then(() => result);
+}
+
+function updateTravis(database: admin.database.Database,
+                      prNumber: string) {
+  return database.ref(FIREBASE_REPORT).child(prNumber).update({
+    commit: process.env['TRAVIS_COMMIT'],
+    sha: process.env['TRAVIS_PULL_REQUEST_SHA'],
+    travis: process.env['TRAVIS_JOB_ID'],
+  });
+}
+
+/** Get a list of filenames from firebase database. */
+function getScreenshotFiles(database: admin.database.Database) {
+  let bucket = openScreenshotsBucket();
+  return bucket.getFiles({ prefix: 'golds/' }).then(function(data: any) {
+    return data[0].filter((file:any) => file.name.endsWith('.screenshot.png'));
+  });
+}
+
+function extractScreenshotName(fileName: string) {
+  return path.basename(fileName, '.screenshot.png');
+}
+
+function getLocalScreenshotFiles(dir: string): string[] {
+  return readdirSync(dir)
+    .filter((fileName: string) => !statSync(path.join(SCREENSHOT_DIR, fileName)).isDirectory())
+    .filter((fileName: string) => fileName.endsWith('.screenshot.png'));
+}
+
+/**
+ * Upload screenshots to google cloud storage.
+ * @param prNumber - The key used in firebase. Here it is the PR number.
+ *   If there's no prNumber, we will upload images to 'golds/' folder
+ * @param mode - Can be 'test' or 'diff' or null.
+ *   If the images are the test results, mode should be 'test'.
+ *   If the images are the diff images generated, mode should be 'diff'.
+ *   For golds mode should be null.
+ */
+function uploadScreenshots(prNumber?: string, mode?: 'test' | 'diff') {
+  let bucket = openScreenshotsBucket();
+
+  let promises: any[] = [];
+  let localDir = mode == 'diff' ? path.join(SCREENSHOT_DIR, 'diff') : SCREENSHOT_DIR;
+  getLocalScreenshotFiles(localDir).forEach((file: string) => {
+    let fileName = path.join(localDir, file);
+    let destination = (mode == null || !prNumber) ?
+      `golds/${file}` : `screenshots/${prNumber}/${mode}/${file}`;
+    promises.push(bucket.upload(fileName, { destination: destination }));
+  });
+  return Promise.all(promises);
+}
+
+/** Download golds screenshots. */
+function downloadAllGoldsAndCompare(
+  files: any[], database: admin.database.Database,
+  prNumber: string) {
+
+  mkdirp(path.join(SCREENSHOT_DIR, `golds`));
+  mkdirp(path.join(SCREENSHOT_DIR, `diff`));
+
+  return Promise.all(files.map((file: any) => {
+    return downloadGold(file).then(() => diffScreenshot(file.name, database, prNumber));
+  })).then((results: boolean[]) => results.every((value: boolean) => value == true));
+}
+
+/** Download one gold screenshot */
+function downloadGold(file: any) {
+  return file.download({
+    destination: path.join(SCREENSHOT_DIR, file.name)
+  });
+}
+
+function diffScreenshot(filename: string, database: admin.database.Database,
+                        prNumber: string) {
+  // TODO(tinayuangao): Run the downloads and diffs in parallel.
+  filename = path.basename(filename);
+  let goldUrl = path.join(SCREENSHOT_DIR, `golds`, filename);
+  let pullRequestUrl = path.join(SCREENSHOT_DIR, filename);
+  let diffUrl = path.join(SCREENSHOT_DIR, `diff`, filename);
+  let filenameKey = extractScreenshotName(filename);
+
+  if (existsSync(goldUrl) && existsSync(pullRequestUrl)) {
+    return new Promise((resolve: any, reject: any) => {
+      imageDiff({
+        actualImage: pullRequestUrl,
+        expectedImage: goldUrl,
+        diffImage: diffUrl,
+      }, (err: any, imagesAreSame: boolean) => {
+        if (err) {
+          console.log(err);
+          imagesAreSame = false;
+          reject(err);
+        }
+        resolve(imagesAreSame);
+        return updateFileResult(database, prNumber, filenameKey, imagesAreSame);
+      });
+    });
+  } else {
+    return updateFileResult(database, prNumber, filenameKey, false).then(() => false);
+  }
+}
+
+/**
+ * Upload a list of filenames to firebase database as gold.
+ * This is necessary for control panel since google-cloud is not available to client side.
+ */
+function setScreenFilenames(database: admin.database.Database,
+                            prNumber?: string) {
+  let filenames: string[] = getLocalScreenshotFiles(SCREENSHOT_DIR);
+  let filelistDatabase = prNumber ?
+    database.ref(FIREBASE_REPORT).child(prNumber).child('filenames') :
+    database.ref(FIREBASE_FILELIST);
+  return filelistDatabase.set(filenames);
+}

tools/gulp/util-functions.ts

@@ -0,0 +1,37 @@
+const request = require('request');
+
+/** Update github pr status to success/failure */
+export function updateGithubStatus(result: boolean, prNumber: string) {
+  let state = result ? 'success' : 'failure';
+  let sha = process.env['TRAVIS_PULL_REQUEST_SHA'];
+  let token = decode(process.env['MATERIAL2_GITHUB_STATUS_TOKEN']);
+
+  let data = JSON.stringify({
+    state: state,
+    target_url: `http://material2-screenshots.firebaseapp.com/${prNumber}`,
+    context: "screenshot-diff",
+    description: `Screenshot test ${state}`
+  });
+
+  let headers =  {
+    'Authorization': `token ${token}`,
+    'User-Agent': 'ScreenshotDiff/1.0.0',
+    'Content-Type': 'application/json',
+    'Content-Length': Buffer.byteLength(data)
+  };
+
+  return new Promise((resolve, reject) => {
+    request({
+      url: `https://api.github.com/repos/angular/material2/statuses/${sha}`,
+      method: 'POST',
+      form: data,
+      headers: headers
+    }, function (error: any, response: any, body: any){
+      resolve(response.statusCode);
+    });
+  });
+}
+
+function decode(value: string): string {
+  return value.split('').reverse().join('');
+}