Skip to content

Commit

Permalink
address comments
Browse files Browse the repository at this point in the history
  • Loading branch information
@tinayuangao
tinayuangao committed Mar 17, 2017
1 parent 5085489 commit fb89d44
Show file tree
Hide file tree
Showing 3 changed files with 134 additions and 85 deletions.
205 changes: 124 additions & 81 deletions functions/index.js
View file
Original file line number Diff line number Diff line change
@@ -1,94 +1,127 @@
'use strict';

const functions = require('firebase-functions');
const firebaseFunctions = require('firebase-functions');
const firebaseAdmin = require('firebase-admin');
const gcs = require('@google-cloud/storage')();
const admin = require('firebase-admin');
const jwt = require('jsonwebtoken');
const fs = require('fs');

admin.initializeApp(functions.config().firebase);
/**
* Data and images handling for Screenshot test
*
* For valid data posted to database /temp/screenshot/reports/$prNumber/$secureToken, move it to
* /screenshot/reports/$prNumber.
* These are data for screenshot results (success or failure), GitHub PR/commit and TravisCI job information
*
* For valid image datas written to database /temp/screenshot/images/$prNumber/$secureToken/, save the image
* data to image files and upload to google cloud storage under location /screenshots/$prNumber
* These are screenshot test result images, and difference images generated from screenshot comparison.
*
* For golden images uploaded to /goldens, read the data from images files and write the data to Firebase database
* under location /screenshot/goldens
* Screenshot tests can only read restricted database data with no credentials, and they cannot access
* Google Cloud Storage. Therefore we copy the image data to database to make it available to screenshot tests.
*
* The JWT is stored in the data path, so every write to database needs a valid JWT to be copied to database/storage.
* All invalid data will be removed.
* The JWT has 3 parts: header, payload and signature. These three parts are joint by '/' in path.
*/

// Initailize the admin app
firebaseAdmin.initializeApp(firebaseFunctions.config().firebase);

/** The valid data types database accepts */
const dataTypes = ['filenames', 'commit', 'result', 'sha', 'travis'];
const repoSlug = functions.config().repo.slug;
const secret = functions.config().secret.key;
const bucket = gcs.bucket(functions.config().firebase.storageBucket);

/** The repo slug. This is used to validate the JWT is sent from correct repo. */
const repoSlug = firebaseFunctions.config().repo.slug;

/** The JWT secret. This is used to validate JWT. */
const secret = firebaseFunctions.config().secret.key;

/** The storage bucket to store the images. The bucket is also used by Firebase Storage. */
const bucket = gcs.bucket(firebaseFunctions.config().firebase.storageBucket);

/** The Json Web Token format. The token is stored in data path. */
const jwtFormat = '{jwtHeader}/{jwtPayload}/{jwtSignature}';

/** The temporary folder name */
const tempFolder = '/temp';

/** Copy valid data from /temp/screenshot/reports/$prNumber/$secureToken/ to /screenshot/reports/$prNumber */
exports.copyData = functions.database.ref('/temp/screenshot/reports/{prNumber}/{token1}/{token2}/{token3}/{dataType}')
.onWrite(event => {
const copyDataPath = `${tempFolder}/screenshot/reports/{prNumber}/${jwtFormat}/{dataType}`;
exports.copyData = firebaseFunctions.database.ref(copyDataPath).onWrite(event => {
const dataType = event.params.dataType;
if (dataTypes.indexOf(dataType) == -1) {
return;
if (dataTypes.includes(dataType)) {
return handleDataChange(event, dataType);
}
return handleDataChange(event, dataType);
return;
});

/** Copy valid data from /temp/screenshot/reports/$prNumber/$secureToken/ to /screenshot/reports/$prNumber */
exports.copyDataResult = functions.database.ref('/temp/screenshot/reports/{prNumber}/{token1}/{token2}/{token3}/results/{filename}')
.onWrite(event => {
const copyDataResultPath = `${tempFolder}/screenshot/reports/{prNumber}/${jwtFormat}/results/{filename}`;
exports.copyDataResult = firebaseFunctions.database.ref(copyDataResultPath).onWrite(event => {
return handleDataChange(event, `results/${event.params.filename}`);
});

/** Copy valid data from database /temp/screenshot/images/$prNumber/$secureToken/ to storage /screenshots/$prNumber */
exports.copyImage = functions.database.ref('/temp/screenshot/images/{prNumber}/{token1}/{token2}/{token3}/{dataType}/{filename}')
.onWrite(event => {
// Only edit data when it is first created. Exit when the data is deleted.
if (event.data.previous.exists() || !event.data.exists()) {
return;
}

const dataType = event.params.dataType;
const prNumber = event.params.prNumber;
const secureToken = `${event.params.token1}.${event.params.token2}.${event.params.token3}`;
const saveFilename = `${event.params.filename}.screenshot.png`;

if (dataType != 'diff' && dataType != 'test') {
return;
}

return validateSecureToken(secureToken, prNumber).then((payload) => {
const tempPath = `/tmp/${dataType}-${saveFilename}`
const filePath = `screenshots/${prNumber}/${dataType}/${saveFilename}`;
const binaryData = new Buffer(event.data.val(), 'base64').toString('binary');
fs.writeFile(tempPath, binaryData, 'binary');
return bucket.upload(tempPath, {
destination: filePath
}).then(() => {
return event.data.ref.parent.set(null);
});
}).catch((error) => {
console.error(`Invalid secure token ${secureToken} ${error}`);
const copyImagePath = `${tempFolder}/screenshot/images/{prNumber}/${jwtFormat}/{dataType}/{filename}`;
exports.copyImage = firebaseFunctions.database.ref(copyImagePath).onWrite(event => {
// Only edit data when it is first created. Exit when the data is deleted.
if (event.data.previous.exists() || !event.data.exists()) {
return;
}

const dataType = event.params.dataType;
const prNumber = event.params.prNumber;
const secureToken = getSecureToken(event);
const saveFilename = `${event.params.filename}.screenshot.png`;

if (dataType != 'diff' && dataType != 'test') {
return;
}

return validateSecureToken(secureToken, prNumber).then((payload) => {
const tempPath = `/tmp/${dataType}-${saveFilename}`
const filePath = `screenshots/${prNumber}/${dataType}/${saveFilename}`;
const binaryData = new Buffer(event.data.val(), 'base64').toString('binary');
fs.writeFile(tempPath, binaryData, 'binary');
return bucket.upload(tempPath, {destination: filePath}).then(() => {
return event.data.ref.parent.set(null);
});
}).catch((error) => {
console.error(`Invalid secure token ${secureToken} ${error}`);
return event.data.ref.parent.set(null);
});
});

/**
* Copy valid goldens from storage /goldens/ to database /screenshot/goldens/
* so we can read the goldens without credentials
* so we can read the goldens without credentials.
*/
exports.copyGoldens = functions.storage.bucket(functions.config().firebase.storageBucket).object().onChange(event => {
const filePath = event.data.name;

// Get the file name.
const fileNames = filePath.split('/');
if (fileNames.length != 2 && fileNames[0] != 'goldens') {
return;
}
const filenameKey = fileNames[1].replace('.screenshot.png', '');

if (event.data.resourceState === 'not_exists') {
return admin.database().ref(`screenshot/goldens/${filenameKey}`).set(null);
}

// Download file from bucket.
const bucket = gcs.bucket(event.data.bucket);
const tempFilePath = `/tmp/${fileNames[1]}`;
return bucket.file(filePath).download({
destination: tempFilePath
}).then(() => {
const data = fs.readFileSync(tempFilePath);
return admin.database().ref(`screenshot/goldens/${filenameKey}`).set(data);
});
exports.copyGoldens = firebaseFunctions.storage.bucket(firebaseFunctions.config().firebase.storageBucket)
.object().onChange(event => {
const filePath = event.data.name;

// Get the file name.
const fileNames = filePath.split('/');
if (fileNames.length != 2 && fileNames[0] != 'goldens') {
return;
}
const filenameKey = fileNames[1].replace('.screenshot.png', '');

// When delete a file, remove the file in database
if (event.data.resourceState === 'not_exists') {
return firebaseAdmin.database().ref(`screenshot/goldens/${filenameKey}`).set(null);
}

// Download file from bucket.
const bucket = gcs.bucket(event.data.bucket);
const tempFilePath = `/tmp/${fileNames[1]}`;
return bucket.file(filePath).download({destination: tempFilePath}).then(() => {
const data = fs.readFileSync(tempFilePath);
return firebaseAdmin.database().ref(`screenshot/goldens/${filenameKey}`).set(data);
});
});

function handleDataChange(event, path) {
Expand All @@ -98,31 +131,41 @@ function handleDataChange(event, path) {
}

const prNumber = event.params.prNumber;
const secureToken = `${event.params.token1}.${event.params.token2}.${event.params.token3}`;
const secureToken = getSecureToken(event);
const original = event.data.val();

return validateSecureToken(secureToken, prNumber).then((payload) => {
return admin.database().ref().child('screenshot/reports').child(prNumber).child(path).set(original).then(() => {
return event.data.ref.parent.set(null);
});
return firebaseAdmin.database().ref().child('screenshot/reports')
.child(prNumber).child(path).set(original).then(() => {
return event.data.ref.parent.set(null);
});
}).catch((error) => {
console.error(`Invalid secure token ${secureToken} ${error}`);
console.error(`Invalid secure token ${secureToken} ${error}`);
return event.data.ref.parent.set(null);
});
}

/**
* Extract the Json Web Token from event params.
* In screenshot gulp task the path we use is {jwtHeader}/{jwtPayload}/{jwtSignature}.
* Replace '/' with '.' to get the token.
*/
function getSecureToken(event) {
return `${event.params.jwtHeader}.${event.params.jwtPayload}.${event.params.jwtSignature}`;
}

function validateSecureToken(token, prNumber) {
return new Promise((resolve, reject) => {
jwt.verify(token, secret, {issuer: 'Travis CI, GmbH'}, (err, payload) => {
if (err) {
reject(err.message || err);
} else if (payload.slug !== repoSlug) {
reject(`jwt slug invalid. expected: ${repoSlug}`);
} else if (payload['pull-request'].toString() !== prNumber) {
reject(`jwt pull-request invalid. expected: ${prNumber} actual: ${payload['pull-request']}`);
} else {
resolve(payload);
}
});
jwt.verify(token, secret, {issuer: 'Travis CI, GmbH'}, (err, payload) => {
if (err) {
reject(err.message || err);
} else if (payload.slug !== repoSlug) {
reject(`jwt slug invalid. expected: ${repoSlug}`);
} else if (payload['pull-request'].toString() !== prNumber) {
reject(`jwt pull-request invalid. expected: ${prNumber} actual: ${payload['pull-request']}`);
} else {
resolve(payload);
}
});
});
}
10 changes: 8 additions & 2 deletions tools/gulp/tasks/screenshots.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,8 +95,14 @@ function getLocalScreenshotFiles(dir: string): string[] {
}

/**
* Get processed secure token. The jwt token is connected by '.', while '.' is not a valid
* path in firebase database. Replace all '.' to '/' to make the path valid
* Get processed secure token. The jwt token has 3 parts: header, payload, signature and has format
* {jwtHeader}.{jwtPayload}.{jwtSignature}
* The three parts is connected by '.', while '.' is not a valid path in firebase database.
* Replace all '.' to '/' to make the path valid
* Output is {jwtHeader}/{jwtPayload}/{jwtSignature}.
* This secure token is used to validate the write access is from our TravisCI under our repo.
* All data is written to /$path/$secureToken/$data and after validated the
* secure token, the data is moved to /$path/$data in database.
*/
function getSecureToken() {
return process.env['FIREBASE_ACCESS_TOKEN'].replace(/[.]/g, '/');
Expand Down
4 changes: 2 additions & 2 deletions tools/gulp/util/firebase.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ const config = require('../../../functions/config.json');

/** Opens a connection to the firebase realtime database. */
export function openFirebaseDashboardDatabase() {
// Initialize the Firebase application with admin credentials.
// Initialize the Firebase application with firebaseAdmin credentials.
// Credentials need to be for a Service Account, which can be created in the Firebase console.
firebaseAdmin.initializeApp({
credential: firebaseAdmin.credential.cert({
Expand Down Expand Up @@ -41,7 +41,7 @@ export function openScreenshotsBucket() {

/** Opens a connection to the firebase database for screenshots. */
export function openFirebaseScreenshotsDatabase() {
// Initialize the Firebase application with admin credentials.
// Initialize the Firebase application with firebaseAdmin credentials.
// Credentials need to be for a Service Account, which can be created in the Firebase console.
let screenshotApp = firebaseAdmin.initializeApp({
credential: firebaseAdmin.credential.cert({
Expand Down

0 comments on commit fb89d44

Please sign in to comment.