Merge pull request #896 from cramforce/integration

Only run integration tests on SauceLabs and run them in more browsers.

.travis.yml

@@ -23,4 +23,4 @@ script:
   - gulp dist
   - gulp presubmit
   - gulp test
-  - gulp test --saucelabs
+  - gulp test --saucelabs --integration

DEVELOPING.md

@@ -76,6 +76,8 @@ export SAUCE_USERNAME=sauce-labs-user-name
 export SAUCE_ACCESS_KEY=access-key
 ```
 
+Also for local testing, download [saucelabs connect](https://docs.saucelabs.com/reference/sauce-connect/) (If you are having trouble, downgrade to 4.3.10) and establish a tunnel by running the `sc` before running tests.
+
 Because of the user name and password requirement pull requests do not directly run on Travis. If your pull request contains JS or CSS changes and it does not change the build system, it will be automatically build by our bot [@ampsauce](https://github.com/ampsauce/amphtml). Builds can be seen on [@ampsauce's Travis](https://travis-ci.org/ampsauce/amphtml/builds) and after they finished their state will be logged to your PR.
 
 If a test flaked on a pull request you can ask for a retry by sending the comment `@ampsauce retry`. This will only be accepted if you are a member of the "ampproject" org. Ping us if you'd like to be added. You may also need to publicly reveal your membership.

build-system/config.js

@@ -18,9 +18,8 @@ var path = require('path');
 
 var karmaConf = path.resolve('karma.conf.js');
 
-var testPaths = [
-  'test/**/*.js',
-  'extensions/**/test/**/*.js',
+var commonTestPaths = [
+  'test/_init_tests.js',
   'test/fixtures/**/*.html',
   {
     pattern: 'dist/**/*.js',
@@ -45,20 +44,28 @@ var testPaths = [
     included: false,
     served: true
   }
-];
+]
+
+var testPaths = commonTestPaths.concat([
+  'test/**/*.js',
+  'extensions/**/test/**/*.js',
+]);
+
+var integrationTestPaths = commonTestPaths.concat([
+  'test/integration/**/*.js',
+  'extensions/**/test/integration/**/*.js',
+]);
 
 var karma = {
   default: {
     configFile: karmaConf,
-    files: testPaths,
     singleRun: true,
     client: {
       captureConsole: false
     }
   },
   firefox: {
     configFile: karmaConf,
-    files: testPaths,
     singleRun: true,
     browsers: ['Firefox'],
     client: {
@@ -70,7 +77,6 @@ var karma = {
   },
   safari: {
     configFile: karmaConf,
-    files: testPaths,
     singleRun: true,
     browsers: ['Safari'],
     client: {
@@ -82,21 +88,34 @@ var karma = {
   },
   saucelabs: {
     configFile: karmaConf,
-    files: testPaths,
     reporters: ['dots', 'saucelabs'],
-    browsers: ['SL_Chrome'],
+    browsers: [
+      'SL_Chrome_android',
+      'SL_Chrome_latest',
+      'SL_Chrome_37',
+      'SL_Firefox_latest',
+      // TODO(#895) Enable these.
+      //'SL_iOS_9_1',
+      //'SL_IE_11',
+      //'SL_Edge_latest',
+    ],
     singleRun: true,
     client: {
-      captureConsole: false
+      mocha: {
+        timeout: 10000
+      },
+      captureConsole: false,
     },
-    browserDisconnectTimeout: 70000,
-    browserNoActivityTimeout: 70000,
+    captureTimeout: 120000,
+    browserDisconnectTimeout: 120000,
+    browserNoActivityTimeout: 120000,
   }
 };
 
 /** @const  */
 module.exports = {
   testPaths: testPaths,
+  integrationTestPaths: integrationTestPaths,
   karma: karma,
   lintGlobs: [
     '**/*.js',

build-system/tasks/test.js

@@ -60,24 +60,35 @@ gulp.task('test', 'Runs tests in chrome', ['build'], function(done) {
     return;
   }
 
-  var config = getConfig();
+  if (!argv.integration && process.env.AMPSAUCE_REPO) {
+    console./*OK*/info('Deactivated for ampsauce repo')
+  }
+
+  var c = getConfig();
   var browsers = [];
 
   if (argv.watch || argv.w) {
-    config.singleRun = false;
+    c.singleRun = false;
   }
 
   if (argv.verbose || argv.v) {
-    config.client.captureConsole = true;
+    c.client.captureConsole = true;
+  }
+
+  if (argv.integration) {
+    c.files = config.integrationTestPaths;
+  } else {
+    c.files = config.testPaths;
   }
 
-  karma.start(config, done);
+  karma.start(c, done);
 }, {
   options: {
     'verbose': '  With logging enabled',
     'watch': '  Watches for changes in files, runs corresponding test(s)',
     'saucelabs': '  Runs test on saucelabs (requires setup)',
     'safari': '  Runs tests in Safari',
-    'firefox': '  Runs tests in Firefox'
+    'firefox': '  Runs tests in Firefox',
+    'integration': 'Run only integration tests.'
   }
 });

karma.conf.js

@@ -62,10 +62,37 @@ module.exports = function(config) {
         base: 'Chrome',
         flags: ['--no-sandbox'],
       },
-      SL_Chrome: {
+      SL_Chrome_android: {
+        base: 'SauceLabs',
+        browserName: 'android',
+      },
+      SL_Chrome_latest: {
         base: 'SauceLabs',
         browserName: 'chrome',
       },
+      SL_Chrome_37: {
+        base: 'SauceLabs',
+        browserName: 'chrome',
+        version: 37,
+      },
+      SL_iOS_9_1: {
+        base: 'SauceLabs',
+        browserName: 'iphone',
+        version: '9.1'
+      },
+      SL_Firefox_latest: {
+        base: 'SauceLabs',
+        browserName: 'firefox',
+      },
+      SL_IE_11: {
+        base: 'SauceLabs',
+        browserName: 'internet explorer',
+        version: 11,
+      },
+      SL_Edge_latest: {
+        base: 'SauceLabs',
+        browserName: 'microsoftedge',
+      },
     },
 
     sauceLabs: {