Fix: don't auto import experimental bento components

packages/optimizer/lib/fetchRuntimeParameters.js

@@ -15,6 +15,7 @@
  */
 'mode strict';
 
+const URL_BENTO_COMPONENT_INFO = 'https://amp.dev/static/bento-components.json';
 const validatorRulesProvider = require('@ampproject/toolbox-validator-rules');
 const {MaxAge} = require('@ampproject/toolbox-core');
 let fallbackRuntime;
@@ -35,6 +36,7 @@ const {
 
 const KEY_VALIDATOR_RULES = 'validator-rules';
 const AMP_RUNTIME_MAX_AGE = 10 * 60; // 10 min
+let cacheErrorLogged = false;
 
 /**
  * Initializes the runtime parameters used by the transformers based on given config and parameter values.
@@ -85,22 +87,48 @@ async function initValidatorRules(runtimeParameters, customRuntimeParameters, co
     config.log.error('Could not fetch validator rules');
     config.log.verbose(error);
   }
+  try {
+    runtimeParameters.bentoComponentInfo =
+      customRuntimeParameters.bentoComponentInfo ||
+      config.bentoComponentInfo ||
+      (await fetchBentoComponentInfoFromCache_(config));
+  } catch (error) {
+    config.log.error('Could not fetch bento component info');
+    config.log.verbose(error);
+    runtimeParameters.bentoComponentInfo = [];
+  }
+}
+async function fetchBentoComponentInfoFromCache_(config) {
+  let bentoComponentInfo = await readFromCache_(config, 'bento-component-info');
+  if (!bentoComponentInfo) {
+    bentoComponentInfo = await fetchBentoComponentInfo_(config);
+    writeToCache_(config, 'bento-component-info', bentoComponentInfo);
+  }
+  return bentoComponentInfo;
+}
+
+async function fetchBentoComponentInfo_(config) {
+  const response = await config.fetch(URL_BENTO_COMPONENT_INFO);
+  if (!response.ok) {
+    config.log.error(
+      `Failed fetching bento component info from ${URL_BENTO_COMPONENT_INFO} with status: ${response.status}`
+    );
+    return [];
+  }
+  return response.json();
 }
 
 /**
  * @private
  */
 async function fetchValidatorRulesFromCache_(config) {
-  if (config.cache === false) {
-    return fetchValidatorRules_(config);
-  }
-  let rawRules = await config.cache.get('validator-rules');
+  let rawRules = await readFromCache_(config, 'validator-rules');
   let validatorRules;
   if (!rawRules) {
     validatorRules = await fetchValidatorRules_(config);
     config.log.debug('Downloaded AMP validation rules');
     // We save the raw rules to make the validation rules JSON serializable
-    config.cache.set(KEY_VALIDATOR_RULES, validatorRules.raw);
+    writeToCache_(config, KEY_VALIDATOR_RULES, validatorRules.raw);
   } else {
     validatorRules = await validatorRulesProvider.fetch({rules: rawRules});
   }
@@ -161,11 +189,8 @@ async function initRuntimeVersion(runtimeParameters, customRuntimeParameters, co
  * @private
  */
 async function fetchAmpRuntimeVersion_(context) {
-  if (context.config.cache === false) {
-    return (await fetchLatestRuntimeData_(context)).version;
-  }
   const versionKey = `version-${context.ampUrlPrefix}-${context.lts}`;
-  let ampRuntimeData = await context.config.cache.get(versionKey);
+  let ampRuntimeData = await readFromCache_(context.config, versionKey);
   if (!ampRuntimeData) {
     ampRuntimeData = await fetchLatestRuntimeData_(context, versionKey);
     context.config.log.debug('Downloaded AMP runtime v' + ampRuntimeData.version);
@@ -200,7 +225,7 @@ async function fetchLatestRuntimeData_({config, ampUrlPrefix, lts}, versionKey =
     // Fallback to built-in runtime version
     ampRuntimeData.version = fallbackRuntime.ampRuntimeVersion;
   } else if (ampRuntimeData.version && versionKey) {
-    config.cache.set(versionKey, ampRuntimeData);
+    writeToCache_(config, versionKey, ampRuntimeData);
   }
   return ampRuntimeData;
 }
@@ -244,7 +269,7 @@ async function fetchAmpRuntimeStyles_(config, ampUrlPrefix, ampRuntimeVersion) {
 async function downloadAmpRuntimeStyles_(config, runtimeCssUrl) {
   let styles;
   if (config.cache !== false) {
-    styles = await config.cache.get(runtimeCssUrl);
+    styles = await readFromCache_(config, runtimeCssUrl);
   }
   if (!styles) {
     const response = await config.fetch(runtimeCssUrl);
@@ -259,7 +284,7 @@ async function downloadAmpRuntimeStyles_(config, runtimeCssUrl) {
     }
     config.log.debug(`Downloaded AMP runtime styles from ${runtimeCssUrl}`);
     if (config.cache !== false) {
-      config.cache.set(runtimeCssUrl, styles);
+      writeToCache_(config, runtimeCssUrl, styles);
     }
   }
   return styles;
@@ -277,4 +302,37 @@ function isAbsoluteUrl_(url) {
   }
 }
 
+/**
+ * @private
+ */
+function readFromCache_(config, key) {
+  if (config.cache === false) {
+    return null;
+  }
+  try {
+    return config.cache.get(key);
+  } catch (e) {
+    if (!cacheErrorLogged) {
+      config.log.warn('Could not read from cache', e);
+      cacheErrorLogged = true;
+    }
+  }
+}
+
+/**
+ * @private
+ */
+function writeToCache_(config, key, value) {
+  if (config.cache === false) {
+    return;
+  }
+  try {
+    config.cache.set(key, value);
+  } catch (e) {
+    if (!cacheErrorLogged) {
+      config.log.warn('Could not write to cache', e);
+      cacheErrorLogged = true;
+    }
+  }
+}
 module.exports = fetchRuntimeParameters;

packages/optimizer/lib/transformers/AutoExtensionImporter.js

@@ -141,6 +141,12 @@ class AutoExtensionImporter {
       this.log_.error('Missing validation rules, cannot auto import extensions');
       return;
     }
+    if (!this.bentoComponentInfo) {
+      this.bentoComponentInfo = {};
+      for (const bentoComponent of params.bentoComponentInfo || []) {
+        this.bentoComponentInfo[bentoComponent.name] = bentoComponent;
+      }
+    }
     if (!this.extensionSpec_) {
       this.extensionSpec_ = this.createExtensionsSpec(params);
     }
@@ -187,13 +193,7 @@ class AutoExtensionImporter {
       const extension = this.extensionSpec_.extensionsMap.get(extensionName.trim());
       this.log_.debug('auto importing', extensionName);
       // Use the latest version by default
-      let version = extension.version[extension.version.length - 1];
-      const customVersion = this.extensionVersions[extensionName];
-      // Let user override default
-      if (customVersion) {
-        this.log_.debug('using custom version for', extensionName, customVersion);
-        version = customVersion;
-      }
+      let version = this.calculateVersion(extension, extensionName);
       const extensionImportAttribs = {
         async: '',
         src: `${host}/v0/${extensionName}-${version}.js`,
@@ -205,6 +205,33 @@ class AutoExtensionImporter {
     }
   }
 
+  calculateVersion(extension, extensionName) {
+    const customVersion = this.extensionVersions[extensionName];
+    // Let user override default
+    if (customVersion) {
+      this.log_.debug('using custom version for', extensionName, customVersion);
+      return customVersion;
+    }
+    // Get latest version as specified by validation rules
+    let version = extension.version[extension.version.length - 1];
+    // Get bento component info as these might still be experimental
+    const bentoComponent = this.bentoComponentInfo[extensionName];
+    if (!bentoComponent) {
+      // No bento component
+      return version;
+    }
+    // Bento component is not experimental, nothing to worry about
+    if (!bentoComponent.experimental) {
+      return version;
+    }
+    // Bento component version is not yet known to the validator
+    if (version != bentoComponent.version) {
+      return version;
+    }
+    // Bento component version is not yet valid, pick the previous one
+    return extension.version[extension.version.length - 2] || '0.1';
+  }
+
   /**
    * @private
    */

packages/optimizer/spec/helpers/TransformerRunner.js

@@ -19,6 +19,7 @@ const {writeFileContents, getFileContents, getDirectories} = require('../helpers
 
 const jsBeautify = require('js-beautify/js/lib/beautify-html.js');
 const validatorRules = require('@ampproject/toolbox-validator-rules').fetch();
+const bentoComponentInfo = require('./bentoComponentInfo.json');
 
 const BEAUTIFY_OPTIONS = {
   'indent_size': 2,
@@ -29,6 +30,7 @@ const BEAUTIFY_OPTIONS = {
 };
 
 const treeParser = require('../../lib/TreeParser.js');
+const {parseDOM} = require('htmlparser2');
 
 const TRANSFORMER_PARAMS = {
   //verbose: true,
@@ -84,6 +86,7 @@ module.exports = function (testConfig) {
         }
         params = testConfig.validAmp ? {} : params;
         params.validatorRules = await validatorRules;
+        params.bentoComponentInfo = bentoComponentInfo;
         await testConfig.transformer.transform(tree, params);
         const actualOutput = serialize(tree, params.__format);
         if (WRITE_SNAPSHOT) {

packages/optimizer/spec/helpers/bentoComponentInfo.json

@@ -0,0 +1,86 @@
+[
+  {
+    "name": "amp-accordion",
+    "experimental": true,
+    "path": "/documentation/components/amp-accordion-v1.0/",
+    "version": "1.0"
+  },
+  {
+    "name": "amp-base-carousel",
+    "experimental": true,
+    "path": "/documentation/components/amp-base-carousel-v1.0/",
+    "version": "1.0"
+  },
+  {
+    "name": "amp-date-countdown",
+    "experimental": true,
+    "path": "/documentation/components/amp-date-countdown-v1.0/",
+    "version": "1.0"
+  },
+  {
+    "name": "amp-date-display",
+    "experimental": false,
+    "path": "/documentation/components/amp-date-display-v1.0/",
+    "version": "1.0"
+  },
+  {
+    "name": "amp-fit-text",
+    "experimental": true,
+    "path": "/documentation/components/amp-fit-text-v1.0/",
+    "version": "1.0"
+  },
+  {
+    "name": "amp-inline-gallery",
+    "experimental": true,
+    "path": "/documentation/components/amp-inline-gallery-v1.0/",
+    "version": "1.0"
+  },
+  {
+    "name": "amp-instagram",
+    "experimental": true,
+    "path": "/documentation/components/amp-instagram-v1.0/",
+    "version": "1.0"
+  },
+  {
+    "name": "amp-lightbox",
+    "experimental": true,
+    "path": "/documentation/components/amp-lightbox-v1.0/",
+    "version": "1.0"
+  },
+  {
+    "name": "amp-selector",
+    "experimental": true,
+    "path": "/documentation/components/amp-selector-v1.0/",
+    "version": "1.0"
+  },
+  {
+    "name": "amp-social-share",
+    "experimental": true,
+    "path": "/documentation/components/amp-social-share-v1.0/",
+    "version": "1.0"
+  },
+  {
+    "name": "amp-stream-gallery",
+    "experimental": true,
+    "path": "/documentation/components/amp-stream-gallery-v1.0/",
+    "version": "1.0"
+  },
+  {
+    "name": "amp-timeago",
+    "experimental": true,
+    "path": "/documentation/components/amp-timeago-v1.0/",
+    "version": "1.0"
+  },
+  {
+    "name": "amp-video",
+    "experimental": true,
+    "path": "/documentation/components/amp-video-v1.0/",
+    "version": "1.0"
+  },
+  {
+    "name": "amp-youtube",
+    "experimental": true,
+    "path": "/documentation/components/amp-youtube-v1.0/",
+    "version": "1.0"
+  }
+]

packages/optimizer/spec/transformers/valid/AutoExtensionImporter/uses-non-experimental-versions/expected_output.html

@@ -0,0 +1,26 @@
+<!doctype html>
+<html ⚡>
+<head>
+  <meta charset="utf-8">
+  <title>My AMP Page</title>
+  <link rel="canonical" href="self.html">
+  <style amp-boilerplate>body{-webkit-animation:-amp-start 8s steps(1,end) 0s 1 normal both;-moz-animation:-amp-start 8s steps(1,end) 0s 1 normal both;-ms-animation:-amp-start 8s steps(1,end) 0s 1 normal both;animation:-amp-start 8s steps(1,end) 0s 1 normal both}@-webkit-keyframes -amp-start{from{visibility:hidden}to{visibility:visible}}@-moz-keyframes -amp-start{from{visibility:hidden}to{visibility:visible}}@-ms-keyframes -amp-start{from{visibility:hidden}to{visibility:visible}}@-o-keyframes -amp-start{from{visibility:hidden}to{visibility:visible}}@keyframes -amp-start{from{visibility:hidden}to{visibility:visible}}</style><noscript><style amp-boilerplate>body{-webkit-animation:none;-moz-animation:none;-ms-animation:none;animation:none}</style></noscript>
+  <script async src="https://cdn.ampproject.org/v0.js"></script>
+  <script async src="https://cdn.ampproject.org/v0/amp-timeago-0.1.js" custom-element="amp-timeago"></script><script async src="https://cdn.ampproject.org/v0/amp-youtube-0.1.js" custom-element="amp-youtube"></script><script async src="https://cdn.ampproject.org/v0/amp-date-display-1.0.js" custom-element="amp-date-display"></script><script async src="https://cdn.ampproject.org/v0/amp-mustache-0.2.js" custom-template="amp-mustache"></script>
+</head>
+<body>
+  <!-- import non-bento component v0.1 instead of experimental bento v1.0 -->
+  <amp-timeago layout="fixed" width="160" height="20" datetime="2017-04-11T00:37:33.809Z" locale="en">
+    Saturday 11 April 2017 00.37
+  </amp-timeago>
+  <amp-youtube data-videoid="mGENRKrdoGY" layout="responsive" width="480" height="270"></amp-youtube>
+  <!-- import non-experimental bento component v1.0 -->
+  <amp-date-display datetime="2017-08-02T15:05:05.000" layout="fixed" width="360" height="20">
+    <template type="amp-mustache">
+      <div>
+        {{dayName}} {{day}} {{monthName}} {{year}}{{hourTwoDigit}}:{{minuteTwoDigit}}:{{secondTwoDigit}}
+      </div>
+    </template>
+  </amp-date-display>
+</body>
+</html>

packages/optimizer/spec/transformers/valid/AutoExtensionImporter/uses-non-experimental-versions/input.html

@@ -0,0 +1,40 @@
+<!doctype html>
+<html ⚡>
+<head>
+  <meta charset="utf-8">
+  <title>My AMP Page</title>
+  <link rel="canonical" href="self.html">
+  <style amp-boilerplate>body{-webkit-animation:-amp-start 8s steps(1,end) 0s 1 normal both;-moz-animation:-amp-start 8s steps(1,end) 0s 1 normal both;-ms-animation:-amp-start 8s steps(1,end) 0s 1 normal both;animation:-amp-start 8s steps(1,end) 0s 1 normal both}@-webkit-keyframes -amp-start{from{visibility:hidden}to{visibility:visible}}@-moz-keyframes -amp-start{from{visibility:hidden}to{visibility:visible}}@-ms-keyframes -amp-start{from{visibility:hidden}to{visibility:visible}}@-o-keyframes -amp-start{from{visibility:hidden}to{visibility:visible}}@keyframes -amp-start{from{visibility:hidden}to{visibility:visible}}</style><noscript><style amp-boilerplate>body{-webkit-animation:none;-moz-animation:none;-ms-animation:none;animation:none}</style></noscript>
+  <script async src="https://cdn.ampproject.org/v0.js"></script>
+</head>
+<body>
+  <!-- import non-bento component v0.1 instead of experimental bento v1.0 -->
+  <amp-timeago
+    layout="fixed"
+    width="160"
+    height="20"
+    datetime="2017-04-11T00:37:33.809Z"
+    locale="en"
+  >
+    Saturday 11 April 2017 00.37
+  </amp-timeago>
+  <amp-youtube
+  data-videoid="mGENRKrdoGY"
+  layout="responsive"
+  width="480"
+  height="270"
+></amp-youtube>
+  <!-- import non-experimental bento component v1.0 -->
+  <amp-date-display
+  datetime="2017-08-02T15:05:05.000"
+  layout="fixed"
+  width="360"
+  height="20"
+>
+  <template type="amp-mustache">
+    <div>
+{{dayName}} {{day}} {{monthName}} {{year}}{{hourTwoDigit}}:{{minuteTwoDigit}}:{{secondTwoDigit}}    </div>
+  </template>
+</amp-date-display>
+</body>
+</html>