Timeout postinstall cache warmup (#835)

Fixes #832
ampproject · Jun 11, 2020 · 0af1812 · 0af1812
1 parent cf1a9ad
commit 0af1812
Show file tree

Hide file tree

Showing 7 changed files with 241 additions and 129 deletions.
diff --git a/package-lock.json b/package-lock.json
diff --git a/packages/optimizer/README.md b/packages/optimizer/README.md
@@ -108,6 +108,29 @@ Automatically import any missing AMP Extensions (e.g. amp-carousel).
 - default: `true`
 - used by: [AutoExtensionImport](lib/transformers/AddMandatoryTags.js)
 
+#### `fetch`
+
+Provide a custom fetch handler. You can use this option to configure a custom proxy server. Example:
+
+```js
+const nodeFetch = require('node-fetch');
+
+const proxyHost = '...';
+const proxyPort = '...';
+
+const fetch = (url, opts={}) => {
+  opts.agent = new HttpsProxyAgent(`${proxyHost}:${proxyPort}');
+  return nodeFetch(url, opts)
+}
+const optimizer = AmpOptimizer.create({
+  fetch,
+});
+```
+
+- name: `fetch`
+- valid options: a [whatwg fetch](https://github.com/whatwg/fetch) compatible fetch implementation. 
+- default: [node-fetch](https://www.npmjs.com/package/node-fetch)
+
 #### `format`
 
 Specifies the AMP format of the input file. Defaults to `AMP`.

diff --git a/packages/optimizer/lib/AmpConstants.js b/packages/optimizer/lib/AmpConstants.js
@@ -20,6 +20,7 @@ const {hasAttribute} = require('./NodeUtils');
 module.exports = {
   AMP_TAGS: ['amp', '⚡', '⚡4ads', 'amp4ads', '⚡4email', 'amp4email'],
   AMP_CACHE_HOST: 'https://cdn.ampproject.org',
+  AMP_VALIDATION_RULES_URL: 'https://cdn.ampproject.org/v0/validator.json',
   AMP_FORMATS: ['AMP', 'AMP4EMAIL', 'AMP4ADS'],
   AMP_RUNTIME_CSS_PATH: '/v0.css',
   appendRuntimeVersion: (prefix, version) => prefix + '/rtv/' + version,

diff --git a/packages/optimizer/lib/fetchRuntimeParameters.js b/packages/optimizer/lib/fetchRuntimeParameters.js
@@ -17,7 +17,12 @@
 
 const validatorRulesProvider = require('@ampproject/toolbox-validator-rules');
 const {MaxAge} = require('@ampproject/toolbox-core');
-const {AMP_CACHE_HOST, AMP_RUNTIME_CSS_PATH, appendRuntimeVersion} = require('./AmpConstants.js');
+const {
+  AMP_CACHE_HOST,
+  AMP_RUNTIME_CSS_PATH,
+  AMP_VALIDATION_RULES_URL,
+  appendRuntimeVersion,
+} = require('./AmpConstants.js');
 
 const KEY_VALIDATOR_RULES = 'validator-rules';
 const AMP_RUNTIME_MAX_AGE = 10 * 60; // 10 min
@@ -35,7 +40,7 @@ const cache = require('./cache.js');
  * @param {Object} customRuntimeParameters - user defined runtime parameters
  * @returns {Promise<Object>} - the runtime parameters
  */
-async function fetchRuntimeParameters(config, customRuntimeParameters) {
+async function fetchRuntimeParameters(config, customRuntimeParameters = {}) {
   const runtimeParameters = Object.assign({}, customRuntimeParameters);
   // Configure the log level
   runtimeParameters.verbose = customRuntimeParameters.verbose || config.verbose || false;
@@ -63,7 +68,7 @@ async function initValidatorRules(runtimeParameters, customRuntimeParameters, co
     runtimeParameters.validatorRules =
       customRuntimeParameters.validatorRules ||
       config.validatorRules ||
-      (await fetchValidatorRules_(config));
+      (await fetchValidatorRulesFromCache_(config));
   } catch (error) {
     config.log.error('Could not fetch validator rules', error);
   }
@@ -72,14 +77,14 @@ async function initValidatorRules(runtimeParameters, customRuntimeParameters, co
 /**
  * @private
  */
-async function fetchValidatorRules_(config) {
+async function fetchValidatorRulesFromCache_(config) {
   if (config.cache === false) {
-    return validatorRulesProvider.fetch();
+    return fetchValidatorRules_(config);
   }
   let rawRules = await cache.get('validator-rules');
   let validatorRules;
   if (!rawRules) {
-    validatorRules = await validatorRulesProvider.fetch();
+    validatorRules = await fetchValidatorRules_(config);
     config.log.debug('Downloaded AMP validation rules');
     // We save the raw rules to make the validation rules JSON serializable
     cache.set(KEY_VALIDATOR_RULES, validatorRules.raw);
@@ -89,6 +94,14 @@ async function fetchValidatorRules_(config) {
   return validatorRules;
 }
 
+async function fetchValidatorRules_(config) {
+  const response = await config.fetch(AMP_VALIDATION_RULES_URL);
+  if (!response.ok) {
+    return null;
+  }
+  return validatorRulesProvider.fetch({rules: await response.json()});
+}
+
 /**
  * Fetch runtime styles based on the runtime version
  *
@@ -164,7 +177,10 @@ async function fetchLatestRuntimeData_({config, ampUrlPrefix, lts}, versionKey =
     config.log.error(
       `Could not download runtime version from ${ampUrlPrefix}. Falling back to ${AMP_CACHE_HOST}`
     );
-    ampRuntimeData = await fetchLatestRuntimeData_({config, AMP_CACHE_HOST, lts}, versionKey);
+    ampRuntimeData = await fetchLatestRuntimeData_(
+      {config, ampUrlPrefix: AMP_CACHE_HOST, lts},
+      versionKey
+    );
   } else if (ampRuntimeData.version && versionKey) {
     cache.set(versionKey, ampRuntimeData);
   }

diff --git a/packages/optimizer/lib/warmup.js b/packages/optimizer/lib/warmup.js
@@ -14,15 +14,48 @@
  * limitations under the License.
  */
 
+const AbortController = require('abort-controller');
+const fetch = require('node-fetch');
+const log = require('@ampproject/toolbox-core').log.tag('AMP OPTIMIZER');
 const AmpOptimizer = require('../');
-const ampOptimizer = AmpOptimizer.create();
 
-async function warmupCaches() {
-  // run a dummy transformation to pre-fill the caches
-  await ampOptimizer.transformHtml('<h1>hello world</h1>', {
-    canonical: '.',
+const DOWNLOAD_TIMEOUT = 2000; // 2 seconds
+const controller = new AbortController();
+
+const fetchRuntimeParameters = require('./fetchRuntimeParameters');
+
+const fetchWithTimout = (url, opts = {}) => {
+  const timeout = setTimeout(() => {
+    controller.abort();
+  }, DOWNLOAD_TIMEOUT);
+  opts.signal = controller.signal;
+  return fetch(url, opts).finally(() => {
+    clearTimeout(timeout);
   });
-  ampOptimizer.config.log.info('Downloading latest AMP runtime data');
+};
+
+const ampOptimizer = AmpOptimizer.create({
+  fetch: fetchWithTimout,
+});
+
+async function warmupCaches() {
+  let success = true;
+  // Hack to avoid error messages in the console during postinstall
+  log.error = (e) => {
+    success = false;
+  };
+  // Re-use config from AMP Optimizer
+  // TODO extract config into it's own class
+  const config = AmpOptimizer.create({log, fetch: fetchWithTimout}).config;
+  // Try to download all runtime data, this will fail if behind a proxy
+  await fetchRuntimeParameters(config);
+  if (success) {
+    log.info('Downloaded latest AMP runtime data.');
+  } else {
+    log.info(
+      'Failed downloading latest AMP runtime data. Proxies need to be configured manually, see https://github.com/ampproject/amp-toolbox/tree/master/packages/optimizer#fetch.'
+    );
+  }
 }
 
 warmupCaches();
diff --git a/packages/optimizer/package-lock.json b/packages/optimizer/package-lock.json
diff --git a/packages/optimizer/package.json b/packages/optimizer/package.json
@@ -33,6 +33,7 @@
     "@ampproject/toolbox-runtime-version": "^2.5.1",
     "@ampproject/toolbox-script-csp": "^2.3.0",
     "@ampproject/toolbox-validator-rules": "^2.3.0",
+    "abort-controller": "3.0.0",
     "cross-fetch": "3.0.4",
     "cssnano": "4.1.10",
     "dom-serializer": "1.0.1",