feat(prerender): hash assets and add version querystring

Author: adamdbradley

src/compiler/optimize/minify-css.ts

@@ -1,15 +1,44 @@
-import { hasError } from '@utils';
+import { CssNode, CssNodeType } from '../style/css-parser/css-parse-declarations'
+import { hasError, isFunction, isString } from '@utils';
 import { parseCss } from '../style/css-parser/parse-css';
 import { serializeCss } from '../style/css-parser/serialize-css';
 
-export const minifyCss = (cssString: string) => {
-  const parseResults = parseCss(cssString);
+export const minifyCss = async (input: { css: string, resolveUrl?: (url: string) => Promise<string> | string; }) => {
+  const parseResults = parseCss(input.css);
 
   if (hasError(parseResults.diagnostics)) {
-    return cssString;
+    return input.css;
+  }
+  if (isFunction(input.resolveUrl) && parseResults.stylesheet && Array.isArray(parseResults.stylesheet.rules)) {
+    await resolveStylesheetUrl(parseResults.stylesheet.rules, input.resolveUrl, new Map());
   }
 
-  const output = serializeCss(parseResults.stylesheet, {});
+  return serializeCss(parseResults.stylesheet, {});
+};
 
-  return output;
+const resolveStylesheetUrl = async (nodes: CssNode[], resolveUrl: (url: string) => Promise<string> | string, resolved: Map<string, string>) => {
+  for (const node of nodes) {
+    if (node.type === CssNodeType.Declaration && isString(node.value) && node.value.includes('url(')) {
+      const urlSplt = node.value.split(',').map(n => n.trim());
+      for (let i = 0; i < urlSplt.length; i++) {
+        const r = /url\((.*?)\)/.exec(urlSplt[i]);
+        if (r) {
+          const orgUrl = r[1].replace(/(\'|\")/g, '');
+          const newUrl = await resolveUrl(orgUrl);
+          urlSplt[i] = urlSplt[i].replace(orgUrl, newUrl);
+        }
+      }
+      node.value = urlSplt.join(',');
+    }
+    if (Array.isArray(node.declarations)) {
+      await resolveStylesheetUrl(node.declarations, resolveUrl, resolved);
+    }
+    if (Array.isArray(node.rules)) {
+      await resolveStylesheetUrl(node.rules, resolveUrl, resolved);
+    }
+    if (Array.isArray(node.keyframes)) {
+      await resolveStylesheetUrl(node.keyframes, resolveUrl, resolved);
+    }
+  }
 };
+

src/compiler/optimize/optimize-css.ts

@@ -15,7 +15,10 @@ export const optimizeCss = async (inputOpts: OptimizeCssInput) => {
     }
   }
   if (inputOpts.minify !== false) {
-    result.output = minifyCss(result.output);
+    result.output = await minifyCss({
+      css: result.output,
+      resolveUrl: inputOpts.resolveUrl
+    });
   }
   return result;
 };

src/compiler/prerender/prerender-hydrate-options.ts

@@ -8,6 +8,7 @@ export const getHydrateOptions = (prerenderConfig: d.PrerenderConfig, url: URL,
     url: prerenderUrl,
     addModulePreloads: true,
     approximateLineWidth: 100,
+    hashAssets: 'querystring',
     inlineExternalStyleSheets: true,
     minifyScriptElements: true,
     minifyStyleElements: true,

src/compiler/prerender/prerender-main.ts

@@ -192,6 +192,7 @@ const runPrerenderOutputTarget = async (
       srcIndexHtmlPath,
       outputTarget,
       hydrateOpts,
+      manager
     );
     if (diagnostics.length > 0 || !templateData || !isString(templateData.html)) {
       return;

src/compiler/prerender/prerender-optimize.ts

@@ -1,12 +1,13 @@
 import type * as d from '../../declarations';
-import { flatOne, unique } from '@utils';
+import { catchError, flatOne, isString, unique } from '@utils';
 import { getScopeId } from '../style/scope-css';
 import { injectModulePreloads } from '../html/inject-module-preloads';
 import { optimizeCss } from '../optimize/optimize-css';
 import { optimizeJs } from '../optimize/optimize-js';
 import { join } from 'path';
+import { minifyCss } from '../optimize/minify-css';
 
-export const inlineExternalStyleSheets = async (config: d.Config, appDir: string, doc: Document) => {
+export const inlineExternalStyleSheets = async (sys: d.CompilerSystem, appDir: string, doc: Document) => {
   const documentLinks = Array.from(doc.querySelectorAll('link[rel=stylesheet]')) as HTMLLinkElement[];
   if (documentLinks.length === 0) {
     return;
@@ -22,7 +23,7 @@ export const inlineExternalStyleSheets = async (config: d.Config, appDir: string
       const fsPath = join(appDir, href);
 
       try {
-        let styles = await config.sys.readFile(fsPath);
+        let styles = await sys.readFile(fsPath);
 
         const optimizeResults = await optimizeCss({
           input: styles,
@@ -94,25 +95,27 @@ export const minifyScriptElements = async (doc: Document, addMinifiedAttr: boole
   );
 };
 
-export const minifyStyleElements = async (doc: Document, addMinifiedAttr: boolean) => {
+export const minifyStyleElements = async (sys: d.CompilerSystem, appDir: string, doc: Document, currentUrl: URL, addMinifiedAttr: boolean) => {
   const styleElms = Array.from(doc.querySelectorAll('style')).filter(styleElm => {
     if (styleElm.hasAttribute(dataMinifiedAttr)) {
       return false;
     }
     return true;
   });
 
-  if (styleElms.length === 0) {
-    return;
-  }
-
   await Promise.all(
     styleElms.map(async styleElm => {
       const content = styleElm.innerHTML.trim();
       if (content.length > 0) {
         const optimizeResults = await optimizeCss({
           input: content,
           minify: true,
+          async resolveUrl(urlProp) {
+            const assetUrl = new URL(urlProp, currentUrl);
+            const hash = await getAssetFileHash(sys, appDir, assetUrl);
+            assetUrl.searchParams.append('v', hash);
+            return assetUrl.pathname + assetUrl.search;
+          }
         });
         if (optimizeResults.diagnostics.length === 0) {
           styleElm.innerHTML = optimizeResults.output;
@@ -191,4 +194,123 @@ export const hasStencilScript = (doc: Document) => {
   return !!doc.querySelector('script[data-stencil]');
 };
 
+export const hashAssets = async (sys: d.CompilerSystem, diagnostics: d.Diagnostic[], hydrateOpts: d.PrerenderHydrateOptions, appDir: string, doc: Document, currentUrl: URL) => {
+  // do one at a time to prevent too many opened files and memory usage issues
+  // hash id is cached in each worker, so shouldn't have to do this for every page
+
+  // update the stylesheet content first so the hash url()s are apart of the file's hash too
+  const links = Array.from(doc.querySelectorAll('link[rel=stylesheet][href]')) as HTMLLinkElement[];
+
+  for (const link of links) {
+    const href = link.getAttribute('href');
+    if (isString(href) && href.length > 0) {
+      const stylesheetUrl = new URL(href, currentUrl);
+      if (currentUrl.host === stylesheetUrl.host) {
+        try {
+          const filePath = join(appDir, stylesheetUrl.pathname);
+          let css = await sys.readFile(filePath);
+          if (isString(css)) {
+            css = await minifyCss({
+              css,
+              async resolveUrl(urlProp) {
+                const assetUrl = new URL(urlProp, stylesheetUrl);
+                const hash = await getAssetFileHash(sys, appDir, assetUrl);
+                assetUrl.searchParams.append('v', hash);
+                return assetUrl.pathname + assetUrl.search;
+              }
+            });
+            await sys.writeFile(filePath, css);
+          }
+        } catch (e) {
+          catchError(diagnostics, e);
+        }
+      }
+    }
+  }
+
+  await hashAsset(sys, hydrateOpts, appDir, doc, currentUrl, 'link[rel="stylesheet"]', ['href']);
+  await hashAsset(sys, hydrateOpts, appDir, doc, currentUrl, 'link[rel="prefetch"]', ['href']);
+  await hashAsset(sys, hydrateOpts, appDir, doc, currentUrl, 'link[rel="preload"]', ['href']);
+  await hashAsset(sys, hydrateOpts, appDir, doc, currentUrl, 'link[rel="modulepreload"]', ['href']);
+  await hashAsset(sys, hydrateOpts, appDir, doc, currentUrl, 'link[rel="icon"]', ['href']);
+  await hashAsset(sys, hydrateOpts, appDir, doc, currentUrl, 'link[rel="apple-touch-icon"]', ['href']);
+  await hashAsset(sys, hydrateOpts, appDir, doc, currentUrl, 'link[rel="manifest"]', ['href']);
+  await hashAsset(sys, hydrateOpts, appDir, doc, currentUrl, 'script', ['src']);
+  await hashAsset(sys, hydrateOpts, appDir, doc, currentUrl, 'img', ['src', 'srcset']);
+  await hashAsset(sys, hydrateOpts, appDir, doc, currentUrl, 'picture > source', ['srcset']);
+}
+
+const hashAsset = async (sys: d.CompilerSystem, hydrateOpts: d.PrerenderHydrateOptions, appDir: string, doc: Document, currentUrl: URL, selector: string, srcAttrs: string[]) => {
+  const elms = Array.from(doc.querySelectorAll(selector));
+
+  // do one at a time to prevent too many opened files and memory usage issues
+  for (const elm of elms) {
+    for (const attrName of srcAttrs) {
+      const srcValues = getAttrUrls(attrName, elm.getAttribute(attrName));
+      for (const srcValue of srcValues) {
+        const assetUrl = new URL(srcValue.src, currentUrl);
+        if (assetUrl.hostname === currentUrl.hostname) {
+          if (hydrateOpts.hashAssets === 'querystring' && !assetUrl.searchParams.has('v')) {
+            const hash = await getAssetFileHash(sys, appDir, assetUrl);
+            if (isString(hash)) {
+              assetUrl.searchParams.append('v', hash);
+              const attrValue = setAttrUrls(assetUrl, srcValue.descriptor);
+              elm.setAttribute(attrName, attrValue);
+            }
+          }
+        }
+      }
+    }
+  }
+}
+
+export const getAttrUrls = (attrName: string, attrValue: string) => {
+  const srcValues: { src: string, descriptor?: string }[] = [];
+  if (isString(attrValue)) {
+    if (attrName.toLowerCase() === 'srcset') {
+      attrValue.split(',').map(a => a.trim()).filter(a => a.length > 0).forEach(src => {
+        const spaceSplt = src.split(' ');
+        if (spaceSplt[0].length > 0) {
+          srcValues.push({ src: spaceSplt[0], descriptor: spaceSplt[1] });
+        }
+      });
+    } else {
+      srcValues.push({ src: attrValue });
+    }
+  }
+  return srcValues;
+}
+
+export const setAttrUrls = (url: URL, descriptor: string) => {
+  let src = url.pathname + url.search;
+  if (isString(descriptor)) {
+    src += ' ' + descriptor;
+  }
+  return src;
+};
+
+const hashedAssets = new Map<string, Promise<string | null>>();
+
+const getAssetFileHash = async (sys: d.CompilerSystem, appDir: string, assetUrl: URL) => {
+  let p = hashedAssets.get(assetUrl.pathname);
+  if (!p) {
+    p = new Promise<string | null>(async resolve => {
+      const assetFilePath = join(appDir, assetUrl.pathname);
+      try {
+        const data = await sys.readFile(assetFilePath, 'binary');
+        if (data != null) {
+          const hash = await sys.generateContentHash(data, 10);
+          resolve(hash);
+          return;
+        }
+      } catch (e) {
+        console.error(e);
+      }
+      resolve(null);
+    });
+    hashedAssets.set(assetUrl.pathname, p);
+  }
+  return p;
+}
+
 const dataMinifiedAttr = 'data-m';

src/compiler/prerender/prerender-queue.ts

@@ -113,6 +113,7 @@ const prerenderUrl = async (results: d.PrerenderResults, manager: d.PrerenderMan
     }
 
     const prerenderRequest: d.PrerenderUrlRequest = {
+      appDir: manager.outputTarget.appDir,
       baseUrl: manager.outputTarget.baseUrl,
       buildId: results.buildId,
       componentGraphPath: manager.componentGraphPath,

src/compiler/prerender/prerender-template-html.ts

@@ -17,6 +17,7 @@ export const generateTemplateHtml = async (
   srcIndexHtmlPath: string,
   outputTarget: d.OutputTargetWww,
   hydrateOpts: d.PrerenderHydrateOptions,
+  manager: d.PrerenderManager
 ) => {
   try {
     if (!isString(srcIndexHtmlPath)) {
@@ -56,7 +57,7 @@ export const generateTemplateHtml = async (
 
     if (hydrateOpts.inlineExternalStyleSheets && !isDebug) {
       try {
-        await inlineExternalStyleSheets(config, outputTarget.appDir, doc);
+        await inlineExternalStyleSheets(config.sys, outputTarget.appDir, doc);
       } catch (e) {
         catchError(diagnostics, e);
       }
@@ -72,7 +73,8 @@ export const generateTemplateHtml = async (
 
     if (hydrateOpts.minifyStyleElements && !isDebug) {
       try {
-        await minifyStyleElements(doc, true);
+        const baseUrl = new URL(outputTarget.baseUrl, manager.devServerHostUrl);
+        await minifyStyleElements(config.sys, outputTarget.appDir, doc, baseUrl, true);
       } catch (e) {
         catchError(diagnostics, e);
       }

src/compiler/prerender/prerender-worker.ts

@@ -2,6 +2,7 @@ import type * as d from '../../declarations';
 import {
   addModulePreloads,
   excludeStaticComponents,
+  hashAssets,
   minifyScriptElements,
   minifyStyleElements,
   removeModulePreloads,
@@ -31,6 +32,7 @@ export const prerenderWorker = async (sys: d.CompilerSystem, prerenderRequest: d
 
   try {
     const url = new URL(prerenderRequest.url, prerenderRequest.devServerHostUrl);
+    const baseUrl = new URL(prerenderRequest.baseUrl);
     const componentGraph = getComponentGraph(sys, prerenderRequest.componentGraphPath);
 
     // webpack work-around/hack
@@ -44,6 +46,7 @@ export const prerenderWorker = async (sys: d.CompilerSystem, prerenderRequest: d
     // create a new window by cloning the cached parsed window
     const win = hydrateApp.createWindowFromHtml(prerenderCtx.templateHtml, prerenderRequest.templateId);
     const doc = win.document;
+    win.location.href = url.href;
 
     // patch this new window
     if (isFunction(sys.applyPrerenderGlobalPatch)) {
@@ -85,6 +88,17 @@ export const prerenderWorker = async (sys: d.CompilerSystem, prerenderRequest: d
     const hydrateResults = (await hydrateApp.hydrateDocument(doc, hydrateOpts)) as d.HydrateResults;
     results.diagnostics.push(...hydrateResults.diagnostics);
 
+    if (typeof prerenderConfig.filePath === 'function') {
+      try {
+        const userWriteToFilePath = prerenderConfig.filePath(url, results.filePath);
+        if (typeof userWriteToFilePath === 'string') {
+          results.filePath = userWriteToFilePath;
+        }
+      } catch (e) {
+        catchError(results.diagnostics, e);
+      }
+    }
+
     if (hydrateOpts.staticDocument) {
       removeStencilScripts(doc);
       removeModulePreloads(doc);
@@ -103,32 +117,28 @@ export const prerenderWorker = async (sys: d.CompilerSystem, prerenderRequest: d
       }
     }
 
-    const minifyPromises: Promise<any>[] = [];
+    const docPromises: Promise<any>[] = [];
     if (hydrateOpts.minifyStyleElements && !prerenderRequest.isDebug) {
-      minifyPromises.push(minifyStyleElements(doc, false));
+      docPromises.push(minifyStyleElements(sys, prerenderRequest.appDir, doc, url, false));
     }
 
     if (hydrateOpts.minifyScriptElements && !prerenderRequest.isDebug) {
-      minifyPromises.push(minifyScriptElements(doc, false));
-    }
-
-    if (minifyPromises.length > 0) {
-      await Promise.all(minifyPromises);
+      docPromises.push(minifyScriptElements(doc, false));
     }
 
-    if (typeof prerenderConfig.filePath === 'function') {
+    if (hydrateOpts.hashAssets && !prerenderRequest.isDebug) {
       try {
-        const userWriteToFilePath = prerenderConfig.filePath(url, results.filePath);
-        if (typeof userWriteToFilePath === 'string') {
-          results.filePath = userWriteToFilePath;
-        }
+        docPromises.push(hashAssets(sys, results.diagnostics, hydrateOpts, prerenderRequest.appDir, doc, url));
       } catch (e) {
         catchError(results.diagnostics, e);
       }
     }
+    
+    if (docPromises.length > 0) {
+      await Promise.all(docPromises);
+    }
 
     if (prerenderConfig.crawlUrls !== false) {
-      const baseUrl = new URL(prerenderRequest.baseUrl);
       results.anchorUrls = crawlAnchorsForNextUrls(
         prerenderConfig,
         results.diagnostics,