Enhancement/issue 971 refactor bundling and optimizations (#974)

* add cloud IDE caveat to puppeteer renderer plugin readme (#967)

* init commit of refactoring for script tags with a src

* initial CSS optimizing

* sync optimized link tags in final output

* refactored for shared reources

* handle inline script bundling

* support serving custom resources using Greenwood plugins in Rollup configuration without needing extra rollup plugin

* non resource related Rollup plugins supported

* custom resource plugins and rollup plugins working together

* handle empty input for Rollup

* updated lock file

* handle inline style tag bundling and optimizing

* default optimization spec passing

* refactor merging app and page templates

* clarifying corrections in spec files

* inline optimization config working

* none optimization support

* none optimization support

* none and static optimization overrides

* refactor html rendering and optimizing

* refactoring and more CLI specs passing

* add missing variable

* SSR specs and optimizing resource bundling

* minor refactoring and logging

* resolving some plugin specs

* restore develop command related GraphQL specs

* custom graphql query spec

* all specs passing

* drop rollup plugin deps from import typescript plugin

* all Greenwood commands and specs passing

* restore static router with custom prerender

* restore postcss-import

* refactor shared resources to a Map and handle dupes

* restore local packages workaround for local Rollup bundling

* better monorepo Rollup facade modules detection

* switch console log

* remove console logging

* update plugin related docs

* local solution for windows support

* refactor errant object assign

* full cross platform URL support

* fix lint

* fix extra bundles when custom prerendering

* clean up stale or already tracked TODOs

* add nested head tag smoke tests

* check for app template validation for HUD display

* misc refactoring and TODOs cleanup

* restore static router (again)

* standardize passing correct reference for prerender scripts

* clean up data-gwd-opt markers from final HTML

packages/cli/src/index.js

@@ -79,7 +79,7 @@ const run = async() => {
       case 'serve':
         process.env.__GWD_COMMAND__ = 'build';
 
-        await (await import('./commands/build.js')).runProductionBuild(Object.assign({}, compilation));
+        await (await import('./commands/build.js')).runProductionBuild(compilation);
         await (await import('./commands/serve.js')).runProdServer(compilation);
 
         break;

packages/cli/src/lib/resource-utils.js

@@ -0,0 +1,50 @@
+import fs from 'fs';
+import { hashString } from '../lib/hashing-utils.js';
+import path from 'path';
+import { pathToFileURL } from 'url';
+
+function modelResource(context, type, src = undefined, contents = undefined, optimizationAttr = undefined, rawAttributes = undefined) {
+  const { projectDirectory, scratchDir, userWorkspace } = context;
+  const extension = type === 'script' ? 'js' : 'css';
+  const windowsDriveRegex = /\/[a-zA-Z]{1}:\//;
+  let sourcePathURL;
+
+  if (src) {
+    sourcePathURL = src.indexOf('/node_modules') === 0
+      ? pathToFileURL(path.join(projectDirectory, src)) // TODO (good first issue) get "real" location of node modules
+      : pathToFileURL(path.join(userWorkspace, src.replace(/\.\.\//g, '').replace('./', '')));
+
+    contents = fs.readFileSync(sourcePathURL, 'utf-8');
+  } else {
+    const scratchFileName = hashString(contents);
+
+    sourcePathURL = pathToFileURL(path.join(scratchDir, `${scratchFileName}.${extension}`));
+    fs.writeFileSync(sourcePathURL, contents);
+  }
+
+  // TODO (good first issue) handle for Windows adding extra / in front of drive letter for whatever reason :(
+  // e.g. turn /C:/... -> C:/...
+  // and also URL is readonly in NodeJS??
+  if (windowsDriveRegex.test(sourcePathURL.pathname)) {
+    const driveMatch = sourcePathURL.pathname.match(windowsDriveRegex)[0];
+
+    sourcePathURL = {
+      ...sourcePathURL,
+      pathname: sourcePathURL.pathname.replace(driveMatch, driveMatch.replace('/', '')),
+      href: sourcePathURL.href.replace(driveMatch, driveMatch.replace('/', ''))
+    };
+  }
+
+  return {
+    src, // if <script src="..."></script> or <link href="..."></link>
+    sourcePathURL, // src as a URL
+    type,
+    contents,
+    optimizedFileName: undefined,
+    optimizedFileContents: undefined,
+    optimizationAttr,
+    rawAttributes
+  };
+}
+
+export { modelResource };

packages/cli/src/lifecycles/bundle.js

@@ -1,21 +1,153 @@
+import fs from 'fs';
 import { getRollupConfig } from '../config/rollup.config.js';
+import { hashString } from '../lib/hashing-utils.js';
+import path from 'path';
 import { rollup } from 'rollup';
 
+async function cleanUpResources(compilation) {
+  const { outputDir } = compilation.context;
+
+  for (const resource of compilation.resources.values()) {
+    const { src, optimizedFileName, optimizationAttr } = resource;
+    const optConfig = ['inline', 'static'].indexOf(compilation.config.optimization) >= 0;
+    const optAttr = ['inline', 'static'].indexOf(optimizationAttr) >= 0;
+
+    if (optimizedFileName && (!src || (optAttr || optConfig))) {
+      fs.unlinkSync(path.join(outputDir, optimizedFileName));
+    }
+  }
+}
+
+async function optimizeStaticPages(compilation, optimizeResources) {
+  const { scratchDir, outputDir } = compilation.context;
+
+  return Promise.all(compilation.graph
+    .filter(page => !page.isSSR || (page.isSSR && page.data.static))
+    .map(async (page) => {
+      const { route, outputPath } = page;
+      const html = await fs.promises.readFile(path.join(scratchDir, outputPath), 'utf-8');
+
+      if (route !== '/404/' && !fs.existsSync(path.join(outputDir, route))) {
+        fs.mkdirSync(path.join(outputDir, route), {
+          recursive: true
+        });
+      }
+
+      let htmlOptimized = await optimizeResources.reduce(async (htmlPromise, resource) => {
+        const contents = await htmlPromise;
+        const shouldOptimize = await resource.shouldOptimize(outputPath, contents);
+
+        return shouldOptimize
+          ? resource.optimize(outputPath, contents)
+          : Promise.resolve(contents);
+      }, Promise.resolve(html));
+
+      // clean up optimization markers
+      htmlOptimized = htmlOptimized.replace(/data-gwd-opt=".*[a-z]"/g, '');
+
+      await fs.promises.writeFile(path.join(outputDir, outputPath), htmlOptimized);
+    })
+  );
+}
+
+async function bundleStyleResources(compilation, optimizationPlugins) {
+  const { outputDir } = compilation.context;
+
+  for (const resource of compilation.resources.values()) {
+    const { contents, optimizationAttr, src = '', type } = resource;
+
+    if (['style', 'link'].includes(type)) {
+      const resourceKey = resource.sourcePathURL.pathname;
+      const srcPath = src && src.replace(/\.\.\//g, '').replace('./', '');
+      let optimizedFileName;
+      let optimizedFileContents;
+
+      if (src) {
+        const basename = path.basename(srcPath);
+        const basenamePieces = path.basename(srcPath).split('.');
+
+        optimizedFileName = srcPath.indexOf('/node_modules') >= 0
+          ? `${basenamePieces[0]}.${hashString(contents)}.css`
+          : srcPath.replace(basename, `${basenamePieces[0]}.${hashString(contents)}.css`);
+      } else {
+        optimizedFileName = `${hashString(contents)}.css`;
+      }
+
+      const outputPathRoot = path.join(outputDir, path.dirname(optimizedFileName));
+
+      if (!fs.existsSync(outputPathRoot)) {
+        fs.mkdirSync(outputPathRoot, {
+          recursive: true
+        });
+      }
+
+      if (compilation.config.optimization === 'none' || optimizationAttr === 'none') {
+        optimizedFileContents = contents;
+      } else {
+        const url = resource.sourcePathURL.pathname;
+        let optimizedStyles = await fs.promises.readFile(url, 'utf-8');
+
+        for (const plugin of optimizationPlugins) {
+          optimizedStyles = await plugin.shouldOptimize(url, optimizedStyles)
+            ? await plugin.optimize(url, optimizedStyles)
+            : optimizedStyles;
+        }
+
+        optimizedFileContents = optimizedStyles;
+      }
+
+      compilation.resources.set(resourceKey, {
+        ...compilation.resources.get(resourceKey),
+        optimizedFileName,
+        optimizedFileContents
+      });
+
+      await fs.promises.writeFile(path.join(outputDir, optimizedFileName), optimizedFileContents);
+    }
+  }
+}
+
+async function bundleScriptResources(compilation) {
+  // https://rollupjs.org/guide/en/#differences-to-the-javascript-api
+  const [rollupConfig] = await getRollupConfig(compilation);
+
+  if (rollupConfig.input.length !== 0) {
+    const bundle = await rollup(rollupConfig);
+    await bundle.write(rollupConfig.output);
+  }
+}
+
 const bundleCompilation = async (compilation) => {
 
   return new Promise(async (resolve, reject) => {
     try {
       compilation.graph = compilation.graph.filter(page => !page.isSSR || (page.isSSR && page.data.static) || (page.isSSR && compilation.config.prerender));
 
-      // https://rollupjs.org/guide/en/#differences-to-the-javascript-api
-      if (compilation.graph.length > 0) {
-        const rollupConfigs = await getRollupConfig({
-          ...compilation
-        });
-        const bundle = await rollup(rollupConfigs[0]);
+      const optimizeResourcePlugins = compilation.config.plugins.filter((plugin) => {
+        return plugin.type === 'resource';
+      }).map((plugin) => {
+        return plugin.provider(compilation);
+      }).filter((provider) => {
+        return provider.shouldOptimize && provider.optimize;
+      });
+      // centrally register all static resources
+      compilation.graph.map((page) => {
+        return page.imports;
+      }).flat().forEach(resource => {
+        compilation.resources.set(resource.sourcePathURL.pathname, resource);
+      });
 
-        await bundle.write(rollupConfigs[0].output);
-      }
+      console.info('bundling static assets...');
+
+      await Promise.all([
+        await bundleScriptResources(compilation),
+        await bundleStyleResources(compilation, optimizeResourcePlugins.filter(plugin => plugin.contentType.includes('text/css')))
+      ]);
+
+      console.info('optimizing static pages....');
+
+      await optimizeStaticPages(compilation, optimizeResourcePlugins.filter(plugin => plugin.contentType.includes('text/html')));
+      await cleanUpResources(compilation);
 
       resolve();
     } catch (err) {

packages/cli/src/lifecycles/compile.js

@@ -9,7 +9,8 @@ const generateCompilation = () => {
       let compilation = {
         graph: [],
         context: {},
-        config: {}
+        config: {},
+        resources: new Map()
       };
 
       console.info('Initializing project config');

packages/cli/src/lifecycles/context.js

@@ -3,7 +3,7 @@ import path from 'path';
 import { fileURLToPath, URL } from 'url';
 
 const initContext = async({ config }) => {
-  const scratchDir = path.join(process.cwd(), './.greenwood/');
+  const scratchDir = path.join(process.cwd(), './.greenwood');
   const outputDir = path.join(process.cwd(), './public');
   const dataDir = fileURLToPath(new URL('../data', import.meta.url));
 

packages/cli/src/lifecycles/graph.js

@@ -1,6 +1,7 @@
 /* eslint-disable complexity, max-depth */
 import fs from 'fs';
 import fm from 'front-matter';
+import { modelResource } from '../lib/resource-utils.js';
 import path from 'path';
 import toc from 'markdown-toc';
 import { Worker } from 'worker_threads';
@@ -133,6 +134,13 @@ const generateGraph = async (compilation) => {
                 });
                 worker.on('message', (result) => {
                   if (result.frontmatter) {
+                    const resources = (result.frontmatter.imports || []).map((resource) => {
+                      const type = path.extname(resource) === '.js' ? 'script' : 'link';
+
+                      return modelResource(compilation.context, type, resource);
+                    });
+
+                    result.frontmatter.imports = resources;
                     ssrFrontmatter = result.frontmatter;
                   }
                   resolve();