Rfc/issue 952 data loading strategies (#1157)

* add support for SSR page loader with runtime request object

* refactor static export from getFrontmatter to export const prerender

* document loaders and prerendering for SSR routes

* adopt and document constructor props pattern for SSR page data loading

* update develop SSR test case for constructor props

* remove desribe.only

* refactor graphql plugin for ESM compat

* add test case for experimental prerendering with custom .gql imports

* upgrade website for breaking changes

* update website documentation and graphql plugin package README

* add test cases for adapter plugins and SSR constructor props

* upgrade wcc to 0.9.0

* misc PR cleanup

packages/cli/package.json

@@ -52,7 +52,7 @@
     "remark-rehype": "^7.0.0",
     "rollup": "^2.58.0",
     "unified": "^9.2.0",
-    "wc-compiler": "~0.8.0"
+    "wc-compiler": "~0.9.0"
   },
   "devDependencies": {
     "@babel/runtime": "^7.10.4",

packages/cli/src/commands/serve.js

@@ -8,7 +8,7 @@ const runProdServer = async (compilation) => {
     try {
       const port = compilation.config.port;
       const hasApisDir = await checkResourceExists(compilation.context.apisDir);
-      const hasDynamicRoutes = compilation.graph.find(page => page.isSSR && !page.data.static);
+      const hasDynamicRoutes = compilation.graph.find(page => page.isSSR && !page.prerender);
       const server = (hasDynamicRoutes && !compilation.config.prerender) || hasApisDir ? getHybridServer : getStaticServer;
 
       (await server(compilation)).listen(port, () => {

packages/cli/src/lib/execute-route-module.js

@@ -1,6 +1,6 @@
 import { renderToString, renderFromHTML } from 'wc-compiler';
 
-async function executeRouteModule({ moduleUrl, compilation, page = {}, prerender = false, htmlContents = null, scripts = [] }) {
+async function executeRouteModule({ moduleUrl, compilation, page = {}, prerender = false, htmlContents = null, scripts = [], request }) {
   const data = {
     template: null,
     body: null,
@@ -15,15 +15,15 @@ async function executeRouteModule({ moduleUrl, compilation, page = {}, prerender
     data.html = html;
   } else {
     const module = await import(moduleUrl).then(module => module);
-    const { getTemplate = null, getBody = null, getFrontmatter = null } = module;
+    const { prerender = false, getTemplate = null, getBody = null, getFrontmatter = null } = module;
 
     if (module.default) {
-      const { html } = await renderToString(new URL(moduleUrl), false);
+      const { html } = await renderToString(new URL(moduleUrl), false, request);
 
       data.body = html;
     } else {
       if (getBody) {
-        data.body = await getBody(compilation, page);
+        data.body = await getBody(compilation, page, request);
       }
     }
 
@@ -34,6 +34,8 @@ async function executeRouteModule({ moduleUrl, compilation, page = {}, prerender
     if (getFrontmatter) {
       data.frontmatter = await getFrontmatter(compilation, page);
     }
+
+    data.prerender = prerender;
   }
 
   return data;

packages/cli/src/lib/resource-utils.js

@@ -208,11 +208,63 @@ function transformKoaRequestIntoStandardRequest(url, request) {
   });
 }
 
+// https://stackoverflow.com/questions/57447685/how-can-i-convert-a-request-object-into-a-stringifiable-object-in-javascript
+async function requestAsObject (_request) {
+  if (!_request instanceof Request) {
+    throw Object.assign(
+      new Error(),
+      { name: 'TypeError', message: 'Argument must be a Request object' }
+    );
+  }
+
+  const request = _request.clone();
+  const contentType = request.headers.get('content-type') || '';
+  let headers = Object.fromEntries(request.headers);
+  let format;
+
+  function stringifiableObject (obj) {
+    const filtered = {};
+    for (const key in obj) {
+      if (['boolean', 'number', 'string'].includes(typeof obj[key]) || obj[key] === null) {
+        filtered[key] = obj[key];
+      }
+    }
+    return filtered;
+  }
+
+  if (contentType.includes('application/x-www-form-urlencoded')) {
+    const formData = await request.formData();
+    const params = {};
+
+    for (const entry of formData.entries()) {
+      params[entry[0]] = entry[1];
+    }
+
+    // when using FormData, let Request set the correct headers
+    // or else it will come out as multipart/form-data
+    // for serialization between route workers, leave a special marker for Greenwood
+    // https://stackoverflow.com/a/43521052/417806
+    headers['content-type'] = 'x-greenwood/www-form-urlencoded';
+    format = JSON.stringify(params);
+  } else if (contentType.includes('application/json')) {
+    format = JSON.stringify(await request.json());
+  } else {
+    format = await request.text();
+  }
+
+  return {
+    ...stringifiableObject(request),
+    body: format,
+    headers
+  };
+}
+
 export {
   checkResourceExists,
   mergeResponse,
   modelResource,
   normalizePathnameForWindows,
+  requestAsObject,
   resolveForRelativeUrl,
   trackResourcesForRoute,
   transformKoaRequestIntoStandardRequest

packages/cli/src/lib/ssr-route-worker.js

@@ -1,9 +1,9 @@
 // https://github.com/nodejs/modules/issues/307#issuecomment-858729422
 import { parentPort } from 'worker_threads';
 
-async function executeModule({ executeModuleUrl, moduleUrl, compilation, page, prerender = false, htmlContents = null, scripts = '[]' }) {
+async function executeModule({ executeModuleUrl, moduleUrl, compilation, page, prerender = false, htmlContents = null, scripts = '[]', request }) {
   const { executeRouteModule } = await import(executeModuleUrl);
-  const data = await executeRouteModule({ moduleUrl, compilation: JSON.parse(compilation), page: JSON.parse(page), prerender, htmlContents, scripts: JSON.parse(scripts) });
+  const data = await executeRouteModule({ moduleUrl, compilation: JSON.parse(compilation), page: JSON.parse(page), prerender, htmlContents, scripts: JSON.parse(scripts), request });
 
   parentPort.postMessage(data);
 }

packages/cli/src/lifecycles/bundle.js

@@ -44,7 +44,7 @@ async function optimizeStaticPages(compilation, plugins) {
   const { scratchDir, outputDir } = compilation.context;
 
   return Promise.all(compilation.graph
-    .filter(page => !page.isSSR || (page.isSSR && page.data.static) || (page.isSSR && compilation.config.prerender))
+    .filter(page => !page.isSSR || (page.isSSR && page.prerender) || (page.isSSR && compilation.config.prerender))
     .map(async (page) => {
       const { route, outputPath } = page;
       const outputDirUrl = new URL(`.${route}`, outputDir);
@@ -189,13 +189,14 @@ async function bundleSsrPages(compilation) {
     const { pagesDir, scratchDir } = compilation.context;
 
     for (const page of compilation.graph) {
-      if (page.isSSR && !page.data.static) {
+      if (page.isSSR && !page.prerender) {
         const { filename, imports, route, template, title } = page;
         const entryFileUrl = new URL(`./_${filename}`, scratchDir);
         const moduleUrl = new URL(`./${filename}`, pagesDir);
+        const request = new Request(moduleUrl); // TODO not really sure how to best no-op this?
         // TODO getTemplate has to be static (for now?)
         // https://github.com/ProjectEvergreen/greenwood/issues/955
-        const data = await executeRouteModule({ moduleUrl, compilation, page, prerender: false, htmlContents: null, scripts: [] });
+        const data = await executeRouteModule({ moduleUrl, compilation, page, prerender: false, htmlContents: null, scripts: [], request });
         let staticHtml = '';
 
         staticHtml = data.template ? data.template : await getPageTemplate(staticHtml, compilation.context, template, []);
@@ -212,7 +213,7 @@ async function bundleSsrPages(compilation) {
             const compilation = JSON.parse('${JSON.stringify(compilation)}');
             const page = JSON.parse('${JSON.stringify(page)}');
             const moduleUrl = '___GWD_ENTRY_FILE_URL=${filename}___';
-            const data = await executeRouteModule({ moduleUrl, compilation, page });
+            const data = await executeRouteModule({ moduleUrl, compilation, page, request });
             let staticHtml = \`${staticHtml}\`;
 
             if (data.body) {

packages/cli/src/lifecycles/graph.js

@@ -1,7 +1,7 @@
 /* eslint-disable complexity, max-depth */
 import fs from 'fs/promises';
 import fm from 'front-matter';
-import { checkResourceExists } from '../lib/resource-utils.js';
+import { checkResourceExists, requestAsObject } from '../lib/resource-utils.js';
 import toc from 'markdown-toc';
 import { Worker } from 'worker_threads';
 
@@ -20,7 +20,8 @@ const generateGraph = async (compilation) => {
         label: 'Index',
         data: {},
         imports: [],
-        resources: []
+        resources: [],
+        prerender: true
       }];
 
       const walkDirectoryForPages = async function(directory, pages = []) {
@@ -46,6 +47,7 @@ const generateGraph = async (compilation) => {
             let imports = [];
             let customData = {};
             let filePath;
+            let prerender = true;
 
             /*
              * check if additional nested directories exist to correctly determine route (minus filename)
@@ -121,14 +123,19 @@ const generateGraph = async (compilation) => {
 
               filePath = route;
 
-              await new Promise((resolve, reject) => {
+              await new Promise(async (resolve, reject) => {
                 const worker = new Worker(new URL('../lib/ssr-route-worker.js', import.meta.url));
+                // TODO "faux" new Request here, a better way?
+                const request = await requestAsObject(new Request(filenameUrl));
 
                 worker.on('message', async (result) => {
+                  prerender = result.prerender;
+
                   if (result.frontmatter) {
                     result.frontmatter.imports = result.frontmatter.imports || [];
                     ssrFrontmatter = result.frontmatter;
                   }
+
                   resolve();
                 });
                 worker.on('error', reject);
@@ -151,7 +158,8 @@ const generateGraph = async (compilation) => {
                       .map((idPart) => {
                         return `${idPart.charAt(0).toUpperCase()}${idPart.substring(1)}`;
                       }).join(' ')
-                  })
+                  }),
+                  request
                 });
               });
 
@@ -190,6 +198,8 @@ const generateGraph = async (compilation) => {
              * route: URL route for a given page on outputFilePath
              * template: page template to use as a base for a generated component
              * title: a default value that can be used for <title></title>
+             * isSSR: if this is a server side route
+             * prerednder: if this should be statically exported
              */
             pages.push({
               data: customData || {},
@@ -208,7 +218,8 @@ const generateGraph = async (compilation) => {
               route,
               template,
               title,
-              isSSR: !isStatic
+              isSSR: !isStatic,
+              prerender
             });
           }
         }

packages/cli/src/lifecycles/prerender.js

@@ -49,7 +49,7 @@ function getPluginInstances (compilation) {
 }
 
 async function preRenderCompilationWorker(compilation, workerPrerender) {
-  const pages = compilation.graph.filter(page => !page.isSSR || (page.isSSR && page.data.static) || (page.isSSR && compilation.config.prerender));
+  const pages = compilation.graph.filter(page => !page.isSSR || (page.isSSR && page.prerender) || (page.isSSR && compilation.config.prerender));
   const { scratchDir } = compilation.context;
   const plugins = getPluginInstances(compilation);
 
@@ -128,7 +128,7 @@ async function preRenderCompilationCustom(compilation, customPrerender) {
 
 async function staticRenderCompilation(compilation) {
   const { scratchDir } = compilation.context;
-  const pages = compilation.graph.filter(page => !page.isSSR || page.isSSR && page.data.static);
+  const pages = compilation.graph.filter(page => !page.isSSR || page.isSSR && page.prerender);
   const plugins = getPluginInstances(compilation);
 
   console.info('pages to generate', `\n ${pages.map(page => page.route).join('\n ')}`);

packages/cli/src/lifecycles/serve.js

@@ -179,7 +179,7 @@ async function getStaticServer(compilation, composable) {
       const matchingRoute = compilation.graph.find(page => page.route === url.pathname);
       const isSPA = compilation.graph.find(page => page.isSPA);
       const { isSSR } = matchingRoute || {};
-      const isStatic = matchingRoute && !isSSR || isSSR && compilation.config.prerender || isSSR && matchingRoute.data.static;
+      const isStatic = matchingRoute && !isSSR || isSSR && compilation.config.prerender || isSSR && matchingRoute.prerender;
 
       if (isSPA || (matchingRoute && isStatic) || url.pathname.split('.').pop() === 'html') {
         const pathname = isSPA
@@ -293,9 +293,8 @@ async function getHybridServer(compilation) {
       const isApiRoute = manifest.apis.has(url.pathname);
       const request = transformKoaRequestIntoStandardRequest(url, ctx.request);
 
-      if (!config.prerender && matchingRoute.isSSR && !matchingRoute.data.static) {
+      if (!config.prerender && matchingRoute.isSSR && !matchingRoute.prerender) {
         const { handler } = await import(new URL(`./__${matchingRoute.filename}`, outputDir));
-        // TODO passing compilation this way too hacky?
         const response = await handler(request, compilation);
 
         ctx.body = Readable.from(response.body);

packages/cli/src/plugins/resource/plugin-api-routes.js

@@ -4,59 +4,9 @@
  *
  */
 import { ResourceInterface } from '../../lib/resource-interface.js';
+import { requestAsObject } from '../../lib/resource-utils.js';
 import { Worker } from 'worker_threads';
 
-// https://stackoverflow.com/questions/57447685/how-can-i-convert-a-request-object-into-a-stringifiable-object-in-javascript
-async function requestAsObject (_request) {
-  if (!_request instanceof Request) {
-    throw Object.assign(
-      new Error(),
-      { name: 'TypeError', message: 'Argument must be a Request object' }
-    );
-  }
-
-  const request = _request.clone();
-  const contentType = request.headers.get('content-type') || '';
-  let headers = Object.fromEntries(request.headers);
-  let format;
-
-  function stringifiableObject (obj) {
-    const filtered = {};
-    for (const key in obj) {
-      if (['boolean', 'number', 'string'].includes(typeof obj[key]) || obj[key] === null) {
-        filtered[key] = obj[key];
-      }
-    }
-    return filtered;
-  }
-
-  if (contentType.includes('application/x-www-form-urlencoded')) {
-    const formData = await request.formData();
-    const params = {};
-
-    for (const entry of formData.entries()) {
-      params[entry[0]] = entry[1];
-    }
-
-    // when using FormData, let Request set the correct headers
-    // or else it will come out as multipart/form-data
-    // for serialization between route workers, leave a special marker for Greenwood
-    // https://stackoverflow.com/a/43521052/417806
-    headers['content-type'] = 'x-greenwood/www-form-urlencoded';
-    format = JSON.stringify(params);
-  } else if (contentType.includes('application/json')) {
-    format = JSON.stringify(await request.json());
-  } else {
-    format = await request.text();
-  }
-
-  return {
-    ...stringifiableObject(request),
-    body: format,
-    headers
-  };
-}
-
 class ApiRoutesResource extends ResourceInterface {
   constructor(compilation, options) {
     super(compilation, options);

packages/cli/src/plugins/resource/plugin-standard-html.js

@@ -14,6 +14,7 @@ import remarkParse from 'remark-parse';
 import remarkRehype from 'remark-rehype';
 import { ResourceInterface } from '../../lib/resource-interface.js';
 import { getUserScripts, getPageTemplate, getAppTemplate } from '../../lib/templating-utils.js';
+import { requestAsObject } from '../../lib/resource-utils.js';
 import unified from 'unified';
 import { Worker } from 'worker_threads';
 
@@ -33,7 +34,7 @@ class StandardHtmlResource extends ResourceInterface {
     return protocol.startsWith('http') && (hasMatchingPageRoute || isSPA);
   }
 
-  async serve(url) {
+  async serve(url, request) {
     const { config, context } = this.compilation;
     const { pagesDir, userWorkspace } = context;
     const { interpolateFrontmatter } = config;
@@ -107,7 +108,7 @@ class StandardHtmlResource extends ResourceInterface {
       const routeModuleLocationUrl = new URL(`./${matchingRoute.filename}`, pagesDir);
       const routeWorkerUrl = this.compilation.config.plugins.find(plugin => plugin.type === 'renderer').provider().executeModuleUrl;
 
-      await new Promise((resolve, reject) => {
+      await new Promise(async (resolve, reject) => {
         const worker = new Worker(new URL('../../lib/ssr-route-worker.js', import.meta.url));
 
         worker.on('message', (result) => {
@@ -146,7 +147,8 @@ class StandardHtmlResource extends ResourceInterface {
           executeModuleUrl: routeWorkerUrl.href,
           moduleUrl: routeModuleLocationUrl.href,
           compilation: JSON.stringify(this.compilation),
-          page: JSON.stringify(matchingRoute)
+          page: JSON.stringify(matchingRoute),
+          request: await requestAsObject(request)
         });
       });
     }

packages/cli/test/cases/build.default.ssr-static-export/src/pages/artists.js

@@ -77,12 +77,13 @@ async function getFrontmatter() {
     ],
     data: {
       author: 'Project Evergreen',
-      date: '01-01-2021',
-      static: true
+      date: '01-01-2021'
     }
   };
 }
 
+export const prerender = true;
+
 export {
   getTemplate,
   getBody,