Server Rendering & Streaming (#8561)

Co-authored-by: Tobbe Lundberg <tobbe@tlundberg.com>

packages/cli/src/commands/buildHandler.js

@@ -11,6 +11,7 @@ import { buildApi } from '@redwoodjs/internal/dist/build/api'
 import { loadAndValidateSdls } from '@redwoodjs/internal/dist/validateSchema'
 import { detectPrerenderRoutes } from '@redwoodjs/prerender/detection'
 import { timedTelemetry } from '@redwoodjs/telemetry'
+import { buildFeServer } from '@redwoodjs/vite'
 
 import { getPaths, getConfig } from '../lib'
 import { generatePrismaCommand } from '../lib/generatePrismaClient'
@@ -105,19 +106,28 @@ export const handler = async ({
       title: 'Building Web...',
       task: async () => {
         if (getConfig().web.bundler !== 'webpack') {
-          // @NOTE: we're using the vite build command here, instead of the
-          // buildWeb function directly because we want the process.cwd to be
-          // the web directory, not the root of the project.
-          // This is important for postcss/tailwind to work correctly
-          // Having a separate binary lets us contain the change of cwd to that
-          // process only. If we changed cwd here, or in the buildWeb function,
-          // it could affect other things that run in parallel while building.
-          // We don't have any parallel tasks right now, but someone might add
-          // one in the future as a performance optimization.
-          await execa(`yarn rw-vite-build --webDir="${rwjsPaths.web.base}"`, {
-            stdio: verbose ? 'inherit' : 'pipe',
-            shell: true,
-          })
+          if (!getConfig().experimental?.streamingSsr?.enabled) {
+            // @NOTE: we're using the vite build command here, instead of the
+            // buildWeb function directly because we want the process.cwd to be
+            // the web directory, not the root of the project.
+            // This is important for postcss/tailwind to work correctly
+            // Having a separate binary lets us contain the change of cwd to that
+            // process only. If we changed cwd here, or in the buildWeb function,
+            // it could affect other things that run in parallel while building.
+            // We don't have any parallel tasks right now, but someone might add
+            // one in the future as a performance optimization.
+            await execa(`yarn rw-vite-build --webDir="${rwjsPaths.web.base}"`, {
+              stdio: verbose ? 'inherit' : 'pipe',
+              shell: true,
+            })
+          } else {
+            // TODO (STREAMING) we need to contain this in a separate binary
+            process.chdir(rwjsPaths.web.base)
+
+            // TODO (STREAMING) we need to use a binary here, so the the cwd is correct
+            // Should merge this with the existing rw-vite-build binary
+            await buildFeServer({ verbose })
+          }
         } else {
           await execa(
             `yarn cross-env NODE_ENV=production webpack --config ${require.resolve(

packages/cli/src/commands/serve.js

@@ -18,6 +18,22 @@ function hasExperimentalServerFile() {
   return fs.existsSync(serverFilePath)
 }
 
+const streamServerErrorHandler = () => {
+  console.error('⚠️  Experimental Render Mode ~ Cannot serve the web side ⚠️')
+  console.log('~'.repeat(50))
+  console.log()
+  console.log()
+  console.log('You can run the new frontend server with: `yarn rw-serve-fe`')
+  console.log('You can run the api server with: yarn rw serve api')
+  console.log()
+  console.log()
+  console.log('~'.repeat(50))
+
+  throw new Error(
+    'You will need to run the FE server and API server separately.'
+  )
+}
+
 export const builder = async (yargs) => {
   yargs
     .usage('usage: $0 <side>')
@@ -41,6 +57,11 @@ export const builder = async (yargs) => {
           socket: argv.socket,
         })
 
+        if (getConfig().experimental?.streamingSsr?.enabled) {
+          streamServerErrorHandler()
+          return
+        }
+
         // Run the experimental server file, if it exists, with web side also
         if (hasExperimentalServerFile()) {
           console.log(
@@ -145,6 +166,11 @@ export const builder = async (yargs) => {
           apiHost: argv.apiHost,
         })
 
+        if (getConfig().experimental?.streamingSsr?.enabled) {
+          streamServerErrorHandler()
+          return
+        }
+
         const { webServerHandler } = await import('./serveHandler.js')
         await webServerHandler(argv)
       },

packages/core/package.json

@@ -16,9 +16,12 @@
     "redwood": "./dist/bins/redwood.js",
     "rw": "./dist/bins/redwood.js",
     "rw-api-server-watch": "./dist/bins/rw-api-server-watch.js",
+    "rw-dev-fe": "./dist/bins/rw-dev-fe.js",
     "rw-gen": "./dist/bins/rw-gen.js",
     "rw-gen-watch": "./dist/bins/rw-gen-watch.js",
     "rw-log-formatter": "./dist/bins/rw-log-formatter.js",
+    "rw-serve-api": "./dist/bins/rw-serve-api.js",
+    "rw-serve-fe": "./dist/bins/rw-serve-fe.js",
     "rwfw": "./dist/bins/rwfw.js"
   },
   "files": [

packages/core/src/bins/rw-dev-fe.ts

@@ -0,0 +1,10 @@
+#!/usr/bin/env node
+import { createRequire } from 'module'
+
+const requireFromRwVite = createRequire(
+  require.resolve('@redwoodjs/vite/package.json')
+)
+
+const bins = requireFromRwVite('./package.json')['bin']
+
+requireFromRwVite(bins['rw-dev-fe'])

packages/core/src/bins/rw-serve-api.ts

@@ -0,0 +1,10 @@
+#!/usr/bin/env node
+import { createRequire } from 'module'
+
+const requireFromApiServer = createRequire(
+  require.resolve('@redwoodjs/api-server/package.json')
+)
+
+const bins = requireFromApiServer('./package.json')['bin']
+
+requireFromApiServer(bins['rw-serve-api'])

packages/core/src/bins/rw-serve-fe.ts

@@ -0,0 +1,10 @@
+#!/usr/bin/env node
+import { createRequire } from 'module'
+
+const requireFromRwVite = createRequire(
+  require.resolve('@redwoodjs/vite/package.json')
+)
+
+const bins = requireFromRwVite('./package.json')['bin']
+
+requireFromRwVite(bins['rw-serve-fe'])

packages/internal/package.json

@@ -43,6 +43,7 @@
     "@graphql-codegen/typescript-resolvers": "3.2.1",
     "@redwoodjs/graphql-server": "5.0.0",
     "@redwoodjs/project-config": "5.0.0",
+    "@redwoodjs/router": "5.0.0",
     "@sdl-codegen/node": "0.0.10",
     "babel-plugin-graphql-tag": "3.3.0",
     "babel-plugin-polyfill-corejs3": "0.8.1",

packages/internal/src/build/babel/api.ts

@@ -228,3 +228,27 @@ export const prebuildApiFile = (
   })
   return result
 }
+
+// TODO (STREAMING) I changed the prebuildApiFile function in https://github.com/redwoodjs/redwood/pull/7672/files
+// but we had to revert. For this branch temporarily, I'm going to add a new function
+// This is used in building routeHooks
+export const transformWithBabel = (
+  srcPath: string,
+  plugins: TransformOptions['plugins']
+) => {
+  const code = fs.readFileSync(srcPath, 'utf-8')
+  const defaultOptions = getApiSideDefaultBabelConfig()
+
+  const result = transform(code, {
+    ...defaultOptions,
+    cwd: getPaths().api.base,
+    filename: srcPath,
+    // we need inline sourcemaps at this level
+    // because this file will eventually be fed to esbuild
+    // when esbuild finds an inline sourcemap, it tries to "combine" it
+    // so the final sourcemap (the one that esbuild generates) combines both mappings
+    sourceMaps: 'inline',
+    plugins,
+  })
+  return result
+}

packages/internal/src/build/babel/common.ts

@@ -6,6 +6,8 @@ import { parseConfigFileTextToJson } from 'typescript'
 
 import { getPaths } from '@redwoodjs/project-config'
 
+import { getWebSideBabelPlugins } from './web'
+
 const pkgJson = require('../../../package.json')
 
 export interface RegisterHookOptions {
@@ -69,6 +71,28 @@ export const getCommonPlugins = () => {
   ]
 }
 
+// TODO (STREAMING) double check this, think about it more carefully please!
+// It's related to yarn workspaces to be or not to be
+export const getRouteHookBabelPlugins = () => {
+  return [
+    ...getWebSideBabelPlugins({
+      forVite: true,
+    }),
+    [
+      'babel-plugin-module-resolver',
+      {
+        alias: {
+          'api/src': './src',
+        },
+        root: [getPaths().api.base],
+        cwd: 'packagejson',
+        loglevel: 'silent', // to silence the unnecessary warnings
+      },
+      'rwjs-api-module-resolver',
+    ],
+  ]
+}
+
 /**
  * Finds, reads and parses the [ts|js]config.json file
  * @returns The config object

packages/internal/src/build/babel/web.ts

@@ -37,6 +37,7 @@ export const getWebSideBabelPlugins = (
             forJest ? rwjsPaths.web.src : './src',
           // adds the paths from [ts|js]config.json to the module resolver
           ...getPathsFromConfig(tsConfigs.web),
+          $api: rwjsPaths.api.base,
         },
         root: [rwjsPaths.web.base],
         cwd: 'packagejson',

packages/internal/src/files.ts

@@ -96,6 +96,13 @@ export const findApiDistFunctions = (cwd: string = getPaths().api.base) => {
   })
 }
 
+export const findRouteHooksSrc = (cwd: string = getPaths().web.src) => {
+  return fg.sync('**/*.routeHooks.{js,ts,tsx,jsx}', {
+    absolute: true,
+    cwd,
+  })
+}
+
 export const findPrerenderedHtml = (cwd = getPaths().web.dist) =>
   fg.sync('**/*.html', { cwd, ignore: ['200.html', '404.html'] })
 

packages/internal/src/generate/templates/api-globImports.d.ts.template

@@ -2,3 +2,8 @@ declare module 'src/services/**/*.{js,ts}'
 declare module 'src/directives/**/*.{js,ts}'
 declare module 'src/graphql/**/*.sdl.{js,ts}'
 declare module 'src/subscriptions/**/*.{js,ts}'
+
+
+declare module 'api/src/services/**/*.{js,ts}'
+declare module 'api/src/directives/**/*.{js,ts}'
+declare module 'api/src/graphql/**/*.sdl.{js,ts}'

packages/internal/src/routes.ts

@@ -1,6 +1,9 @@
+import path from 'path'
+
 import chalk from 'chalk'
 
-import { getPaths } from '@redwoodjs/project-config'
+import { getPaths, getRouteHookForPage } from '@redwoodjs/project-config'
+import { getRouteRegexAndParams } from '@redwoodjs/router'
 
 // Circular dependency when trying to use the standard import
 const { getProject } = require('@redwoodjs/structure/dist/index')
@@ -63,3 +66,48 @@ export function warningForDuplicateRoutes() {
   }
   return message.trimEnd()
 }
+
+export interface RouteSpec {
+  name: string
+  path: string
+  hasParams: boolean
+  id: string
+  isNotFound: boolean
+  filePath: string | undefined
+  relativeFilePath: string | undefined
+  routeHooks: string | undefined | null
+  matchRegexString: string | null
+  redirect: { to: string; permanent: boolean } | null
+  renderMode: 'stream' | 'html'
+}
+
+export const getProjectRoutes = (): RouteSpec[] => {
+  const rwProject = getProject(getPaths().base)
+  const routes = rwProject.getRouter().routes
+
+  return routes.map((route: any) => {
+    const { matchRegexString, routeParams } = route.isNotFound
+      ? { matchRegexString: null, routeParams: null }
+      : getRouteRegexAndParams(route.path)
+
+    return {
+      name: route.isNotFound ? 'NotFoundPage' : route.name,
+      path: route.isNotFound ? 'notfound' : route.path,
+      hasParams: route.hasParameters,
+      id: route.id,
+      isNotFound: route.isNotFound,
+      filePath: route.page?.filePath,
+      relativeFilePath: route.page?.filePath
+        ? path.relative(getPaths().web.src, route.page?.filePath)
+        : undefined,
+      routeHooks: getRouteHookForPage(route.page?.filePath),
+      renderMode: route.renderMode,
+      matchRegexString: matchRegexString,
+      paramNames: routeParams,
+      // TODO (STREAMING) deal with permanent/temp later
+      redirect: route.redirect
+        ? { to: route.redirect, permanent: false }
+        : null,
+    }
+  })
+}

packages/internal/tsconfig.json

@@ -10,5 +10,6 @@
   "references": [
     { "path": "../graphql-server" }, // ODD, but we do this so we dont have to have internal as a runtime dependency
     { "path": "../project-config" },
+    { "path": "../router" },
   ]
 }