Fix custom server React resolution with app dir and pages both presen…

…ted (vercel#49805)

If running Next.js in the custom server with both app and pages directories presented, a standalone server needs to serve all traffic and the server running in main thread should be a no-op. (cc @ijjk).

Fixes the problem described here: vercel#49355 (comment).

packages/next/src/server/lib/render-server-standalone.ts

@@ -11,11 +11,13 @@ export const createServerHandler = async ({
   port,
   hostname,
   dir,
+  dev = false,
   minimalMode,
 }: {
   port: number
   hostname: string
   dir: string
+  dev?: boolean
   minimalMode: boolean
 }) => {
   const routerWorker = new Worker(renderServerPath, {
@@ -60,7 +62,7 @@ export const createServerHandler = async ({
   const { port: routerPort } = await routerWorker.initialize({
     dir,
     port,
-    dev: false,
+    dev,
     hostname,
     minimalMode,
     workerType: 'router',

packages/next/src/server/next.ts

@@ -2,21 +2,26 @@ import type { Options as DevServerOptions } from './dev/next-dev-server'
 import type { NodeRequestHandler } from './next-server'
 import type { UrlWithParsedQuery } from 'url'
 import type { NextConfigComplete } from './config-shared'
+import type { IncomingMessage, ServerResponse } from 'http'
+import type { NextParsedUrlQuery, NextUrlWithParsedQuery } from './request-meta'
 
 import './require-hook'
 import './node-polyfill-fetch'
 import './node-polyfill-crypto'
 import { default as Server } from './next-server'
 import * as log from '../build/output/log'
 import loadConfig from './config'
-import { resolve } from 'path'
+import { join, resolve } from 'path'
 import { NON_STANDARD_NODE_ENV } from '../lib/constants'
-import { PHASE_DEVELOPMENT_SERVER } from '../shared/lib/constants'
+import {
+  PHASE_DEVELOPMENT_SERVER,
+  SERVER_DIRECTORY,
+} from '../shared/lib/constants'
 import { PHASE_PRODUCTION_SERVER } from '../shared/lib/constants'
-import { IncomingMessage, ServerResponse } from 'http'
-import { NextUrlWithParsedQuery } from './request-meta'
 import { getTracer } from './lib/trace/tracer'
 import { NextServerSpan } from './lib/trace/constants'
+import { formatUrl } from '../shared/lib/router/utils/format-url'
+import { findDir } from '../lib/find-pages-dir'
 
 let ServerImpl: typeof Server
 
@@ -29,6 +34,7 @@ const getServerImpl = async () => {
 
 export type NextServerOptions = Partial<DevServerOptions> & {
   preloadedConfig?: NextConfigComplete
+  internal_setStandaloneConfig?: boolean
 }
 
 export interface RequestHandler {
@@ -39,11 +45,17 @@ export interface RequestHandler {
   ): Promise<void>
 }
 
+const SYMBOL_SET_STANDALONE_MODE = Symbol('next.set_standalone_mode')
+const SYMBOL_LOAD_CONFIG = Symbol('next.load_config')
+
 export class NextServer {
   private serverPromise?: Promise<Server>
   private server?: Server
   private reqHandlerPromise?: Promise<NodeRequestHandler>
   private preparedAssetPrefix?: string
+
+  private standaloneMode?: boolean
+
   public options: NextServerOptions
 
   constructor(options: NextServerOptions) {
@@ -58,6 +70,10 @@ export class NextServer {
     return this.options.port
   }
 
+  [SYMBOL_SET_STANDALONE_MODE]() {
+    this.standaloneMode = true
+  }
+
   getRequestHandler(): RequestHandler {
     return async (
       req: IncomingMessage,
@@ -127,6 +143,8 @@ export class NextServer {
   async prepare() {
     const server = await this.getServer()
 
+    if (this.standaloneMode) return
+
     // We shouldn't prepare the server in production,
     // because this code won't be executed when deployed
     if (this.options.dev) {
@@ -151,7 +169,7 @@ export class NextServer {
     return server
   }
 
-  private async loadConfig() {
+  private async [SYMBOL_LOAD_CONFIG]() {
     return (
       this.options.preloadedConfig ||
       loadConfig(
@@ -166,7 +184,11 @@ export class NextServer {
 
   private async getServer() {
     if (!this.serverPromise) {
-      this.serverPromise = this.loadConfig().then(async (conf) => {
+      this.serverPromise = this[SYMBOL_LOAD_CONFIG]().then(async (conf) => {
+        if (this.standaloneMode) {
+          process.env.__NEXT_PRIVATE_STANDALONE_CONFIG = JSON.stringify(conf)
+        }
+
         if (!this.options.dev) {
           if (conf.output === 'standalone') {
             if (!process.env.__NEXT_PRIVATE_STANDALONE_CONFIG) {
@@ -181,17 +203,6 @@ export class NextServer {
           }
         }
 
-        if (this.options.customServer !== false) {
-          // When running as a custom server with app dir, we must set this env
-          // to correctly alias the React versions.
-          if (conf.experimental.appDir) {
-            process.env.__NEXT_PRIVATE_PREBUNDLED_REACT = conf.experimental
-              .serverActions
-              ? 'experimental'
-              : 'next'
-          }
-        }
-
         this.server = await this.createServer({
           ...this.options,
           conf,
@@ -250,6 +261,107 @@ function createServer(options: NextServerOptions): NextServer {
     )
   }
 
+  if (options.customServer !== false) {
+    // If the `app` dir exists, we'll need to run the standalone server to have
+    // both types of renderers (pages, app) running in separated processes,
+    // instead of having the Next server only.
+    let shouldUseStandaloneMode = false
+
+    const dir = resolve(options.dir || '.')
+    const server = new NextServer(options)
+
+    const { createServerHandler } =
+      require('./lib/render-server-standalone') as typeof import('./lib/render-server-standalone')
+
+    let handlerPromise: Promise<ReturnType<typeof createServerHandler>>
+
+    return new Proxy(
+      {},
+      {
+        get: function (_, propKey) {
+          switch (propKey) {
+            case 'prepare':
+              return async () => {
+                // Instead of running Next Server's `prepare`, we'll run the loadConfig first to determine
+                // if we should run the standalone server or not.
+                const config = await server[SYMBOL_LOAD_CONFIG]()
+
+                // Check if the application has app dir or not. This depends on the mode (dev or prod).
+                // For dev, `app` should be existing in the sources and for prod it should be existing
+                // in the dist folder.
+                const distDir =
+                  process.env.NEXT_RUNTIME === 'edge'
+                    ? config.distDir
+                    : join(dir, config.distDir)
+                const serverDistDir = join(distDir, SERVER_DIRECTORY)
+                const hasAppDir = !!findDir(
+                  options.dev ? dir : serverDistDir,
+                  'app'
+                )
+
+                if (hasAppDir) {
+                  shouldUseStandaloneMode = true
+                  server[SYMBOL_SET_STANDALONE_MODE]()
+
+                  handlerPromise =
+                    handlerPromise ||
+                    createServerHandler({
+                      port: options.port || 3000,
+                      dev: options.dev,
+                      dir,
+                      hostname: options.hostname || 'localhost',
+                      minimalMode: false,
+                    })
+                } else {
+                  return server.prepare()
+                }
+              }
+            case 'getRequestHandler': {
+              return () => {
+                let handler: RequestHandler
+                return async (req: IncomingMessage, res: ServerResponse) => {
+                  if (shouldUseStandaloneMode) {
+                    const standaloneHandler = await handlerPromise
+                    return standaloneHandler(req, res)
+                  }
+                  handler = handler || server.getRequestHandler()
+                  return handler(req, res)
+                }
+              }
+            }
+            case 'render': {
+              return async (
+                req: IncomingMessage,
+                res: ServerResponse,
+                pathname: string,
+                query?: NextParsedUrlQuery,
+                parsedUrl?: NextUrlWithParsedQuery
+              ) => {
+                if (shouldUseStandaloneMode) {
+                  const handler = await handlerPromise
+                  req.url = formatUrl({
+                    ...parsedUrl,
+                    pathname,
+                    query,
+                  })
+                  return handler(req, res)
+                }
+
+                return server.render(req, res, pathname, query, parsedUrl)
+              }
+            }
+            default: {
+              const method = server[propKey as keyof NextServer]
+              if (typeof method === 'function') {
+                return method.bind(server)
+              }
+            }
+          }
+        },
+      }
+    ) as any
+  }
+
   return new NextServer(options)
 }
 

test/production/custom-server/app/1/page.js

@@ -0,0 +1,5 @@
+import { version } from 'react'
+
+export default function Page() {
+  return <div>app: {version}</div>
+}

test/production/custom-server/app/layout.js

@@ -0,0 +1,12 @@
+export const metadata = {
+  title: 'Next.js',
+  description: 'Generated by Next.js',
+}
+
+export default function RootLayout({ children }) {
+  return (
+    <html lang="en">
+      <body>{children}</body>
+    </html>
+  )
+}

test/production/custom-server/custom-server.test.ts

@@ -11,5 +11,18 @@ createNextDescribe(
       const $ = await next.render$(`/${page}`)
       expect($('p').text()).toBe(`Page ${page}`)
     })
+
+    describe('with app dir', () => {
+      it('should render app with react canary', async () => {
+        const $ = await next.render$(`/1`)
+        expect($('body').text()).toMatch(/app: .+-canary/)
+      })
+
+      it('should render pages with react stable', async () => {
+        const $ = await next.render$(`/2`)
+        expect($('body').text()).toMatch(/pages:/)
+        expect($('body').text()).not.toMatch(/canary/)
+      })
+    })
   }
 )

test/production/custom-server/pages/2.js

@@ -0,0 +1,5 @@
+import { version } from 'react'
+
+export default function Page() {
+  return <div>pages: {version}</div>
+}