Memory improvements to static workers

packages/next/src/build/index.ts

@@ -11,6 +11,7 @@ import chalk from 'next/dist/compiled/chalk'
 import crypto from 'crypto'
 import { isMatch, makeRe } from 'next/dist/compiled/micromatch'
 import { promises, writeFileSync } from 'fs'
+import os from 'os'
 import { Worker as JestWorker } from 'next/dist/compiled/jest-worker'
 import { Worker } from '../lib/worker'
 import devalue from 'next/dist/compiled/devalue'
@@ -1239,6 +1240,17 @@ export default async function build(
 
       process.env.NEXT_PHASE = PHASE_PRODUCTION_BUILD
 
+      // We limit the number of workers used based on the number of CPUs and
+      // the current available memory. This is to prevent the system from
+      // running out of memory as well as maximize speed. We assume that
+      // each worker will consume ~1GB of memory in a production build.
+      // For example, if the system has 10 CPU cores and 8GB of remaining memory
+      // we will use 8 workers.
+      const numWorkers = Math.min(
+        config.experimental.cpus || 1,
+        Math.floor(os.freemem() / 1e9)
+      )
+
       const staticWorkers = new Worker(staticWorker, {
         timeout: timeout * 1000,
         onRestart: (method, [arg], attempts) => {
@@ -1270,7 +1282,7 @@ export default async function build(
             infoPrinted = true
           }
         },
-        numWorkers: config.experimental.cpus,
+        numWorkers,
         enableWorkerThreads: config.experimental.workerThreads,
         computeWorkerKey(method, ...args) {
           if (method === 'exportPage') {

packages/next/src/build/webpack-build.ts → packages/next/src/build/webpack-build/impl.ts

@@ -1,37 +1,36 @@
 import type { webpack } from 'next/dist/compiled/webpack/webpack'
 import chalk from 'next/dist/compiled/chalk'
-import formatWebpackMessages from '../client/dev/error-overlay/format-webpack-messages'
-import { nonNullable } from '../lib/non-nullable'
+import formatWebpackMessages from '../../client/dev/error-overlay/format-webpack-messages'
+import { nonNullable } from '../../lib/non-nullable'
 import {
   COMPILER_NAMES,
   CLIENT_STATIC_FILES_RUNTIME_MAIN_APP,
   APP_CLIENT_INTERNALS,
   PHASE_PRODUCTION_BUILD,
   COMPILER_INDEXES,
-} from '../shared/lib/constants'
-import { runCompiler } from './compiler'
-import * as Log from './output/log'
-import getBaseWebpackConfig, { loadProjectInfo } from './webpack-config'
-import { NextError } from '../lib/is-error'
-import { TelemetryPlugin } from './webpack/plugins/telemetry-plugin'
+} from '../../shared/lib/constants'
+import { runCompiler } from '../compiler'
+import * as Log from '../output/log'
+import getBaseWebpackConfig, { loadProjectInfo } from '../webpack-config'
+import { NextError } from '../../lib/is-error'
+import { TelemetryPlugin } from '../webpack/plugins/telemetry-plugin'
 import {
   NextBuildContext,
   resumePluginState,
   getPluginState,
-} from './build-context'
-import { createEntrypoints } from './entries'
-import loadConfig from '../server/config'
-import { trace } from '../trace'
-import { WEBPACK_LAYERS } from '../lib/constants'
+} from '../build-context'
+import { createEntrypoints } from '../entries'
+import loadConfig from '../../server/config'
+import { trace } from '../../trace'
+import { WEBPACK_LAYERS } from '../../lib/constants'
 import {
   TraceEntryPointsPlugin,
   TurbotraceContext,
-} from './webpack/plugins/next-trace-entrypoints-plugin'
-import { UnwrapPromise } from '../lib/coalesced-function'
-import * as pagesPluginModule from './webpack/plugins/pages-manifest-plugin'
-import { Worker } from 'next/dist/compiled/jest-worker'
+} from '../webpack/plugins/next-trace-entrypoints-plugin'
+import { UnwrapPromise } from '../../lib/coalesced-function'
+import * as pagesPluginModule from '../webpack/plugins/pages-manifest-plugin'
+
 import origDebug from 'next/dist/compiled/debug'
-import { ChildProcess } from 'child_process'
 
 const debug = origDebug('next:build:webpack-build')
 
@@ -57,7 +56,7 @@ function isTraceEntryPointsPlugin(
   return plugin instanceof TraceEntryPointsPlugin
 }
 
-async function webpackBuildImpl(
+export async function webpackBuildImpl(
   compilerName?: keyof typeof COMPILER_INDEXES
 ): Promise<{
   duration: number
@@ -367,108 +366,3 @@ export async function workerMain(workerData: {
   }
   return result
 }
-
-async function webpackBuildWithWorker() {
-  const {
-    config,
-    telemetryPlugin,
-    buildSpinner,
-    nextBuildSpan,
-    ...prunedBuildContext
-  } = NextBuildContext
-
-  const getWorker = (compilerName: string) => {
-    const _worker = new Worker(__filename, {
-      exposedMethods: ['workerMain'],
-      numWorkers: 1,
-      maxRetries: 0,
-      forkOptions: {
-        env: {
-          ...process.env,
-          NEXT_PRIVATE_BUILD_WORKER: '1',
-        },
-      },
-    }) as Worker & { workerMain: typeof workerMain }
-    _worker.getStderr().pipe(process.stderr)
-    _worker.getStdout().pipe(process.stdout)
-
-    for (const worker of ((_worker as any)._workerPool?._workers || []) as {
-      _child: ChildProcess
-    }[]) {
-      worker._child.on('exit', (code, signal) => {
-        if (code || signal) {
-          console.error(
-            `Compiler ${compilerName} unexpectedly exited with code: ${code} and signal: ${signal}`
-          )
-        }
-      })
-    }
-
-    return _worker
-  }
-
-  const combinedResult = {
-    duration: 0,
-    turbotraceContext: {} as TurbotraceContext,
-  }
-  // order matters here
-  const ORDERED_COMPILER_NAMES = [
-    'server',
-    'edge-server',
-    'client',
-  ] as (keyof typeof COMPILER_INDEXES)[]
-
-  for (const compilerName of ORDERED_COMPILER_NAMES) {
-    const worker = getWorker(compilerName)
-
-    const curResult = await worker.workerMain({
-      buildContext: prunedBuildContext,
-      compilerName,
-    })
-    // destroy worker so it's not sticking around using memory
-    await worker.end()
-
-    // Update plugin state
-    prunedBuildContext.pluginState = curResult.pluginState
-
-    prunedBuildContext.serializedPagesManifestEntries = {
-      edgeServerAppPaths:
-        curResult.serializedPagesManifestEntries?.edgeServerAppPaths,
-      edgeServerPages:
-        curResult.serializedPagesManifestEntries?.edgeServerPages,
-      nodeServerAppPaths:
-        curResult.serializedPagesManifestEntries?.nodeServerAppPaths,
-      nodeServerPages:
-        curResult.serializedPagesManifestEntries?.nodeServerPages,
-    }
-
-    combinedResult.duration += curResult.duration
-
-    if (curResult.turbotraceContext?.entriesTrace) {
-      combinedResult.turbotraceContext = curResult.turbotraceContext
-
-      const { entryNameMap } = combinedResult.turbotraceContext.entriesTrace!
-      if (entryNameMap) {
-        combinedResult.turbotraceContext.entriesTrace!.entryNameMap = new Map(
-          entryNameMap
-        )
-      }
-    }
-  }
-  buildSpinner?.stopAndPersist()
-  Log.info('Compiled successfully')
-
-  return combinedResult
-}
-
-export async function webpackBuild() {
-  const config = NextBuildContext.config!
-
-  if (config.experimental.webpackBuildWorker) {
-    debug('using separate compiler workers')
-    return await webpackBuildWithWorker()
-  } else {
-    debug('building all compilers in same process')
-    return await webpackBuildImpl()
-  }
-}

packages/next/src/build/webpack-build/index.ts

@@ -0,0 +1,116 @@
+import { COMPILER_INDEXES } from '../../shared/lib/constants'
+import * as Log from '../output/log'
+import { NextBuildContext } from '../build-context'
+import type { TurbotraceContext } from '../webpack/plugins/next-trace-entrypoints-plugin'
+import { Worker } from 'next/dist/compiled/jest-worker'
+import origDebug from 'next/dist/compiled/debug'
+import { ChildProcess } from 'child_process'
+import path from 'path'
+
+const debug = origDebug('next:build:webpack-build')
+
+async function webpackBuildWithWorker() {
+  const {
+    config,
+    telemetryPlugin,
+    buildSpinner,
+    nextBuildSpan,
+    ...prunedBuildContext
+  } = NextBuildContext
+
+  const getWorker = (compilerName: string) => {
+    const _worker = new Worker(path.join(__dirname, 'impl.js'), {
+      exposedMethods: ['workerMain'],
+      numWorkers: 1,
+      maxRetries: 0,
+      forkOptions: {
+        env: {
+          ...process.env,
+          NEXT_PRIVATE_BUILD_WORKER: '1',
+        },
+      },
+    }) as Worker & typeof import('./impl')
+    _worker.getStderr().pipe(process.stderr)
+    _worker.getStdout().pipe(process.stdout)
+
+    for (const worker of ((_worker as any)._workerPool?._workers || []) as {
+      _child: ChildProcess
+    }[]) {
+      worker._child.on('exit', (code, signal) => {
+        if (code || signal) {
+          console.error(
+            `Compiler ${compilerName} unexpectedly exited with code: ${code} and signal: ${signal}`
+          )
+        }
+      })
+    }
+
+    return _worker
+  }
+
+  const combinedResult = {
+    duration: 0,
+    turbotraceContext: {} as TurbotraceContext,
+  }
+  // order matters here
+  const ORDERED_COMPILER_NAMES = [
+    'server',
+    'edge-server',
+    'client',
+  ] as (keyof typeof COMPILER_INDEXES)[]
+
+  for (const compilerName of ORDERED_COMPILER_NAMES) {
+    const worker = getWorker(compilerName)
+
+    const curResult = await worker.workerMain({
+      buildContext: prunedBuildContext,
+      compilerName,
+    })
+    // destroy worker so it's not sticking around using memory
+    await worker.end()
+
+    // Update plugin state
+    prunedBuildContext.pluginState = curResult.pluginState
+
+    prunedBuildContext.serializedPagesManifestEntries = {
+      edgeServerAppPaths:
+        curResult.serializedPagesManifestEntries?.edgeServerAppPaths,
+      edgeServerPages:
+        curResult.serializedPagesManifestEntries?.edgeServerPages,
+      nodeServerAppPaths:
+        curResult.serializedPagesManifestEntries?.nodeServerAppPaths,
+      nodeServerPages:
+        curResult.serializedPagesManifestEntries?.nodeServerPages,
+    }
+
+    combinedResult.duration += curResult.duration
+
+    if (curResult.turbotraceContext?.entriesTrace) {
+      combinedResult.turbotraceContext = curResult.turbotraceContext
+
+      const { entryNameMap } = combinedResult.turbotraceContext.entriesTrace!
+      if (entryNameMap) {
+        combinedResult.turbotraceContext.entriesTrace!.entryNameMap = new Map(
+          entryNameMap
+        )
+      }
+    }
+  }
+  buildSpinner?.stopAndPersist()
+  Log.info('Compiled successfully')
+
+  return combinedResult
+}
+
+export async function webpackBuild() {
+  const config = NextBuildContext.config!
+
+  if (config.experimental.webpackBuildWorker) {
+    debug('using separate compiler workers')
+    return await webpackBuildWithWorker()
+  } else {
+    debug('building all compilers in same process')
+    const webpackBuildImpl = require('./impl').webpackBuildImpl
+    return await webpackBuildImpl()
+  }
+}