-
Notifications
You must be signed in to change notification settings - Fork 27.3k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Memory improvements to static workers (#47823)
Continue the work to improve #32314. This PR splits `webpack-build` into an `index` module and an `impl` module. The `index` module invokes `impl` based on that if it should use workers or in the same process. Another notable change is that I modified the number of static workers, to make it also based on remaining memory. I did some benchmarks and found that using the number of CPUs might not be ideal. After the build process (if `webpackBuildWorker` **isn't** enabled), the memory usage was already very high. While the memory usage of the main process didn't increase in the static optimization step, the overall number went high again because we created 10 static workers:  As of today, a MacBook Pro has 10 CPU cores and 16 GB memory. The idle memory of the system might be 8 GB, and the remaining memory when starting the static workers might be 4~6 GB only. Here's a benchmark of `next build` on `vercel-site`:  The memory dropped there quickly because it has `webpackBuildWorker` enabled, but then it increased again with static workers (note that we already have `cpus: Math.round(require('node:os').cpus().length / 2) || 1` configured in that repo). It makes more sense to limit that inside Next.js natively.
- Loading branch information
Showing
3 changed files
with
151 additions
and
125 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,116 @@ | ||
| import { COMPILER_INDEXES } from '../../shared/lib/constants' | ||
| import * as Log from '../output/log' | ||
| import { NextBuildContext } from '../build-context' | ||
| import type { TurbotraceContext } from '../webpack/plugins/next-trace-entrypoints-plugin' | ||
| import { Worker } from 'next/dist/compiled/jest-worker' | ||
| import origDebug from 'next/dist/compiled/debug' | ||
| import { ChildProcess } from 'child_process' | ||
| import path from 'path' | ||
|
|
||
| const debug = origDebug('next:build:webpack-build') | ||
|
|
||
| async function webpackBuildWithWorker() { | ||
| const { | ||
| config, | ||
| telemetryPlugin, | ||
| buildSpinner, | ||
| nextBuildSpan, | ||
| ...prunedBuildContext | ||
| } = NextBuildContext | ||
|
|
||
| const getWorker = (compilerName: string) => { | ||
| const _worker = new Worker(path.join(__dirname, 'impl.js'), { | ||
| exposedMethods: ['workerMain'], | ||
| numWorkers: 1, | ||
| maxRetries: 0, | ||
| forkOptions: { | ||
| env: { | ||
| ...process.env, | ||
| NEXT_PRIVATE_BUILD_WORKER: '1', | ||
| }, | ||
| }, | ||
| }) as Worker & typeof import('./impl') | ||
| _worker.getStderr().pipe(process.stderr) | ||
| _worker.getStdout().pipe(process.stdout) | ||
|
|
||
| for (const worker of ((_worker as any)._workerPool?._workers || []) as { | ||
| _child: ChildProcess | ||
| }[]) { | ||
| worker._child.on('exit', (code, signal) => { | ||
| if (code || signal) { | ||
| console.error( | ||
| `Compiler ${compilerName} unexpectedly exited with code: ${code} and signal: ${signal}` | ||
| ) | ||
| } | ||
| }) | ||
| } | ||
|
|
||
| return _worker | ||
| } | ||
|
|
||
| const combinedResult = { | ||
| duration: 0, | ||
| turbotraceContext: {} as TurbotraceContext, | ||
| } | ||
| // order matters here | ||
| const ORDERED_COMPILER_NAMES = [ | ||
| 'server', | ||
| 'edge-server', | ||
| 'client', | ||
| ] as (keyof typeof COMPILER_INDEXES)[] | ||
|
|
||
| for (const compilerName of ORDERED_COMPILER_NAMES) { | ||
| const worker = getWorker(compilerName) | ||
|
|
||
| const curResult = await worker.workerMain({ | ||
| buildContext: prunedBuildContext, | ||
| compilerName, | ||
| }) | ||
| // destroy worker so it's not sticking around using memory | ||
| await worker.end() | ||
|
|
||
| // Update plugin state | ||
| prunedBuildContext.pluginState = curResult.pluginState | ||
|
|
||
| prunedBuildContext.serializedPagesManifestEntries = { | ||
| edgeServerAppPaths: | ||
| curResult.serializedPagesManifestEntries?.edgeServerAppPaths, | ||
| edgeServerPages: | ||
| curResult.serializedPagesManifestEntries?.edgeServerPages, | ||
| nodeServerAppPaths: | ||
| curResult.serializedPagesManifestEntries?.nodeServerAppPaths, | ||
| nodeServerPages: | ||
| curResult.serializedPagesManifestEntries?.nodeServerPages, | ||
| } | ||
|
|
||
| combinedResult.duration += curResult.duration | ||
|
|
||
| if (curResult.turbotraceContext?.entriesTrace) { | ||
| combinedResult.turbotraceContext = curResult.turbotraceContext | ||
|
|
||
| const { entryNameMap } = combinedResult.turbotraceContext.entriesTrace! | ||
| if (entryNameMap) { | ||
| combinedResult.turbotraceContext.entriesTrace!.entryNameMap = new Map( | ||
| entryNameMap | ||
| ) | ||
| } | ||
| } | ||
| } | ||
| buildSpinner?.stopAndPersist() | ||
| Log.info('Compiled successfully') | ||
|
|
||
| return combinedResult | ||
| } | ||
|
|
||
| export async function webpackBuild() { | ||
| const config = NextBuildContext.config! | ||
|
|
||
| if (config.experimental.webpackBuildWorker) { | ||
| debug('using separate compiler workers') | ||
| return await webpackBuildWithWorker() | ||
| } else { | ||
| debug('building all compilers in same process') | ||
| const webpackBuildImpl = require('./impl').webpackBuildImpl | ||
| return await webpackBuildImpl() | ||
| } | ||
| } |