diff --git a/packages/next/build/index.ts b/packages/next/build/index.ts index 444ab05e95229..2a2179e519b09 100644 --- a/packages/next/build/index.ts +++ b/packages/next/build/index.ts @@ -1447,32 +1447,36 @@ export default async function build( } const root = path.parse(dir).root - const serverResult = await nodeFileTrace( - [require.resolve('next/dist/server/next-server')], - { - base: root, - processCwd: dir, - ignore: [ - '**/next/dist/pages/**/*', - '**/next/dist/compiled/webpack/(bundle4|bundle5).js', - '**/node_modules/webpack5/**/*', - '**/next/dist/server/lib/squoosh/**/*.wasm', - ...(ciEnvironment.hasNextSupport - ? [ - // only ignore image-optimizer code when - // this is being handled outside of next-server - '**/next/dist/server/image-optimizer.js', - '**/node_modules/sharp/**/*', - ] - : []), - ...(!hasSsrAmpPages - ? [ - '**/next/dist/compiled/@ampproject/toolbox-optimizer/**/*', - ] - : []), - ], - } - ) + const toTrace = [require.resolve('next/dist/server/next-server')] + + // ensure we trace any dependencies needed for custom + // incremental cache handler + if (config.experimental.incrementalCacheHandlerPath) { + toTrace.push( + require.resolve(config.experimental.incrementalCacheHandlerPath) + ) + } + const serverResult = await nodeFileTrace(toTrace, { + base: root, + processCwd: dir, + ignore: [ + '**/next/dist/pages/**/*', + '**/next/dist/compiled/webpack/(bundle4|bundle5).js', + '**/node_modules/webpack5/**/*', + '**/next/dist/server/lib/squoosh/**/*.wasm', + ...(ciEnvironment.hasNextSupport + ? [ + // only ignore image-optimizer code when + // this is being handled outside of next-server + '**/next/dist/server/image-optimizer.js', + '**/node_modules/sharp/**/*', + ] + : []), + ...(!hasSsrAmpPages + ? ['**/next/dist/compiled/@ampproject/toolbox-optimizer/**/*'] + : []), + ], + }) const tracedFiles = new Set() diff --git a/packages/next/server/base-server.ts b/packages/next/server/base-server.ts index 94c819e700b8a..77fdc077da26d 100644 --- a/packages/next/server/base-server.ts +++ b/packages/next/server/base-server.ts @@ -45,7 +45,7 @@ import { isTargetLikeServerless } from './utils' import Router from './router' import { getPathMatch } from '../shared/lib/router/utils/path-match' import { setRevalidateHeaders } from './send-payload/revalidate-headers' -import { IncrementalCache } from './incremental-cache' +import { IncrementalCache } from './lib/incremental-cache' import { execOnce } from '../shared/lib/utils' import { isBlockedPage, isBot } from './utils' import RenderResult from './render-result' @@ -71,6 +71,7 @@ import { getLocaleRedirect } from '../shared/lib/i18n/get-locale-redirect' import { getHostname } from '../shared/lib/get-hostname' import { parseUrl as parseUrlUtil } from '../shared/lib/router/utils/parse-url' import { getNextPathnameInfo } from '../shared/lib/router/utils/get-next-pathname-info' +import { normalizePagePath } from '../shared/lib/page-path/normalize-page-path' export type FindComponentsResult = { components: LoadComponentsReturnType @@ -352,8 +353,7 @@ export default abstract class Server { dev, distDir: this.distDir, pagesDir: join(this.serverDistDir, 'pages'), - locales: this.nextConfig.i18n?.locales, - max: this.nextConfig.experimental.isrMemoryCacheSize, + maxMemoryCacheSize: this.nextConfig.experimental.isrMemoryCacheSize, flushToDisk: !minimalMode && this.nextConfig.experimental.isrFlushToDisk, getPrerenderManifest: () => { if (dev) { @@ -679,6 +679,12 @@ export default abstract class Server { return Object.assign(customRoutes, { rewrites }) } + protected getFallback(page: string): Promise { + page = normalizePagePath(page) + const cacheFs = this.getCacheFilesystem() + return cacheFs.readFile(join(this.serverDistDir, 'pages', `${page}.html`)) + } + protected getPreviewProps(): __ApiPreviewProps { return this.getPrerenderManifest().preview } @@ -1566,7 +1572,7 @@ export default abstract class Server { if (!isDataReq) { // Production already emitted the fallback as static HTML. if (isProduction) { - const html = await this.incrementalCache.getFallback( + const html = await this.getFallback( locale ? `/${locale}${pathname}` : pathname ) return { diff --git a/packages/next/server/config-shared.ts b/packages/next/server/config-shared.ts index 92ba9bdc97216..756dd7c5a4570 100644 --- a/packages/next/server/config-shared.ts +++ b/packages/next/server/config-shared.ts @@ -83,6 +83,8 @@ export interface ExperimentalConfig { browsersListForSwc?: boolean manualClientBasePath?: boolean newNextLinkBehavior?: boolean + // custom path to a cache handler to use + incrementalCacheHandlerPath?: string disablePostcssPresetEnv?: boolean swcMinify?: boolean swcFileReading?: boolean diff --git a/packages/next/server/lib/incremental-cache/file-system-cache.ts b/packages/next/server/lib/incremental-cache/file-system-cache.ts new file mode 100644 index 0000000000000..1a9c25477e40c --- /dev/null +++ b/packages/next/server/lib/incremental-cache/file-system-cache.ts @@ -0,0 +1,36 @@ +import { CacheFs } from '../../../shared/lib/utils' +import path from '../../../shared/lib/isomorphic/path' +import type { CacheHandler, CacheHandlerContext } from './' + +export default class FileSystemCache implements CacheHandler { + private flushToDisk?: boolean + private pagesDir: string + private fs: CacheFs + + constructor(ctx: CacheHandlerContext) { + this.flushToDisk = ctx.flushToDisk + this.pagesDir = ctx.pagesDir + this.fs = ctx.fs + } + + public async get(key: string) { + return this.fs.readFile(this.getSeedPath(key)) + } + public async getMeta(key: string) { + const stat = await this.fs.stat(this.getSeedPath(key)) + return { + mtime: stat.mtime.getTime(), + } + } + + public async set(key: string, data: string) { + if (!this.flushToDisk) return + const pathname = this.getSeedPath(key) + await this.fs.mkdir(path.dirname(pathname)) + return this.fs.writeFile(pathname, data) + } + + private getSeedPath(pathname: string): string { + return path.join(this.pagesDir, pathname) + } +} diff --git a/packages/next/server/incremental-cache.ts b/packages/next/server/lib/incremental-cache/index.ts similarity index 65% rename from packages/next/server/incremental-cache.ts rename to packages/next/server/lib/incremental-cache/index.ts index bb309aed2a11b..ee362a14f92d7 100644 --- a/packages/next/server/incremental-cache.ts +++ b/packages/next/server/lib/incremental-cache/index.ts @@ -1,66 +1,93 @@ -import type { CacheFs } from '../shared/lib/utils' +import type { CacheFs } from '../../../shared/lib/utils' +import FileSystemCache from './file-system-cache' import LRUCache from 'next/dist/compiled/lru-cache' -import path from '../shared/lib/isomorphic/path' -import { PrerenderManifest } from '../build' -import { normalizePagePath } from '../shared/lib/page-path/normalize-page-path' -import { IncrementalCacheValue, IncrementalCacheEntry } from './response-cache' +import path from '../../../shared/lib/isomorphic/path' +import { PrerenderManifest } from '../../../build' +import { normalizePagePath } from '../../../shared/lib/page-path/normalize-page-path' +import { + IncrementalCacheValue, + IncrementalCacheEntry, +} from '../../response-cache' function toRoute(pathname: string): string { return pathname.replace(/\/$/, '').replace(/\/index$/, '') || '/' } -export class IncrementalCache { - incrementalOptions: { - flushToDisk?: boolean - pagesDir?: string - distDir?: string - dev?: boolean +export interface CacheHandlerContext { + flushToDisk?: boolean + pagesDir: string + distDir: string + dev?: boolean + fs: CacheFs +} + +export class CacheHandler { + // eslint-disable-next-line + constructor(_ctx: CacheHandlerContext) {} + + public async get(_key: string): Promise { + return '' + } + public async getMeta(_key: string): Promise<{ + // time in epoch e.g. Date.now() + mtime: number + }> { + return {} as any } + public async set(_key: string, _data: string): Promise {} +} +export class IncrementalCache { prerenderManifest: PrerenderManifest cache?: LRUCache - locales?: string[] - fs: CacheFs + dev?: boolean + cacheHandler: CacheHandler constructor({ fs, - max, dev, distDir, pagesDir, flushToDisk, - locales, + maxMemoryCacheSize, getPrerenderManifest, + incrementalCacheHandlerPath, }: { fs: CacheFs dev: boolean - max?: number distDir: string pagesDir: string flushToDisk?: boolean - locales?: string[] + maxMemoryCacheSize?: number + incrementalCacheHandlerPath?: string getPrerenderManifest: () => PrerenderManifest }) { - this.fs = fs - this.incrementalOptions = { + let cacheHandlerMod: any = FileSystemCache + + if (incrementalCacheHandlerPath) { + cacheHandlerMod = require(incrementalCacheHandlerPath) + cacheHandlerMod = cacheHandlerMod.default || cacheHandlerMod + } + this.cacheHandler = new (cacheHandlerMod as typeof CacheHandler)({ dev, distDir, + fs, pagesDir, - flushToDisk: - !dev && (typeof flushToDisk !== 'undefined' ? flushToDisk : true), - } - this.locales = locales + flushToDisk, + }) + + this.dev = dev this.prerenderManifest = getPrerenderManifest() if (process.env.__NEXT_TEST_MAX_ISR_CACHE) { // Allow cache size to be overridden for testing purposes - max = parseInt(process.env.__NEXT_TEST_MAX_ISR_CACHE, 10) + maxMemoryCacheSize = parseInt(process.env.__NEXT_TEST_MAX_ISR_CACHE, 10) } - if (max) { + if (maxMemoryCacheSize) { this.cache = new LRUCache({ - max, + max: maxMemoryCacheSize, length({ value }) { if (!value) { return 25 @@ -76,10 +103,6 @@ export class IncrementalCache { } } - private getSeedPath(pathname: string, ext: string): string { - return path.join(this.incrementalOptions.pagesDir!, `${pathname}.${ext}`) - } - private calculateRevalidate( pathname: string, fromTime: number @@ -88,7 +111,7 @@ export class IncrementalCache { // in development we don't have a prerender-manifest // and default to always revalidating to allow easier debugging - if (this.incrementalOptions.dev) return new Date().getTime() - 1000 + if (this.dev) return new Date().getTime() - 1000 const { initialRevalidateSeconds } = this.prerenderManifest.routes[ pathname @@ -103,14 +126,9 @@ export class IncrementalCache { return revalidateAfter } - getFallback(page: string): Promise { - page = normalizePagePath(page) - return this.fs.readFile(this.getSeedPath(page, 'html')) - } - // get data from cache if available async get(pathname: string): Promise { - if (this.incrementalOptions.dev) return null + if (this.dev) return null pathname = normalizePagePath(pathname) let data = this.cache && this.cache.get(pathname) @@ -127,14 +145,15 @@ export class IncrementalCache { } try { - const htmlPath = this.getSeedPath(pathname, 'html') - const jsonPath = this.getSeedPath(pathname, 'json') - const html = await this.fs.readFile(htmlPath) - const pageData = JSON.parse(await this.fs.readFile(jsonPath)) - const { mtime } = await this.fs.stat(htmlPath) + const htmlPath = `${pathname}.html` + const html = await this.cacheHandler.get(htmlPath) + const pageData = JSON.parse( + await this.cacheHandler.get(`${pathname}.json`) + ) + const { mtime } = await this.cacheHandler.getMeta(htmlPath) data = { - revalidateAfter: this.calculateRevalidate(pathname, mtime.getTime()), + revalidateAfter: this.calculateRevalidate(pathname, mtime), value: { kind: 'PAGE', html, @@ -175,10 +194,8 @@ export class IncrementalCache { data: IncrementalCacheValue | null, revalidateSeconds?: number | false ) { - if (this.incrementalOptions.dev) return + if (this.dev) return if (typeof revalidateSeconds !== 'undefined') { - // TODO: Update this to not mutate the manifest from the - // build. this.prerenderManifest.routes[pathname] = { dataRoute: path.posix.join( '/_next/data', @@ -200,17 +217,15 @@ export class IncrementalCache { }) } - // TODO: This option needs to cease to exist unless it stops mutating the - // `next build` output's manifest. - if (this.incrementalOptions.flushToDisk && data?.kind === 'PAGE') { + if (data?.kind === 'PAGE') { try { - const seedHtmlPath = this.getSeedPath(pathname, 'html') - const seedJsonPath = this.getSeedPath(pathname, 'json') - await this.fs.mkdir(path.dirname(seedHtmlPath)) - await this.fs.writeFile(seedHtmlPath, data.html) - await this.fs.writeFile(seedJsonPath, JSON.stringify(data.pageData)) + await this.cacheHandler.set(`${pathname}.html`, data.html) + await this.cacheHandler.set( + `${pathname}.json`, + JSON.stringify(data.pageData) + ) } catch (error) { - // failed to flush to disk + // failed to set to cache handler console.warn('Failed to update prerender files for', pathname, error) } }