From 68058c43fb0786aaf3b1845c62f9bf79e089161b Mon Sep 17 00:00:00 2001 From: sebastien Date: Thu, 5 Sep 2024 18:00:04 +0200 Subject: [PATCH 1/7] simpler mdxLoader code --- packages/docusaurus-mdx-loader/src/loader.ts | 72 ++++++++++++-------- 1 file changed, 44 insertions(+), 28 deletions(-) diff --git a/packages/docusaurus-mdx-loader/src/loader.ts b/packages/docusaurus-mdx-loader/src/loader.ts index 7a7f63c63074..46b5d69d1622 100644 --- a/packages/docusaurus-mdx-loader/src/loader.ts +++ b/packages/docusaurus-mdx-loader/src/loader.ts @@ -18,11 +18,8 @@ import { createAssetsExportCode, extractContentTitleData, } from './utils'; -import type { - SimpleProcessors, - MDXOptions, - SimpleProcessorResult, -} from './processor'; +import type {WebpackCompilerName} from '@docusaurus/utils'; +import type {SimpleProcessors, MDXOptions} from './processor'; import type {ResolveMarkdownLink} from './remark/resolveMarkdownLinks'; import type {MarkdownConfig} from '@docusaurus/types'; @@ -53,15 +50,17 @@ export type Options = Partial & { processors?: SimpleProcessors; }; -export async function mdxLoader( - this: LoaderContext, - fileContent: string, -): Promise { - const compilerName = getWebpackLoaderCompilerName(this); - const callback = this.async(); - const filePath = this.resourcePath; - const options: Options = this.getOptions(); - +async function loadMDX({ + fileContent, + filePath, + options, + compilerName, +}: { + fileContent: string; + filePath: string; + options: Options; + compilerName: WebpackCompilerName; +}): Promise { const {frontMatter} = await options.markdownConfig.parseFrontMatter({ filePath, fileContent, @@ -70,18 +69,13 @@ export async function mdxLoader( const hasFrontMatter = Object.keys(frontMatter).length > 0; - let result: SimpleProcessorResult; - try { - result = await compileToJSX({ - fileContent, - filePath, - frontMatter, - options, - compilerName, - }); - } catch (error) { - return callback(error as Error); - } + const result = await compileToJSX({ + fileContent, + filePath, + frontMatter, + options, + compilerName, + }); const contentTitle = extractContentTitleData(result.data); @@ -97,7 +91,7 @@ ${JSON.stringify(frontMatter, null, 2)}`; if (!options.isMDXPartialFrontMatterWarningDisabled) { const shouldError = process.env.NODE_ENV === 'test' || process.env.CI; if (shouldError) { - return callback(new Error(errorMessage)); + throw new Error(errorMessage); } logger.warn(errorMessage); } @@ -146,5 +140,27 @@ ${exportsCode} ${result.content} `; - return callback(null, code); + return code; +} + +export async function mdxLoader( + this: LoaderContext, + fileContent: string, +): Promise { + const compilerName = getWebpackLoaderCompilerName(this); + const callback = this.async(); + const filePath = this.resourcePath; + const options: Options = this.getOptions(); + + try { + const result = await loadMDX({ + fileContent, + filePath, + options, + compilerName, + }); + return callback(null, result); + } catch (error) { + return callback(error as Error); + } } From f3c6b151d6e6ee4bc442f54644e3748e5c040fc1 Mon Sep 17 00:00:00 2001 From: sebastien Date: Thu, 5 Sep 2024 18:26:55 +0200 Subject: [PATCH 2/7] crossCompilerCache --- .../src/createMDXLoader.ts | 6 +++- packages/docusaurus-mdx-loader/src/loader.ts | 35 +++++++++++++++++-- 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/packages/docusaurus-mdx-loader/src/createMDXLoader.ts b/packages/docusaurus-mdx-loader/src/createMDXLoader.ts index 523c6d240122..f279b9ca24fb 100644 --- a/packages/docusaurus-mdx-loader/src/createMDXLoader.ts +++ b/packages/docusaurus-mdx-loader/src/createMDXLoader.ts @@ -19,7 +19,11 @@ async function enhancedOptions(options: Options): Promise { // Lazy creation messes-up with Rsdoctor ability to measure mdx-loader perf const newOptions: Options = options.processors ? options - : {...options, processors: await createProcessors({options})}; + : { + ...options, + processors: await createProcessors({options}), + crossCompilerCache: new Map(), + }; return newOptions; } diff --git a/packages/docusaurus-mdx-loader/src/loader.ts b/packages/docusaurus-mdx-loader/src/loader.ts index 46b5d69d1622..a5fa46627fa9 100644 --- a/packages/docusaurus-mdx-loader/src/loader.ts +++ b/packages/docusaurus-mdx-loader/src/loader.ts @@ -48,6 +48,7 @@ export type Options = Partial & { // Will usually be created by "createMDXLoaderItem" processors?: SimpleProcessors; + crossCompilerCache?: Map>; // MDX => Promise cache }; async function loadMDX({ @@ -143,6 +144,37 @@ ${result.content} return code; } +async function loadMDXWithCaching({ + fileContent, + filePath, + options, + compilerName, +}: { + fileContent: string; + filePath: string; + options: Options; + compilerName: WebpackCompilerName; +}): Promise { + // Note: we cache promises instead of strings + // This is because client/server compilations might be triggered in parallel + // When this happens for the same file, we don't want to compile it twice + const cachedPromise = options.crossCompilerCache?.get(fileContent); + if (cachedPromise) { + // We can clean up the cache and free memory here + // We know there are only 2 compilations for the same file + options.crossCompilerCache?.delete(fileContent); + return cachedPromise; + } + const promise = loadMDX({ + fileContent, + filePath, + options, + compilerName, + }); + options.crossCompilerCache?.set(fileContent, promise); + return promise; +} + export async function mdxLoader( this: LoaderContext, fileContent: string, @@ -151,9 +183,8 @@ export async function mdxLoader( const callback = this.async(); const filePath = this.resourcePath; const options: Options = this.getOptions(); - try { - const result = await loadMDX({ + const result = await loadMDXWithCaching({ fileContent, filePath, options, From 92c971f4c9f1fcf997ee99aa63ab6a46416c27aa Mon Sep 17 00:00:00 2001 From: sebastien Date: Thu, 5 Sep 2024 19:11:29 +0200 Subject: [PATCH 3/7] add faster.mdxCrossCompilerCache option --- packages/docusaurus-types/src/config.d.ts | 1 + .../__snapshots__/config.test.ts.snap | 10 +++ .../__tests__/__snapshots__/site.test.ts.snap | 1 + .../server/__tests__/configValidation.test.ts | 76 +++++++++++++++++++ .../docusaurus/src/server/configValidation.ts | 5 ++ 5 files changed, 93 insertions(+) diff --git a/packages/docusaurus-types/src/config.d.ts b/packages/docusaurus-types/src/config.d.ts index 6be5dac20379..ceb154c01729 100644 --- a/packages/docusaurus-types/src/config.d.ts +++ b/packages/docusaurus-types/src/config.d.ts @@ -126,6 +126,7 @@ export type StorageConfig = { export type FasterConfig = { swcJsLoader: boolean; swcJsMinimizer: boolean; + mdxCrossCompilerCache: boolean; }; export type FutureConfig = { diff --git a/packages/docusaurus/src/server/__tests__/__snapshots__/config.test.ts.snap b/packages/docusaurus/src/server/__tests__/__snapshots__/config.test.ts.snap index 7071ed4edf2f..00135ff98e3a 100644 --- a/packages/docusaurus/src/server/__tests__/__snapshots__/config.test.ts.snap +++ b/packages/docusaurus/src/server/__tests__/__snapshots__/config.test.ts.snap @@ -9,6 +9,7 @@ exports[`loadSiteConfig website with .cjs siteConfig 1`] = ` "customFields": {}, "future": { "experimental_faster": { + "mdxCrossCompilerCache": false, "swcJsLoader": false, "swcJsMinimizer": false, }, @@ -74,6 +75,7 @@ exports[`loadSiteConfig website with ts + js config 1`] = ` "customFields": {}, "future": { "experimental_faster": { + "mdxCrossCompilerCache": false, "swcJsLoader": false, "swcJsMinimizer": false, }, @@ -139,6 +141,7 @@ exports[`loadSiteConfig website with valid JS CJS config 1`] = ` "customFields": {}, "future": { "experimental_faster": { + "mdxCrossCompilerCache": false, "swcJsLoader": false, "swcJsMinimizer": false, }, @@ -204,6 +207,7 @@ exports[`loadSiteConfig website with valid JS ESM config 1`] = ` "customFields": {}, "future": { "experimental_faster": { + "mdxCrossCompilerCache": false, "swcJsLoader": false, "swcJsMinimizer": false, }, @@ -269,6 +273,7 @@ exports[`loadSiteConfig website with valid TypeScript CJS config 1`] = ` "customFields": {}, "future": { "experimental_faster": { + "mdxCrossCompilerCache": false, "swcJsLoader": false, "swcJsMinimizer": false, }, @@ -334,6 +339,7 @@ exports[`loadSiteConfig website with valid TypeScript ESM config 1`] = ` "customFields": {}, "future": { "experimental_faster": { + "mdxCrossCompilerCache": false, "swcJsLoader": false, "swcJsMinimizer": false, }, @@ -399,6 +405,7 @@ exports[`loadSiteConfig website with valid async config 1`] = ` "customFields": {}, "future": { "experimental_faster": { + "mdxCrossCompilerCache": false, "swcJsLoader": false, "swcJsMinimizer": false, }, @@ -466,6 +473,7 @@ exports[`loadSiteConfig website with valid async config creator function 1`] = ` "customFields": {}, "future": { "experimental_faster": { + "mdxCrossCompilerCache": false, "swcJsLoader": false, "swcJsMinimizer": false, }, @@ -533,6 +541,7 @@ exports[`loadSiteConfig website with valid config creator function 1`] = ` "customFields": {}, "future": { "experimental_faster": { + "mdxCrossCompilerCache": false, "swcJsLoader": false, "swcJsMinimizer": false, }, @@ -603,6 +612,7 @@ exports[`loadSiteConfig website with valid siteConfig 1`] = ` "favicon": "img/docusaurus.ico", "future": { "experimental_faster": { + "mdxCrossCompilerCache": false, "swcJsLoader": false, "swcJsMinimizer": false, }, diff --git a/packages/docusaurus/src/server/__tests__/__snapshots__/site.test.ts.snap b/packages/docusaurus/src/server/__tests__/__snapshots__/site.test.ts.snap index 404e2c330cdb..73fd5dc8d58f 100644 --- a/packages/docusaurus/src/server/__tests__/__snapshots__/site.test.ts.snap +++ b/packages/docusaurus/src/server/__tests__/__snapshots__/site.test.ts.snap @@ -79,6 +79,7 @@ exports[`load loads props for site with custom i18n path 1`] = ` "customFields": {}, "future": { "experimental_faster": { + "mdxCrossCompilerCache": false, "swcJsLoader": false, "swcJsMinimizer": false, }, diff --git a/packages/docusaurus/src/server/__tests__/configValidation.test.ts b/packages/docusaurus/src/server/__tests__/configValidation.test.ts index 671e19d6b4d4..49d9860bcbe1 100644 --- a/packages/docusaurus/src/server/__tests__/configValidation.test.ts +++ b/packages/docusaurus/src/server/__tests__/configValidation.test.ts @@ -48,6 +48,7 @@ describe('normalizeConfig', () => { experimental_faster: { swcJsLoader: true, swcJsMinimizer: true, + mdxCrossCompilerCache: true, }, experimental_storage: { type: 'sessionStorage', @@ -743,6 +744,7 @@ describe('future', () => { experimental_faster: { swcJsLoader: true, swcJsMinimizer: true, + mdxCrossCompilerCache: true, }, experimental_storage: { type: 'sessionStorage', @@ -1091,6 +1093,8 @@ describe('future', () => { it('accepts faster - full', () => { const faster: FasterConfig = { swcJsLoader: true, + swcJsMinimizer: true, + mdxCrossCompilerCache: true, }; expect( normalizeConfig({ @@ -1202,6 +1206,7 @@ describe('future', () => { `); }); }); + describe('swcJsMinimizer', () => { it('accepts - undefined', () => { const faster: Partial = { @@ -1272,5 +1277,76 @@ describe('future', () => { `); }); }); + + describe('mdxCrossCompilerCache', () => { + it('accepts - undefined', () => { + const faster: Partial = { + mdxCrossCompilerCache: undefined, + }; + expect( + normalizeConfig({ + future: { + experimental_faster: faster, + }, + }), + ).toEqual(fasterContaining({mdxCrossCompilerCache: false})); + }); + + it('accepts - true', () => { + const faster: Partial = { + mdxCrossCompilerCache: true, + }; + expect( + normalizeConfig({ + future: { + experimental_faster: faster, + }, + }), + ).toEqual(fasterContaining({mdxCrossCompilerCache: true})); + }); + + it('accepts - false', () => { + const faster: Partial = { + mdxCrossCompilerCache: false, + }; + expect( + normalizeConfig({ + future: { + experimental_faster: faster, + }, + }), + ).toEqual(fasterContaining({mdxCrossCompilerCache: false})); + }); + + it('rejects - null', () => { + // @ts-expect-error: invalid + const faster: Partial = {mdxCrossCompilerCache: 42}; + expect(() => + normalizeConfig({ + future: { + experimental_faster: faster, + }, + }), + ).toThrowErrorMatchingInlineSnapshot(` + ""future.experimental_faster.mdxCrossCompilerCache" must be a boolean + " + `); + }); + + it('rejects - number', () => { + // @ts-expect-error: invalid + const faster: Partial = {mdxCrossCompilerCache: 42}; + expect(() => + normalizeConfig({ + future: { + experimental_faster: faster, + }, + }), + ).toThrowErrorMatchingInlineSnapshot(` + ""future.experimental_faster.mdxCrossCompilerCache" must be a boolean + " + `); + }); + }); }); }); diff --git a/packages/docusaurus/src/server/configValidation.ts b/packages/docusaurus/src/server/configValidation.ts index 2cb6c63a8810..c066d5ce955a 100644 --- a/packages/docusaurus/src/server/configValidation.ts +++ b/packages/docusaurus/src/server/configValidation.ts @@ -44,12 +44,14 @@ export const DEFAULT_STORAGE_CONFIG: StorageConfig = { export const DEFAULT_FASTER_CONFIG: FasterConfig = { swcJsLoader: false, swcJsMinimizer: false, + mdxCrossCompilerCache: false, }; // When using the "faster: true" shortcut export const DEFAULT_FASTER_CONFIG_TRUE: FasterConfig = { swcJsLoader: true, swcJsMinimizer: true, + mdxCrossCompilerCache: true, }; export const DEFAULT_FUTURE_CONFIG: FutureConfig = { @@ -217,6 +219,9 @@ const FASTER_CONFIG_SCHEMA = Joi.alternatives() swcJsMinimizer: Joi.boolean().default( DEFAULT_FASTER_CONFIG.swcJsMinimizer, ), + mdxCrossCompilerCache: Joi.boolean().default( + DEFAULT_FASTER_CONFIG.mdxCrossCompilerCache, + ), }), Joi.boolean() .required() From 95f0dbf7f4d85e4b7bd557a86eff29100d8e34c8 Mon Sep 17 00:00:00 2001 From: sebastien Date: Thu, 5 Sep 2024 19:31:22 +0200 Subject: [PATCH 4/7] wire mdx cross-compiler cache --- .../src/createMDXLoader.ts | 45 ++++++--- packages/docusaurus-mdx-loader/src/index.ts | 3 +- packages/docusaurus-mdx-loader/src/loader.ts | 37 +++---- packages/docusaurus-mdx-loader/src/options.ts | 29 ++++++ .../src/__tests__/index.test.ts | 3 +- .../src/index.ts | 16 ++- .../src/index.ts | 97 ++++++++++--------- .../src/index.ts | 60 ++++++------ .../server/plugins/__tests__/plugins.test.ts | 2 + .../src/server/plugins/synthetic.ts | 2 + 10 files changed, 164 insertions(+), 130 deletions(-) create mode 100644 packages/docusaurus-mdx-loader/src/options.ts diff --git a/packages/docusaurus-mdx-loader/src/createMDXLoader.ts b/packages/docusaurus-mdx-loader/src/createMDXLoader.ts index f279b9ca24fb..9bbafad5f7da 100644 --- a/packages/docusaurus-mdx-loader/src/createMDXLoader.ts +++ b/packages/docusaurus-mdx-loader/src/createMDXLoader.ts @@ -6,34 +6,49 @@ */ import {createProcessors} from './processor'; -import type {Options} from './loader'; +import type {Options} from './options'; import type {RuleSetRule, RuleSetUseItem} from 'webpack'; -async function enhancedOptions(options: Options): Promise { +type CreateOptions = { + useCrossCompilerCache?: boolean; +}; + +async function normalizeOptions( + optionsInput: Options & CreateOptions, +): Promise { // Because Jest doesn't like ESM / createProcessors() if (process.env.N0DE_ENV === 'test' || process.env.JEST_WORKER_ID) { - return options; + return optionsInput; } + let options = optionsInput; + // We create the processor earlier here, to avoid the lazy processor creating // Lazy creation messes-up with Rsdoctor ability to measure mdx-loader perf - const newOptions: Options = options.processors - ? options - : { - ...options, - processors: await createProcessors({options}), - crossCompilerCache: new Map(), - }; - - return newOptions; + if (!options.processors) { + options = {...options, processors: await createProcessors({options})}; + } + + // Cross-compiler cache permits to compile client/server MDX only once + // We don't want to cache in dev mode (docusaurus start) + // We only have multiple compilers in production mode (docusaurus build) + // TODO wrong but good enough for now (example: "docusaurus build --dev") + if (options.useCrossCompilerCache && process.env.NODE_ENV === 'production') { + options = { + ...options, + crossCompilerCache: new Map(), + }; + } + + return options; } export async function createMDXLoaderItem( - options: Options, + options: Options & CreateOptions, ): Promise { return { loader: require.resolve('@docusaurus/mdx-loader'), - options: await enhancedOptions(options), + options: await normalizeOptions(options), }; } @@ -42,7 +57,7 @@ export async function createMDXLoaderRule({ options, }: { include: RuleSetRule['include']; - options: Options; + options: Options & CreateOptions; }): Promise { return { test: /\.mdx?$/i, diff --git a/packages/docusaurus-mdx-loader/src/index.ts b/packages/docusaurus-mdx-loader/src/index.ts index d8e5ffa2e94a..59f2ad19159e 100644 --- a/packages/docusaurus-mdx-loader/src/index.ts +++ b/packages/docusaurus-mdx-loader/src/index.ts @@ -37,5 +37,6 @@ export type LoadedMDXContent = { (): JSX.Element; }; -export type {Options, MDXPlugin} from './loader'; +export type {MDXPlugin} from './loader'; export type {MDXOptions} from './processor'; +export type {Options} from './options'; diff --git a/packages/docusaurus-mdx-loader/src/loader.ts b/packages/docusaurus-mdx-loader/src/loader.ts index a5fa46627fa9..ab60427f3776 100644 --- a/packages/docusaurus-mdx-loader/src/loader.ts +++ b/packages/docusaurus-mdx-loader/src/loader.ts @@ -19,10 +19,7 @@ import { extractContentTitleData, } from './utils'; import type {WebpackCompilerName} from '@docusaurus/utils'; -import type {SimpleProcessors, MDXOptions} from './processor'; -import type {ResolveMarkdownLink} from './remark/resolveMarkdownLinks'; - -import type {MarkdownConfig} from '@docusaurus/types'; +import type {Options} from './options'; import type {LoaderContext} from 'webpack'; // TODO as of April 2023, no way to import/re-export this ESM type easily :/ @@ -32,25 +29,6 @@ type Pluggable = any; // TODO fix this asap export type MDXPlugin = Pluggable; -export type Options = Partial & { - markdownConfig: MarkdownConfig; - staticDirs: string[]; - siteDir: string; - isMDXPartial?: (filePath: string) => boolean; - isMDXPartialFrontMatterWarningDisabled?: boolean; - removeContentTitle?: boolean; - metadataPath?: (filePath: string) => string; - createAssets?: (metadata: { - filePath: string; - frontMatter: {[key: string]: unknown}; - }) => {[key: string]: unknown}; - resolveMarkdownLink?: ResolveMarkdownLink; - - // Will usually be created by "createMDXLoaderItem" - processors?: SimpleProcessors; - crossCompilerCache?: Map>; // MDX => Promise cache -}; - async function loadMDX({ fileContent, filePath, @@ -144,6 +122,13 @@ ${result.content} return code; } +// Note: we cache promises instead of strings +// This is because client/server compilations might be triggered in parallel +// When this happens for the same file, we don't want to compile it twice +// Note we use fileContent instead of filePath as cache key +// This is because the same file can be compiled with different options +// This is notably the case for blog posts that can be truncated +// An alternative would be to use this.resource (including ?query#hash) async function loadMDXWithCaching({ fileContent, filePath, @@ -155,13 +140,13 @@ async function loadMDXWithCaching({ options: Options; compilerName: WebpackCompilerName; }): Promise { - // Note: we cache promises instead of strings - // This is because client/server compilations might be triggered in parallel - // When this happens for the same file, we don't want to compile it twice const cachedPromise = options.crossCompilerCache?.get(fileContent); if (cachedPromise) { // We can clean up the cache and free memory here // We know there are only 2 compilations for the same file + // Note: once we introduce RSCs we'll probably have 3 compilations + // Note: we can't use string keys in WeakMap + // But we could eventually use WeakRef for the values options.crossCompilerCache?.delete(fileContent); return cachedPromise; } diff --git a/packages/docusaurus-mdx-loader/src/options.ts b/packages/docusaurus-mdx-loader/src/options.ts new file mode 100644 index 000000000000..0c8c7fe4ceeb --- /dev/null +++ b/packages/docusaurus-mdx-loader/src/options.ts @@ -0,0 +1,29 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +import type {MDXOptions, SimpleProcessors} from './processor'; +import type {MarkdownConfig} from '@docusaurus/types'; +import type {ResolveMarkdownLink} from './remark/resolveMarkdownLinks'; + +export type Options = Partial & { + markdownConfig: MarkdownConfig; + staticDirs: string[]; + siteDir: string; + isMDXPartial?: (filePath: string) => boolean; + isMDXPartialFrontMatterWarningDisabled?: boolean; + removeContentTitle?: boolean; + metadataPath?: (filePath: string) => string; + createAssets?: (metadata: { + filePath: string; + frontMatter: {[key: string]: unknown}; + }) => {[key: string]: unknown}; + resolveMarkdownLink?: ResolveMarkdownLink; + + // Will usually be created by "createMDXLoaderItem" + processors?: SimpleProcessors; + crossCompilerCache?: Map>; // MDX => Promise cache +}; diff --git a/packages/docusaurus-plugin-content-blog/src/__tests__/index.test.ts b/packages/docusaurus-plugin-content-blog/src/__tests__/index.test.ts index 35c695d8655c..5d8cff5778b1 100644 --- a/packages/docusaurus-plugin-content-blog/src/__tests__/index.test.ts +++ b/packages/docusaurus-plugin-content-blog/src/__tests__/index.test.ts @@ -13,6 +13,7 @@ import { getFileCommitDate, LAST_UPDATE_FALLBACK, } from '@docusaurus/utils'; +import {DEFAULT_FUTURE_CONFIG} from '@docusaurus/core/src/server/configValidation'; import pluginContentBlog from '../index'; import {validateOptions} from '../options'; import type { @@ -106,7 +107,7 @@ const getPlugin = async ( baseUrl: '/', url: 'https://docusaurus.io', markdown, - future: {}, + future: DEFAULT_FUTURE_CONFIG, staticDirectories: ['static'], } as DocusaurusConfig; return pluginContentBlog( diff --git a/packages/docusaurus-plugin-content-blog/src/index.ts b/packages/docusaurus-plugin-content-blog/src/index.ts index bacaaad45185..e22f2e54f2cb 100644 --- a/packages/docusaurus-plugin-content-blog/src/index.ts +++ b/packages/docusaurus-plugin-content-blog/src/index.ts @@ -21,10 +21,7 @@ import { resolveMarkdownLinkPathname, } from '@docusaurus/utils'; import {getTagsFilePathsToWatch} from '@docusaurus/utils-validation'; -import { - createMDXLoaderItem, - type Options as MDXLoaderOptions, -} from '@docusaurus/mdx-loader'; +import {createMDXLoaderItem} from '@docusaurus/mdx-loader'; import { getBlogTags, paginateBlogPosts, @@ -114,7 +111,9 @@ export default async function pluginContentBlog( const contentDirs = getContentPathList(contentPaths); - const loaderOptions: MDXLoaderOptions = { + const mdxLoaderItem = await createMDXLoaderItem({ + useCrossCompilerCache: + siteConfig.future.experimental_faster.mdxCrossCompilerCache, admonitions, remarkPlugins, rehypePlugins, @@ -168,7 +167,7 @@ export default async function pluginContentBlog( } return permalink; }, - }; + }); function createBlogMarkdownLoader(): RuleSetUseItem { const markdownLoaderOptions: BlogMarkdownLoaderOptions = { @@ -185,10 +184,7 @@ export default async function pluginContentBlog( include: contentDirs // Trailing slash is important, see https://github.com/facebook/docusaurus/pull/3970 .map(addTrailingPathSeparator), - use: [ - await createMDXLoaderItem(loaderOptions), - createBlogMarkdownLoader(), - ], + use: [mdxLoaderItem, createBlogMarkdownLoader()], }; } diff --git a/packages/docusaurus-plugin-content-docs/src/index.ts b/packages/docusaurus-plugin-content-docs/src/index.ts index 747018318667..fe72e070bb17 100644 --- a/packages/docusaurus-plugin-content-docs/src/index.ts +++ b/packages/docusaurus-plugin-content-docs/src/index.ts @@ -25,10 +25,7 @@ import { getTagsFile, getTagsFilePathsToWatch, } from '@docusaurus/utils-validation'; -import { - createMDXLoaderRule, - type Options as MDXLoaderOptions, -} from '@docusaurus/mdx-loader'; +import {createMDXLoaderRule} from '@docusaurus/mdx-loader'; import {loadSidebars, resolveSidebarPathOption} from './sidebars'; import {CategoryMetadataFilenamePattern} from './sidebars/generator'; import { @@ -107,50 +104,56 @@ export default async function pluginContentDocs( // Trailing slash is important, see https://github.com/facebook/docusaurus/pull/3970 .map(addTrailingPathSeparator); - const loaderOptions: MDXLoaderOptions = { - admonitions: options.admonitions, - remarkPlugins, - rehypePlugins, - recmaPlugins, - beforeDefaultRehypePlugins, - beforeDefaultRemarkPlugins, - staticDirs: siteConfig.staticDirectories.map((dir) => - path.resolve(siteDir, dir), - ), - siteDir, - isMDXPartial: createAbsoluteFilePathMatcher(options.exclude, contentDirs), - metadataPath: (mdxPath: string) => { - // Note that metadataPath must be the same/in-sync as - // the path from createData for each MDX. - const aliasedPath = aliasedSitePath(mdxPath, siteDir); - return path.join(dataDir, `${docuHash(aliasedPath)}.json`); - }, - // createAssets converts relative paths to require() calls - createAssets: ({frontMatter}: {frontMatter: DocFrontMatter}) => ({ - image: frontMatter.image, - }), - markdownConfig: siteConfig.markdown, - resolveMarkdownLink: ({linkPathname, sourceFilePath}) => { - const version = getVersionFromSourceFilePath( - sourceFilePath, - versionsMetadata, - ); - const permalink = resolveMarkdownLinkPathname(linkPathname, { - sourceFilePath, - sourceToPermalink: contentHelpers.sourceToPermalink, - siteDir, - contentPaths: version, - }); - if (permalink === null) { - logger.report( - siteConfig.onBrokenMarkdownLinks, - )`Docs markdown link couldn't be resolved: (url=${linkPathname}) in source file path=${sourceFilePath} for version number=${version.versionName}`; - } - return permalink; + return createMDXLoaderRule({ + include: contentDirs, + options: { + useCrossCompilerCache: + siteConfig.future.experimental_faster.mdxCrossCompilerCache, + admonitions: options.admonitions, + remarkPlugins, + rehypePlugins, + recmaPlugins, + beforeDefaultRehypePlugins, + beforeDefaultRemarkPlugins, + staticDirs: siteConfig.staticDirectories.map((dir) => + path.resolve(siteDir, dir), + ), + siteDir, + isMDXPartial: createAbsoluteFilePathMatcher( + options.exclude, + contentDirs, + ), + metadataPath: (mdxPath: string) => { + // Note that metadataPath must be the same/in-sync as + // the path from createData for each MDX. + const aliasedPath = aliasedSitePath(mdxPath, siteDir); + return path.join(dataDir, `${docuHash(aliasedPath)}.json`); + }, + // createAssets converts relative paths to require() calls + createAssets: ({frontMatter}: {frontMatter: DocFrontMatter}) => ({ + image: frontMatter.image, + }), + markdownConfig: siteConfig.markdown, + resolveMarkdownLink: ({linkPathname, sourceFilePath}) => { + const version = getVersionFromSourceFilePath( + sourceFilePath, + versionsMetadata, + ); + const permalink = resolveMarkdownLinkPathname(linkPathname, { + sourceFilePath, + sourceToPermalink: contentHelpers.sourceToPermalink, + siteDir, + contentPaths: version, + }); + if (permalink === null) { + logger.report( + siteConfig.onBrokenMarkdownLinks, + )`Docs markdown link couldn't be resolved: (url=${linkPathname}) in source file path=${sourceFilePath} for version number=${version.versionName}`; + } + return permalink; + }, }, - }; - - return createMDXLoaderRule({include: contentDirs, options: loaderOptions}); + }); } const docsMDXLoaderRule = await createDocsMDXLoaderRule(); diff --git a/packages/docusaurus-plugin-content-pages/src/index.ts b/packages/docusaurus-plugin-content-pages/src/index.ts index e4e0e6bd8f3e..bb06e3adcd8a 100644 --- a/packages/docusaurus-plugin-content-pages/src/index.ts +++ b/packages/docusaurus-plugin-content-pages/src/index.ts @@ -14,10 +14,7 @@ import { createAbsoluteFilePathMatcher, DEFAULT_PLUGIN_ID, } from '@docusaurus/utils'; -import { - createMDXLoaderRule, - type Options as MDXLoaderOptions, -} from '@docusaurus/mdx-loader'; +import {createMDXLoaderRule} from '@docusaurus/mdx-loader'; import {createAllRoutes} from './routes'; import { createPagesContentPaths, @@ -57,36 +54,39 @@ export default async function pluginContentPages( } = options; const contentDirs = getContentPathList(contentPaths); - const loaderOptions: MDXLoaderOptions = { - admonitions, - remarkPlugins, - rehypePlugins, - recmaPlugins, - beforeDefaultRehypePlugins, - beforeDefaultRemarkPlugins, - staticDirs: siteConfig.staticDirectories.map((dir) => - path.resolve(siteDir, dir), - ), - siteDir, - isMDXPartial: createAbsoluteFilePathMatcher(options.exclude, contentDirs), - metadataPath: (mdxPath: string) => { - // Note that metadataPath must be the same/in-sync as - // the path from createData for each MDX. - const aliasedSource = aliasedSitePath(mdxPath, siteDir); - return path.join(dataDir, `${docuHash(aliasedSource)}.json`); - }, - // createAssets converts relative paths to require() calls - createAssets: ({frontMatter}: {frontMatter: PageFrontMatter}) => ({ - image: frontMatter.image, - }), - markdownConfig: siteConfig.markdown, - }; - return createMDXLoaderRule({ include: contentDirs // Trailing slash is important, see https://github.com/facebook/docusaurus/pull/3970 .map(addTrailingPathSeparator), - options: loaderOptions, + options: { + useCrossCompilerCache: + siteConfig.future.experimental_faster.mdxCrossCompilerCache, + admonitions, + remarkPlugins, + rehypePlugins, + recmaPlugins, + beforeDefaultRehypePlugins, + beforeDefaultRemarkPlugins, + staticDirs: siteConfig.staticDirectories.map((dir) => + path.resolve(siteDir, dir), + ), + siteDir, + isMDXPartial: createAbsoluteFilePathMatcher( + options.exclude, + contentDirs, + ), + metadataPath: (mdxPath: string) => { + // Note that metadataPath must be the same/in-sync as + // the path from createData for each MDX. + const aliasedSource = aliasedSitePath(mdxPath, siteDir); + return path.join(dataDir, `${docuHash(aliasedSource)}.json`); + }, + // createAssets converts relative paths to require() calls + createAssets: ({frontMatter}: {frontMatter: PageFrontMatter}) => ({ + image: frontMatter.image, + }), + markdownConfig: siteConfig.markdown, + }, }); } diff --git a/packages/docusaurus/src/server/plugins/__tests__/plugins.test.ts b/packages/docusaurus/src/server/plugins/__tests__/plugins.test.ts index 9e753643e4d0..39c12daa0bc8 100644 --- a/packages/docusaurus/src/server/plugins/__tests__/plugins.test.ts +++ b/packages/docusaurus/src/server/plugins/__tests__/plugins.test.ts @@ -8,6 +8,7 @@ import path from 'path'; import {fromPartial} from '@total-typescript/shoehorn'; import {loadPlugins, reloadPlugin} from '../plugins'; +import {DEFAULT_FUTURE_CONFIG} from '../../configValidation'; import type {LoadContext, Plugin, PluginConfig} from '@docusaurus/types'; async function testLoad({ @@ -27,6 +28,7 @@ async function testLoad({ siteConfig: { baseUrl: '/', trailingSlash: true, + future: DEFAULT_FUTURE_CONFIG, themeConfig: {}, staticDirectories: [], presets: [], diff --git a/packages/docusaurus/src/server/plugins/synthetic.ts b/packages/docusaurus/src/server/plugins/synthetic.ts index 500c8249217a..a79125c634ed 100644 --- a/packages/docusaurus/src/server/plugins/synthetic.ts +++ b/packages/docusaurus/src/server/plugins/synthetic.ts @@ -80,6 +80,8 @@ export async function createMDXFallbackPlugin({ siteConfig, }: LoadContext): Promise { const mdxLoaderItem = await createMDXLoaderItem({ + useCrossCompilerCache: + siteConfig.future.experimental_faster.mdxCrossCompilerCache, admonitions: true, staticDirs: siteConfig.staticDirectories.map((dir) => path.resolve(siteDir, dir), From 59a3255ee7f7a0f2c36ba9a6be5dc050df9f51df Mon Sep 17 00:00:00 2001 From: sebastien Date: Thu, 5 Sep 2024 19:33:05 +0200 Subject: [PATCH 5/7] fix imports --- packages/docusaurus-mdx-loader/src/preprocessor.ts | 2 +- packages/docusaurus-mdx-loader/src/processor.ts | 2 +- packages/docusaurus-mdx-loader/src/utils.ts | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/docusaurus-mdx-loader/src/preprocessor.ts b/packages/docusaurus-mdx-loader/src/preprocessor.ts index 0581e6bcd9cc..4d5d707311e5 100644 --- a/packages/docusaurus-mdx-loader/src/preprocessor.ts +++ b/packages/docusaurus-mdx-loader/src/preprocessor.ts @@ -11,7 +11,7 @@ import { admonitionTitleToDirectiveLabel, } from '@docusaurus/utils'; import {normalizeAdmonitionOptions} from './remark/admonitions'; -import type {Options} from './loader'; +import type {Options} from './options'; /** * Preprocess the string before passing it to MDX diff --git a/packages/docusaurus-mdx-loader/src/processor.ts b/packages/docusaurus-mdx-loader/src/processor.ts index 09f76bab320d..3cf6cdef9b61 100644 --- a/packages/docusaurus-mdx-loader/src/processor.ts +++ b/packages/docusaurus-mdx-loader/src/processor.ts @@ -20,7 +20,7 @@ import codeCompatPlugin from './remark/mdx1Compat/codeCompatPlugin'; import {getFormat} from './format'; import type {WebpackCompilerName} from '@docusaurus/utils'; import type {MDXFrontMatter} from './frontMatter'; -import type {Options} from './loader'; +import type {Options} from './options'; import type {AdmonitionOptions} from './remark/admonitions'; // @ts-expect-error: TODO see https://github.com/microsoft/TypeScript/issues/49721 diff --git a/packages/docusaurus-mdx-loader/src/utils.ts b/packages/docusaurus-mdx-loader/src/utils.ts index 36c74322fb43..8941e2e90846 100644 --- a/packages/docusaurus-mdx-loader/src/utils.ts +++ b/packages/docusaurus-mdx-loader/src/utils.ts @@ -10,7 +10,7 @@ import {escapePath, type WebpackCompilerName} from '@docusaurus/utils'; import {getProcessor, type SimpleProcessorResult} from './processor'; import {validateMDXFrontMatter} from './frontMatter'; import preprocessor from './preprocessor'; -import type {Options} from './loader'; +import type {Options} from './options'; /** * Converts assets an object with Webpack require calls code. From 91edbd4c7150c60047df592ff4bdae344358928e Mon Sep 17 00:00:00 2001 From: sebastien Date: Fri, 6 Sep 2024 08:48:55 +0200 Subject: [PATCH 6/7] use resource as cache key --- packages/docusaurus-mdx-loader/src/createMDXLoader.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/docusaurus-mdx-loader/src/createMDXLoader.ts b/packages/docusaurus-mdx-loader/src/createMDXLoader.ts index 9bbafad5f7da..f9de7c6f868a 100644 --- a/packages/docusaurus-mdx-loader/src/createMDXLoader.ts +++ b/packages/docusaurus-mdx-loader/src/createMDXLoader.ts @@ -33,7 +33,11 @@ async function normalizeOptions( // We don't want to cache in dev mode (docusaurus start) // We only have multiple compilers in production mode (docusaurus build) // TODO wrong but good enough for now (example: "docusaurus build --dev") - if (options.useCrossCompilerCache && process.env.NODE_ENV === 'production') { + if ( + options.useCrossCompilerCache && + process.env.NODE_ENV === 'production' && + process.env.DOCUSAURUS_AB_BENCHMARK === 'true' + ) { options = { ...options, crossCompilerCache: new Map(), From 7a345688fe60e26a378cc83acedff19090fda3d6 Mon Sep 17 00:00:00 2001 From: sebastien Date: Fri, 6 Sep 2024 09:05:29 +0200 Subject: [PATCH 7/7] use resource as cache key --- .../src/createMDXLoader.ts | 6 +---- packages/docusaurus-mdx-loader/src/loader.ts | 24 +++++++++++-------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/packages/docusaurus-mdx-loader/src/createMDXLoader.ts b/packages/docusaurus-mdx-loader/src/createMDXLoader.ts index f9de7c6f868a..9bbafad5f7da 100644 --- a/packages/docusaurus-mdx-loader/src/createMDXLoader.ts +++ b/packages/docusaurus-mdx-loader/src/createMDXLoader.ts @@ -33,11 +33,7 @@ async function normalizeOptions( // We don't want to cache in dev mode (docusaurus start) // We only have multiple compilers in production mode (docusaurus build) // TODO wrong but good enough for now (example: "docusaurus build --dev") - if ( - options.useCrossCompilerCache && - process.env.NODE_ENV === 'production' && - process.env.DOCUSAURUS_AB_BENCHMARK === 'true' - ) { + if (options.useCrossCompilerCache && process.env.NODE_ENV === 'production') { options = { ...options, crossCompilerCache: new Map(), diff --git a/packages/docusaurus-mdx-loader/src/loader.ts b/packages/docusaurus-mdx-loader/src/loader.ts index ab60427f3776..546d4390dccf 100644 --- a/packages/docusaurus-mdx-loader/src/loader.ts +++ b/packages/docusaurus-mdx-loader/src/loader.ts @@ -125,29 +125,33 @@ ${result.content} // Note: we cache promises instead of strings // This is because client/server compilations might be triggered in parallel // When this happens for the same file, we don't want to compile it twice -// Note we use fileContent instead of filePath as cache key -// This is because the same file can be compiled with different options -// This is notably the case for blog posts that can be truncated -// An alternative would be to use this.resource (including ?query#hash) async function loadMDXWithCaching({ + resource, fileContent, filePath, options, compilerName, }: { + resource: string; // path?query#hash + filePath: string; // path fileContent: string; - filePath: string; options: Options; compilerName: WebpackCompilerName; }): Promise { - const cachedPromise = options.crossCompilerCache?.get(fileContent); + // Note we "resource" as cache key, not "filePath" nor "fileContent" + // This is because: + // - the same file can be compiled in different variants (blog.mdx?truncated) + // - the same content can be processed differently (versioned docs links) + const cacheKey = resource; + + const cachedPromise = options.crossCompilerCache?.get(cacheKey); if (cachedPromise) { // We can clean up the cache and free memory here // We know there are only 2 compilations for the same file // Note: once we introduce RSCs we'll probably have 3 compilations // Note: we can't use string keys in WeakMap // But we could eventually use WeakRef for the values - options.crossCompilerCache?.delete(fileContent); + options.crossCompilerCache?.delete(cacheKey); return cachedPromise; } const promise = loadMDX({ @@ -156,7 +160,7 @@ async function loadMDXWithCaching({ options, compilerName, }); - options.crossCompilerCache?.set(fileContent, promise); + options.crossCompilerCache?.set(cacheKey, promise); return promise; } @@ -166,12 +170,12 @@ export async function mdxLoader( ): Promise { const compilerName = getWebpackLoaderCompilerName(this); const callback = this.async(); - const filePath = this.resourcePath; const options: Options = this.getOptions(); try { const result = await loadMDXWithCaching({ + resource: this.resource, + filePath: this.resourcePath, fileContent, - filePath, options, compilerName, });