From cad432fba9f64f4dfb54233ec86e3b514a948c6d Mon Sep 17 00:00:00 2001 From: hai-x Date: Sun, 22 Dec 2024 03:28:22 +0800 Subject: [PATCH] perf: improve FlagDependencyExportsPlugin for large JSON by depth --- crates/node_binding/binding.d.ts | 7 +- .../src/raw_options/raw_module/mod.rs | 31 ++++++-- crates/rspack_core/src/options/module.rs | 8 +++ .../src/json_exports_dependency.rs | 37 +++++++--- crates/rspack_plugin_json/src/lib.rs | 19 ++++- packages/rspack/etc/core.api.md | 8 +++ packages/rspack/src/config/adapter.ts | 70 ++++++++++++------- packages/rspack/src/config/defaults.ts | 17 ++++- packages/rspack/src/config/types.ts | 10 +++ .../bailout-flag-dep-export-perf/data.json | 27 +++++++ .../bailout-flag-dep-export-perf/index.js | 11 +++ .../webpack.config.js | 11 +++ .../json/flag-dep-export-perf/data.json | 27 +++++++ .../json/flag-dep-export-perf/index.js | 11 +++ .../flag-dep-export-perf/webpack.config.js | 4 ++ 15 files changed, 249 insertions(+), 49 deletions(-) create mode 100644 tests/webpack-test/configCases/json/bailout-flag-dep-export-perf/data.json create mode 100644 tests/webpack-test/configCases/json/bailout-flag-dep-export-perf/index.js create mode 100644 tests/webpack-test/configCases/json/bailout-flag-dep-export-perf/webpack.config.js create mode 100644 tests/webpack-test/configCases/json/flag-dep-export-perf/data.json create mode 100644 tests/webpack-test/configCases/json/flag-dep-export-perf/index.js create mode 100644 tests/webpack-test/configCases/json/flag-dep-export-perf/webpack.config.js diff --git a/crates/node_binding/binding.d.ts b/crates/node_binding/binding.d.ts index 6aadbe376b0..cea98c381d7 100644 --- a/crates/node_binding/binding.d.ts +++ b/crates/node_binding/binding.d.ts @@ -1569,6 +1569,10 @@ export interface RawJavascriptParserOptions { importDynamic?: boolean } +export interface RawJsonParserOptions { + exportsDepth?: number +} + export interface RawLazyCompilationOption { module: ((err: Error | null, arg: RawModuleArg) => RawModuleInfo) test?: RawLazyCompilationTest @@ -1804,12 +1808,13 @@ export interface RawOutputOptions { } export interface RawParserOptions { - type: "asset" | "css" | "css/auto" | "css/module" | "javascript" | "javascript/auto" | "javascript/dynamic" | "javascript/esm" + type: "asset" | "css" | "css/auto" | "css/module" | "javascript" | "javascript/auto" | "javascript/dynamic" | "javascript/esm" | "json" asset?: RawAssetParserOptions css?: RawCssParserOptions cssAuto?: RawCssAutoParserOptions cssModule?: RawCssModuleParserOptions javascript?: RawJavascriptParserOptions + json?: RawJsonParserOptions } export interface RawPathData { diff --git a/crates/rspack_binding_values/src/raw_options/raw_module/mod.rs b/crates/rspack_binding_values/src/raw_options/raw_module/mod.rs index da924f94b08..8697622ac73 100644 --- a/crates/rspack_binding_values/src/raw_options/raw_module/mod.rs +++ b/crates/rspack_binding_values/src/raw_options/raw_module/mod.rs @@ -12,10 +12,10 @@ use rspack_core::{ CssAutoGeneratorOptions, CssAutoParserOptions, CssGeneratorOptions, CssModuleGeneratorOptions, CssModuleParserOptions, CssParserOptions, DescriptionData, DynamicImportFetchPriority, DynamicImportMode, ExportPresenceMode, FuncUseCtx, GeneratorOptions, GeneratorOptionsMap, - JavascriptParserOptions, JavascriptParserOrder, JavascriptParserUrl, ModuleNoParseRule, - ModuleNoParseRules, ModuleNoParseTestFn, ModuleOptions, ModuleRule, ModuleRuleEffect, - ModuleRuleEnforce, ModuleRuleUse, ModuleRuleUseLoader, OverrideStrict, ParserOptions, - ParserOptionsMap, + JavascriptParserOptions, JavascriptParserOrder, JavascriptParserUrl, JsonParserOptions, + ModuleNoParseRule, ModuleNoParseRules, ModuleNoParseTestFn, ModuleOptions, ModuleRule, + ModuleRuleEffect, ModuleRuleEnforce, ModuleRuleUse, ModuleRuleUseLoader, OverrideStrict, + ParserOptions, ParserOptionsMap, }; use rspack_error::error; use rspack_napi::threadsafe_function::ThreadsafeFunction; @@ -183,7 +183,7 @@ pub struct RawModuleRule { #[napi(object)] pub struct RawParserOptions { #[napi( - ts_type = r#""asset" | "css" | "css/auto" | "css/module" | "javascript" | "javascript/auto" | "javascript/dynamic" | "javascript/esm""# + ts_type = r#""asset" | "css" | "css/auto" | "css/module" | "javascript" | "javascript/auto" | "javascript/dynamic" | "javascript/esm" | "json""# )] pub r#type: String, pub asset: Option, @@ -191,6 +191,7 @@ pub struct RawParserOptions { pub css_auto: Option, pub css_module: Option, pub javascript: Option, + pub json: Option, } impl From for ParserOptions { @@ -246,6 +247,12 @@ impl From for ParserOptions { .expect("should have an \"css_module\" when RawParserOptions.type is \"css/module\"") .into(), ), + "json" => Self::Json( + value + .json + .expect("should have an \"json\" when RawParserOptions.type is \"json\"") + .into(), + ), _ => panic!( "Failed to resolve the RawParserOptions.type {}.", value.r#type @@ -425,6 +432,20 @@ impl From for CssModuleParserOptions { } } +#[derive(Debug, Default)] +#[napi(object)] +pub struct RawJsonParserOptions { + pub exports_depth: Option, +} + +impl From for JsonParserOptions { + fn from(value: RawJsonParserOptions) -> Self { + Self { + exports_depth: value.exports_depth, + } + } +} + #[derive(Debug, Default)] #[napi(object, object_to_js = false)] pub struct RawGeneratorOptions { diff --git a/crates/rspack_core/src/options/module.rs b/crates/rspack_core/src/options/module.rs index 9579dc6c5b1..5ac9e43ef94 100644 --- a/crates/rspack_core/src/options/module.rs +++ b/crates/rspack_core/src/options/module.rs @@ -41,6 +41,7 @@ pub enum ParserOptions { JavascriptAuto(JavascriptParserOptions), JavascriptEsm(JavascriptParserOptions), JavascriptDynamic(JavascriptParserOptions), + Json(JsonParserOptions), Unknown, } @@ -68,6 +69,7 @@ impl ParserOptions { JavascriptDynamic, JavascriptParserOptions ); + get_variant!(get_json, Json, JsonParserOptions); } #[cacheable] @@ -287,6 +289,12 @@ pub struct CssModuleParserOptions { pub named_exports: Option, } +#[cacheable] +#[derive(Debug, Clone, MergeFrom)] +pub struct JsonParserOptions { + pub exports_depth: Option, +} + #[derive(Debug)] pub struct GeneratorOptionsMap(HashMap); diff --git a/crates/rspack_plugin_json/src/json_exports_dependency.rs b/crates/rspack_plugin_json/src/json_exports_dependency.rs index 629b3d6899d..12c469fc45b 100644 --- a/crates/rspack_plugin_json/src/json_exports_dependency.rs +++ b/crates/rspack_plugin_json/src/json_exports_dependency.rs @@ -13,13 +13,15 @@ pub struct JsonExportsDependency { id: DependencyId, #[cacheable(with=AsPreset)] data: JsonValue, + exports_depth: f64, } impl JsonExportsDependency { - pub fn new(data: JsonValue) -> Self { + pub fn new(data: JsonValue, exports_depth: f64) -> Self { Self { data, id: DependencyId::new(), + exports_depth, } } } @@ -31,8 +33,10 @@ impl Dependency for JsonExportsDependency { } fn get_exports(&self, _mg: &ModuleGraph) -> Option { + let exports_depth: u64 = self.exports_depth as u64; Some(ExportsSpec { - exports: get_exports_from_data(&self.data).unwrap_or(ExportsOfExportsSpec::Null), + exports: get_exports_from_data(&self.data, exports_depth, 1) + .unwrap_or(ExportsOfExportsSpec::Null), ..Default::default() }) } @@ -68,7 +72,14 @@ impl DependencyTemplate for JsonExportsDependency { } } -fn get_exports_from_data(data: &JsonValue) -> Option { +fn get_exports_from_data( + data: &JsonValue, + exports_depth: u64, + cur_depth: u64, +) -> Option { + if cur_depth > exports_depth { + return None; + } let ret = match data { JsonValue::Null | JsonValue::Short(_) @@ -84,11 +95,13 @@ fn get_exports_from_data(data: &JsonValue) -> Option { ExportNameOrSpec::ExportSpec(ExportSpec { name: k.into(), can_mangle: Some(true), - exports: get_exports_from_data(v).map(|item| match item { - ExportsOfExportsSpec::True => unreachable!(), - ExportsOfExportsSpec::Null => unreachable!(), - ExportsOfExportsSpec::Array(arr) => arr, - }), + exports: get_exports_from_data(v, exports_depth, cur_depth + 1).map( + |item| match item { + ExportsOfExportsSpec::True => unreachable!(), + ExportsOfExportsSpec::Null => unreachable!(), + ExportsOfExportsSpec::Array(arr) => arr, + }, + ), ..Default::default() }) }) @@ -106,9 +119,11 @@ fn get_exports_from_data(data: &JsonValue) -> Option { ExportNameOrSpec::ExportSpec(ExportSpec { name: itoa!(i).into(), can_mangle: Some(true), - exports: get_exports_from_data(item).map(|item| match item { - ExportsOfExportsSpec::True | ExportsOfExportsSpec::Null => unreachable!(), - ExportsOfExportsSpec::Array(arr) => arr, + exports: get_exports_from_data(item, exports_depth, cur_depth + 1).map(|item| { + match item { + ExportsOfExportsSpec::True | ExportsOfExportsSpec::Null => unreachable!(), + ExportsOfExportsSpec::Array(arr) => arr, + } }), ..Default::default() }) diff --git a/crates/rspack_plugin_json/src/lib.rs b/crates/rspack_plugin_json/src/lib.rs index f9abb2bef1e..a611a5a7a39 100644 --- a/crates/rspack_plugin_json/src/lib.rs +++ b/crates/rspack_plugin_json/src/lib.rs @@ -32,7 +32,9 @@ mod utils; #[cacheable] #[derive(Debug)] -struct JsonParserAndGenerator; +struct JsonParserAndGenerator { + pub exports_depth: f64, +} #[cacheable_dyn] impl ParserAndGenerator for JsonParserAndGenerator { @@ -119,7 +121,10 @@ impl ParserAndGenerator for JsonParserAndGenerator { rspack_core::ParseResult { presentational_dependencies: vec![], dependencies: if let Some(data) = data { - vec![Box::new(JsonExportsDependency::new(data))] + vec![Box::new(JsonExportsDependency::new( + data, + self.exports_depth, + ))] } else { vec![] }, @@ -224,7 +229,15 @@ impl Plugin for JsonPlugin { ) -> Result<()> { ctx.context.register_parser_and_generator_builder( rspack_core::ModuleType::Json, - Box::new(|_, _| Box::new(JsonParserAndGenerator {})), + Box::new(|p, _| { + let p = p + .and_then(|p| p.get_json()) + .expect("should have JsonParserOptions"); + + Box::new(JsonParserAndGenerator { + exports_depth: p.exports_depth.expect("should have exports_depth"), + }) + }), ); Ok(()) diff --git a/packages/rspack/etc/core.api.md b/packages/rspack/etc/core.api.md index 785a9a494e6..27f21bc3977 100644 --- a/packages/rspack/etc/core.api.md +++ b/packages/rspack/etc/core.api.md @@ -2747,6 +2747,11 @@ type JsonObject_2 = { [Key in string]?: JsonValue_2 | undefined; }; +// @public (undocumented) +export type JsonParserOptions = { + exportsDepth?: number; +}; + // @public (undocumented) type JsonPrimitive = string | number | boolean | null; @@ -4460,6 +4465,7 @@ export type ParserOptionsByModuleTypeKnown = { "javascript/auto"?: JavascriptParserOptions; "javascript/dynamic"?: JavascriptParserOptions; "javascript/esm"?: JavascriptParserOptions; + json?: JsonParserOptions; }; // @public @@ -5308,6 +5314,7 @@ declare namespace rspackExports { CssAutoParserOptions, CssModuleParserOptions, JavascriptParserOptions, + JsonParserOptions, ParserOptionsByModuleTypeKnown, ParserOptionsByModuleTypeUnknown, ParserOptionsByModuleType, @@ -10453,6 +10460,7 @@ declare namespace t { CssAutoParserOptions, CssModuleParserOptions, JavascriptParserOptions, + JsonParserOptions, ParserOptionsByModuleTypeKnown, ParserOptionsByModuleTypeUnknown, ParserOptionsByModuleType, diff --git a/packages/rspack/src/config/adapter.ts b/packages/rspack/src/config/adapter.ts index 72c1c88683b..af27a8ab9ba 100644 --- a/packages/rspack/src/config/adapter.ts +++ b/packages/rspack/src/config/adapter.ts @@ -15,6 +15,7 @@ import { type RawFuncUseCtx, type RawGeneratorOptions, type RawJavascriptParserOptions, + type RawJsonParserOptions, type RawModuleRule, type RawModuleRuleUse, type RawOptions, @@ -54,6 +55,7 @@ import type { CssParserOptions, GeneratorOptionsByModuleType, JavascriptParserOptions, + JsonParserOptions, Node, Optimization, ParserOptionsByModuleType, @@ -261,19 +263,19 @@ const getRawModuleRule = ( : undefined, descriptionData: rule.descriptionData ? Object.fromEntries( - Object.entries(rule.descriptionData).map(([k, v]) => [ - k, - getRawRuleSetCondition(v) - ]) - ) + Object.entries(rule.descriptionData).map(([k, v]) => [ + k, + getRawRuleSetCondition(v) + ]) + ) : undefined, with: rule.with ? Object.fromEntries( - Object.entries(rule.with).map(([k, v]) => [ - k, - getRawRuleSetCondition(v) - ]) - ) + Object.entries(rule.with).map(([k, v]) => [ + k, + getRawRuleSetCondition(v) + ]) + ) : undefined, resource: rule.resource ? getRawRuleSetCondition(rule.resource) : undefined, resourceQuery: rule.resourceQuery @@ -300,27 +302,27 @@ const getRawModuleRule = ( resolve: rule.resolve ? getRawResolve(rule.resolve) : undefined, oneOf: rule.oneOf ? rule.oneOf - .filter(Boolean) - .map((rule, index) => - getRawModuleRule( - rule as RuleSetRule, - `${path}.oneOf[${index}]`, - options, - (rule as RuleSetRule).type ?? upperType - ) + .filter(Boolean) + .map((rule, index) => + getRawModuleRule( + rule as RuleSetRule, + `${path}.oneOf[${index}]`, + options, + (rule as RuleSetRule).type ?? upperType ) + ) : undefined, rules: rule.rules ? rule.rules - .filter(Boolean) - .map((rule, index) => - getRawModuleRule( - rule as RuleSetRule, - `${path}.rules[${index}]`, - options, - (rule as RuleSetRule).type ?? upperType - ) + .filter(Boolean) + .map((rule, index) => + getRawModuleRule( + rule as RuleSetRule, + `${path}.rules[${index}]`, + options, + (rule as RuleSetRule).type ?? upperType ) + ) : undefined, enforce: rule.enforce }; @@ -490,6 +492,14 @@ function getRawParserOptions( cssModule: getRawCssParserOptions(parser) }; } + + if (type === "json") { + return { + type: "json", + json: getRawJsonParserOptions(parser) + }; + } + // FIXME: shouldn't depend on module type, for example: `rules: [{ test: /\.css/, generator: {..} }]` will error throw new Error(`unreachable: unknow module type: ${type}`); } @@ -566,6 +576,14 @@ function getRawCssParserOptions( }; } +function getRawJsonParserOptions( + parser: JsonParserOptions +): RawJsonParserOptions { + return { + exportsDepth: parser.exportsDepth + }; +} + function getRawGeneratorOptions( generator: { [k: string]: any }, type: string diff --git a/packages/rspack/src/config/defaults.ts b/packages/rspack/src/config/defaults.ts index af221c0653a..a7209e13641 100644 --- a/packages/rspack/src/config/defaults.ts +++ b/packages/rspack/src/config/defaults.ts @@ -12,7 +12,7 @@ import assert from "node:assert"; import fs from "node:fs"; import path from "node:path"; -import { ASSET_MODULE_TYPE } from "../ModuleTypeConstants"; +import { ASSET_MODULE_TYPE, JSON_MODULE_TYPE } from "../ModuleTypeConstants"; import { Template } from "../Template"; import { LightningCssMinimizerRspackPlugin, @@ -102,7 +102,8 @@ export const applyRspackOptionsDefaults = ( applyModuleDefaults(options.module, { asyncWebAssembly: options.experiments.asyncWebAssembly!, css: options.experiments.css, - targetProperties + targetProperties, + mode: options.mode }); applyOutputDefaults(options.output, { @@ -285,11 +286,13 @@ const applyModuleDefaults = ( { asyncWebAssembly, css, - targetProperties + targetProperties, + mode }: { asyncWebAssembly: boolean; css?: boolean; targetProperties: any; + mode?: Mode; } ) => { assertNotNill(module.parser); @@ -307,6 +310,14 @@ const applyModuleDefaults = ( assertNotNill(module.parser.javascript); applyJavascriptParserOptionsDefaults(module.parser.javascript); + F(module.parser, JSON_MODULE_TYPE, () => ({})); + assertNotNill(module.parser[JSON_MODULE_TYPE]); + D( + module.parser[JSON_MODULE_TYPE], + "exportsDepth", + mode === "development" ? 1 : Number.POSITIVE_INFINITY + ); + if (css) { F(module.parser, "css", () => ({})); assertNotNill(module.parser.css); diff --git a/packages/rspack/src/config/types.ts b/packages/rspack/src/config/types.ts index 879843399e4..0a94af85056 100644 --- a/packages/rspack/src/config/types.ts +++ b/packages/rspack/src/config/types.ts @@ -1068,6 +1068,13 @@ export type JavascriptParserOptions = { importDynamic?: boolean; }; +export type JsonParserOptions = { + /** + * The depth of json dependency flagged as `exportInfo`. + */ + exportsDepth?: number; +}; + /** Configure all parsers' options in one place with module.parser. */ export type ParserOptionsByModuleTypeKnown = { /** Parser options for `asset` modules. */ @@ -1093,6 +1100,9 @@ export type ParserOptionsByModuleTypeKnown = { /** Parser options for `javascript/esm` modules. */ "javascript/esm"?: JavascriptParserOptions; + + /** Parser options for `json` modules. */ + json?: JsonParserOptions; }; /** Configure all parsers' options in one place with module.parser. */ diff --git a/tests/webpack-test/configCases/json/bailout-flag-dep-export-perf/data.json b/tests/webpack-test/configCases/json/bailout-flag-dep-export-perf/data.json new file mode 100644 index 00000000000..bee9f9b8d88 --- /dev/null +++ b/tests/webpack-test/configCases/json/bailout-flag-dep-export-perf/data.json @@ -0,0 +1,27 @@ +{ + "depth_1": { + "depth_2": { + "depth_3": { + "depth_4": { + "depth_5": { + "depth_6": "depth_6" + } + } + } + } + }, + "_depth_1": { + "_depth_2": { + "_depth_3": { + "_depth_4": { + "_depth_5": { + "_depth_6": "_depth_6" + } + } + } + } + }, + "__depth_1": [ + { "__depth_3": [{ "__depth_5": [{ "__depth_7": ["__depth_8"] }] }] } + ] +} diff --git a/tests/webpack-test/configCases/json/bailout-flag-dep-export-perf/index.js b/tests/webpack-test/configCases/json/bailout-flag-dep-export-perf/index.js new file mode 100644 index 00000000000..df6ffa8072b --- /dev/null +++ b/tests/webpack-test/configCases/json/bailout-flag-dep-export-perf/index.js @@ -0,0 +1,11 @@ +export * from './data.json'; + +it("should compile and run", () => { + expect(__webpack_exports_info__.depth_1.provideInfo).toBe(true) + expect(__webpack_exports_info__._depth_1.provideInfo).toBe(true) + expect(__webpack_exports_info__.__depth_1.provideInfo).toBe(true) + + expect(__webpack_exports_info__.depth_1.depth_2.provideInfo).toBe(true) + expect(__webpack_exports_info__._depth_1._depth_2._depth_3._depth_4.provideInfo).toBe(true) + expect(__webpack_exports_info__.__depth_1[0].__depth_3[0].__depth_5.provideInfo).toBe(true) +}); diff --git a/tests/webpack-test/configCases/json/bailout-flag-dep-export-perf/webpack.config.js b/tests/webpack-test/configCases/json/bailout-flag-dep-export-perf/webpack.config.js new file mode 100644 index 00000000000..9b83c43996c --- /dev/null +++ b/tests/webpack-test/configCases/json/bailout-flag-dep-export-perf/webpack.config.js @@ -0,0 +1,11 @@ +/** @type {import("@rspack/core").Configuration} */ +module.exports = { + mode: "development", + module: { + parser: { + json: { + exportsDepth: Infinity + } + } + } +}; diff --git a/tests/webpack-test/configCases/json/flag-dep-export-perf/data.json b/tests/webpack-test/configCases/json/flag-dep-export-perf/data.json new file mode 100644 index 00000000000..bee9f9b8d88 --- /dev/null +++ b/tests/webpack-test/configCases/json/flag-dep-export-perf/data.json @@ -0,0 +1,27 @@ +{ + "depth_1": { + "depth_2": { + "depth_3": { + "depth_4": { + "depth_5": { + "depth_6": "depth_6" + } + } + } + } + }, + "_depth_1": { + "_depth_2": { + "_depth_3": { + "_depth_4": { + "_depth_5": { + "_depth_6": "_depth_6" + } + } + } + } + }, + "__depth_1": [ + { "__depth_3": [{ "__depth_5": [{ "__depth_7": ["__depth_8"] }] }] } + ] +} diff --git a/tests/webpack-test/configCases/json/flag-dep-export-perf/index.js b/tests/webpack-test/configCases/json/flag-dep-export-perf/index.js new file mode 100644 index 00000000000..6ecc00cfec0 --- /dev/null +++ b/tests/webpack-test/configCases/json/flag-dep-export-perf/index.js @@ -0,0 +1,11 @@ +export * from './data.json'; + +it("should compile and run", () => { + expect(__webpack_exports_info__.depth_1.provideInfo).toBe(true) + expect(__webpack_exports_info__._depth_1.provideInfo).toBe(true) + expect(__webpack_exports_info__.__depth_1.provideInfo).toBe(true) + + expect(__webpack_exports_info__.depth_1.depth_2.provideInfo).toBe(undefined) + expect(__webpack_exports_info__._depth_1._depth_2._depth_3._depth_4.provideInfo).toBe(undefined) + expect(__webpack_exports_info__.__depth_1[0].__depth_3[0].__depth_5.provideInfo).toBe(undefined) +}); diff --git a/tests/webpack-test/configCases/json/flag-dep-export-perf/webpack.config.js b/tests/webpack-test/configCases/json/flag-dep-export-perf/webpack.config.js new file mode 100644 index 00000000000..45f6191a8d4 --- /dev/null +++ b/tests/webpack-test/configCases/json/flag-dep-export-perf/webpack.config.js @@ -0,0 +1,4 @@ +/** @type {import("@rspack/core").Configuration} */ +module.exports = { + mode: "development" +};