Skip to content

Commit

Permalink
perf: improve FlagDependencyExportsPlugin for large JSON by depth
Browse files Browse the repository at this point in the history
  • Loading branch information
@hai-x
hai-x committed Dec 22, 2024
1 parent ed67e97 commit cad432f
Show file tree
Hide file tree
Showing 15 changed files with 249 additions and 49 deletions.
7 changes: 6 additions & 1 deletion crates/node_binding/binding.d.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1569,6 +1569,10 @@ export interface RawJavascriptParserOptions {
importDynamic?: boolean
}

export interface RawJsonParserOptions {
exportsDepth?: number
}

export interface RawLazyCompilationOption {
module: ((err: Error | null, arg: RawModuleArg) => RawModuleInfo)
test?: RawLazyCompilationTest
Expand Down Expand Up @@ -1804,12 +1808,13 @@ export interface RawOutputOptions {
}

export interface RawParserOptions {
type: "asset" | "css" | "css/auto" | "css/module" | "javascript" | "javascript/auto" | "javascript/dynamic" | "javascript/esm"
type: "asset" | "css" | "css/auto" | "css/module" | "javascript" | "javascript/auto" | "javascript/dynamic" | "javascript/esm" | "json"
asset?: RawAssetParserOptions
css?: RawCssParserOptions
cssAuto?: RawCssAutoParserOptions
cssModule?: RawCssModuleParserOptions
javascript?: RawJavascriptParserOptions
json?: RawJsonParserOptions
}

export interface RawPathData {
Expand Down
31 changes: 26 additions & 5 deletions crates/rspack_binding_values/src/raw_options/raw_module/mod.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,10 @@ use rspack_core::{
CssAutoGeneratorOptions, CssAutoParserOptions, CssGeneratorOptions, CssModuleGeneratorOptions,
CssModuleParserOptions, CssParserOptions, DescriptionData, DynamicImportFetchPriority,
DynamicImportMode, ExportPresenceMode, FuncUseCtx, GeneratorOptions, GeneratorOptionsMap,
JavascriptParserOptions, JavascriptParserOrder, JavascriptParserUrl, ModuleNoParseRule,
ModuleNoParseRules, ModuleNoParseTestFn, ModuleOptions, ModuleRule, ModuleRuleEffect,
ModuleRuleEnforce, ModuleRuleUse, ModuleRuleUseLoader, OverrideStrict, ParserOptions,
ParserOptionsMap,
JavascriptParserOptions, JavascriptParserOrder, JavascriptParserUrl, JsonParserOptions,
ModuleNoParseRule, ModuleNoParseRules, ModuleNoParseTestFn, ModuleOptions, ModuleRule,
ModuleRuleEffect, ModuleRuleEnforce, ModuleRuleUse, ModuleRuleUseLoader, OverrideStrict,
ParserOptions, ParserOptionsMap,
};
use rspack_error::error;
use rspack_napi::threadsafe_function::ThreadsafeFunction;
Expand Down Expand Up @@ -183,14 +183,15 @@ pub struct RawModuleRule {
#[napi(object)]
pub struct RawParserOptions {
#[napi(
ts_type = r#""asset" | "css" | "css/auto" | "css/module" | "javascript" | "javascript/auto" | "javascript/dynamic" | "javascript/esm""#
ts_type = r#""asset" | "css" | "css/auto" | "css/module" | "javascript" | "javascript/auto" | "javascript/dynamic" | "javascript/esm" | "json""#
)]
pub r#type: String,
pub asset: Option<RawAssetParserOptions>,
pub css: Option<RawCssParserOptions>,
pub css_auto: Option<RawCssAutoParserOptions>,
pub css_module: Option<RawCssModuleParserOptions>,
pub javascript: Option<RawJavascriptParserOptions>,
pub json: Option<RawJsonParserOptions>,
}

impl From<RawParserOptions> for ParserOptions {
Expand Down Expand Up @@ -246,6 +247,12 @@ impl From<RawParserOptions> for ParserOptions {
.expect("should have an \"css_module\" when RawParserOptions.type is \"css/module\"")
.into(),
),
"json" => Self::Json(
value
.json
.expect("should have an \"json\" when RawParserOptions.type is \"json\"")
.into(),
),
_ => panic!(
"Failed to resolve the RawParserOptions.type {}.",
value.r#type
Expand Down Expand Up @@ -425,6 +432,20 @@ impl From<RawCssModuleParserOptions> for CssModuleParserOptions {
}
}

#[derive(Debug, Default)]
#[napi(object)]
pub struct RawJsonParserOptions {
pub exports_depth: Option<f64>,
}

impl From<RawJsonParserOptions> for JsonParserOptions {
fn from(value: RawJsonParserOptions) -> Self {
Self {
exports_depth: value.exports_depth,
}
}
}

#[derive(Debug, Default)]
#[napi(object, object_to_js = false)]
pub struct RawGeneratorOptions {
Expand Down
8 changes: 8 additions & 0 deletions crates/rspack_core/src/options/module.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ pub enum ParserOptions {
JavascriptAuto(JavascriptParserOptions),
JavascriptEsm(JavascriptParserOptions),
JavascriptDynamic(JavascriptParserOptions),
Json(JsonParserOptions),
Unknown,
}

Expand Down Expand Up @@ -68,6 +69,7 @@ impl ParserOptions {
JavascriptDynamic,
JavascriptParserOptions
);
get_variant!(get_json, Json, JsonParserOptions);
}

#[cacheable]
Expand Down Expand Up @@ -287,6 +289,12 @@ pub struct CssModuleParserOptions {
pub named_exports: Option<bool>,
}

#[cacheable]
#[derive(Debug, Clone, MergeFrom)]
pub struct JsonParserOptions {
pub exports_depth: Option<f64>,
}

#[derive(Debug)]
pub struct GeneratorOptionsMap(HashMap<String, GeneratorOptions>);

Expand Down
37 changes: 26 additions & 11 deletions crates/rspack_plugin_json/src/json_exports_dependency.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,13 +13,15 @@ pub struct JsonExportsDependency {
id: DependencyId,
#[cacheable(with=AsPreset)]
data: JsonValue,
exports_depth: f64,
}

impl JsonExportsDependency {
pub fn new(data: JsonValue) -> Self {
pub fn new(data: JsonValue, exports_depth: f64) -> Self {
Self {
data,
id: DependencyId::new(),
exports_depth,
}
}
}
Expand All @@ -31,8 +33,10 @@ impl Dependency for JsonExportsDependency {
}

fn get_exports(&self, _mg: &ModuleGraph) -> Option<ExportsSpec> {
let exports_depth: u64 = self.exports_depth as u64;
Some(ExportsSpec {
exports: get_exports_from_data(&self.data).unwrap_or(ExportsOfExportsSpec::Null),
exports: get_exports_from_data(&self.data, exports_depth, 1)
.unwrap_or(ExportsOfExportsSpec::Null),
..Default::default()
})
}
Expand Down Expand Up @@ -68,7 +72,14 @@ impl DependencyTemplate for JsonExportsDependency {
}
}

fn get_exports_from_data(data: &JsonValue) -> Option<ExportsOfExportsSpec> {
fn get_exports_from_data(
data: &JsonValue,
exports_depth: u64,
cur_depth: u64,
) -> Option<ExportsOfExportsSpec> {
if cur_depth > exports_depth {
return None;
}
let ret = match data {
JsonValue::Null
| JsonValue::Short(_)
Expand All @@ -84,11 +95,13 @@ fn get_exports_from_data(data: &JsonValue) -> Option<ExportsOfExportsSpec> {
ExportNameOrSpec::ExportSpec(ExportSpec {
name: k.into(),
can_mangle: Some(true),
exports: get_exports_from_data(v).map(|item| match item {
ExportsOfExportsSpec::True => unreachable!(),
ExportsOfExportsSpec::Null => unreachable!(),
ExportsOfExportsSpec::Array(arr) => arr,
}),
exports: get_exports_from_data(v, exports_depth, cur_depth + 1).map(
|item| match item {
ExportsOfExportsSpec::True => unreachable!(),
ExportsOfExportsSpec::Null => unreachable!(),
ExportsOfExportsSpec::Array(arr) => arr,
},
),
..Default::default()
})
})
Expand All @@ -106,9 +119,11 @@ fn get_exports_from_data(data: &JsonValue) -> Option<ExportsOfExportsSpec> {
ExportNameOrSpec::ExportSpec(ExportSpec {
name: itoa!(i).into(),
can_mangle: Some(true),
exports: get_exports_from_data(item).map(|item| match item {
ExportsOfExportsSpec::True | ExportsOfExportsSpec::Null => unreachable!(),
ExportsOfExportsSpec::Array(arr) => arr,
exports: get_exports_from_data(item, exports_depth, cur_depth + 1).map(|item| {
match item {
ExportsOfExportsSpec::True | ExportsOfExportsSpec::Null => unreachable!(),
ExportsOfExportsSpec::Array(arr) => arr,
}
}),
..Default::default()
})
Expand Down
19 changes: 16 additions & 3 deletions crates/rspack_plugin_json/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,9 @@ mod utils;

#[cacheable]
#[derive(Debug)]
struct JsonParserAndGenerator;
struct JsonParserAndGenerator {
pub exports_depth: f64,
}

#[cacheable_dyn]
impl ParserAndGenerator for JsonParserAndGenerator {
Expand Down Expand Up @@ -119,7 +121,10 @@ impl ParserAndGenerator for JsonParserAndGenerator {
rspack_core::ParseResult {
presentational_dependencies: vec![],
dependencies: if let Some(data) = data {
vec![Box::new(JsonExportsDependency::new(data))]
vec![Box::new(JsonExportsDependency::new(
data,
self.exports_depth,
))]
} else {
vec![]
},
Expand Down Expand Up @@ -224,7 +229,15 @@ impl Plugin for JsonPlugin {
) -> Result<()> {
ctx.context.register_parser_and_generator_builder(
rspack_core::ModuleType::Json,
Box::new(|_, _| Box::new(JsonParserAndGenerator {})),
Box::new(|p, _| {
let p = p
.and_then(|p| p.get_json())
.expect("should have JsonParserOptions");

Box::new(JsonParserAndGenerator {
exports_depth: p.exports_depth.expect("should have exports_depth"),
})
}),
);

Ok(())
Expand Down
8 changes: 8 additions & 0 deletions packages/rspack/etc/core.api.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2747,6 +2747,11 @@ type JsonObject_2 = {
[Key in string]?: JsonValue_2 | undefined;
};

// @public (undocumented)
export type JsonParserOptions = {
exportsDepth?: number;
};

// @public (undocumented)
type JsonPrimitive = string | number | boolean | null;

Expand Down Expand Up @@ -4460,6 +4465,7 @@ export type ParserOptionsByModuleTypeKnown = {
"javascript/auto"?: JavascriptParserOptions;
"javascript/dynamic"?: JavascriptParserOptions;
"javascript/esm"?: JavascriptParserOptions;
json?: JsonParserOptions;
};

// @public
Expand Down Expand Up @@ -5308,6 +5314,7 @@ declare namespace rspackExports {
CssAutoParserOptions,
CssModuleParserOptions,
JavascriptParserOptions,
JsonParserOptions,
ParserOptionsByModuleTypeKnown,
ParserOptionsByModuleTypeUnknown,
ParserOptionsByModuleType,
Expand Down Expand Up @@ -10453,6 +10460,7 @@ declare namespace t {
CssAutoParserOptions,
CssModuleParserOptions,
JavascriptParserOptions,
JsonParserOptions,
ParserOptionsByModuleTypeKnown,
ParserOptionsByModuleTypeUnknown,
ParserOptionsByModuleType,
Expand Down
70 changes: 44 additions & 26 deletions packages/rspack/src/config/adapter.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import {
type RawFuncUseCtx,
type RawGeneratorOptions,
type RawJavascriptParserOptions,
type RawJsonParserOptions,
type RawModuleRule,
type RawModuleRuleUse,
type RawOptions,
Expand Down Expand Up @@ -54,6 +55,7 @@ import type {
CssParserOptions,
GeneratorOptionsByModuleType,
JavascriptParserOptions,
JsonParserOptions,
Node,
Optimization,
ParserOptionsByModuleType,
Expand Down Expand Up @@ -261,19 +263,19 @@ const getRawModuleRule = (
: undefined,
descriptionData: rule.descriptionData
? Object.fromEntries(
Object.entries(rule.descriptionData).map(([k, v]) => [
k,
getRawRuleSetCondition(v)
])
)
Object.entries(rule.descriptionData).map(([k, v]) => [
k,
getRawRuleSetCondition(v)
])
)
: undefined,
with: rule.with
? Object.fromEntries(
Object.entries(rule.with).map(([k, v]) => [
k,
getRawRuleSetCondition(v)
])
)
Object.entries(rule.with).map(([k, v]) => [
k,
getRawRuleSetCondition(v)
])
)
: undefined,
resource: rule.resource ? getRawRuleSetCondition(rule.resource) : undefined,
resourceQuery: rule.resourceQuery
Expand All @@ -300,27 +302,27 @@ const getRawModuleRule = (
resolve: rule.resolve ? getRawResolve(rule.resolve) : undefined,
oneOf: rule.oneOf
? rule.oneOf
.filter(Boolean)
.map((rule, index) =>
getRawModuleRule(
rule as RuleSetRule,
`${path}.oneOf[${index}]`,
options,
(rule as RuleSetRule).type ?? upperType
)
.filter(Boolean)
.map((rule, index) =>
getRawModuleRule(
rule as RuleSetRule,
`${path}.oneOf[${index}]`,
options,
(rule as RuleSetRule).type ?? upperType
)
)
: undefined,
rules: rule.rules
? rule.rules
.filter(Boolean)
.map((rule, index) =>
getRawModuleRule(
rule as RuleSetRule,
`${path}.rules[${index}]`,
options,
(rule as RuleSetRule).type ?? upperType
)
.filter(Boolean)
.map((rule, index) =>
getRawModuleRule(
rule as RuleSetRule,
`${path}.rules[${index}]`,
options,
(rule as RuleSetRule).type ?? upperType
)
)
: undefined,
enforce: rule.enforce
};
Expand Down Expand Up @@ -490,6 +492,14 @@ function getRawParserOptions(
cssModule: getRawCssParserOptions(parser)
};
}

if (type === "json") {
return {
type: "json",
json: getRawJsonParserOptions(parser)
};
}

// FIXME: shouldn't depend on module type, for example: `rules: [{ test: /\.css/, generator: {..} }]` will error
throw new Error(`unreachable: unknow module type: ${type}`);
}
Expand Down Expand Up @@ -566,6 +576,14 @@ function getRawCssParserOptions(
};
}

function getRawJsonParserOptions(
parser: JsonParserOptions
): RawJsonParserOptions {
return {
exportsDepth: parser.exportsDepth
};
}

function getRawGeneratorOptions(
generator: { [k: string]: any },
type: string
Expand Down
Loading

0 comments on commit cad432f

Please sign in to comment.