Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor(new_split_chunks): remove unused code and add more comments #2947

Merged
merged 1 commit into from
Apr 27, 2023
+92 −87
Merged

refactor(new_split_chunks): remove unused code and add more comments #2947

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 0 additions & 2 deletions crates/rspack_plugin_split_chunks_new/src/common.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,5 +57,3 @@ pub fn create_module_filter(re: Option<String>) -> ModuleFilter {
}

pub(crate) type SplitChunkSizes = FxHashMap<SourceType, f64>;

// pub type CacheGroupNameGetter = Arc<dyn Fn(&dyn Module) -> String + Send + Sync>;
9 changes: 7 additions & 2 deletions crates/rspack_plugin_split_chunks_new/src/module_group.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,9 @@ use crate::common::SplitChunkSizes;
///
/// `ModuleGroup` captures/contains a bunch of modules due to the `optimization.splitChunks` configuration.
///
/// `ModuleGroup` would be transform into `Chunk` in the end.
/// `ModuleGroup` would be transform into `Chunk`s in the end.
///
/// A `ModuleGroup` would be transform into multiple `Chunk`s if the `name` dynamic computed
///
/// The original name of `ModuleGroup` is `ChunkInfoItem` borrowed from Webpack
#[derive(Derivative)]
Expand All @@ -19,8 +21,11 @@ pub(crate) struct ModuleGroup {
pub modules: IdentifierSet,
pub cache_group_index: usize,
pub cache_group_priority: f64,
pub name: String,
/// If the `ModuleGroup` is going to create a chunk, which will be named using `chunk_name`
/// A module
pub chunk_name: String,
pub sizes: SplitChunkSizes,
/// `Chunk`s which `Module`s in this ModuleGroup belong to
#[derivative(Debug = "ignore")]
pub chunks: FxHashSet<ChunkUkey>,
}
Expand Down
168 changes: 85 additions & 83 deletions crates/rspack_plugin_split_chunks_new/src/plugin.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,29 +2,21 @@ use std::fmt::Debug;

use dashmap::DashMap;
use rayon::prelude::*;
use rspack_core::{ChunkUkey, Compilation, Module, Plugin};
use rspack_core::{Chunk, ChunkUkey, Compilation, Module, Plugin};
use rustc_hash::{FxHashMap, FxHashSet};

use crate::{
cache_group::CacheGroup,
module_group::{compare_entries, ModuleGroup},
};

struct _OverallOptions {
pub cache_groups: Vec<CacheGroup>,
pub min_chunks: u32,
pub max_size: f64,
pub min_size: f64,
}

type ModuleGroupMap = FxHashMap<String, ModuleGroup>;

pub struct PluginOptions {
pub cache_groups: Vec<CacheGroup>,
}

pub struct SplitChunksPlugin {
// overall: OverallOptions,
cache_groups: Vec<CacheGroup>,
}

Expand All @@ -36,46 +28,54 @@ impl SplitChunksPlugin {
}

fn inner_impl(&self, compilation: &mut Compilation) {
let mut module_group_map = self.prepare_module_and_chunks_info_map(compilation);
let mut module_group_map = self.prepare_module_group_map(compilation);

while !module_group_map.is_empty() {
let (_module_group_key, module_group) = self.find_best_module_group(&mut module_group_map);

let new_chunk = if let Some(chunk) = compilation.named_chunks.get(&module_group.name) {
*chunk
} else {
let chunk = Compilation::add_named_chunk(
module_group.name.clone(),
&mut compilation.chunk_by_ukey,
&mut compilation.named_chunks,
);

chunk
.chunk_reasons
.push("Create by split chunks".to_string());
chunk.ukey
};
compilation.chunk_graph.add_chunk(new_chunk);

let used_chunks = &module_group.chunks;
let new_chunk = self.get_corresponding_chunk(compilation, &module_group);

let original_chunks = &module_group.chunks;

self.move_modules_to_new_chunk_and_remove_from_old_chunks(
&module_group,
new_chunk,
used_chunks,
original_chunks,
compilation,
);
self.split_from_original_chunks(&module_group, used_chunks, new_chunk, compilation);

self.split_from_original_chunks(&module_group, original_chunks, new_chunk, compilation);

self.remove_all_modules_from_other_module_groups(
&module_group,
&mut module_group_map,
used_chunks,
original_chunks,
compilation,
)
}
}

fn _ensure_max_size_fit(&self, _compilation: &mut Compilation) {}
fn get_corresponding_chunk(
&self,
compilation: &mut Compilation,
module_group: &ModuleGroup,
) -> ChunkUkey {
if let Some(chunk) = compilation.named_chunks.get(&module_group.chunk_name) {
*chunk
} else {
let chunk = Compilation::add_named_chunk(
module_group.chunk_name.clone(),
&mut compilation.chunk_by_ukey,
&mut compilation.named_chunks,
);

chunk
.chunk_reasons
.push("Create by split chunks".to_string());
compilation.chunk_graph.add_chunk(chunk.ukey);
chunk.ukey
}
}

fn remove_all_modules_from_other_module_groups(
&self,
Expand Down Expand Up @@ -122,14 +122,14 @@ impl SplitChunksPlugin {
&self,
item: &ModuleGroup,
new_chunk: ChunkUkey,
used_chunks: &FxHashSet<ChunkUkey>,
original_chunks: &FxHashSet<ChunkUkey>,
compilation: &mut Compilation,
) {
for module_identifier in &item.modules {
// First, we remove modules from old chunks

// Remove module from old chunks
for used_chunk in used_chunks {
for used_chunk in original_chunks {
compilation
.chunk_graph
.disconnect_chunk_and_module(used_chunk, *module_identifier);
Expand All @@ -155,19 +155,20 @@ impl SplitChunksPlugin {
) {
let new_chunk_ukey = new_chunk;
for original_chunk in original_chunks {
debug_assert!(&new_chunk_ukey != original_chunk);
let [new_chunk, original_chunk] = compilation
.chunk_by_ukey
._todo_should_remove_this_method_inner_mut()
.get_many_mut([&new_chunk_ukey, original_chunk])
.expect("TODO:");
.expect("split_from_original_chunks failed");
original_chunk.split(new_chunk, &mut compilation.chunk_group_by_ukey);
}
}

fn find_best_module_group(&self, module_group_map: &mut ModuleGroupMap) -> (String, ModuleGroup) {
// perf(hyf): I wonder if we could use BinaryHeap to avoid sorting for find_best_module_group call
debug_assert!(!module_group_map.is_empty());
let mut iter = module_group_map.iter();
let mut iter: std::collections::hash_map::Iter<String, ModuleGroup> = module_group_map.iter();
let (key, mut best_module_group) = iter.next().expect("at least have one item");

let mut best_entry_key = key.clone();
Expand All @@ -184,11 +185,20 @@ impl SplitChunksPlugin {
(best_entry_key, best_module_group)
}

fn prepare_module_and_chunks_info_map(&self, compilation: &mut Compilation) -> ModuleGroupMap {
fn prepare_module_group_map(&self, compilation: &mut Compilation) -> ModuleGroupMap {
let chunk_db = &compilation.chunk_by_ukey;
let chunk_group_db = &compilation.chunk_group_by_ukey;

let module_and_corresponding_cache_group = compilation
/// If a module meets requirements of a `ModuleGroup`. We consider the `Module` and the `CacheGroup`
/// to be a `MatchedItem`, which are consumed later to calculate `ModuleGroup`.
struct MatchedItem<'a> {
module: &'a dyn Module,
cache_group_index: usize,
cache_group: &'a CacheGroup,
selected_chunks: Box<[&'a Chunk]>,
}

let matched_items = compilation
.module_graph
.modules()
.values()
Expand All @@ -201,7 +211,8 @@ impl SplitChunksPlugin {
// A module may match multiple CacheGroups
self.cache_groups.par_iter().enumerate().filter_map(
move |(cache_group_index, cache_group)| {
let is_match_the_test = (cache_group.test)(module);
// Filter by `splitChunks.cacheGroups.{cacheGroup}.test`
let is_match_the_test: bool = (cache_group.test)(module);

if !is_match_the_test {
return None;
Expand All @@ -210,63 +221,54 @@ impl SplitChunksPlugin {
let selected_chunks = belong_to_chunks
.iter()
.map(|c| chunk_db.get(c).expect("Should have a chunk here"))
// Filter by `splitChunks.cacheGroups.{cacheGroup}.chunks`
.filter(|c| (cache_group.chunk_filter)(c, chunk_group_db))
.collect::<Vec<_>>();
.collect::<Box<[_]>>();

// Filter by `splitChunks.cacheGroups.{cacheGroup}.minChunks`
if selected_chunks.len() < cache_group.min_chunks as usize {
return None;
}

Some((module, cache_group_index, cache_group, selected_chunks))
Some(MatchedItem {
module: &**module,
cache_group,
cache_group_index,
selected_chunks,
})
},
)
});

let chunks_info_map: DashMap<String, ModuleGroup> = DashMap::default();

module_and_corresponding_cache_group.for_each(
|(module, cache_group_index, cache_group, selected_chunks)| {
let key = ["name: ", &cache_group.name].join("");

let mut chunks_info_item = chunks_info_map.entry(key).or_insert_with(|| ModuleGroup {
// The ChunkInfoItem is not existed. Initialize it.
modules: Default::default(),
cache_group_index,
cache_group_priority: cache_group.priority,
sizes: Default::default(),
chunks: Default::default(),
name: cache_group.name.clone(),
});

chunks_info_item.add_module(module);
chunks_info_item
.chunks
.extend(selected_chunks.iter().map(|c| c.ukey))
},
);

chunks_info_map.into_iter().collect()
}
}
let module_group_map: DashMap<String, ModuleGroup> = DashMap::default();

matched_items.for_each(|matched_item| {
let MatchedItem {
module,
cache_group_index,
cache_group,
selected_chunks,
} = matched_item;

// Merge the `Module` of `MatchedItem` into the `ModuleGroup` which has the same `key`/`cache_group.name`
let key = ["name: ", &cache_group.name].join("");

let mut module_group = module_group_map.entry(key).or_insert_with(|| ModuleGroup {
modules: Default::default(),
cache_group_index,
cache_group_priority: cache_group.priority,
sizes: Default::default(),
chunks: Default::default(),
chunk_name: cache_group.name.clone(),
});

trait EstimatedSize {
fn estimated_size(&self, source_type: &rspack_core::SourceType) -> f64;
}
module_group.add_module(module);
module_group
.chunks
.extend(selected_chunks.iter().map(|c| c.ukey))
});

impl<T: Module> EstimatedSize for T {
fn estimated_size(&self, source_type: &rspack_core::SourceType) -> f64 {
use rspack_core::ModuleType;
let coefficient: f64 = match self.module_type() {
// 5.0 is a number in practice
rspack_core::ModuleType::Jsx
| ModuleType::JsxDynamic
| ModuleType::JsxEsm
| ModuleType::Tsx => 7.5,
ModuleType::Js | ModuleType::JsDynamic => 1.5,
_ => 1.0,
};

self.size(source_type) * coefficient
module_group_map.into_iter().collect()
}
}

Expand Down