From ca8aeedd780244aa74003aebd7d3f1e0b9b7765e Mon Sep 17 00:00:00 2001 From: jerrykingxyz Date: Thu, 18 Apr 2024 14:14:25 +0800 Subject: [PATCH] refactor: split make queue --- crates/rspack_core/src/compiler/make/mod.rs | 653 ++---------------- crates/rspack_core/src/compiler/make/queue.rs | 500 -------------- .../src/compiler/make/tasks/add.rs | 105 +++ .../src/compiler/make/tasks/build.rs | 230 ++++++ .../src/compiler/make/tasks/clean.rs | 43 ++ .../src/compiler/make/tasks/factorize.rs | 285 ++++++++ .../src/compiler/make/tasks/mod.rs | 130 ++++ .../make/tasks/process_dependencies.rs | 127 ++++ crates/rspack_core/src/compiler/mod.rs | 2 +- 9 files changed, 962 insertions(+), 1113 deletions(-) delete mode 100644 crates/rspack_core/src/compiler/make/queue.rs create mode 100644 crates/rspack_core/src/compiler/make/tasks/add.rs create mode 100644 crates/rspack_core/src/compiler/make/tasks/build.rs create mode 100644 crates/rspack_core/src/compiler/make/tasks/clean.rs create mode 100644 crates/rspack_core/src/compiler/make/tasks/factorize.rs create mode 100644 crates/rspack_core/src/compiler/make/tasks/mod.rs create mode 100644 crates/rspack_core/src/compiler/make/tasks/process_dependencies.rs diff --git a/crates/rspack_core/src/compiler/make/mod.rs b/crates/rspack_core/src/compiler/make/mod.rs index 3e08e33c16e..96ce30e13b7 100644 --- a/crates/rspack_core/src/compiler/make/mod.rs +++ b/crates/rspack_core/src/compiler/make/mod.rs @@ -1,34 +1,20 @@ -mod queue; mod rebuild_deps_builder; +mod tasks; -use std::{ - collections::VecDeque, - path::PathBuf, - sync::{ - atomic::{AtomicBool, Ordering}, - Arc, - }, -}; +use std::path::PathBuf; use rayon::prelude::*; use rspack_error::Result; use rspack_identifier::Identifier; use rustc_hash::{FxHashMap as HashMap, FxHashSet as HashSet}; -use tokio::sync::mpsc; -use tokio::sync::mpsc::error::TryRecvError; -pub use self::queue::FactorizeTask; -use self::queue::{ - AddQueue, AddTask, AddTaskResult, BuildQueue, BuildTask, BuildTaskResult, CleanQueue, CleanTask, - CleanTaskResult, FactorizeQueue, FactorizeTaskResult, ProcessDependenciesQueue, - ProcessDependenciesResult, ProcessDependenciesTask, TaskResult, WorkerTask, -}; pub use self::rebuild_deps_builder::RebuildDepsBuilder; +use self::tasks::{clean::CleanTask, factorize::FactorizeTask, MakeTaskContext}; use crate::{ - logger::Logger, tree_shaking::BailoutFlag, AsyncDependenciesBlock, - AsyncDependenciesBlockIdentifier, BoxDependency, BuildDependency, CacheOptions, Compilation, - Context, ContextDependency, DependencyId, DependencyParents, DependencyType, GroupOptions, - Module, ModuleFactoryResult, ModuleGraph, ModuleGraphModule, ModuleIdentifier, ModuleIssuer, + tree_shaking::BailoutFlag, + utils::task_loop::{run_task_loop, Task}, + AsyncDependenciesBlockIdentifier, BuildDependency, Compilation, Context, DependencyId, + DependencyType, GroupOptions, Module, ModuleGraphPartial, ModuleIdentifier, ModuleIssuer, ModuleProfile, NormalModuleSource, Resolve, }; @@ -62,47 +48,13 @@ type ModuleDeps = ( Vec<(AsyncDependenciesBlockIdentifier, Option)>, ); +#[derive(Default)] struct UpdateModuleGraph { origin_module_deps: HashMap, /// Rebuild module issuer mappings origin_module_issuers: HashMap, - factorize_queue: FactorizeQueue, - add_queue: AddQueue, - build_queue: BuildQueue, - process_dependencies_queue: ProcessDependenciesQueue, - - make_failed_dependencies: HashSet, - make_failed_module: HashSet, - need_check_isolated_module_ids: HashSet, - - active_task_count: usize, - is_expected_shutdown: Arc, - - result_tx: mpsc::UnboundedSender>, - result_rx: mpsc::UnboundedReceiver>, -} - -impl Default for UpdateModuleGraph { - fn default() -> Self { - let (result_tx, result_rx) = mpsc::unbounded_channel::>(); - Self { - origin_module_deps: HashMap::default(), - origin_module_issuers: HashMap::default(), - factorize_queue: FactorizeQueue::new(), - add_queue: AddQueue::new(), - build_queue: BuildQueue::new(), - process_dependencies_queue: ProcessDependenciesQueue::new(), - make_failed_dependencies: HashSet::default(), - make_failed_module: HashSet::default(), - need_check_isolated_module_ids: HashSet::default(), - active_task_count: 0, - is_expected_shutdown: Arc::new(AtomicBool::new(false)), - result_tx, - result_rx, - } - } } impl UpdateModuleGraph { @@ -152,573 +104,58 @@ impl UpdateModuleGraph { compilation: &mut Compilation, build_dependencies: HashSet, ) -> Result<()> { - let logger = compilation.get_logger("rspack.Compilation"); - - let mut errored = None; - let module_graph = compilation.get_module_graph(); - build_dependencies + let init_tasks = build_dependencies .into_iter() - .for_each(|(id, parent_module_identifier)| { + .filter_map(|(id, parent_module_identifier)| { let dependency = module_graph .dependency_by_id(&id) .expect("dependency not found"); if dependency.as_module_dependency().is_none() && dependency.as_context_dependency().is_none() { - return; + return None; } let parent_module = parent_module_identifier.and_then(|id| module_graph.module_by_identifier(&id)); if parent_module_identifier.is_some() && parent_module.is_none() { - return; + return None; } - self.handle_module_creation( - compilation, - parent_module_identifier, - parent_module.and_then(|m| m.get_context()), - vec![id], - parent_module_identifier.is_none(), - parent_module.and_then(|module| module.get_resolve_options()), - compilation.lazy_visit_modules.clone(), - parent_module - .and_then(|m| m.as_normal_module()) - .and_then(|module| module.name_for_condition()), - true, - ); - }); - - let mut add_time = logger.time_aggregate("module add task"); - let mut process_deps_time = logger.time_aggregate("module process dependencies task"); - let mut factorize_time = logger.time_aggregate("module factorize task"); - let mut build_time = logger.time_aggregate("module build task"); - - let mut build_cache_counter = None; - let mut factorize_cache_counter = None; - - if !(matches!(compilation.options.cache, CacheOptions::Disabled)) { - build_cache_counter = Some(logger.cache("module build cache")); - factorize_cache_counter = Some(logger.cache("module factorize cache")); - } - - tokio::task::block_in_place(|| loop { - let start = factorize_time.start(); - - while let Some(task) = self.factorize_queue.get_task(compilation) { - self.active_task_count += 1; - - // TODO: change when we insert dependency to module_graph - compilation - .get_module_graph_mut() - .add_dependency(task.dependency.clone()); - - tokio::spawn({ - let result_tx = self.result_tx.clone(); - let is_expected_shutdown = self.is_expected_shutdown.clone(); - - async move { - if is_expected_shutdown.load(Ordering::Relaxed) { - return; - } - - let result = task.run().await; - if !is_expected_shutdown.load(Ordering::Relaxed) { - result_tx - .send(result) - .expect("Failed to send factorize result"); - } - } - }); - } - factorize_time.end(start); - - let start = build_time.start(); - while let Some(task) = self.build_queue.get_task(compilation) { - self.active_task_count += 1; - tokio::spawn({ - let result_tx = self.result_tx.clone(); - let is_expected_shutdown = self.is_expected_shutdown.clone(); - - async move { - if is_expected_shutdown.load(Ordering::Relaxed) { - return; - } - - let result = task.run().await; - if !is_expected_shutdown.load(Ordering::Relaxed) { - result_tx.send(result).expect("Failed to send build result"); - } - } - }); - } - build_time.end(start); - - let start = add_time.start(); - while let Some(task) = self.add_queue.get_task(compilation) { - self.active_task_count += 1; - let result = task.run(compilation); - self - .result_tx - .send(result) - .expect("Failed to send add result"); - } - add_time.end(start); - - let start = process_deps_time.start(); - while let Some(task) = self.process_dependencies_queue.get_task(compilation) { - self.active_task_count += 1; - - let mut sorted_dependencies = HashMap::default(); - let module_graph = compilation.get_module_graph(); - - task.dependencies.into_iter().for_each(|dependency_id| { - let dependency = module_graph - .dependency_by_id(&dependency_id) - .expect("should have dependency"); - // FIXME: now only module/context dependency can put into resolve queue. - // FIXME: should align webpack - let resource_identifier = - if let Some(module_dependency) = dependency.as_module_dependency() { - // TODO need implement more dependency `resource_identifier()` - // https://github.com/webpack/webpack/blob/main/lib/Compilation.js#L1621 - let id = if let Some(resource_identifier) = module_dependency.resource_identifier() { - resource_identifier.to_string() - } else { - format!( - "{}|{}", - module_dependency.dependency_type(), - module_dependency.request() - ) - }; - Some(id) - } else { - dependency - .as_context_dependency() - .map(|d| ContextDependency::resource_identifier(d).to_string()) - }; - - if let Some(resource_identifier) = resource_identifier { - sorted_dependencies - .entry(resource_identifier) - .or_insert(vec![]) - .push(dependency_id); - } - }); - - let original_module_identifier = &task.original_module_identifier; - let module = module_graph - .module_by_identifier(original_module_identifier) - .expect("Module expected"); - - for dependencies in sorted_dependencies.into_values() { + Some( self.handle_module_creation( compilation, - Some(module.identifier()), - module.get_context(), - dependencies, - false, - task.resolve_options.clone(), - compilation.lazy_visit_modules.clone(), - module - .as_normal_module() + parent_module_identifier, + parent_module.and_then(|m| m.get_context()), + vec![id], + parent_module_identifier.is_none(), + parent_module.and_then(|module| module.get_resolve_options()), + parent_module + .and_then(|m| m.as_normal_module()) .and_then(|module| module.name_for_condition()), - true, - ); - } - - tokio::spawn({ - let tx = self.result_tx.clone(); - let is_expected_shutdown = self.is_expected_shutdown.clone(); - async move { - if is_expected_shutdown.load(Ordering::Relaxed) { - return; - } - - tx.send(Ok(TaskResult::ProcessDependencies(Box::new( - ProcessDependenciesResult { - module_identifier: task.original_module_identifier, - }, - )))) - .expect("Failed to send process dependencies result"); - } - }); - } - process_deps_time.end(start); - - match self.result_rx.try_recv() { - Ok(item) => { - if let Ok(item) = &item { - match item { - TaskResult::Factorize(result) => { - if let Some(ModuleFactoryResult { - module: Some(module), - .. - }) = &result.factory_result - { - self.factorize_queue.complete_task( - result.dependency, - module.identifier(), - compilation, - ) - } - } - TaskResult::Add(result) => { - let module = match result.as_ref() { - AddTaskResult::ModuleReused { module } => module.identifier(), - AddTaskResult::ModuleAdded { module, .. } => module.identifier(), - }; - - self.add_queue.complete_task(module, module, compilation) - } - TaskResult::Build(result) => { - let id = result.module.identifier(); - self.build_queue.complete_task(id, id, compilation); - } - TaskResult::ProcessDependencies(result) => { - self.process_dependencies_queue.complete_task( - result.module_identifier, - result.module_identifier, - compilation, - ); - } - } - } - - match item { - Ok(TaskResult::Factorize(box task_result)) => { - let FactorizeTaskResult { - original_module_identifier, - factory_result, - dependencies, - is_entry, - current_profile, - exports_info_related, - file_dependencies, - context_dependencies, - missing_dependencies, - diagnostics, - connect_origin, - .. - } = task_result; - if !diagnostics.is_empty() { - if let Some(id) = original_module_identifier { - self.make_failed_module.insert(id); - } else { - self - .make_failed_dependencies - .insert((dependencies[0], None)); - } - } - - compilation.push_batch_diagnostic( - diagnostics - .into_iter() - .map(|d| d.with_module_identifier(original_module_identifier)) - .collect(), - ); - - compilation.file_dependencies.extend(file_dependencies); - compilation - .context_dependencies - .extend(context_dependencies); - compilation - .missing_dependencies - .extend(missing_dependencies); - - if let Some(factory_result) = factory_result { - if let Some(counter) = &mut factorize_cache_counter { - if factory_result.from_cache { - counter.hit(); - } else { - counter.miss(); - } - } - - if let Some(module) = factory_result.module { - let module_identifier = module.identifier(); - let mut mgm = ModuleGraphModule::new( - module.identifier(), - *module.module_type(), - exports_info_related.exports_info.id, - ); - mgm.set_issuer_if_unset(original_module_identifier); - - let mut module_graph = compilation.get_module_graph_mut(); - module_graph.set_exports_info( - exports_info_related.exports_info.id, - exports_info_related.exports_info, - ); - module_graph.set_export_info( - exports_info_related.side_effects_info.id, - exports_info_related.side_effects_info, - ); - module_graph.set_export_info( - exports_info_related.other_exports_info.id, - exports_info_related.other_exports_info, - ); - - self.add_queue.add_task(AddTask { - original_module_identifier, - module, - module_graph_module: Box::new(mgm), - dependencies, - is_entry, - current_profile, - connect_origin, - }); - tracing::trace!("Module created: {}", &module_identifier); - } else { - let module_graph = compilation.get_module_graph(); - let dep = module_graph - .dependency_by_id(&dependencies[0]) - .expect("dep should available"); - tracing::trace!("Module ignored: {dep:?}") - } - } else { - let module_graph = compilation.get_module_graph(); - let dep = module_graph - .dependency_by_id(&dependencies[0]) - .expect("dep should available"); - tracing::trace!("Module created with failure, but without bailout: {dep:?}"); - } - } - Ok(TaskResult::Add(box task_result)) => match task_result { - AddTaskResult::ModuleAdded { - module, - current_profile, - } => { - tracing::trace!("Module added: {}", module.identifier()); - self.build_queue.add_task(BuildTask { - module, - resolver_factory: compilation.resolver_factory.clone(), - compiler_options: compilation.options.clone(), - plugin_driver: compilation.plugin_driver.clone(), - cache: compilation.cache.clone(), - current_profile, - }); - } - AddTaskResult::ModuleReused { module, .. } => { - tracing::trace!("Module reused: {}, skipping build", module.identifier()); - - let module_identifier = module.identifier(); - if compilation - .get_module_graph() - .module_by_identifier(&module_identifier) - .is_some() - { - self.active_task_count += 1; - self - .result_tx - .send(Ok(TaskResult::ProcessDependencies(Box::new( - ProcessDependenciesResult { - module_identifier: module.identifier(), - }, - )))) - .expect("Failed to send factorize result"); - } - } - }, - Ok(TaskResult::Build(box task_result)) => { - let BuildTaskResult { - mut module, - build_result, - diagnostics, - current_profile, - from_cache, - } = task_result; - - if let Some(counter) = &mut build_cache_counter { - if from_cache { - counter.hit(); - } else { - counter.miss(); - } - } - - if compilation.options.builtins.tree_shaking.enable() { - compilation - .optimize_analyze_result_map - .insert(module.identifier(), build_result.analyze_result); - } - - if !diagnostics.is_empty() { - self.make_failed_module.insert(module.identifier()); - } - - tracing::trace!("Module built: {}", module.identifier()); - compilation.push_batch_diagnostic(diagnostics); - compilation - .get_module_graph_mut() - .get_optimization_bailout_mut(&module.identifier()) - .extend(build_result.optimization_bailouts); - compilation - .file_dependencies - .extend(build_result.build_info.file_dependencies.clone()); - compilation - .context_dependencies - .extend(build_result.build_info.context_dependencies.clone()); - compilation - .missing_dependencies - .extend(build_result.build_info.missing_dependencies.clone()); - compilation - .build_dependencies - .extend(build_result.build_info.build_dependencies.clone()); - - let mut queue = VecDeque::new(); - let mut all_dependencies = vec![]; - let mut handle_block = - |dependencies: Vec, - blocks: Vec, - queue: &mut VecDeque, - module_graph: &mut ModuleGraph, - current_block: Option| { - for dependency in dependencies { - let dependency_id = *dependency.id(); - if current_block.is_none() { - module.add_dependency_id(dependency_id); - } - all_dependencies.push(dependency_id); - module_graph.set_parents( - dependency_id, - DependencyParents { - block: current_block.as_ref().map(|block| block.identifier()), - module: module.identifier(), - }, - ); - module_graph.add_dependency(dependency); - } - if let Some(current_block) = current_block { - module.add_block_id(current_block.identifier()); - module_graph.add_block(current_block); - } - for block in blocks { - queue.push_back(block); - } - }; - handle_block( - build_result.dependencies, - build_result.blocks, - &mut queue, - &mut compilation.get_module_graph_mut(), - None, - ); - while let Some(mut block) = queue.pop_front() { - let dependencies = block.take_dependencies(); - let blocks = block.take_blocks(); - handle_block( - dependencies, - blocks, - &mut queue, - &mut compilation.get_module_graph_mut(), - Some(block), - ); - } - - { - let mut module_graph = compilation.get_module_graph_mut(); - let mgm = module_graph - .module_graph_module_by_identifier_mut(&module.identifier()) - .expect("Failed to get mgm"); - mgm.__deprecated_all_dependencies = all_dependencies.clone(); - if let Some(current_profile) = current_profile { - mgm.set_profile(current_profile); - } - } - - let module_identifier = module.identifier(); - - module.set_build_info(build_result.build_info); - module.set_build_meta(build_result.build_meta); - - let mut mg = compilation.get_module_graph_mut(); - - let resolve_options = module.get_resolve_options(); - mg.add_module(module); - - self - .process_dependencies_queue - .add_task(ProcessDependenciesTask { - dependencies: all_dependencies, - original_module_identifier: module_identifier, - resolve_options, - }); - } - Ok(TaskResult::ProcessDependencies(task_result)) => { - tracing::trace!( - "Processing dependencies of {} finished", - task_result.module_identifier - ); - } - Err(err) => { - // Severe internal error encountered, we should end the compiling here. - errored = Some(err); - self.is_expected_shutdown.store(true, Ordering::Relaxed); - break; - } - } - - self.active_task_count -= 1; - } - Err(TryRecvError::Disconnected) => { - self.is_expected_shutdown.store(true, Ordering::Relaxed); - break; - } - Err(TryRecvError::Empty) => { - if self.active_task_count == 0 { - self.is_expected_shutdown.store(true, Ordering::Relaxed); - break; - } - } - } - }); - logger.time_aggregate_end(add_time); - logger.time_aggregate_end(process_deps_time); - logger.time_aggregate_end(factorize_time); - logger.time_aggregate_end(build_time); + ), + ) + }) + .collect::>(); - if let Some(counter) = build_cache_counter { - logger.cache_end(counter); - } - if let Some(counter) = factorize_cache_counter { - logger.cache_end(counter); - } + let mut make_module_graph = ModuleGraphPartial::new(true); + compilation.swap_make_module_graph(&mut make_module_graph); + let mut ctx = MakeTaskContext::new(compilation, make_module_graph); + let res = run_task_loop(&mut ctx, init_tasks); - compilation - .make_failed_dependencies - .extend(self.make_failed_dependencies.drain()); - compilation - .make_failed_module - .extend(self.make_failed_module.drain()); tracing::debug!("All task is finished"); // clean isolated module - let mut clean_queue = CleanQueue::new(); - clean_queue.add_tasks( - self - .need_check_isolated_module_ids - .drain() - .map(|module_identifier| CleanTask { module_identifier }), - ); - - while let Some(task) = clean_queue.get_task(compilation) { - match task.run(compilation) { - CleanTaskResult::ModuleIsUsed { module_identifier } => { - tracing::trace!("Module is used: {}", module_identifier); - } - CleanTaskResult::ModuleIsCleaned { - module_identifier, - dependent_module_identifiers, - } => { - tracing::trace!("Module is cleaned: {}", module_identifier); - clean_queue.add_tasks( - dependent_module_identifiers - .into_iter() - .map(|module_identifier| CleanTask { module_identifier }), - ); - } - }; + let mut clean_tasks: Vec>> = + Vec::with_capacity(self.need_check_isolated_module_ids.len()); + for module_identifier in &self.need_check_isolated_module_ids { + clean_tasks.push(Box::new(CleanTask { + module_identifier: *module_identifier, + })); } + run_task_loop(&mut ctx, clean_tasks)?; + + ctx.emit_data_to_compilation(compilation); tracing::debug!("All clean task is finished"); // set origin module issues @@ -821,11 +258,7 @@ impl UpdateModuleGraph { .collect(); } - if let Some(err) = errored { - Err(err) - } else { - Ok(()) - } + res } #[allow(clippy::too_many_arguments)] @@ -837,10 +270,8 @@ impl UpdateModuleGraph { dependencies: Vec, is_entry: bool, resolve_options: Option>, - lazy_visit_modules: std::collections::HashSet, issuer: Option>, - connect_origin: bool, - ) { + ) -> Box> { let current_profile = compilation .options .profile @@ -861,7 +292,7 @@ impl UpdateModuleGraph { None } }); - self.factorize_queue.add_task(FactorizeTask { + Box::new(FactorizeTask { module_factory: compilation.get_dependency_factory(&dependency), original_module_identifier, original_module_source, @@ -871,15 +302,13 @@ impl UpdateModuleGraph { dependencies, is_entry, resolve_options, - lazy_visit_modules, resolver_factory: compilation.resolver_factory.clone(), loader_resolver_factory: compilation.loader_resolver_factory.clone(), options: compilation.options.clone(), plugin_driver: compilation.plugin_driver.clone(), cache: compilation.cache.clone(), current_profile, - connect_origin, - }); + }) } fn module_deps(compilation: &Compilation, module_identifier: &ModuleIdentifier) -> ModuleDeps { diff --git a/crates/rspack_core/src/compiler/make/queue.rs b/crates/rspack_core/src/compiler/make/queue.rs deleted file mode 100644 index f969020fb8f..00000000000 --- a/crates/rspack_core/src/compiler/make/queue.rs +++ /dev/null @@ -1,500 +0,0 @@ -use std::path::PathBuf; -use std::sync::Arc; - -use derivative::Derivative; -use rspack_error::{Diagnostic, IntoTWithDiagnosticArray, Result}; -use rspack_sources::BoxSource; -use rustc_hash::FxHashSet as HashSet; - -use crate::{ - cache::Cache, BoxDependency, BuildContext, BuildResult, Compilation, CompilerContext, - CompilerOptions, Context, Module, ModuleFactory, ModuleFactoryCreateData, ModuleFactoryResult, - ModuleGraph, ModuleGraphModule, ModuleIdentifier, ModuleProfile, Resolve, ResolverFactory, - SharedPluginDriver, WorkerQueue, -}; -use crate::{DependencyId, ExportInfo, ExportsInfo, UsageState}; - -pub type CleanQueue = WorkerQueue; - -#[derive(Debug)] -pub enum TaskResult { - Factorize(Box), - Add(Box), - Build(Box), - ProcessDependencies(Box), -} - -#[async_trait::async_trait] -pub trait WorkerTask { - async fn run(self) -> Result; -} - -#[derive(Derivative)] -#[derivative(Debug)] -pub struct FactorizeTask { - pub module_factory: Arc, - pub original_module_identifier: Option, - pub original_module_source: Option, - pub original_module_context: Option>, - pub issuer: Option>, - pub dependency: BoxDependency, - pub dependencies: Vec, - pub is_entry: bool, - pub resolve_options: Option>, - pub resolver_factory: Arc, - pub loader_resolver_factory: Arc, - pub options: Arc, - pub lazy_visit_modules: std::collections::HashSet, - pub plugin_driver: SharedPluginDriver, - pub cache: Arc, - pub current_profile: Option>, - pub connect_origin: bool, -} - -/// a struct temporarily used creating ExportsInfo -#[derive(Debug)] -pub struct ExportsInfoRelated { - pub exports_info: ExportsInfo, - pub other_exports_info: ExportInfo, - pub side_effects_info: ExportInfo, -} - -#[derive(Derivative)] -#[derivative(Debug)] -pub struct FactorizeTaskResult { - pub dependency: DependencyId, - pub original_module_identifier: Option, - /// Result will be available if [crate::ModuleFactory::create] returns `Ok`. - pub factory_result: Option, - pub dependencies: Vec, - pub is_entry: bool, - pub current_profile: Option>, - pub exports_info_related: ExportsInfoRelated, - - pub file_dependencies: HashSet, - pub context_dependencies: HashSet, - pub missing_dependencies: HashSet, - pub diagnostics: Vec, - pub connect_origin: bool, -} - -impl FactorizeTaskResult { - fn with_factory_result(mut self, factory_result: Option) -> Self { - self.factory_result = factory_result; - self - } - - fn with_diagnostics(mut self, diagnostics: Vec) -> Self { - self.diagnostics = diagnostics; - self - } - - fn with_file_dependencies(mut self, files: impl IntoIterator) -> Self { - self.file_dependencies = files.into_iter().collect(); - self - } - - fn with_context_dependencies(mut self, contexts: impl IntoIterator) -> Self { - self.context_dependencies = contexts.into_iter().collect(); - self - } - - fn with_missing_dependencies(mut self, missing: impl IntoIterator) -> Self { - self.missing_dependencies = missing.into_iter().collect(); - self - } -} - -#[async_trait::async_trait] -impl WorkerTask for FactorizeTask { - async fn run(self) -> Result { - if let Some(current_profile) = &self.current_profile { - current_profile.mark_factory_start(); - } - let dependency = self.dependency; - let dep_id = *dependency.id(); - - let context = if let Some(context) = dependency.get_context() { - context - } else if let Some(context) = &self.original_module_context { - context - } else { - &self.options.context - } - .clone(); - - let other_exports_info = ExportInfo::new(None, UsageState::Unknown, None); - let side_effects_only_info = ExportInfo::new( - Some("*side effects only*".into()), - UsageState::Unknown, - None, - ); - let exports_info = ExportsInfo::new(other_exports_info.id, side_effects_only_info.id); - let factorize_task_result = FactorizeTaskResult { - dependency: dep_id, - original_module_identifier: self.original_module_identifier, - factory_result: None, - dependencies: self.dependencies, - is_entry: self.is_entry, - current_profile: self.current_profile, - exports_info_related: ExportsInfoRelated { - exports_info, - other_exports_info, - side_effects_info: side_effects_only_info, - }, - file_dependencies: Default::default(), - context_dependencies: Default::default(), - missing_dependencies: Default::default(), - diagnostics: Default::default(), - connect_origin: self.connect_origin, - // callback: self.callback, - }; - - // Error and result are not mutually exclusive in webpack module factorization. - // Rspack puts results that need to be shared in both error and ok in [ModuleFactoryCreateData]. - let mut create_data = ModuleFactoryCreateData { - resolve_options: self.resolve_options, - context, - dependency, - issuer: self.issuer, - issuer_identifier: self.original_module_identifier, - - file_dependencies: Default::default(), - missing_dependencies: Default::default(), - context_dependencies: Default::default(), - diagnostics: Default::default(), - }; - - match self.module_factory.create(&mut create_data).await { - Ok(result) => { - if let Some(current_profile) = &factorize_task_result.current_profile { - current_profile.mark_factory_end(); - } - let diagnostics = create_data.diagnostics.drain(..).collect(); - Ok(TaskResult::Factorize(Box::new( - factorize_task_result - .with_factory_result(Some(result)) - .with_diagnostics(diagnostics) - .with_file_dependencies(create_data.file_dependencies.drain()) - .with_missing_dependencies(create_data.missing_dependencies.drain()) - .with_context_dependencies(create_data.context_dependencies.drain()), - ))) - } - Err(mut e) => { - if let Some(current_profile) = &factorize_task_result.current_profile { - current_profile.mark_factory_end(); - } - // Wrap source code if available - if let Some(s) = self.original_module_source { - e = e.with_source_code(s.source().to_string()); - } - // Bail out if `options.bail` set to `true`, - // which means 'Fail out on the first error instead of tolerating it.' - if self.options.bail { - return Err(e); - } - let mut diagnostics = Vec::with_capacity(create_data.diagnostics.len() + 1); - diagnostics.push(e.into()); - diagnostics.append(&mut create_data.diagnostics); - // Continue bundling if `options.bail` set to `false`. - Ok(TaskResult::Factorize(Box::new( - factorize_task_result - .with_diagnostics(diagnostics) - .with_file_dependencies(create_data.file_dependencies.drain()) - .with_missing_dependencies(create_data.missing_dependencies.drain()) - .with_context_dependencies(create_data.context_dependencies.drain()), - ))) - } - } - } -} - -pub type FactorizeQueue = WorkerQueue; - -#[derive(Derivative)] -#[derivative(Debug)] -pub struct AddTask { - pub original_module_identifier: Option, - pub module: Box, - pub module_graph_module: Box, - pub dependencies: Vec, - pub is_entry: bool, - pub current_profile: Option>, - pub connect_origin: bool, -} - -#[derive(Debug)] -pub enum AddTaskResult { - ModuleReused { - module: Box, - }, - ModuleAdded { - module: Box, - current_profile: Option>, - }, -} - -impl AddTask { - pub fn run(self, compilation: &mut Compilation) -> Result { - if let Some(current_profile) = &self.current_profile { - current_profile.mark_integration_start(); - } - - if self.module.as_self_module().is_some() && self.connect_origin { - let issuer = self - .module_graph_module - .get_issuer() - .identifier() - .expect("self module should have issuer"); - - set_resolved_module( - &mut compilation.get_module_graph_mut(), - self.original_module_identifier, - self.dependencies, - *issuer, - )?; - - return Ok(TaskResult::Add(Box::new(AddTaskResult::ModuleReused { - module: self.module, - }))); - } - - let module_identifier = self.module.identifier(); - - if self.connect_origin - && compilation - .get_module_graph() - .module_graph_module_by_identifier(&module_identifier) - .is_some() - { - set_resolved_module( - &mut compilation.get_module_graph_mut(), - self.original_module_identifier, - self.dependencies, - module_identifier, - )?; - - // if let Some(callback) = self.callback { - // callback(&self.module); - // } - - return Ok(TaskResult::Add(Box::new(AddTaskResult::ModuleReused { - module: self.module, - }))); - } - - compilation - .get_module_graph_mut() - .add_module_graph_module(*self.module_graph_module); - - if self.connect_origin { - set_resolved_module( - &mut compilation.get_module_graph_mut(), - self.original_module_identifier, - self.dependencies, - module_identifier, - )?; - } - - if self.is_entry { - compilation - .entry_module_identifiers - .insert(module_identifier); - } - - if let Some(current_profile) = &self.current_profile { - current_profile.mark_integration_end(); - } - - // if let Some(callback) = self.callback { - // callback(&self.module); - // } - - Ok(TaskResult::Add(Box::new(AddTaskResult::ModuleAdded { - module: self.module, - current_profile: self.current_profile, - }))) - } -} - -fn set_resolved_module( - module_graph: &mut ModuleGraph, - original_module_identifier: Option, - dependencies: Vec, - module_identifier: ModuleIdentifier, -) -> Result<()> { - for dependency in dependencies { - module_graph.set_resolved_module(original_module_identifier, dependency, module_identifier)?; - } - Ok(()) -} - -pub type AddQueue = WorkerQueue; - -#[derive(Debug)] -pub struct BuildTask { - pub module: Box, - pub resolver_factory: Arc, - pub compiler_options: Arc, - pub plugin_driver: SharedPluginDriver, - pub cache: Arc, - pub current_profile: Option>, -} - -#[derive(Debug)] -pub struct BuildTaskResult { - pub module: Box, - pub build_result: Box, - pub diagnostics: Vec, - pub current_profile: Option>, - pub from_cache: bool, -} - -#[async_trait::async_trait] -impl WorkerTask for BuildTask { - async fn run(self) -> Result { - if let Some(current_profile) = &self.current_profile { - current_profile.mark_building_start(); - } - - let mut module = self.module; - let compiler_options = self.compiler_options; - let resolver_factory = self.resolver_factory; - let cache = self.cache; - let plugin_driver = self.plugin_driver; - - let (build_result, is_cache_valid) = cache - .build_module_occasion - .use_cache(&mut module, |module| async { - plugin_driver - .compilation_hooks - .build_module - .call(module) - .await?; - - let result = module - .build( - BuildContext { - compiler_context: CompilerContext { - options: compiler_options.clone(), - resolver_factory: resolver_factory.clone(), - module: module.identifier(), - module_context: module.as_normal_module().and_then(|m| m.get_context()), - module_source_map_kind: module.get_source_map_kind().clone(), - plugin_driver: plugin_driver.clone(), - cache: cache.clone(), - }, - plugin_driver: plugin_driver.clone(), - compiler_options: &compiler_options, - }, - None, - ) - .await; - - plugin_driver - .compilation_hooks - .succeed_module - .call(module) - .await?; - - result.map(|t| { - let diagnostics = module - .clone_diagnostics() - .into_iter() - .map(|d| d.with_module_identifier(Some(module.identifier()))) - .collect(); - (t.with_diagnostic(diagnostics), module) - }) - }) - .await?; - - if is_cache_valid { - plugin_driver - .compilation_hooks - .still_valid_module - .call(&mut module) - .await?; - } - - if let Some(current_profile) = &self.current_profile { - current_profile.mark_building_end(); - } - - build_result.map(|build_result| { - let (build_result, diagnostics) = build_result.split_into_parts(); - - TaskResult::Build(Box::new(BuildTaskResult { - module, - build_result: Box::new(build_result), - diagnostics, - current_profile: self.current_profile, - from_cache: is_cache_valid, - })) - }) - } -} - -pub type BuildQueue = WorkerQueue; - -#[derive(Debug)] -pub struct ProcessDependenciesTask { - pub original_module_identifier: ModuleIdentifier, - pub dependencies: Vec, - pub resolve_options: Option>, -} - -#[derive(Debug)] -pub struct ProcessDependenciesResult { - pub module_identifier: ModuleIdentifier, -} - -pub type ProcessDependenciesQueue = WorkerQueue; - -pub struct CleanTask { - pub module_identifier: ModuleIdentifier, -} - -#[derive(Debug)] -pub enum CleanTaskResult { - ModuleIsUsed { - module_identifier: ModuleIdentifier, - }, - ModuleIsCleaned { - module_identifier: ModuleIdentifier, - dependent_module_identifiers: Vec, - }, -} - -impl CleanTask { - pub fn run(self, compilation: &mut Compilation) -> CleanTaskResult { - let module_identifier = self.module_identifier; - let module_graph = compilation.get_module_graph(); - let mgm = match module_graph.module_graph_module_by_identifier(&module_identifier) { - Some(mgm) => mgm, - None => { - return CleanTaskResult::ModuleIsCleaned { - module_identifier, - dependent_module_identifiers: vec![], - } - } - }; - - if !mgm.incoming_connections().is_empty() { - return CleanTaskResult::ModuleIsUsed { module_identifier }; - } - - let dependent_module_identifiers: Vec = module_graph - .get_module_all_depended_modules(&module_identifier) - .expect("should have module") - .into_iter() - .copied() - .collect(); - compilation - .get_module_graph_mut() - .revoke_module(&module_identifier); - CleanTaskResult::ModuleIsCleaned { - module_identifier, - dependent_module_identifiers, - } - } -} diff --git a/crates/rspack_core/src/compiler/make/tasks/add.rs b/crates/rspack_core/src/compiler/make/tasks/add.rs new file mode 100644 index 00000000000..d58ff830b16 --- /dev/null +++ b/crates/rspack_core/src/compiler/make/tasks/add.rs @@ -0,0 +1,105 @@ +use rspack_error::Result; + +use super::{build::BuildTask, MakeTaskContext}; +use crate::{ + module_graph::{ModuleGraph, ModuleGraphModule}, + utils::task_loop::{Task, TaskResult, TaskType}, + DependencyId, Module, ModuleIdentifier, ModuleProfile, +}; + +#[derive(Debug)] +pub struct AddTask { + pub original_module_identifier: Option, + pub module: Box, + pub module_graph_module: Box, + pub dependencies: Vec, + pub is_entry: bool, + pub current_profile: Option>, +} + +impl Task for AddTask { + fn get_task_type(&self) -> TaskType { + TaskType::Sync + } + fn sync_run(self: Box, context: &mut MakeTaskContext) -> TaskResult { + if let Some(current_profile) = &self.current_profile { + current_profile.mark_integration_start(); + } + + let module_identifier = self.module.identifier(); + let module_graph = &mut MakeTaskContext::get_module_graph(&mut context.module_graph_partial); + + if self.module.as_self_module().is_some() { + let issuer = self + .module_graph_module + .get_issuer() + .identifier() + .expect("self module should have issuer"); + + set_resolved_module( + module_graph, + self.original_module_identifier, + self.dependencies, + *issuer, + )?; + + // reused module + return Ok(vec![]); + } + + if module_graph + .module_graph_module_by_identifier(&module_identifier) + .is_some() + { + set_resolved_module( + module_graph, + self.original_module_identifier, + self.dependencies, + module_identifier, + )?; + + // reused module + return Ok(vec![]); + } + + module_graph.add_module_graph_module(*self.module_graph_module); + + set_resolved_module( + module_graph, + self.original_module_identifier, + self.dependencies, + module_identifier, + )?; + + if self.is_entry { + context.entry_module_identifiers.insert(module_identifier); + } + + if let Some(current_profile) = &self.current_profile { + current_profile.mark_integration_end(); + } + + tracing::trace!("Module added: {}", self.module.identifier()); + + Ok(vec![Box::new(BuildTask { + module: self.module, + current_profile: self.current_profile, + resolver_factory: context.resolver_factory.clone(), + compiler_options: context.compiler_options.clone(), + plugin_driver: context.plugin_driver.clone(), + cache: context.cache.clone(), + })]) + } +} + +fn set_resolved_module( + module_graph: &mut ModuleGraph, + original_module_identifier: Option, + dependencies: Vec, + module_identifier: ModuleIdentifier, +) -> Result<()> { + for dependency in dependencies { + module_graph.set_resolved_module(original_module_identifier, dependency, module_identifier)?; + } + Ok(()) +} diff --git a/crates/rspack_core/src/compiler/make/tasks/build.rs b/crates/rspack_core/src/compiler/make/tasks/build.rs new file mode 100644 index 00000000000..781bd6997f4 --- /dev/null +++ b/crates/rspack_core/src/compiler/make/tasks/build.rs @@ -0,0 +1,230 @@ +use std::{collections::VecDeque, sync::Arc}; + +use rspack_error::{Diagnostic, IntoTWithDiagnosticArray}; + +use super::{process_dependencies::ProcessDependenciesTask, MakeTaskContext}; +use crate::{ + cache::Cache, + utils::task_loop::{Task, TaskResult, TaskType}, + AsyncDependenciesBlock, BoxDependency, BuildContext, BuildResult, CompilerContext, + CompilerOptions, DependencyParents, Module, ModuleProfile, ResolverFactory, SharedPluginDriver, +}; + +#[derive(Debug)] +pub struct BuildTask { + pub module: Box, + pub current_profile: Option>, + pub resolver_factory: Arc, + pub compiler_options: Arc, + pub plugin_driver: SharedPluginDriver, + pub cache: Arc, +} + +#[async_trait::async_trait] +impl Task for BuildTask { + fn get_task_type(&self) -> TaskType { + TaskType::Async + } + async fn async_run(self: Box) -> TaskResult { + let Self { + compiler_options, + resolver_factory, + plugin_driver, + cache, + current_profile, + mut module, + } = *self; + if let Some(current_profile) = ¤t_profile { + current_profile.mark_building_start(); + } + + let (build_result, is_cache_valid) = cache + .build_module_occasion + .use_cache(&mut module, |module| async { + plugin_driver + .compilation_hooks + .build_module + .call(module) + .await?; + + let result = module + .build( + BuildContext { + compiler_context: CompilerContext { + options: compiler_options.clone(), + resolver_factory: resolver_factory.clone(), + module: module.identifier(), + module_context: module.as_normal_module().and_then(|m| m.get_context()), + module_source_map_kind: module.get_source_map_kind().clone(), + plugin_driver: plugin_driver.clone(), + cache: cache.clone(), + }, + plugin_driver: plugin_driver.clone(), + compiler_options: &compiler_options, + }, + None, + ) + .await; + + plugin_driver + .compilation_hooks + .succeed_module + .call(module) + .await?; + + result.map(|t| { + let diagnostics = module + .clone_diagnostics() + .into_iter() + .map(|d| d.with_module_identifier(Some(module.identifier()))) + .collect(); + (t.with_diagnostic(diagnostics), module) + }) + }) + .await?; + + if is_cache_valid { + plugin_driver + .compilation_hooks + .still_valid_module + .call(&mut module) + .await?; + } + + if let Some(current_profile) = ¤t_profile { + current_profile.mark_building_end(); + } + + build_result.map::>>, _>(|build_result| { + let (build_result, diagnostics) = build_result.split_into_parts(); + vec![Box::new(BuildResultTask { + module, + build_result: Box::new(build_result), + diagnostics, + current_profile, + from_cache: is_cache_valid, + })] + }) + } +} + +#[derive(Debug)] +struct BuildResultTask { + pub module: Box, + pub build_result: Box, + pub diagnostics: Vec, + pub current_profile: Option>, + pub from_cache: bool, +} + +impl Task for BuildResultTask { + fn get_task_type(&self) -> TaskType { + TaskType::Sync + } + fn sync_run(self: Box, context: &mut MakeTaskContext) -> TaskResult { + let BuildResultTask { + mut module, + build_result, + diagnostics, + current_profile, + from_cache, + } = *self; + + if let Some(counter) = &mut context.build_cache_counter { + if from_cache { + counter.hit(); + } else { + counter.miss(); + } + } + + let module_graph = &mut MakeTaskContext::get_module_graph(&mut context.module_graph_partial); + if context.compiler_options.builtins.tree_shaking.enable() { + context + .optimize_analyze_result_map + .insert(module.identifier(), build_result.analyze_result); + } + + if !diagnostics.is_empty() { + context.make_failed_module.insert(module.identifier()); + } + + tracing::trace!("Module built: {}", module.identifier()); + context.diagnostics.extend(diagnostics); + module_graph + .get_optimization_bailout_mut(&module.identifier()) + .extend(build_result.optimization_bailouts); + context + .file_dependencies + .extend(build_result.build_info.file_dependencies.clone()); + context + .context_dependencies + .extend(build_result.build_info.context_dependencies.clone()); + context + .missing_dependencies + .extend(build_result.build_info.missing_dependencies.clone()); + context + .build_dependencies + .extend(build_result.build_info.build_dependencies.clone()); + + let mut queue = VecDeque::new(); + let mut all_dependencies = vec![]; + let mut handle_block = |dependencies: Vec, + blocks: Vec, + current_block: Option| + -> Vec { + for dependency in dependencies { + let dependency_id = *dependency.id(); + if current_block.is_none() { + module.add_dependency_id(dependency_id); + } + all_dependencies.push(dependency_id); + module_graph.set_parents( + dependency_id, + DependencyParents { + block: current_block.as_ref().map(|block| block.identifier()), + module: module.identifier(), + }, + ); + module_graph.add_dependency(dependency); + } + if let Some(current_block) = current_block { + module.add_block_id(current_block.identifier()); + module_graph.add_block(current_block); + } + blocks + }; + let blocks = handle_block(build_result.dependencies, build_result.blocks, None); + queue.extend(blocks); + + while let Some(mut block) = queue.pop_front() { + let dependencies = block.take_dependencies(); + let blocks = handle_block(dependencies, block.take_blocks(), Some(block)); + queue.extend(blocks); + } + + { + let mgm = module_graph + .module_graph_module_by_identifier_mut(&module.identifier()) + .expect("Failed to get mgm"); + mgm.__deprecated_all_dependencies = all_dependencies.clone(); + if let Some(current_profile) = current_profile { + mgm.set_profile(current_profile); + } + } + + let module_identifier = module.identifier(); + + module.set_build_info(build_result.build_info); + module.set_build_meta(build_result.build_meta); + + let resolve_options = module.get_resolve_options(); + module_graph.add_module(module); + + Ok(vec![Box::new(ProcessDependenciesTask { + dependencies: all_dependencies, + original_module_identifier: module_identifier, + resolve_options, + })]) + } +} diff --git a/crates/rspack_core/src/compiler/make/tasks/clean.rs b/crates/rspack_core/src/compiler/make/tasks/clean.rs new file mode 100644 index 00000000000..dba2cb18556 --- /dev/null +++ b/crates/rspack_core/src/compiler/make/tasks/clean.rs @@ -0,0 +1,43 @@ +use super::MakeTaskContext; +use crate::{ + utils::task_loop::{Task, TaskResult, TaskType}, + ModuleIdentifier, +}; + +pub struct CleanTask { + pub module_identifier: ModuleIdentifier, +} + +impl Task for CleanTask { + fn get_task_type(&self) -> TaskType { + TaskType::Sync + } + fn sync_run(self: Box, context: &mut MakeTaskContext) -> TaskResult { + let module_identifier = self.module_identifier; + let module_graph = &mut MakeTaskContext::get_module_graph(&mut context.module_graph_partial); + let Some(mgm) = module_graph.module_graph_module_by_identifier(&module_identifier) else { + tracing::trace!("Module is cleaned: {}", module_identifier); + return Ok(vec![]); + }; + + if !mgm.incoming_connections().is_empty() { + tracing::trace!("Module is used: {}", module_identifier); + return Ok(vec![]); + } + + let dependent_module_identifiers: Vec = module_graph + .get_module_all_depended_modules(&module_identifier) + .expect("should have module") + .into_iter() + .copied() + .collect(); + module_graph.revoke_module(&module_identifier); + + let mut res: Vec>> = + Vec::with_capacity(dependent_module_identifiers.len()); + for module_identifier in dependent_module_identifiers { + res.push(Box::new(CleanTask { module_identifier })) + } + Ok(res) + } +} diff --git a/crates/rspack_core/src/compiler/make/tasks/factorize.rs b/crates/rspack_core/src/compiler/make/tasks/factorize.rs new file mode 100644 index 00000000000..70d19d0ef3f --- /dev/null +++ b/crates/rspack_core/src/compiler/make/tasks/factorize.rs @@ -0,0 +1,285 @@ +use std::path::PathBuf; +use std::sync::Arc; + +use rspack_error::Diagnostic; +use rspack_sources::BoxSource; +use rustc_hash::FxHashSet as HashSet; + +use super::{add::AddTask, MakeTaskContext}; +use crate::{ + cache::Cache, + module_graph::ModuleGraphModule, + utils::task_loop::{Task, TaskResult, TaskType}, + BoxDependency, CompilerOptions, Context, DependencyId, ExportInfo, ExportsInfo, ModuleFactory, + ModuleFactoryCreateData, ModuleFactoryResult, ModuleIdentifier, ModuleProfile, Resolve, + ResolverFactory, SharedPluginDriver, UsageState, +}; + +#[derive(Debug)] +pub struct FactorizeTask { + pub module_factory: Arc, + pub original_module_identifier: Option, + pub original_module_source: Option, + pub original_module_context: Option>, + pub issuer: Option>, + pub dependency: BoxDependency, + pub dependencies: Vec, + pub is_entry: bool, + pub resolve_options: Option>, + pub resolver_factory: Arc, + pub loader_resolver_factory: Arc, + pub options: Arc, + pub plugin_driver: SharedPluginDriver, + pub cache: Arc, + pub current_profile: Option>, +} + +#[async_trait::async_trait] +impl Task for FactorizeTask { + fn get_task_type(&self) -> TaskType { + TaskType::Async + } + async fn async_run(self: Box) -> TaskResult { + if let Some(current_profile) = &self.current_profile { + current_profile.mark_factory_start(); + } + let dependency = self.dependency; + // let dep_id = *dependency.id(); + + let context = if let Some(context) = dependency.get_context() { + context + } else if let Some(context) = &self.original_module_context { + context + } else { + &self.options.context + } + .clone(); + + let other_exports_info = ExportInfo::new(None, UsageState::Unknown, None); + let side_effects_only_info = ExportInfo::new( + Some("*side effects only*".into()), + UsageState::Unknown, + None, + ); + let exports_info = ExportsInfo::new(other_exports_info.id, side_effects_only_info.id); + let factorize_result_task = FactorizeResultTask { + // dependency: dep_id, + original_module_identifier: self.original_module_identifier, + factory_result: None, + dependencies: self.dependencies, + is_entry: self.is_entry, + current_profile: self.current_profile, + exports_info_related: ExportsInfoRelated { + exports_info, + other_exports_info, + side_effects_info: side_effects_only_info, + }, + file_dependencies: Default::default(), + context_dependencies: Default::default(), + missing_dependencies: Default::default(), + diagnostics: Default::default(), + }; + + // Error and result are not mutually exclusive in webpack module factorization. + // Rspack puts results that need to be shared in both error and ok in [ModuleFactoryCreateData]. + let mut create_data = ModuleFactoryCreateData { + resolve_options: self.resolve_options, + context, + dependency, + issuer: self.issuer, + issuer_identifier: self.original_module_identifier, + + file_dependencies: Default::default(), + missing_dependencies: Default::default(), + context_dependencies: Default::default(), + diagnostics: Default::default(), + }; + match self.module_factory.create(&mut create_data).await { + Ok(result) => { + if let Some(current_profile) = &factorize_result_task.current_profile { + current_profile.mark_factory_end(); + } + let diagnostics = create_data.diagnostics.drain(..).collect(); + Ok(vec![Box::new( + factorize_result_task + .with_factory_result(Some(result)) + .with_diagnostics(diagnostics) + .with_file_dependencies(create_data.file_dependencies.drain()) + .with_missing_dependencies(create_data.missing_dependencies.drain()) + .with_context_dependencies(create_data.context_dependencies.drain()), + )]) + } + Err(mut e) => { + if let Some(current_profile) = &factorize_result_task.current_profile { + current_profile.mark_factory_end(); + } + // Wrap source code if available + if let Some(s) = self.original_module_source { + e = e.with_source_code(s.source().to_string()); + } + // Bail out if `options.bail` set to `true`, + // which means 'Fail out on the first error instead of tolerating it.' + if self.options.bail { + return Err(e); + } + let mut diagnostics = Vec::with_capacity(create_data.diagnostics.len() + 1); + diagnostics.push(e.into()); + diagnostics.append(&mut create_data.diagnostics); + // Continue bundling if `options.bail` set to `false`. + Ok(vec![Box::new( + factorize_result_task + .with_diagnostics(diagnostics) + .with_file_dependencies(create_data.file_dependencies.drain()) + .with_missing_dependencies(create_data.missing_dependencies.drain()) + .with_context_dependencies(create_data.context_dependencies.drain()), + )]) + } + } + } +} + +/// a struct temporarily used creating ExportsInfo +#[derive(Debug)] +pub struct ExportsInfoRelated { + pub exports_info: ExportsInfo, + pub other_exports_info: ExportInfo, + pub side_effects_info: ExportInfo, +} + +#[derive(Debug)] +struct FactorizeResultTask { + // pub dependency: DependencyId, + pub original_module_identifier: Option, + /// Result will be available if [crate::ModuleFactory::create] returns `Ok`. + pub factory_result: Option, + pub dependencies: Vec, + pub is_entry: bool, + pub current_profile: Option>, + pub exports_info_related: ExportsInfoRelated, + + pub file_dependencies: HashSet, + pub context_dependencies: HashSet, + pub missing_dependencies: HashSet, + pub diagnostics: Vec, +} + +impl FactorizeResultTask { + fn with_factory_result(mut self, factory_result: Option) -> Self { + self.factory_result = factory_result; + self + } + + fn with_diagnostics(mut self, diagnostics: Vec) -> Self { + self.diagnostics = diagnostics; + self + } + + fn with_file_dependencies(mut self, files: impl IntoIterator) -> Self { + self.file_dependencies = files.into_iter().collect(); + self + } + + fn with_context_dependencies(mut self, contexts: impl IntoIterator) -> Self { + self.context_dependencies = contexts.into_iter().collect(); + self + } + + fn with_missing_dependencies(mut self, missing: impl IntoIterator) -> Self { + self.missing_dependencies = missing.into_iter().collect(); + self + } +} + +impl Task for FactorizeResultTask { + fn get_task_type(&self) -> TaskType { + TaskType::Sync + } + fn sync_run(self: Box, context: &mut MakeTaskContext) -> TaskResult { + let FactorizeResultTask { + original_module_identifier, + factory_result, + dependencies, + is_entry, + current_profile, + exports_info_related, + file_dependencies, + context_dependencies, + missing_dependencies, + diagnostics, + .. + } = *self; + if !diagnostics.is_empty() { + if let Some(id) = original_module_identifier { + context.make_failed_module.insert(id); + } else { + context + .make_failed_dependencies + .insert((dependencies[0], None)); + } + } + + context.diagnostics.extend( + diagnostics + .into_iter() + .map(|d| d.with_module_identifier(original_module_identifier)), + ); + + context.file_dependencies.extend(file_dependencies); + context.context_dependencies.extend(context_dependencies); + context.missing_dependencies.extend(missing_dependencies); + let module_graph = &mut MakeTaskContext::get_module_graph(&mut context.module_graph_partial); + let Some(factory_result) = factory_result else { + let dep = module_graph + .dependency_by_id(&dependencies[0]) + .expect("dep should available"); + tracing::trace!("Module created with failure, but without bailout: {dep:?}"); + return Ok(vec![]); + }; + + if let Some(counter) = &mut context.factorize_cache_counter { + if factory_result.from_cache { + counter.hit(); + } else { + counter.miss(); + } + } + + let Some(module) = factory_result.module else { + let dep = module_graph + .dependency_by_id(&dependencies[0]) + .expect("dep should available"); + tracing::trace!("Module ignored: {dep:?}"); + return Ok(vec![]); + }; + let module_identifier = module.identifier(); + let mut mgm = ModuleGraphModule::new( + module.identifier(), + *module.module_type(), + exports_info_related.exports_info.id, + ); + mgm.set_issuer_if_unset(original_module_identifier); + + module_graph.set_exports_info( + exports_info_related.exports_info.id, + exports_info_related.exports_info, + ); + module_graph.set_export_info( + exports_info_related.side_effects_info.id, + exports_info_related.side_effects_info, + ); + module_graph.set_export_info( + exports_info_related.other_exports_info.id, + exports_info_related.other_exports_info, + ); + tracing::trace!("Module created: {}", &module_identifier); + + Ok(vec![Box::new(AddTask { + original_module_identifier, + module, + module_graph_module: Box::new(mgm), + dependencies, + is_entry, + current_profile, + })]) + } +} diff --git a/crates/rspack_core/src/compiler/make/tasks/mod.rs b/crates/rspack_core/src/compiler/make/tasks/mod.rs new file mode 100644 index 00000000000..666e5c9342e --- /dev/null +++ b/crates/rspack_core/src/compiler/make/tasks/mod.rs @@ -0,0 +1,130 @@ +mod add; +mod build; +pub mod clean; +pub mod factorize; +mod process_dependencies; + +use std::{hash::BuildHasherDefault, path::PathBuf, sync::Arc}; + +use indexmap::IndexSet; +use rspack_error::Diagnostic; +use rspack_identifier::{IdentifierMap, IdentifierSet}; +use rustc_hash::{FxHashMap as HashMap, FxHashSet as HashSet, FxHasher}; + +use crate::{ + cache::Cache, module_graph::ModuleGraph, tree_shaking::visitor::OptimizeAnalyzeResult, + BuildDependency, CacheCount, CacheOptions, Compilation, CompilationLogger, CompilerOptions, + DependencyType, Logger, ModuleFactory, ModuleGraphPartial, ModuleIdentifier, ResolverFactory, + SharedPluginDriver, +}; + +pub struct MakeTaskContext { + plugin_driver: SharedPluginDriver, + compiler_options: Arc, + resolver_factory: Arc, + loader_resolver_factory: Arc, + cache: Arc, + dependency_factories: HashMap>, + + module_graph_partial: ModuleGraphPartial, + make_failed_dependencies: HashSet, + make_failed_module: HashSet, + entry_module_identifiers: IdentifierSet, + diagnostics: Vec, + + // TODO move outof context + logger: CompilationLogger, + build_cache_counter: Option, + factorize_cache_counter: Option, + // add_timer: StartTimeAggregate, + // process_deps_timer: StartTimeAggregate, + // factorize_timer: StartTimeAggregate, + // build_timer: StartTimeAggregate, + /// Collecting all module that need to skip in tree-shaking ast modification phase + // bailout_module_identifiers: IdentifierMap, + optimize_analyze_result_map: IdentifierMap, + + file_dependencies: IndexSet>, + context_dependencies: IndexSet>, + missing_dependencies: IndexSet>, + build_dependencies: IndexSet>, +} + +impl MakeTaskContext { + pub fn new(compilation: &Compilation, make_module_graph_partial: ModuleGraphPartial) -> Self { + let logger = compilation.get_logger("rspack.Compilation"); + let mut build_cache_counter = None; + let mut factorize_cache_counter = None; + if !(matches!(compilation.options.cache, CacheOptions::Disabled)) { + build_cache_counter = Some(logger.cache("module build cache")); + factorize_cache_counter = Some(logger.cache("module factorize cache")); + } + + Self { + plugin_driver: compilation.plugin_driver.clone(), + compiler_options: compilation.options.clone(), + resolver_factory: compilation.resolver_factory.clone(), + loader_resolver_factory: compilation.loader_resolver_factory.clone(), + cache: compilation.cache.clone(), + dependency_factories: compilation.dependency_factories.clone(), + + module_graph_partial: make_module_graph_partial, + make_failed_dependencies: Default::default(), + make_failed_module: Default::default(), + entry_module_identifiers: Default::default(), + diagnostics: Default::default(), + optimize_analyze_result_map: Default::default(), + + // TODO use timer in tasks + logger, + build_cache_counter, + factorize_cache_counter, + // add_timer: logger.time_aggregate("module add task"), + // process_deps_timer: logger.time_aggregate("module process dependencies task"), + // factorize_timer: logger.time_aggregate("module factorize task"), + // build_timer: logger.time_aggregate("module build task"), + file_dependencies: Default::default(), + context_dependencies: Default::default(), + missing_dependencies: Default::default(), + build_dependencies: Default::default(), + } + } + + pub fn emit_data_to_compilation(mut self, compilation: &mut Compilation) { + if let Some(counter) = self.build_cache_counter { + self.logger.cache_end(counter); + } + if let Some(counter) = self.factorize_cache_counter { + self.logger.cache_end(counter); + } + + compilation + .make_failed_dependencies + .extend(self.make_failed_dependencies.drain()); + compilation + .make_failed_module + .extend(self.make_failed_module.drain()); + compilation.file_dependencies.extend(self.file_dependencies); + compilation + .context_dependencies + .extend(self.context_dependencies); + compilation + .missing_dependencies + .extend(self.missing_dependencies); + compilation + .build_dependencies + .extend(self.build_dependencies); + + compilation.push_batch_diagnostic(self.diagnostics); + compilation + .entry_module_identifiers + .extend(self.entry_module_identifiers); + compilation.swap_make_module_graph(&mut self.module_graph_partial); + compilation.optimize_analyze_result_map = self.optimize_analyze_result_map; + } + + // TODO use module graph with make artifact + pub fn get_module_graph(module_graph_partial: &mut ModuleGraphPartial) -> ModuleGraph { + ModuleGraph::new(vec![], Some(module_graph_partial)) + } +} diff --git a/crates/rspack_core/src/compiler/make/tasks/process_dependencies.rs b/crates/rspack_core/src/compiler/make/tasks/process_dependencies.rs new file mode 100644 index 00000000000..07de4bb038a --- /dev/null +++ b/crates/rspack_core/src/compiler/make/tasks/process_dependencies.rs @@ -0,0 +1,127 @@ +use rustc_hash::FxHashMap as HashMap; + +use super::{factorize::FactorizeTask, MakeTaskContext}; +use crate::{ + utils::task_loop::{Task, TaskResult, TaskType}, + ContextDependency, DependencyId, DependencyType, ErrorSpan, Module, ModuleIdentifier, + ModuleProfile, NormalModuleSource, Resolve, +}; + +#[derive(Debug)] +pub struct ProcessDependenciesTask { + pub original_module_identifier: ModuleIdentifier, + pub dependencies: Vec, + pub resolve_options: Option>, +} + +impl Task for ProcessDependenciesTask { + fn get_task_type(&self) -> TaskType { + TaskType::Sync + } + + fn sync_run(self: Box, context: &mut MakeTaskContext) -> TaskResult { + let Self { + original_module_identifier, + dependencies, + resolve_options, + } = *self; + let mut sorted_dependencies = HashMap::default(); + let module_graph = &mut MakeTaskContext::get_module_graph(&mut context.module_graph_partial); + + dependencies.into_iter().for_each(|dependency_id| { + let dependency = module_graph + .dependency_by_id(&dependency_id) + .expect("should have dependency"); + // FIXME: now only module/context dependency can put into resolve queue. + // FIXME: should align webpack + let resource_identifier = if let Some(module_dependency) = dependency.as_module_dependency() { + // TODO need implement more dependency `resource_identifier()` + // https://github.com/webpack/webpack/blob/main/lib/Compilation.js#L1621 + let id = if let Some(resource_identifier) = module_dependency.resource_identifier() { + resource_identifier.to_string() + } else { + format!( + "{}|{}", + module_dependency.dependency_type(), + module_dependency.request() + ) + }; + Some(id) + } else { + dependency + .as_context_dependency() + .map(|d| ContextDependency::resource_identifier(d).to_string()) + }; + + if let Some(resource_identifier) = resource_identifier { + sorted_dependencies + .entry(resource_identifier) + .or_insert(vec![]) + .push(dependency_id); + } + }); + + let module = module_graph + .module_by_identifier(&original_module_identifier) + .expect("Module expected"); + + let mut res: Vec>> = vec![]; + for dependencies in sorted_dependencies.into_values() { + let current_profile = context + .compiler_options + .profile + .then(Box::::default); + let dependency = module_graph + .dependency_by_id(&dependencies[0]) + .expect("should have dependency") + .clone(); + let original_module_source = module_graph + .module_by_identifier(&original_module_identifier) + .and_then(|m| m.as_normal_module()) + .and_then(|m| { + if let NormalModuleSource::BuiltSucceed(s) = m.source() { + Some(s.clone()) + } else { + None + } + }); + let dependency_type = dependency.dependency_type(); + // TODO move module_factory calculate to dependency factories + let module_factory = context + .dependency_factories + .get(&match dependency_type { + DependencyType::EsmImport(_) => DependencyType::EsmImport(ErrorSpan::default()), + DependencyType::EsmExport(_) => DependencyType::EsmExport(ErrorSpan::default()), + _ => dependency_type.clone(), + }) + .unwrap_or_else(|| { + panic!( + "No module factory available for dependency type: {}, resourceIdentifier: {:?}", + dependency_type, + dependency.resource_identifier() + ) + }) + .clone(); + res.push(Box::new(FactorizeTask { + module_factory, + original_module_identifier: Some(module.identifier()), + original_module_context: module.get_context(), + original_module_source, + issuer: module + .as_normal_module() + .and_then(|module| module.name_for_condition()), + dependency, + dependencies, + is_entry: false, + resolve_options: resolve_options.clone(), + resolver_factory: context.resolver_factory.clone(), + loader_resolver_factory: context.loader_resolver_factory.clone(), + options: context.compiler_options.clone(), + plugin_driver: context.plugin_driver.clone(), + cache: context.cache.clone(), + current_profile, + })); + } + Ok(res) + } +} diff --git a/crates/rspack_core/src/compiler/mod.rs b/crates/rspack_core/src/compiler/mod.rs index fb2f45d02d9..903d8e8ca38 100644 --- a/crates/rspack_core/src/compiler/mod.rs +++ b/crates/rspack_core/src/compiler/mod.rs @@ -20,7 +20,7 @@ use tracing::instrument; pub use self::compilation::*; pub use self::hmr::{collect_changed_modules, CompilationRecords}; -pub use self::make::{FactorizeTask, MakeParam}; +pub use self::make::MakeParam; pub use self::module_executor::{ExecuteModuleId, ModuleExecutor}; use crate::cache::Cache; use crate::tree_shaking::symbol::{IndirectType, StarSymbolKind, DEFAULT_JS_WORD};