From 9ecd1a2b2f5176e0f63acf06e1ba6a25be608ab6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?lichu=20acu=C3=B1a?= Date: Fri, 23 Aug 2024 14:33:39 -0700 Subject: [PATCH] Refactored module IDs optimization (#68846) ### What? Refactored module ID strategies implementation to walk modules graph individually for each endpoint. Based on feedback from [next.js#68408](https://github.com/vercel/next.js/pull/68408) and [turbo#8912](https://github.com/vercel/turbo/pull/8912). Comments marked with `NOTE(LichuAcu)` are intended to make reviewing easier and will be removed before merging. --- crates/next-api/src/app.rs | 76 ++++-- .../next-api/src/global_module_id_strategy.rs | 83 +++++++ crates/next-api/src/instrumentation.rs | 7 +- crates/next-api/src/lib.rs | 1 + crates/next-api/src/middleware.rs | 7 +- crates/next-api/src/pages.rs | 223 ++++++++++++------ crates/next-api/src/project.rs | 39 ++- crates/next-api/src/route.rs | 2 + crates/next-core/src/next_client/context.rs | 6 +- crates/next-core/src/next_edge/context.rs | 6 +- crates/next-core/src/next_server/context.rs | 5 + .../node_modules/external-package/index.js | 2 +- .../opted-out-external-package/index.js | 2 +- .../externals-pages-bundle/test/index.test.js | 12 +- .../turbopack-browser/src/chunking_context.rs | 17 ++ .../crates/turbopack-core/src/changed.rs | 2 +- .../src/chunk/chunking_context.rs | 6 +- .../src/chunk/global_module_id_strategy.rs | 102 ++++++++ .../crates/turbopack-core/src/chunk/mod.rs | 2 + .../src/chunk/module_id_strategies.rs | 59 +++++ .../turbopack-nodejs/src/chunking_context.rs | 17 ++ 21 files changed, 558 insertions(+), 118 deletions(-) create mode 100644 crates/next-api/src/global_module_id_strategy.rs create mode 100644 turbopack/crates/turbopack-core/src/chunk/global_module_id_strategy.rs create mode 100644 turbopack/crates/turbopack-core/src/chunk/module_id_strategies.rs diff --git a/crates/next-api/src/app.rs b/crates/next-api/src/app.rs index 87a1762cd240a..cae78afa761b8 100644 --- a/crates/next-api/src/app.rs +++ b/crates/next-api/src/app.rs @@ -53,12 +53,15 @@ use turbopack_core::{ }, file_source::FileSource, ident::AssetIdent, - module::Module, + issue::IssueSeverity, + module::{Module, Modules}, output::{OutputAsset, OutputAssets}, raw_output::RawOutput, + resolve::{origin::PlainResolveOrigin, parse::Request, pattern::Pattern}, source::Source, virtual_output::VirtualOutputAsset, }; +use turbopack_ecmascript::resolve::cjs_resolve; use crate::{ dynamic_imports::{ @@ -554,6 +557,30 @@ impl AppProject { .collect(), )) } + + #[turbo_tasks::function] + pub async fn client_main_module(self: Vc) -> Result>> { + let client_module_context = Vc::upcast(self.client_module_context()); + + let client_main_module = cjs_resolve( + Vc::upcast(PlainResolveOrigin::new( + client_module_context, + self.project().project_path().join("_".into()), + )), + Request::parse(Value::new(Pattern::Constant( + "next/dist/client/app-next-turbopack.js".into(), + ))), + None, + IssueSeverity::Error.cell(), + ) + .resolve() + .await? + .first_module() + .await? + .context("expected Next.js client runtime to resolve to a module")?; + + Ok(client_main_module) + } } #[turbo_tasks::function] @@ -719,6 +746,24 @@ impl AppEndpoint { )) } + #[turbo_tasks::function] + async fn app_endpoint_entry(self: Vc) -> Result> { + let this = self.await?; + + let next_config = self.await?.app_project.project().next_config(); + let app_entry = match this.ty { + AppEndpointType::Page { loader_tree, .. } => self.app_page_entry(loader_tree), + AppEndpointType::Route { path, root_layouts } => { + self.app_route_entry(path, root_layouts, next_config) + } + AppEndpointType::Metadata { metadata } => { + self.app_metadata_entry(metadata, next_config) + } + }; + + Ok(app_entry) + } + #[turbo_tasks::function] fn output_assets(self: Vc) -> Vc { self.output().output_assets() @@ -728,24 +773,15 @@ impl AppEndpoint { async fn output(self: Vc) -> Result> { let this = self.await?; - let next_config = self.await?.app_project.project().next_config(); - let (app_entry, process_client, process_ssr) = match this.ty { - AppEndpointType::Page { ty, loader_tree } => ( - self.app_page_entry(loader_tree), - true, - matches!(ty, AppPageEndpointType::Html), - ), + let app_entry = self.app_endpoint_entry().await?; + + let (process_client, process_ssr) = match this.ty { + AppEndpointType::Page { ty, .. } => (true, matches!(ty, AppPageEndpointType::Html)), // NOTE(alexkirsz) For routes, technically, a lot of the following code is not needed, // as we know we won't have any client references. However, for now, for simplicity's // sake, we just do the same thing as for pages. - AppEndpointType::Route { path, root_layouts } => ( - self.app_route_entry(path, root_layouts, next_config), - false, - false, - ), - AppEndpointType::Metadata { metadata } => { - (self.app_metadata_entry(metadata, next_config), false, false) - } + AppEndpointType::Route { .. } => (false, false), + AppEndpointType::Metadata { .. } => (false, false), }; let node_root = this.app_project.project().node_root(); @@ -760,8 +796,6 @@ impl AppEndpoint { // assets to add to the middleware manifest (to be loaded in the edge runtime). let mut middleware_assets = vec![]; - let app_entry = app_entry.await?; - let runtime = app_entry.config.await?.runtime.unwrap_or_default(); let rsc_entry = app_entry.rsc_entry; @@ -1334,6 +1368,12 @@ impl Endpoint for AppEndpoint { .project() .client_changed(self.output().client_assets())) } + + #[turbo_tasks::function] + async fn root_modules(self: Vc) -> Result> { + let rsc_entry = self.app_endpoint_entry().await?.rsc_entry; + Ok(Vc::cell(vec![rsc_entry])) + } } #[turbo_tasks::value] diff --git a/crates/next-api/src/global_module_id_strategy.rs b/crates/next-api/src/global_module_id_strategy.rs new file mode 100644 index 0000000000000..720f779ed03e8 --- /dev/null +++ b/crates/next-api/src/global_module_id_strategy.rs @@ -0,0 +1,83 @@ +use anyhow::Result; +use turbo_tasks::Vc; +use turbopack_core::chunk::{ + global_module_id_strategy::{ + children_modules_idents, merge_preprocessed_module_ids, PreprocessedChildrenIdents, + }, + module_id_strategies::{GlobalModuleIdStrategy, ModuleIdStrategy}, +}; + +use crate::{ + project::Project, + route::{Endpoint, Route}, +}; + +#[turbo_tasks::value] +pub struct GlobalModuleIdStrategyBuilder; + +// NOTE(LichuAcu) To access all entrypoints, we need to access an instance of `Project`, but +// `Project` is not available in `turbopack-core`, so we need need this +// `GlobalModuleIdStrategyBuilder` in `next-api`. +#[turbo_tasks::value_impl] +impl GlobalModuleIdStrategyBuilder { + #[turbo_tasks::function] + pub async fn build(project: Vc) -> Result>> { + let mut preprocessed_module_ids = Vec::new(); + + preprocessed_module_ids.push(children_modules_idents(project.client_main_modules())); + + let entrypoints = project.entrypoints().await?; + + preprocessed_module_ids.push(preprocess_module_ids(entrypoints.pages_error_endpoint)); + preprocessed_module_ids.push(preprocess_module_ids(entrypoints.pages_app_endpoint)); + preprocessed_module_ids.push(preprocess_module_ids(entrypoints.pages_document_endpoint)); + + for (_, route) in entrypoints.routes.iter() { + match route { + Route::Page { + html_endpoint, + data_endpoint, + } => { + preprocessed_module_ids.push(preprocess_module_ids(*html_endpoint)); + preprocessed_module_ids.push(preprocess_module_ids(*data_endpoint)); + } + Route::PageApi { endpoint } => { + preprocessed_module_ids.push(preprocess_module_ids(*endpoint)); + } + Route::AppPage(page_routes) => { + for page_route in page_routes { + preprocessed_module_ids + .push(preprocess_module_ids(page_route.html_endpoint)); + preprocessed_module_ids + .push(preprocess_module_ids(page_route.rsc_endpoint)); + } + } + Route::AppRoute { + original_name: _, + endpoint, + } => { + preprocessed_module_ids.push(preprocess_module_ids(*endpoint)); + } + Route::Conflict => { + tracing::info!("WARN: conflict"); + } + } + } + + let module_id_map = merge_preprocessed_module_ids(preprocessed_module_ids).await?; + + Ok(Vc::upcast( + GlobalModuleIdStrategy::new(module_id_map).await?, + )) + } +} + +// NOTE(LichuAcu) We can't move this function to `turbopack-core` because we need access to +// `Endpoint`, which is not available there. +#[turbo_tasks::function] +async fn preprocess_module_ids( + endpoint: Vc>, +) -> Result> { + let root_modules = endpoint.root_modules(); + Ok(children_modules_idents(root_modules)) +} diff --git a/crates/next-api/src/instrumentation.rs b/crates/next-api/src/instrumentation.rs index 84c8ae8d1073d..11e139d362722 100644 --- a/crates/next-api/src/instrumentation.rs +++ b/crates/next-api/src/instrumentation.rs @@ -15,7 +15,7 @@ use turbopack_core::{ EntryChunkGroupResult, }, context::AssetContext, - module::Module, + module::{Module, Modules}, output::{OutputAsset, OutputAssets}, reference_type::{EntryReferenceSubType, ReferenceType}, source::Source, @@ -236,4 +236,9 @@ impl Endpoint for InstrumentationEndpoint { fn client_changed(self: Vc) -> Vc { Completion::immutable() } + + #[turbo_tasks::function] + fn root_modules(self: Vc) -> Result> { + Err(anyhow::anyhow!("Not implemented.")) + } } diff --git a/crates/next-api/src/lib.rs b/crates/next-api/src/lib.rs index f92ce3e532d8d..e16935972fd9d 100644 --- a/crates/next-api/src/lib.rs +++ b/crates/next-api/src/lib.rs @@ -6,6 +6,7 @@ mod app; mod dynamic_imports; pub mod entrypoints; mod font; +pub mod global_module_id_strategy; mod instrumentation; mod loadable_manifest; mod middleware; diff --git a/crates/next-api/src/middleware.rs b/crates/next-api/src/middleware.rs index b413d649be7ec..4b69a392d7a88 100644 --- a/crates/next-api/src/middleware.rs +++ b/crates/next-api/src/middleware.rs @@ -14,7 +14,7 @@ use turbopack_core::{ asset::AssetContent, chunk::{availability_info::AvailabilityInfo, ChunkingContextExt}, context::AssetContext, - module::Module, + module::{Module, Modules}, output::OutputAssets, reference_type::{EntryReferenceSubType, ReferenceType}, source::Source, @@ -237,4 +237,9 @@ impl Endpoint for MiddlewareEndpoint { fn client_changed(self: Vc) -> Vc { Completion::immutable() } + + #[turbo_tasks::function] + fn root_modules(self: Vc) -> Result> { + Err(anyhow::anyhow!("Not implemented.")) + } } diff --git a/crates/next-api/src/pages.rs b/crates/next-api/src/pages.rs index 54aa952018a7f..df0d5aecf5789 100644 --- a/crates/next-api/src/pages.rs +++ b/crates/next-api/src/pages.rs @@ -46,7 +46,7 @@ use turbopack_core::{ context::AssetContext, file_source::FileSource, issue::IssueSeverity, - module::Module, + module::{Module, Modules}, output::{OutputAsset, OutputAssets}, reference_type::{EcmaScriptModulesReferenceSubType, EntryReferenceSubType, ReferenceType}, resolve::{origin::PlainResolveOrigin, parse::Request, pattern::Pattern}, @@ -555,6 +555,33 @@ impl PagesProject { let ssr_data_runtime_entries = self.data_runtime_entries(); ssr_data_runtime_entries.resolve_entries(Vc::upcast(self.edge_ssr_module_context())) } + + #[turbo_tasks::function] + pub async fn client_main_module(self: Vc) -> Result>> { + let client_module_context = self.client_module_context(); + + let client_main_module = esm_resolve( + Vc::upcast(PlainResolveOrigin::new( + client_module_context, + self.project().project_path().join("_".into()), + )), + Request::parse(Value::new(Pattern::Constant( + match *self.project().next_mode().await? { + NextMode::Development => "next/dist/client/next-dev-turbopack.js", + NextMode::Build => "next/dist/client/next-turbopack.js", + } + .into(), + ))), + Value::new(EcmaScriptModulesReferenceSubType::Undefined), + IssueSeverity::Error.cell(), + None, + ) + .first_module() + .await? + .context("expected Next.js client runtime to resolve to a module")?; + + Ok(client_main_module) + } } #[turbo_tasks::value] @@ -614,17 +641,23 @@ impl PageEndpoint { Ok(Vc::upcast(FileSource::new(this.page.project_path()))) } + #[turbo_tasks::function] + async fn client_module(self: Vc) -> Result>> { + let this = self.await?; + Ok(create_page_loader_entry_module( + this.pages_project.client_module_context(), + self.source(), + this.pathname, + )) + } + #[turbo_tasks::function] async fn client_chunks(self: Vc) -> Result> { async move { let this = self.await?; - let client_module_context = this.pages_project.client_module_context(); - let client_module = create_page_loader_entry_module( - client_module_context, - self.source(), - this.pathname, - ); + let client_module = self.client_module(); + let client_main_module = this.pages_project.client_main_module(); let Some(client_module) = Vc::try_resolve_sidecast::>(client_module).await? @@ -632,26 +665,6 @@ impl PageEndpoint { bail!("expected an ECMAScript module asset"); }; - let client_main_module = esm_resolve( - Vc::upcast(PlainResolveOrigin::new( - client_module_context, - this.pages_project.project().project_path().join("_".into()), - )), - Request::parse(Value::new(Pattern::Constant( - match *this.pages_project.project().next_mode().await? { - NextMode::Development => "next/dist/client/next-dev-turbopack.js", - NextMode::Build => "next/dist/client/next-turbopack.js", - } - .into(), - ))), - Value::new(EcmaScriptModulesReferenceSubType::Undefined), - IssueSeverity::Error.cell(), - None, - ) - .first_module() - .await? - .context("expected Next.js client runtime to resolve to a module")?; - let Some(client_main_module) = Vc::try_resolve_sidecast::>(client_main_module).await? else { @@ -693,15 +706,84 @@ impl PageEndpoint { Ok(Vc::upcast(page_loader)) } + #[turbo_tasks::function] + async fn internal_ssr_chunk_module(self: Vc) -> Result> { + let this = self.await?; + + let (reference_type, project_root, module_context, edge_module_context) = match this.ty { + PageEndpointType::Html | PageEndpointType::SsrOnly => ( + Value::new(ReferenceType::Entry(EntryReferenceSubType::Page)), + this.pages_project.project().project_path(), + this.pages_project.ssr_module_context(), + this.pages_project.edge_ssr_module_context(), + ), + PageEndpointType::Data => ( + Value::new(ReferenceType::Entry(EntryReferenceSubType::Page)), + this.pages_project.project().project_path(), + this.pages_project.ssr_data_module_context(), + this.pages_project.edge_ssr_data_module_context(), + ), + PageEndpointType::Api => ( + Value::new(ReferenceType::Entry(EntryReferenceSubType::PagesApi)), + this.pages_project.project().project_path(), + this.pages_project.api_module_context(), + this.pages_project.edge_api_module_context(), + ), + }; + + let ssr_module = module_context + .process(self.source(), reference_type.clone()) + .module(); + + let config = parse_config_from_source(ssr_module).await?; + let is_edge = matches!(config.runtime, NextRuntime::Edge); + + let ssr_module = if is_edge { + create_page_ssr_entry_module( + this.pathname, + reference_type, + project_root, + Vc::upcast(edge_module_context), + self.source(), + this.original_name, + this.pages_structure, + config.runtime, + this.pages_project.project().next_config(), + ) + } else { + let pathname = &**this.pathname.await?; + + // `/_app` and `/_document` never get rendered directly so they don't need to be + // wrapped in the route module. + if pathname == "/_app" || pathname == "/_document" { + ssr_module + } else { + create_page_ssr_entry_module( + this.pathname, + reference_type, + project_root, + Vc::upcast(module_context), + self.source(), + this.original_name, + this.pages_structure, + config.runtime, + this.pages_project.project().next_config(), + ) + } + }; + + Ok(InternalSsrChunkModule { + ssr_module, + runtime: config.runtime, + } + .cell()) + } + #[turbo_tasks::function] async fn internal_ssr_chunk( self: Vc, ty: SsrChunkType, - reference_type: Value, node_path: Vc, - project_root: Vc, - module_context: Vc, - edge_module_context: Vc, chunking_context: Vc, edge_chunking_context: Vc>, runtime_entries: Vc, @@ -710,26 +792,14 @@ impl PageEndpoint { async move { let this = self.await?; - let ssr_module = module_context - .process(self.source(), reference_type.clone()) - .module(); + let InternalSsrChunkModule { + ssr_module, + runtime, + } = *self.internal_ssr_chunk_module().await?; - let config = parse_config_from_source(ssr_module).await?; - let is_edge = matches!(config.runtime, NextRuntime::Edge); + let is_edge = matches!(runtime, NextRuntime::Edge); if is_edge { - let ssr_module = create_page_ssr_entry_module( - this.pathname, - reference_type, - project_root, - Vc::upcast(edge_module_context), - self.source(), - this.original_name, - this.pages_structure, - config.runtime, - this.pages_project.project().next_config(), - ); - let mut evaluatable_assets = edge_runtime_entries.await?.clone_value(); let evaluatable = Vc::try_resolve_sidecast(ssr_module) .await? @@ -763,24 +833,6 @@ impl PageEndpoint { } else { let pathname = &**this.pathname.await?; - // `/_app` and `/_document` never get rendered directly so they don't need to be - // wrapped in the route module. - let ssr_module = if pathname == "/_app" || pathname == "/_document" { - ssr_module - } else { - create_page_ssr_entry_module( - this.pathname, - reference_type, - project_root, - Vc::upcast(module_context), - self.source(), - this.original_name, - this.pages_structure, - config.runtime, - this.pages_project.project().next_config(), - ) - }; - let asset_path = get_asset_path_from_pathname(pathname, ".js"); let ssr_entry_chunk_path_string: RcStr = format!("pages{asset_path}").into(); @@ -832,14 +884,10 @@ impl PageEndpoint { let this = self.await?; Ok(self.internal_ssr_chunk( SsrChunkType::Page, - Value::new(ReferenceType::Entry(EntryReferenceSubType::Page)), this.pages_project .project() .node_root() .join("server".into()), - this.pages_project.project().project_path(), - this.pages_project.ssr_module_context(), - this.pages_project.edge_ssr_module_context(), this.pages_project.project().server_chunking_context(true), this.pages_project.project().edge_chunking_context(true), this.pages_project.ssr_runtime_entries(), @@ -852,14 +900,10 @@ impl PageEndpoint { let this = self.await?; Ok(self.internal_ssr_chunk( SsrChunkType::Data, - Value::new(ReferenceType::Entry(EntryReferenceSubType::Page)), this.pages_project .project() .node_root() .join("server/data".into()), - this.pages_project.project().project_path(), - this.pages_project.ssr_data_module_context(), - this.pages_project.edge_ssr_data_module_context(), this.pages_project.project().server_chunking_context(true), this.pages_project.project().edge_chunking_context(true), this.pages_project.ssr_data_runtime_entries(), @@ -872,14 +916,10 @@ impl PageEndpoint { let this = self.await?; Ok(self.internal_ssr_chunk( SsrChunkType::Api, - Value::new(ReferenceType::Entry(EntryReferenceSubType::PagesApi)), this.pages_project .project() .node_root() .join("server".into()), - this.pages_project.project().project_path(), - this.pages_project.api_module_context(), - this.pages_project.edge_api_module_context(), this.pages_project.project().server_chunking_context(false), this.pages_project.project().edge_chunking_context(false), this.pages_project.ssr_runtime_entries(), @@ -1129,6 +1169,18 @@ impl PageEndpoint { } } +#[turbo_tasks::value] +pub struct InternalSsrChunkModule { + pub ssr_module: Vc>, + pub runtime: NextRuntime, +} + +#[turbo_tasks::value] +pub struct ClientChunksModules { + pub client_module: Vc>, + pub client_main_module: Vc>, +} + #[turbo_tasks::value_impl] impl Endpoint for PageEndpoint { #[turbo_tasks::function] @@ -1212,6 +1264,21 @@ impl Endpoint for PageEndpoint { .project() .client_changed(self.output().client_assets())) } + + #[turbo_tasks::function] + async fn root_modules(self: Vc) -> Result> { + let this = self.await?; + let mut modules = vec![]; + + let ssr_chunk_module = self.internal_ssr_chunk_module().await?; + modules.push(ssr_chunk_module.ssr_module); + + if let PageEndpointType::Html = this.ty { + modules.push(self.client_module()); + } + + Ok(Vc::cell(modules)) + } } #[turbo_tasks::value] diff --git a/crates/next-api/src/project.rs b/crates/next-api/src/project.rs index 6f4e00899a291..d0c624c0e2564 100644 --- a/crates/next-api/src/project.rs +++ b/crates/next-api/src/project.rs @@ -34,12 +34,16 @@ use turbo_tasks_fs::{DiskFileSystem, FileSystem, FileSystemPath, VirtualFileSyst use turbopack::{evaluate_context::node_build_environment, ModuleAssetContext}; use turbopack_core::{ changed::content_changed, - chunk::ChunkingContext, + chunk::{ + module_id_strategies::{DevModuleIdStrategy, ModuleIdStrategy}, + ChunkingContext, + }, compile_time_info::CompileTimeInfo, context::AssetContext, diagnostics::DiagnosticExt, file_source::FileSource, issue::{Issue, IssueExt, IssueSeverity, IssueStage, OptionStyledString, StyledString}, + module::Modules, output::{OutputAsset, OutputAssets}, resolve::{find_context_file, FindContextFileResult}, source::Source, @@ -54,6 +58,7 @@ use crate::{ app::{AppProject, OptionAppProject, ECMASCRIPT_CLIENT_TRANSITION_NAME}, build, entrypoints::Entrypoints, + global_module_id_strategy::GlobalModuleIdStrategyBuilder, instrumentation::InstrumentationEndpoint, middleware::MiddlewareEndpoint, pages::PagesProject, @@ -443,7 +448,7 @@ impl Issue for ConflictIssue { #[turbo_tasks::value_impl] impl Project { #[turbo_tasks::function] - async fn app_project(self: Vc) -> Result> { + pub async fn app_project(self: Vc) -> Result> { let app_dir = find_app_dir(self.project_path()).await?; Ok(Vc::cell( @@ -452,7 +457,7 @@ impl Project { } #[turbo_tasks::function] - async fn pages_project(self: Vc) -> Result> { + pub async fn pages_project(self: Vc) -> Result> { Ok(PagesProject::new(self)) } @@ -621,6 +626,7 @@ impl Project { self.next_config().computed_asset_prefix(), self.client_compile_time_info().environment(), self.next_mode(), + self.module_id_strategy(), )) } @@ -637,6 +643,7 @@ impl Project { self.client_relative_path(), self.next_config().computed_asset_prefix(), self.server_compile_time_info().environment(), + self.module_id_strategy(), ) } else { get_server_chunking_context( @@ -644,6 +651,7 @@ impl Project { self.project_path(), self.node_root(), self.server_compile_time_info().environment(), + self.module_id_strategy(), ) } } @@ -661,6 +669,7 @@ impl Project { self.client_relative_path(), self.next_config().computed_asset_prefix(), self.edge_compile_time_info().environment(), + self.module_id_strategy(), ) } else { get_edge_chunking_context( @@ -668,6 +677,7 @@ impl Project { self.project_path(), self.node_root(), self.edge_compile_time_info().environment(), + self.module_id_strategy(), ) } } @@ -1161,6 +1171,29 @@ impl Project { let path = self.client_root(); any_output_changed(roots, path, false) } + + #[turbo_tasks::function] + pub async fn client_main_modules(self: Vc) -> Result> { + let pages_project = self.pages_project(); + let mut modules = vec![pages_project.client_main_module()]; + + if let Some(app_project) = *self.app_project().await? { + modules.push(app_project.client_main_module()); + } + + Ok(Vc::cell(modules)) + } + + /// Get the module id strategy for the project. + /// In production mode, we use the global module id strategy with optimized ids. + /// In development mode, we use a standard module id strategy with no modifications. + #[turbo_tasks::function] + pub async fn module_id_strategy(self: Vc) -> Result>> { + match *self.next_mode().await? { + NextMode::Build => Ok(Vc::upcast(GlobalModuleIdStrategyBuilder::build(self))), + NextMode::Development => Ok(Vc::upcast(DevModuleIdStrategy::new())), + } + } } #[turbo_tasks::function] diff --git a/crates/next-api/src/route.rs b/crates/next-api/src/route.rs index 9736f507adc92..8b48b574eb786 100644 --- a/crates/next-api/src/route.rs +++ b/crates/next-api/src/route.rs @@ -2,6 +2,7 @@ use anyhow::Result; use indexmap::IndexMap; use serde::{Deserialize, Serialize}; use turbo_tasks::{debug::ValueDebugFormat, trace::TraceRawVcs, Completion, RcStr, Vc}; +use turbopack_core::module::Modules; use crate::paths::ServerPath; @@ -75,6 +76,7 @@ pub trait Endpoint { fn write_to_disk(self: Vc) -> Vc; fn server_changed(self: Vc) -> Vc; fn client_changed(self: Vc) -> Vc; + fn root_modules(self: Vc) -> Vc; } #[turbo_tasks::value(shared)] diff --git a/crates/next-core/src/next_client/context.rs b/crates/next-core/src/next_client/context.rs index 440e7a2ba655b..5d7b6b0722fed 100644 --- a/crates/next-core/src/next_client/context.rs +++ b/crates/next-core/src/next_client/context.rs @@ -14,7 +14,7 @@ use turbopack::{ }; use turbopack_browser::{react_refresh::assert_can_resolve_react_refresh, BrowserChunkingContext}; use turbopack_core::{ - chunk::ChunkingContext, + chunk::{module_id_strategies::ModuleIdStrategy, ChunkingContext}, compile_time_info::{ CompileTimeDefineValue, CompileTimeDefines, CompileTimeInfo, FreeVarReference, FreeVarReferences, @@ -353,6 +353,7 @@ pub async fn get_client_chunking_context( asset_prefix: Vc>, environment: Vc, mode: Vc, + module_id_strategy: Vc>, ) -> Result>> { let next_mode = mode.await?; let mut builder = BrowserChunkingContext::builder( @@ -366,7 +367,8 @@ pub async fn get_client_chunking_context( ) .chunk_base_path(asset_prefix) .minify_type(next_mode.minify_type()) - .asset_base_path(asset_prefix); + .asset_base_path(asset_prefix) + .module_id_strategy(module_id_strategy); if next_mode.is_development() { builder = builder.hot_module_replacement(); diff --git a/crates/next-core/src/next_edge/context.rs b/crates/next-core/src/next_edge/context.rs index bf30c76b66018..9dbd307893d43 100644 --- a/crates/next-core/src/next_edge/context.rs +++ b/crates/next-core/src/next_edge/context.rs @@ -6,7 +6,7 @@ use turbo_tasks_fs::FileSystemPath; use turbopack::resolve_options_context::ResolveOptionsContext; use turbopack_browser::BrowserChunkingContext; use turbopack_core::{ - chunk::ChunkingContext, + chunk::{module_id_strategies::ModuleIdStrategy, ChunkingContext}, compile_time_info::{ CompileTimeDefineValue, CompileTimeDefines, CompileTimeInfo, FreeVarReference, FreeVarReferences, @@ -178,6 +178,7 @@ pub async fn get_edge_chunking_context_with_client_assets( client_root: Vc, asset_prefix: Vc>, environment: Vc, + module_id_strategy: Vc>, ) -> Result>> { let output_root = node_root.join("server/edge".into()); let next_mode = mode.await?; @@ -193,6 +194,7 @@ pub async fn get_edge_chunking_context_with_client_assets( ) .asset_base_path(asset_prefix) .minify_type(next_mode.minify_type()) + .module_id_strategy(module_id_strategy) .build(), )) } @@ -203,6 +205,7 @@ pub async fn get_edge_chunking_context( project_path: Vc, node_root: Vc, environment: Vc, + module_id_strategy: Vc>, ) -> Result>> { let output_root = node_root.join("server/edge".into()); let next_mode = mode.await?; @@ -222,6 +225,7 @@ pub async fn get_edge_chunking_context( // asset from the output directory. .asset_base_path(Vc::cell(Some("blob:server/edge/".into()))) .minify_type(next_mode.minify_type()) + .module_id_strategy(module_id_strategy) .build(), )) } diff --git a/crates/next-core/src/next_server/context.rs b/crates/next-core/src/next_server/context.rs index 73d78bdd22444..e297c93f319ba 100644 --- a/crates/next-core/src/next_server/context.rs +++ b/crates/next-core/src/next_server/context.rs @@ -14,6 +14,7 @@ use turbopack::{ transition::Transition, }; use turbopack_core::{ + chunk::module_id_strategies::ModuleIdStrategy, compile_time_info::{ CompileTimeDefineValue, CompileTimeDefines, CompileTimeInfo, FreeVarReferences, }, @@ -922,6 +923,7 @@ pub async fn get_server_chunking_context_with_client_assets( client_root: Vc, asset_prefix: Vc>, environment: Vc, + module_id_strategy: Vc>, ) -> Result> { let next_mode = mode.await?; // TODO(alexkirsz) This should return a trait that can be implemented by the @@ -938,6 +940,7 @@ pub async fn get_server_chunking_context_with_client_assets( ) .asset_prefix(asset_prefix) .minify_type(next_mode.minify_type()) + .module_id_strategy(module_id_strategy) .build()) } @@ -947,6 +950,7 @@ pub async fn get_server_chunking_context( project_path: Vc, node_root: Vc, environment: Vc, + module_id_strategy: Vc>, ) -> Result> { let next_mode = mode.await?; // TODO(alexkirsz) This should return a trait that can be implemented by the @@ -962,5 +966,6 @@ pub async fn get_server_chunking_context( next_mode.runtime_type(), ) .minify_type(next_mode.minify_type()) + .module_id_strategy(module_id_strategy) .build()) } diff --git a/test/integration/externals-pages-bundle/node_modules/external-package/index.js b/test/integration/externals-pages-bundle/node_modules/external-package/index.js index ca285d93526b1..37b5c8e6d64a6 100644 --- a/test/integration/externals-pages-bundle/node_modules/external-package/index.js +++ b/test/integration/externals-pages-bundle/node_modules/external-package/index.js @@ -1,3 +1,3 @@ module.exports = { - foo: 'bar', + foo: 'external-package content', } diff --git a/test/integration/externals-pages-bundle/node_modules/opted-out-external-package/index.js b/test/integration/externals-pages-bundle/node_modules/opted-out-external-package/index.js index 2d853a9c0b310..1d0aa9c5b8e38 100644 --- a/test/integration/externals-pages-bundle/node_modules/opted-out-external-package/index.js +++ b/test/integration/externals-pages-bundle/node_modules/opted-out-external-package/index.js @@ -1,3 +1,3 @@ module.exports = { - bar: 'baz', + bar: 'opted-out-external-package content', } diff --git a/test/integration/externals-pages-bundle/test/index.test.js b/test/integration/externals-pages-bundle/test/index.test.js index b7f1e75e9ddac..71f3bcac2907c 100644 --- a/test/integration/externals-pages-bundle/test/index.test.js +++ b/test/integration/externals-pages-bundle/test/index.test.js @@ -27,10 +27,8 @@ describe('bundle pages externals with config.bundlePagesRouterDependencies', () allBundles += output } - // we don't know the name of the minified `__turbopack_external_require__`, so we just check the arguments. - expect(allBundles).not.toContain( - '"[externals]/ [external] (external-package, cjs)"' - ) + // we don't know the name of the minified `__turbopack_external_require__`, so we just check the content. + expect(allBundles).toContain('"external-package content"') } else { const output = await fs.readFile( join(appDir, '.next/server/pages/index.js'), @@ -53,9 +51,9 @@ describe('bundle pages externals with config.bundlePagesRouterDependencies', () allBundles += output } - // we don't know the name of the minified `__turbopack_external_require__`, so we just check the arguments. - expect(allBundles).toContain( - '"[externals]/ [external] (opted-out-external-package, cjs)"' + // we don't know the name of the minified `__turbopack_external_require__`, so we just check the content. + expect(allBundles).not.toContain( + '"opted-out-external-package content"' ) } else { const output = await fs.readFile( diff --git a/turbopack/crates/turbopack-browser/src/chunking_context.rs b/turbopack/crates/turbopack-browser/src/chunking_context.rs index 616a30e01590c..0718c962e99d2 100644 --- a/turbopack/crates/turbopack-browser/src/chunking_context.rs +++ b/turbopack/crates/turbopack-browser/src/chunking_context.rs @@ -6,6 +6,7 @@ use turbopack_core::{ chunk::{ availability_info::AvailabilityInfo, chunk_group::{make_chunk_group, MakeChunkGroupResult}, + module_id_strategies::{DevModuleIdStrategy, ModuleIdStrategy}, Chunk, ChunkGroupResult, ChunkItem, ChunkableModule, ChunkingContext, EntryChunkGroupResult, EvaluatableAssets, MinifyType, ModuleId, }, @@ -77,6 +78,11 @@ impl BrowserChunkingContextBuilder { self } + pub fn module_id_strategy(mut self, module_id_strategy: Vc>) -> Self { + self.chunking_context.module_id_strategy = module_id_strategy; + self + } + pub fn build(self) -> Vc { BrowserChunkingContext::new(Value::new(self.chunking_context)) } @@ -122,6 +128,8 @@ pub struct BrowserChunkingContext { minify_type: MinifyType, /// Whether to use manifest chunks for lazy compilation manifest_chunks: bool, + /// The module id strategy to use + module_id_strategy: Vc>, } impl BrowserChunkingContext { @@ -151,6 +159,7 @@ impl BrowserChunkingContext { runtime_type, minify_type: MinifyType::NoMinify, manifest_chunks: false, + module_id_strategy: Vc::upcast(DevModuleIdStrategy::new()), }, } } @@ -479,6 +488,14 @@ impl ChunkingContext for BrowserChunkingContext { bail!("Browser chunking context does not support entry chunk groups") } + #[turbo_tasks::function] + async fn chunk_item_id_from_ident( + self: Vc, + ident: Vc, + ) -> Result> { + Ok(self.await?.module_id_strategy.get_module_id(ident)) + } + #[turbo_tasks::function] async fn async_loader_chunk_item( self: Vc, diff --git a/turbopack/crates/turbopack-core/src/changed.rs b/turbopack/crates/turbopack-core/src/changed.rs index 159d40f4d06a6..7a03c9dce12d9 100644 --- a/turbopack/crates/turbopack-core/src/changed.rs +++ b/turbopack/crates/turbopack-core/src/changed.rs @@ -17,7 +17,7 @@ async fn get_referenced_output_assets( Ok(parent.references().await?.clone_value().into_iter()) } -async fn get_referenced_modules( +pub async fn get_referenced_modules( parent: Vc>, ) -> Result>> + Send> { Ok(primary_referenced_modules(parent) diff --git a/turbopack/crates/turbopack-core/src/chunk/chunking_context.rs b/turbopack/crates/turbopack-core/src/chunk/chunking_context.rs index 2b2c1eb3ce4fe..6019122deb4f0 100644 --- a/turbopack/crates/turbopack-core/src/chunk/chunking_context.rs +++ b/turbopack/crates/turbopack-core/src/chunk/chunking_context.rs @@ -1,6 +1,6 @@ use anyhow::Result; use serde::{Deserialize, Serialize}; -use turbo_tasks::{trace::TraceRawVcs, RcStr, TaskInput, Upcast, Value, ValueToString, Vc}; +use turbo_tasks::{trace::TraceRawVcs, RcStr, TaskInput, Upcast, Value, Vc}; use turbo_tasks_fs::FileSystemPath; use turbo_tasks_hash::DeterministicHash; @@ -116,9 +116,7 @@ pub trait ChunkingContext { async fn chunk_item_id_from_ident( self: Vc, ident: Vc, - ) -> Result> { - Ok(ModuleId::String(ident.to_string().await?.clone_value()).cell()) - } + ) -> Result>; fn chunk_item_id(self: Vc, chunk_item: Vc>) -> Vc { self.chunk_item_id_from_ident(chunk_item.asset_ident()) diff --git a/turbopack/crates/turbopack-core/src/chunk/global_module_id_strategy.rs b/turbopack/crates/turbopack-core/src/chunk/global_module_id_strategy.rs new file mode 100644 index 0000000000000..7e23fdbe6b3ed --- /dev/null +++ b/turbopack/crates/turbopack-core/src/chunk/global_module_id_strategy.rs @@ -0,0 +1,102 @@ +use std::collections::{HashMap, HashSet}; + +use anyhow::Result; +use turbo_tasks::{ + graph::{AdjacencyMap, GraphTraversal}, + ValueToString, Vc, +}; +use turbo_tasks_hash::hash_xxh3_hash64; + +use crate::{ + changed::get_referenced_modules, + chunk::ModuleId, + ident::AssetIdent, + module::{Module, Modules}, +}; + +#[turbo_tasks::value] +pub struct PreprocessedChildrenIdents { + // module_id -> full hash + // We save the full hash to avoid re-hashing in `merge_preprocessed_module_ids` + // if this endpoint did not change. + modules_idents: HashMap, +} + +// NOTE(LichuAcu) Called on endpoint.root_modules(). It would probably be better if this was called +// directly on `Endpoint`, but such struct is not available in turbopack-core. The whole function +// could be moved to `next-api`, but it would require adding turbo-tasks-hash to `next-api`, +// making it heavier. +#[turbo_tasks::function] +pub async fn children_modules_idents( + root_modules: Vc, +) -> Result> { + let children_modules_iter = AdjacencyMap::new() + .skip_duplicates() + .visit(root_modules.await?.iter().copied(), get_referenced_modules) + .await + .completed()? + .into_inner() + .into_reverse_topological(); + + // module_id -> full hash + let mut modules_idents = HashMap::new(); + + for module in children_modules_iter { + let module_ident = module.ident(); + let hash = hash_xxh3_hash64(module_ident.to_string().await?); + modules_idents.insert(module_ident.await?.clone_value(), hash); + } + + Ok(PreprocessedChildrenIdents { modules_idents }.cell()) +} + +// Note(LichuAcu): This could be split into two functions: one that merges the preprocessed module +// ids and another that generates the final, optimized module ids. Thoughts? +pub async fn merge_preprocessed_module_ids( + prepared_module_ids: Vec>, +) -> Result>> { + let mut module_id_map: HashMap> = HashMap::new(); + let mut used_ids: HashSet = HashSet::new(); + + for prepared_module_ids in prepared_module_ids { + for (module_ident, full_hash) in prepared_module_ids.await?.modules_idents.iter() { + process_module( + module_ident.clone(), + *full_hash, + &mut module_id_map, + &mut used_ids, + ) + .await?; + } + } + + Ok(module_id_map) +} + +pub async fn process_module( + module_ident: AssetIdent, + full_hash: u64, + id_map: &mut HashMap>, + used_ids: &mut HashSet, +) -> Result<()> { + if id_map.contains_key(&module_ident) { + return Ok(()); + } + + let mut masked_hash = full_hash & 0xF; + let mut mask = 0xF; + while used_ids.contains(&masked_hash) { + if mask == 0xFFFFFFFFFFFFFFFF { + return Err(anyhow::anyhow!("This is a... 64-bit hash collision?")); + } + mask = (mask << 4) | 0xF; + masked_hash = full_hash & mask; + } + + let hashed_module_id = ModuleId::String(masked_hash.to_string().into()); + + id_map.insert(module_ident, hashed_module_id.cell()); + used_ids.insert(masked_hash); + + Ok(()) +} diff --git a/turbopack/crates/turbopack-core/src/chunk/mod.rs b/turbopack/crates/turbopack-core/src/chunk/mod.rs index bfc69139cb57e..5942a3f14a24b 100644 --- a/turbopack/crates/turbopack-core/src/chunk/mod.rs +++ b/turbopack/crates/turbopack-core/src/chunk/mod.rs @@ -6,6 +6,8 @@ pub(crate) mod chunking_context; pub(crate) mod containment_tree; pub(crate) mod data; pub(crate) mod evaluate; +pub mod global_module_id_strategy; +pub mod module_id_strategies; pub mod optimize; use std::{ diff --git a/turbopack/crates/turbopack-core/src/chunk/module_id_strategies.rs b/turbopack/crates/turbopack-core/src/chunk/module_id_strategies.rs new file mode 100644 index 0000000000000..3b03ed9e38a50 --- /dev/null +++ b/turbopack/crates/turbopack-core/src/chunk/module_id_strategies.rs @@ -0,0 +1,59 @@ +use std::collections::HashMap; + +use anyhow::Result; +use turbo_tasks::{ValueToString, Vc}; +use turbo_tasks_hash::hash_xxh3_hash64; + +use super::ModuleId; +use crate::ident::AssetIdent; + +#[turbo_tasks::value_trait] +pub trait ModuleIdStrategy { + fn get_module_id(self: Vc, ident: Vc) -> Vc; +} + +#[turbo_tasks::value] +pub struct DevModuleIdStrategy; + +impl DevModuleIdStrategy { + pub fn new() -> Vc { + DevModuleIdStrategy {}.cell() + } +} + +#[turbo_tasks::value_impl] +impl ModuleIdStrategy for DevModuleIdStrategy { + #[turbo_tasks::function] + async fn get_module_id(self: Vc, ident: Vc) -> Result> { + Ok(ModuleId::String(ident.to_string().await?.clone_value()).cell()) + } +} + +#[turbo_tasks::value] +pub struct GlobalModuleIdStrategy { + module_id_map: HashMap>, +} + +impl GlobalModuleIdStrategy { + pub async fn new(module_id_map: HashMap>) -> Result> { + Ok(GlobalModuleIdStrategy { module_id_map }.cell()) + } +} + +#[turbo_tasks::value_impl] +impl ModuleIdStrategy for GlobalModuleIdStrategy { + #[turbo_tasks::function] + async fn get_module_id(self: Vc, ident: Vc) -> Result> { + if let Some(module_id) = self.await?.module_id_map.get(&ident.await?.clone_value()) { + // dbg!(format!("Hit {}", ident.to_string().await?)); + return Ok(*module_id); + } + // dbg!(format!("Miss {}", ident.to_string().await?)); + Ok(ModuleId::String( + hash_xxh3_hash64(ident.to_string().await?) + .to_string() + .into(), + ) + .cell()) + } +} diff --git a/turbopack/crates/turbopack-nodejs/src/chunking_context.rs b/turbopack/crates/turbopack-nodejs/src/chunking_context.rs index 74dadf468b223..bd420f5c6ab20 100644 --- a/turbopack/crates/turbopack-nodejs/src/chunking_context.rs +++ b/turbopack/crates/turbopack-nodejs/src/chunking_context.rs @@ -8,6 +8,7 @@ use turbopack_core::{ chunk::{ availability_info::AvailabilityInfo, chunk_group::{make_chunk_group, MakeChunkGroupResult}, + module_id_strategies::{DevModuleIdStrategy, ModuleIdStrategy}, Chunk, ChunkGroupResult, ChunkItem, ChunkableModule, ChunkingContext, EntryChunkGroupResult, EvaluatableAssets, MinifyType, ModuleId, }, @@ -53,6 +54,11 @@ impl NodeJsChunkingContextBuilder { self } + pub fn module_id_strategy(mut self, module_id_strategy: Vc>) -> Self { + self.chunking_context.module_id_strategy = module_id_strategy; + self + } + /// Builds the chunking context. pub fn build(self) -> Vc { NodeJsChunkingContext::new(Value::new(self.chunking_context)) @@ -84,6 +90,8 @@ pub struct NodeJsChunkingContext { minify_type: MinifyType, /// Whether to use manifest chunks for lazy compilation manifest_chunks: bool, + /// The strategy to use for generating module ids + module_id_strategy: Vc>, } impl NodeJsChunkingContext { @@ -109,6 +117,7 @@ impl NodeJsChunkingContext { runtime_type, minify_type: MinifyType::NoMinify, manifest_chunks: false, + module_id_strategy: Vc::upcast(DevModuleIdStrategy::new()), }, } } @@ -349,6 +358,14 @@ impl ChunkingContext for NodeJsChunkingContext { bail!("the build chunking context does not support evaluated chunk groups") } + #[turbo_tasks::function] + async fn chunk_item_id_from_ident( + self: Vc, + ident: Vc, + ) -> Result> { + Ok(self.await?.module_id_strategy.get_module_id(ident)) + } + #[turbo_tasks::function] async fn async_loader_chunk_item( self: Vc,