Refactored module IDs optimization (#68846)

### What?

Refactored module ID strategies implementation to walk modules graph individually for each endpoint.

Based on feedback from [next.js#68408](#68408) and [turbo#8912](vercel/turborepo#8912).

Comments marked with `NOTE(LichuAcu)` are intended to make reviewing easier and will be removed before merging.

crates/next-api/src/app.rs

@@ -53,12 +53,15 @@ use turbopack_core::{
     },
     file_source::FileSource,
     ident::AssetIdent,
-    module::Module,
+    issue::IssueSeverity,
+    module::{Module, Modules},
     output::{OutputAsset, OutputAssets},
     raw_output::RawOutput,
+    resolve::{origin::PlainResolveOrigin, parse::Request, pattern::Pattern},
     source::Source,
     virtual_output::VirtualOutputAsset,
 };
+use turbopack_ecmascript::resolve::cjs_resolve;
 
 use crate::{
     dynamic_imports::{
@@ -554,6 +557,30 @@ impl AppProject {
                 .collect(),
         ))
     }
+
+    #[turbo_tasks::function]
+    pub async fn client_main_module(self: Vc<Self>) -> Result<Vc<Box<dyn Module>>> {
+        let client_module_context = Vc::upcast(self.client_module_context());
+
+        let client_main_module = cjs_resolve(
+            Vc::upcast(PlainResolveOrigin::new(
+                client_module_context,
+                self.project().project_path().join("_".into()),
+            )),
+            Request::parse(Value::new(Pattern::Constant(
+                "next/dist/client/app-next-turbopack.js".into(),
+            ))),
+            None,
+            IssueSeverity::Error.cell(),
+        )
+        .resolve()
+        .await?
+        .first_module()
+        .await?
+        .context("expected Next.js client runtime to resolve to a module")?;
+
+        Ok(client_main_module)
+    }
 }
 
 #[turbo_tasks::function]
@@ -719,6 +746,24 @@ impl AppEndpoint {
         ))
     }
 
+    #[turbo_tasks::function]
+    async fn app_endpoint_entry(self: Vc<Self>) -> Result<Vc<AppEntry>> {
+        let this = self.await?;
+
+        let next_config = self.await?.app_project.project().next_config();
+        let app_entry = match this.ty {
+            AppEndpointType::Page { loader_tree, .. } => self.app_page_entry(loader_tree),
+            AppEndpointType::Route { path, root_layouts } => {
+                self.app_route_entry(path, root_layouts, next_config)
+            }
+            AppEndpointType::Metadata { metadata } => {
+                self.app_metadata_entry(metadata, next_config)
+            }
+        };
+
+        Ok(app_entry)
+    }
+
     #[turbo_tasks::function]
     fn output_assets(self: Vc<Self>) -> Vc<OutputAssets> {
         self.output().output_assets()
@@ -728,24 +773,15 @@ impl AppEndpoint {
     async fn output(self: Vc<Self>) -> Result<Vc<AppEndpointOutput>> {
         let this = self.await?;
 
-        let next_config = self.await?.app_project.project().next_config();
-        let (app_entry, process_client, process_ssr) = match this.ty {
-            AppEndpointType::Page { ty, loader_tree } => (
-                self.app_page_entry(loader_tree),
-                true,
-                matches!(ty, AppPageEndpointType::Html),
-            ),
+        let app_entry = self.app_endpoint_entry().await?;
+
+        let (process_client, process_ssr) = match this.ty {
+            AppEndpointType::Page { ty, .. } => (true, matches!(ty, AppPageEndpointType::Html)),
             // NOTE(alexkirsz) For routes, technically, a lot of the following code is not needed,
             // as we know we won't have any client references. However, for now, for simplicity's
             // sake, we just do the same thing as for pages.
-            AppEndpointType::Route { path, root_layouts } => (
-                self.app_route_entry(path, root_layouts, next_config),
-                false,
-                false,
-            ),
-            AppEndpointType::Metadata { metadata } => {
-                (self.app_metadata_entry(metadata, next_config), false, false)
-            }
+            AppEndpointType::Route { .. } => (false, false),
+            AppEndpointType::Metadata { .. } => (false, false),
         };
 
         let node_root = this.app_project.project().node_root();
@@ -760,8 +796,6 @@ impl AppEndpoint {
         // assets to add to the middleware manifest (to be loaded in the edge runtime).
         let mut middleware_assets = vec![];
 
-        let app_entry = app_entry.await?;
-
         let runtime = app_entry.config.await?.runtime.unwrap_or_default();
 
         let rsc_entry = app_entry.rsc_entry;
@@ -1334,6 +1368,12 @@ impl Endpoint for AppEndpoint {
             .project()
             .client_changed(self.output().client_assets()))
     }
+
+    #[turbo_tasks::function]
+    async fn root_modules(self: Vc<Self>) -> Result<Vc<Modules>> {
+        let rsc_entry = self.app_endpoint_entry().await?.rsc_entry;
+        Ok(Vc::cell(vec![rsc_entry]))
+    }
 }
 
 #[turbo_tasks::value]

crates/next-api/src/global_module_id_strategy.rs

@@ -0,0 +1,83 @@
+use anyhow::Result;
+use turbo_tasks::Vc;
+use turbopack_core::chunk::{
+    global_module_id_strategy::{
+        children_modules_idents, merge_preprocessed_module_ids, PreprocessedChildrenIdents,
+    },
+    module_id_strategies::{GlobalModuleIdStrategy, ModuleIdStrategy},
+};
+
+use crate::{
+    project::Project,
+    route::{Endpoint, Route},
+};
+
+#[turbo_tasks::value]
+pub struct GlobalModuleIdStrategyBuilder;
+
+// NOTE(LichuAcu) To access all entrypoints, we need to access an instance of `Project`, but
+// `Project` is not available in `turbopack-core`, so we need need this
+// `GlobalModuleIdStrategyBuilder` in `next-api`.
+#[turbo_tasks::value_impl]
+impl GlobalModuleIdStrategyBuilder {
+    #[turbo_tasks::function]
+    pub async fn build(project: Vc<Project>) -> Result<Vc<Box<dyn ModuleIdStrategy>>> {
+        let mut preprocessed_module_ids = Vec::new();
+
+        preprocessed_module_ids.push(children_modules_idents(project.client_main_modules()));
+
+        let entrypoints = project.entrypoints().await?;
+
+        preprocessed_module_ids.push(preprocess_module_ids(entrypoints.pages_error_endpoint));
+        preprocessed_module_ids.push(preprocess_module_ids(entrypoints.pages_app_endpoint));
+        preprocessed_module_ids.push(preprocess_module_ids(entrypoints.pages_document_endpoint));
+
+        for (_, route) in entrypoints.routes.iter() {
+            match route {
+                Route::Page {
+                    html_endpoint,
+                    data_endpoint,
+                } => {
+                    preprocessed_module_ids.push(preprocess_module_ids(*html_endpoint));
+                    preprocessed_module_ids.push(preprocess_module_ids(*data_endpoint));
+                }
+                Route::PageApi { endpoint } => {
+                    preprocessed_module_ids.push(preprocess_module_ids(*endpoint));
+                }
+                Route::AppPage(page_routes) => {
+                    for page_route in page_routes {
+                        preprocessed_module_ids
+                            .push(preprocess_module_ids(page_route.html_endpoint));
+                        preprocessed_module_ids
+                            .push(preprocess_module_ids(page_route.rsc_endpoint));
+                    }
+                }
+                Route::AppRoute {
+                    original_name: _,
+                    endpoint,
+                } => {
+                    preprocessed_module_ids.push(preprocess_module_ids(*endpoint));
+                }
+                Route::Conflict => {
+                    tracing::info!("WARN: conflict");
+                }
+            }
+        }
+
+        let module_id_map = merge_preprocessed_module_ids(preprocessed_module_ids).await?;
+
+        Ok(Vc::upcast(
+            GlobalModuleIdStrategy::new(module_id_map).await?,
+        ))
+    }
+}
+
+// NOTE(LichuAcu) We can't move this function to `turbopack-core` because we need access to
+// `Endpoint`, which is not available there.
+#[turbo_tasks::function]
+async fn preprocess_module_ids(
+    endpoint: Vc<Box<dyn Endpoint>>,
+) -> Result<Vc<PreprocessedChildrenIdents>> {
+    let root_modules = endpoint.root_modules();
+    Ok(children_modules_idents(root_modules))
+}

crates/next-api/src/instrumentation.rs

@@ -15,7 +15,7 @@ use turbopack_core::{
         EntryChunkGroupResult,
     },
     context::AssetContext,
-    module::Module,
+    module::{Module, Modules},
     output::{OutputAsset, OutputAssets},
     reference_type::{EntryReferenceSubType, ReferenceType},
     source::Source,
@@ -236,4 +236,9 @@ impl Endpoint for InstrumentationEndpoint {
     fn client_changed(self: Vc<Self>) -> Vc<Completion> {
         Completion::immutable()
     }
+
+    #[turbo_tasks::function]
+    fn root_modules(self: Vc<Self>) -> Result<Vc<Modules>> {
+        Err(anyhow::anyhow!("Not implemented."))
+    }
 }

crates/next-api/src/lib.rs

@@ -6,6 +6,7 @@ mod app;
 mod dynamic_imports;
 pub mod entrypoints;
 mod font;
+pub mod global_module_id_strategy;
 mod instrumentation;
 mod loadable_manifest;
 mod middleware;

crates/next-api/src/middleware.rs

@@ -14,7 +14,7 @@ use turbopack_core::{
     asset::AssetContent,
     chunk::{availability_info::AvailabilityInfo, ChunkingContextExt},
     context::AssetContext,
-    module::Module,
+    module::{Module, Modules},
     output::OutputAssets,
     reference_type::{EntryReferenceSubType, ReferenceType},
     source::Source,
@@ -237,4 +237,9 @@ impl Endpoint for MiddlewareEndpoint {
     fn client_changed(self: Vc<Self>) -> Vc<Completion> {
         Completion::immutable()
     }
+
+    #[turbo_tasks::function]
+    fn root_modules(self: Vc<Self>) -> Result<Vc<Modules>> {
+        Err(anyhow::anyhow!("Not implemented."))
+    }
 }