Module ID global optimization

crates/turbopack-browser/src/chunking_context.rs

@@ -514,4 +514,9 @@ impl ChunkingContext for BrowserChunkingContext {
             self.chunk_item_id_from_ident(AsyncLoaderModule::asset_ident_for(module))
         })
     }
+
+    #[turbo_tasks::function]
+    async fn global_information(self: Vc<Self>) -> Result<Vc<OptionGlobalInformation>> {
+        Ok(self.await?.global_information)
+    }
 }

crates/turbopack-core/src/changed.rs

@@ -17,7 +17,7 @@ async fn get_referenced_output_assets(
     Ok(parent.references().await?.clone_value().into_iter())
 }
 
-async fn get_referenced_modules(
+pub async fn get_referenced_modules(
     parent: Vc<Box<dyn Module>>,
 ) -> Result<impl Iterator<Item = Vc<Box<dyn Module>>> + Send> {
     Ok(primary_referenced_modules(parent)

crates/turbopack-core/src/chunk/chunking_context.rs

@@ -4,7 +4,10 @@
 use turbo_tasks_fs::FileSystemPath;
 use turbo_tasks_hash::DeterministicHash;
 
-use super::{availability_info::AvailabilityInfo, ChunkableModule, EvaluatableAssets};
+use super::{
+    availability_info::AvailabilityInfo, global_information::OptionGlobalInformation,
+    ChunkableModule, EvaluatableAssets,
+};
 use crate::{
     chunk::{ChunkItem, ModuleId},
     environment::Environment,
@@ -114,10 +117,15 @@
         availability_info: Value<AvailabilityInfo>,
     ) -> Result<Vc<EntryChunkGroupResult>>;
 
+    fn global_information(self: Vc<Self>) -> Vc<OptionGlobalInformation>;
+
     async fn chunk_item_id_from_ident(
         self: Vc<Self>,
         ident: Vc<AssetIdent>,
     ) -> Result<Vc<ModuleId>> {
+        if let Some(global_information) = &*self.global_information().await? {
+            return Ok(global_information.get_module_id(ident).await?);
+        }
         Ok(ModuleId::String(ident.to_string().await?.clone_value()).cell())
     }
 

crates/turbopack-core/src/chunk/global_information.rs

@@ -1,6 +1,67 @@
+use std::collections::{HashMap, HashSet};
+
+use anyhow::Result;
+use turbo_tasks::{ValueToString, Vc};
+use turbo_tasks_hash::hash_xxh3_hash64;
+
+use super::ModuleId;
+use crate::{ident::AssetIdent, module::Module};
+
 #[turbo_tasks::value]
 #[derive(Clone, Debug)]
-pub struct GlobalInformation {}
+pub struct GlobalInformation {
+    pub module_id_map: HashMap<AssetIdent, ModuleId>,
+}
+
+impl GlobalInformation {
+    pub async fn get_module_id(&self, asset_ident: Vc<AssetIdent>) -> Result<Vc<ModuleId>> {
+        let ident_str = asset_ident.to_string().await?;
+        let ident = asset_ident.await?;
+        let hashed_module_id = self.module_id_map.get(&ident);
+        if let Some(hashed_module_id) = hashed_module_id {
+            dbg!("Hashed module ID found", &ident_str, hashed_module_id);
+            return Ok(hashed_module_id.clone().cell());
+        }
+        dbg!("Hashed module ID not found", &ident_str);
+        return Ok(ModuleId::String(ident_str.clone_value()).cell());
+    }
+}
 
 #[turbo_tasks::value(transparent)]
 pub struct OptionGlobalInformation(Option<GlobalInformation>);
+
+// NOTE(LichuAcu): This is a temporary location for this function,
+// it could later be moved to a `module_id_optimization.rs` file with
+// other module ID optimization logic.
+pub async fn process_module(
+    module: Vc<Box<dyn Module>>,
+    id_map: &mut HashMap<AssetIdent, ModuleId>,
+    used_ids: &mut HashSet<u64>,
+) -> Result<()> {
+    let ident = module.ident();
+
+    let hash = hash_xxh3_hash64(ident.to_string().await?);
+    let mut masked_hash = hash & 0xF;
+    let mut mask = 0xF;
+    while used_ids.contains(&masked_hash) {
+        if mask == 0xFFFFFFFFFFFFFFFF {
+            return Err(anyhow::anyhow!("This is a... 64-bit hash collision?"));
+        }
+        mask = (mask << 4) | 0xF;
+        masked_hash = hash & mask;
+    }
+
+    let hashed_module_id = ModuleId::String(masked_hash.to_string().into());
+
+    dbg!(
+        "process_module",
+        ident.await?.clone_value(),
+        &hashed_module_id,
+        mask
+    );
+
+    id_map.insert(ident.await?.clone_value(), hashed_module_id);
+    used_ids.insert(masked_hash);
+
+    Ok(())
+}

crates/turbopack-nodejs/src/chunking_context.rs

@@ -136,14 +136,6 @@ impl NodeJsChunkingContext {
 
 #[turbo_tasks::value_impl]
 impl NodeJsChunkingContext {
-    #[turbo_tasks::function]
-    async fn chunk_item_id_from_ident(
-        self: Vc<Self>,
-        ident: Vc<AssetIdent>,
-    ) -> Result<Vc<ModuleId>> {
-        Ok(ModuleId::String(ident.to_string().await?.clone_value()).cell())
-    }
-
     #[turbo_tasks::function]
     fn new(this: Value<NodeJsChunkingContext>) -> Vc<Self> {
         this.into_value().cell()
@@ -393,4 +385,9 @@ impl ChunkingContext for NodeJsChunkingContext {
             self.chunk_item_id_from_ident(AsyncLoaderModule::asset_ident_for(module))
         })
     }
+
+    #[turbo_tasks::function]
+    async fn global_information(self: Vc<Self>) -> Result<Vc<OptionGlobalInformation>> {
+        Ok(self.await?.global_information)
+    }
 }