feat: Enable tree shaking in next.js (#8755)

.gitattributes

@@ -1,4 +1,5 @@
 .config/ast-grep/rule-tests/__snapshots__/** linguist-generated=true
 crates/turbo-tasks-macros-tests/tests/**/*.stderr linguist-generated=true
+crates/turbopack-ecmascript/tests/tree-shaker/analyzer/**/output.md linguist-generated=true
 crates/turbopack-tests/tests/snapshot/**/output/** linguist-generated=true
 crates/turborepo-lockfiles/fixtures/*.lock text eol=lf

crates/turbopack-ecmascript/benches/analyzer.rs

@@ -56,7 +56,7 @@ pub fn benchmark(c: &mut Criterion) {
  program.visit_mut_with(&mut resolver(unresolved_mark, top_level_mark, false));
 
  let eval_context =
- EvalContext::new(&program, unresolved_mark, top_level_mark, false, None);
+ EvalContext::new(&program, unresolved_mark, top_level_mark, None);
  let var_graph = create_graph(&program, &eval_context);
 
  let input = BenchInput {

crates/turbopack-ecmascript/src/analyzer/graph.rs

@@ -270,13 +270,12 @@ impl EvalContext {
  module: &Program,
  unresolved_mark: Mark,
  top_level_mark: Mark,
- skip_namespace: bool,
  source: Option<Vc<Box<dyn Source>>>,
  ) -> Self {
  Self {
  unresolved_mark,
  top_level_mark,
- imports: ImportMap::analyze(module, skip_namespace, source),
+ imports: ImportMap::analyze(module, source),
  }
  }
 

crates/turbopack-ecmascript/src/analyzer/imports.rs

@@ -139,8 +139,6 @@ pub(crate) struct ImportMap {
 pub(crate) enum ImportedSymbol {
  ModuleEvaluation,
  Symbol(JsWord),
- /// User requested the whole module
- Namespace,
  Exports,
  Part(u32),
 }
@@ -202,16 +200,11 @@ impl ImportMap {
  }
 
  /// Analyze ES import
- pub(super) fn analyze(
- m: &Program,
- skip_namespace: bool,
- source: Option<Vc<Box<dyn Source>>>,
- ) -> Self {
+ pub(super) fn analyze(m: &Program, source: Option<Vc<Box<dyn Source>>>) -> Self {
  let mut data = ImportMap::default();
 
  m.visit_with(&mut Analyzer {
  data: &mut data,
- skip_namespace,
  source,
  });
 
@@ -221,7 +214,6 @@ impl ImportMap {
 
 struct Analyzer<'a> {
  data: &'a mut ImportMap,
- skip_namespace: bool,
  source: Option<Vc<Box<dyn Source>>>,
 }
 
@@ -233,10 +225,6 @@ impl<'a> Analyzer<'a> {
  imported_symbol: ImportedSymbol,
  annotations: ImportAnnotations,
  ) -> Option<usize> {
- if self.skip_namespace && matches!(imported_symbol, ImportedSymbol::Namespace) {
- return None;
- }
-
  let issue_source = self
  .source
  .map(|s| IssueSource::from_swc_offsets(s, span.lo.to_usize(), span.hi.to_usize()));
@@ -339,7 +327,7 @@ impl Visit for Analyzer<'_> {
  let i = self.ensure_reference(
  export.span,
  export.src.value.clone(),
- symbol.unwrap_or(ImportedSymbol::Namespace),
+ symbol.unwrap_or(ImportedSymbol::Exports),
  annotations,
  );
  if let Some(i) = i {
@@ -455,7 +443,7 @@ fn get_import_symbol_from_import(specifier: &ImportSpecifier) -> ImportedSymbol
  _ => local.sym.clone(),
  }),
  ImportSpecifier::Default(..) => ImportedSymbol::Symbol(js_word!("default")),
- ImportSpecifier::Namespace(..) => ImportedSymbol::Namespace,
+ ImportSpecifier::Namespace(..) => ImportedSymbol::Exports,
  }
 }
 
@@ -465,6 +453,6 @@ fn get_import_symbol_from_export(specifier: &ExportSpecifier) -> ImportedSymbol
  ImportedSymbol::Symbol(orig_name(orig))
  }
  ExportSpecifier::Default(..) => ImportedSymbol::Symbol(js_word!("default")),
- ExportSpecifier::Namespace(..) => ImportedSymbol::Namespace,
+ ExportSpecifier::Namespace(..) => ImportedSymbol::Exports,
  }
 }

crates/turbopack-ecmascript/src/analyzer/mod.rs

@@ -3588,8 +3588,7 @@ mod tests {
  let top_level_mark = Mark::new();
  m.visit_mut_with(&mut resolver(unresolved_mark, top_level_mark, false));
 
- let eval_context =
- EvalContext::new(&m, unresolved_mark, top_level_mark, false, None);
+ let eval_context = EvalContext::new(&m, unresolved_mark, top_level_mark, None);
 
  let mut var_graph = create_graph(&m, &eval_context);
 

crates/turbopack-ecmascript/src/lib.rs

@@ -120,6 +120,9 @@ pub enum TreeShakingMode {
  ReexportsOnly,
 }
 
+#[turbo_tasks::value(transparent)]
+pub struct OptionTreeShaking(pub Option<TreeShakingMode>);
+
 #[turbo_tasks::value(shared, serialization = "auto_for_input")]
 #[derive(PartialOrd, Ord, Hash, Debug, Default, Copy, Clone)]
 pub struct EcmascriptOptions {
@@ -139,6 +142,11 @@ pub struct EcmascriptOptions {
  /// If false, they will reference the whole directory. If true, they won't
  /// reference anything and lead to an runtime error instead.
  pub ignore_dynamic_requests: bool,
+
+ /// The list of export names that should make tree shaking bail off. This is
+ /// required because tree shaking can split imports like `export const
+ /// runtime = 'edge'` as a separate module.
+ pub special_exports: Vc<Vec<RcStr>>,
 }
 
 #[turbo_tasks::value(serialization = "auto_for_input")]

crates/turbopack-ecmascript/src/minify.rs

@@ -54,9 +54,12 @@ pub async fn minify(path: Vc<FileSystemPath>, code: Vc<Code>) -> Result<Vc<Code>
  GLOBALS.set(&Default::default(), || {
  let program = match parser.parse_program() {
  Ok(program) => program,
- Err(_error) => {
+ Err(err) => {
  // TODO should emit an issue
- bail!("failed to parse source code")
+ bail!(
+ "failed to parse source code\n{err:?}\n{}",
+ code.source_code().to_str()?
+ )
  }
  };
  let comments = SingleThreadedComments::default();

crates/turbopack-ecmascript/src/parse.rs

@@ -432,7 +432,6 @@ async fn parse_content(
  &parsed_program,
  unresolved_mark,
  top_level_mark,
- false,
  Some(source),
  );
 

crates/turbopack-ecmascript/src/references/esm/base.rs

@@ -96,6 +96,7 @@ pub struct EsmAssetReference {
  pub issue_source: Option<Vc<IssueSource>>,
  pub export_name: Option<Vc<ModulePart>>,
  pub import_externals: bool,
+ pub special_exports: Vc<Vec<RcStr>>,
 }
 
 /// A list of [EsmAssetReference]s
@@ -121,6 +122,7 @@ impl EsmAssetReference {
  issue_source: Option<Vc<IssueSource>>,
  annotations: Value<ImportAnnotations>,
  export_name: Option<Vc<ModulePart>>,
+ special_exports: Vc<Vec<RcStr>>,
  import_externals: bool,
  ) -> Vc<Self> {
  Self::cell(EsmAssetReference {
@@ -130,6 +132,7 @@ impl EsmAssetReference {
  annotations: annotations.into_value(),
  export_name,
  import_externals,
+ special_exports,
  })
  }
 

crates/turbopack-ecmascript/src/references/mod.rs

@@ -353,6 +353,7 @@ struct AnalysisState<'a> {
  // the object allocation.
  first_import_meta: bool,
  tree_shaking_mode: Option<TreeShakingMode>,
+ special_exports: Vc<Vec<RcStr>>,
  import_externals: bool,
  ignore_dynamic_requests: bool,
 }
@@ -412,6 +413,7 @@ pub(crate) async fn analyse_ecmascript_module_internal(
  let options = raw_module.options;
  let compile_time_info = raw_module.compile_time_info;
  let options = options.await?;
+ let special_exports = options.special_exports;
  let import_externals = options.import_externals;
 
  let origin = Vc::upcast::<Box<dyn ResolveOrigin>>(module);
@@ -428,7 +430,7 @@ pub(crate) async fn analyse_ecmascript_module_internal(
 
  let parsed = if let Some(part) = part {
  let parsed = parse(source, ty, transforms);
- let split_data = split(source.ident(), source, parsed);
+ let split_data = split(source.ident(), source, parsed, special_exports);
  part_of_module(split_data, part)
  } else {
  parse(source, ty, transforms)
@@ -576,7 +578,6 @@ pub(crate) async fn analyse_ecmascript_module_internal(
  ImportedSymbol::Symbol(name) => Some(ModulePart::export((&**name).into())),
  ImportedSymbol::Part(part_id) => Some(ModulePart::internal(*part_id)),
  ImportedSymbol::Exports => Some(ModulePart::exports()),
- ImportedSymbol::Namespace => None,
  },
  Some(TreeShakingMode::ReexportsOnly) => match &r.imported_symbol {
  ImportedSymbol::ModuleEvaluation => {
@@ -586,10 +587,10 @@ pub(crate) async fn analyse_ecmascript_module_internal(
  ImportedSymbol::Symbol(name) => Some(ModulePart::export((&**name).into())),
  ImportedSymbol::Part(part_id) => Some(ModulePart::internal(*part_id)),
  ImportedSymbol::Exports => None,
- ImportedSymbol::Namespace => None,
  },
  None => None,
  },
+ special_exports,
  import_externals,
  );
 
@@ -814,6 +815,7 @@ pub(crate) async fn analyse_ecmascript_module_internal(
  first_import_meta: true,
  tree_shaking_mode: options.tree_shaking_mode,
  import_externals: options.import_externals,
+ special_exports: options.special_exports,
  ignore_dynamic_requests: options.ignore_dynamic_requests,
  };
 
@@ -1972,6 +1974,7 @@ async fn handle_free_var_reference(
  .map(|export| ModulePart::export(export.clone())),
  None => None,
  },
+ state.special_exports,
  state.import_externals,
  )
  .resolve()

crates/turbopack-ecmascript/src/tree_shake/asset.rs

@@ -23,7 +23,7 @@ use crate::{
 /// This type is used for an advanced tree shkaing.
 #[turbo_tasks::value]
 pub struct EcmascriptModulePartAsset {
- pub(crate) full_module: Vc<EcmascriptModuleAsset>,
+ pub full_module: Vc<EcmascriptModuleAsset>,
  pub(crate) part: Vc<ModulePart>,
  pub(crate) import_externals: bool,
 }