Add compiler support for namespaced crates

compiler/rustc_resolve/src/build_reduced_graph.rs

@@ -23,8 +23,8 @@ use rustc_middle::metadata::ModChild;
 use rustc_middle::ty::Feed;
 use rustc_middle::{bug, ty};
 use rustc_span::hygiene::{ExpnId, LocalExpnId, MacroKind};
-use rustc_span::{Ident, Span, Symbol, kw, sym};
-use tracing::debug;
+use rustc_span::{DUMMY_SP, Ident, Span, Symbol, kw, sym};
+use tracing::{debug, instrument};
 
 use crate::Namespace::{MacroNS, TypeNS, ValueNS};
 use crate::def_collector::collect_definitions;
@@ -70,6 +70,7 @@ impl<'ra, Id: Into<DefId>> ToNameBinding<'ra> for (Res, ty::Visibility<Id>, Span
 impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
     /// Defines `name` in namespace `ns` of module `parent` to be `def` if it is not yet defined;
     /// otherwise, reports an error.
+    #[instrument(level = "debug", skip(self, def))]
     pub(crate) fn define<T>(&mut self, parent: Module<'ra>, ident: Ident, ns: Namespace, def: T)
     where
         T: ToNameBinding<'ra>,
@@ -118,6 +119,13 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
             return module.copied();
         }
 
+        //if def_id.is_crate_root() && !def_id.is_local() {
+        //    bug!(
+        //        "expected module for external crate {:?} to be created via `create_module_for_external_crate`",
+        //        def_id
+        //    );
+        //}
+
         if !def_id.is_local() {
             // Query `def_kind` is not used because query system overhead is too expensive here.
             let def_kind = self.cstore().def_kind_untracked(def_id);
@@ -143,6 +151,52 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
         None
     }
 
+    /// Creates `Module` instance for an external crate.
+    /// `Module`s are usually created via `get_module`, but namespaced crates depend
+    /// on the crate name passed via `--extern`, whereas `get_module` uses the metadata's
+    /// crate name. We need a way to pass that name to a `Module` and this method does that.
+    #[instrument(level = "debug", skip(self))]
+    pub(crate) fn create_module_for_external_crate(
+        &mut self,
+        def_id: DefId,
+        ident: Ident,
+        finalize: bool,
+    ) -> Module<'ra> {
+        if let Some(module) = self.module_map.get(&def_id) {
+            return *module;
+        }
+
+        if finalize {
+            self.crate_loader(|c| c.process_path_extern(ident.name, ident.span));
+        }
+
+        let def_kind = self.cstore().def_kind_untracked(def_id);
+        match def_kind {
+            DefKind::Mod => {
+                let parent = self
+                    .tcx
+                    .opt_parent(def_id)
+                    .map(|parent_id| self.get_nearest_non_block_module(parent_id));
+                // Query `expn_that_defined` is not used because
+                // hashing spans in its result is expensive.
+                let expn_id = self.cstore().expn_that_defined_untracked(def_id, self.tcx.sess);
+                let module = self.new_module(
+                    parent,
+                    ModuleKind::Def(def_kind, def_id, Some(ident.name)),
+                    expn_id,
+                    self.def_span(def_id),
+                    // FIXME: Account for `#[no_implicit_prelude]` attributes.
+                    parent.is_some_and(|module| module.no_implicit_prelude),
+                );
+
+                return module;
+            }
+            _ => {
+                bug!("expected DefKind::Mod for external crate");
+            }
+        }
+    }
+
     pub(crate) fn expn_def_scope(&mut self, expn_id: ExpnId) -> Module<'ra> {
         match expn_id.expn_data().macro_def_id {
             Some(def_id) => self.macro_def_scope(def_id),
@@ -196,7 +250,33 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
         visitor.parent_scope.macro_rules
     }
 
+    // Builds the reduced graph for an external crate.
+    // It builds the graph for each child module of the crate and it also checks whether
+    // namespaced crates with the same base name exist. If any exist it builds a graph for
+    // each of those crates.
+    #[instrument(level = "debug", skip(self))]
     pub(crate) fn build_reduced_graph_external(&mut self, module: Module<'ra>) {
+        // Check if `module`'s name is the base crate name for a namespaced crate
+        // (i.e. `my_api` for the namespaced crates `my_api::utils` and `my_api::core`),
+        // and process the extern crate for the namespaced crates if they exist.
+        if let Some(module_name) = module.kind.name() {
+            let namespace_crate_name = self
+                .namespaced_crate_names
+                .get(module_name.as_str())
+                .map(|names| names.iter().map(|s| s.to_string().clone()).collect::<Vec<String>>());
+
+            if let Some(namespaced_crate_names) = namespace_crate_name {
+                debug!(?namespaced_crate_names);
+                for namespaced_crate_name in namespaced_crate_names {
+                    let parent_scope = ParentScope::module(module, self);
+                    self.build_reduced_graph_for_namespaced_crate(
+                        &namespaced_crate_name,
+                        parent_scope,
+                    );
+                }
+            }
+        }
+
         for child in self.tcx.module_children(module.def_id()) {
             let parent_scope = ParentScope::module(module, self);
             self.build_reduced_graph_for_external_crate_res(child, parent_scope)
@@ -273,6 +353,43 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
             | Res::Err => bug!("unexpected resolution: {:?}", res),
         }
     }
+
+    // Builds the reduced graph for a namespaced external crate (crate names like `foo::bar`).
+    #[instrument(level = "debug", skip(self))]
+    fn build_reduced_graph_for_namespaced_crate(
+        &mut self,
+        name: &str,
+        parent_scope: ParentScope<'ra>,
+    ) {
+        let crate_id = self.crate_loader(|c| {
+            c.maybe_process_path_extern(Symbol::intern(&name))
+                .expect(&format!("no crate_num for namespaced crate {}", name))
+        });
+
+        let module = self.create_module_for_external_crate(
+            crate_id.as_def_id(),
+            Ident::from_str(name),
+            false,
+        );
+        let ident = Ident::from_str(
+            name.split("::").nth(1).expect("namespaced crate name has unexpected form"),
+        );
+        let parent = parent_scope.module;
+        let res = module.res().expect("namespaced crate has no Res").expect_non_local();
+        let expansion = parent_scope.expansion;
+
+        match res {
+            Res::Def(DefKind::Mod, _) => {
+                self.define(
+                    parent,
+                    ident,
+                    TypeNS,
+                    (module, ty::Visibility::<DefId>::Public, DUMMY_SP, expansion),
+                );
+            }
+            _ => bug!("expected namespaced crate to have Res Def(DefKind::Mod)"),
+        }
+    }
 }
 
 struct BuildReducedGraphVisitor<'a, 'ra, 'tcx> {
@@ -469,6 +586,7 @@ impl<'a, 'ra, 'tcx> BuildReducedGraphVisitor<'a, 'ra, 'tcx> {
         }
     }
 
+    #[instrument(level = "debug", skip(self))]
     fn build_reduced_graph_for_use_tree(
         &mut self,
         // This particular use tree
@@ -482,11 +600,6 @@ impl<'a, 'ra, 'tcx> BuildReducedGraphVisitor<'a, 'ra, 'tcx> {
         vis: ty::Visibility,
         root_span: Span,
     ) {
-        debug!(
-            "build_reduced_graph_for_use_tree(parent_prefix={:?}, use_tree={:?}, nested={})",
-            parent_prefix, use_tree, nested
-        );
-
         // Top level use tree reuses the item's id and list stems reuse their parent
         // use tree's ids, so in both cases their visibilities are already filled.
         if nested && !list_stem {
@@ -589,7 +702,7 @@ impl<'a, 'ra, 'tcx> BuildReducedGraphVisitor<'a, 'ra, 'tcx> {
                         let crate_root = self.r.resolve_crate_root(source.ident);
                         let crate_name = match crate_root.kind {
                             ModuleKind::Def(.., name) => name,
-                            ModuleKind::Block => unreachable!(),
+                            ModuleKind::Block | ModuleKind::NamespaceCrate(..) => unreachable!(),
                         };
                         // HACK(eddyb) unclear how good this is, but keeping `$crate`
                         // in `source` breaks `tests/ui/imports/import-crate-var.rs`,
@@ -897,6 +1010,7 @@ impl<'a, 'ra, 'tcx> BuildReducedGraphVisitor<'a, 'ra, 'tcx> {
         }
     }
 
+    #[instrument(level = "debug", skip(self))]
     fn build_reduced_graph_for_extern_crate(
         &mut self,
         orig_name: Option<Symbol>,
@@ -922,7 +1036,7 @@ impl<'a, 'ra, 'tcx> BuildReducedGraphVisitor<'a, 'ra, 'tcx> {
             });
             crate_id.map(|crate_id| {
                 self.r.extern_crate_map.insert(local_def_id, crate_id);
-                self.r.expect_module(crate_id.as_def_id())
+                self.r.create_module_for_external_crate(crate_id.as_def_id(), ident, false)
             })
         }
         .map(|module| {

compiler/rustc_resolve/src/diagnostics.rs

@@ -222,6 +222,10 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
             // indirectly *calls* the resolver, and would cause a query cycle.
             ModuleKind::Def(kind, _, _) => kind.descr(parent.def_id()),
             ModuleKind::Block => "block",
+            ModuleKind::NamespaceCrate(..) => {
+                // `NamespaceCrate`s are virtual modules
+                unreachable!();
+            }
         };
 
         let (name, span) =

compiler/rustc_resolve/src/ident.rs

@@ -575,7 +575,11 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
                         None => Err(Determinacy::Determined),
                     },
                     Scope::ExternPrelude => {
-                        match this.extern_prelude_get(ident, finalize.is_some()) {
+                        match this.extern_prelude_get(
+                            ident,
+                            finalize.is_some(),
+                            Some(*parent_scope),
+                        ) {
                             Some(binding) => Ok((binding, Flags::empty())),
                             None => Err(Determinacy::determined(
                                 this.graph_root.unexpanded_invocations.borrow().is_empty(),
@@ -796,8 +800,6 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
         )
     }
 
-    /// Attempts to resolve `ident` in namespaces `ns` of `module`.
-    /// Invariant: if `finalize` is `Some`, expansion and import resolution must be complete.
     #[instrument(level = "debug", skip(self))]
     fn resolve_ident_in_module_unadjusted(
         &mut self,
@@ -831,7 +833,9 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
                 assert_eq!(shadowing, Shadowing::Unrestricted);
                 return if ns != TypeNS {
                     Err((Determined, Weak::No))
-                } else if let Some(binding) = self.extern_prelude_get(ident, finalize.is_some()) {
+                } else if let Some(binding) =
+                    self.extern_prelude_get(ident, finalize.is_some(), Some(*parent_scope))
+                {
                     Ok(binding)
                 } else if !self.graph_root.unexpanded_invocations.borrow().is_empty() {
                     // Macro-expanded `extern crate` items can add names to extern prelude.
@@ -866,9 +870,79 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
             }
         };
 
+        let result = self.try_resolve_ident_in_module_unadjusted(
+            module,
+            ident,
+            ns,
+            parent_scope,
+            shadowing,
+            finalize,
+            ignore_binding,
+            ignore_import,
+        );
+
+        // If resolution failed for the module, check if we can resolve ident to a
+        // namespaced crate. Suppose we have the crate `my_api` and the namespaced
+        // crate `my_api::utils` as dependencies and `my_api` doesn't have a module
+        // named `utils`. The `Ident` `utils` will be unresolvable for the `Module`
+        // corresponding to `my_api`. If there's a crate named `my_api::utils` in the
+        // extern prelude we resolve `my_api::utils` to that crate then. If `my_api`
+        // does have a module named `utils` and a namespaced crate `my_api::utils`
+        // exists, then we report a name conflict error (via `build_reduced_graph_external`).
+        if let Err(err) = result {
+            if let Some(module_name) = module.kind.name() {
+                let Some(namespaced_crate_names) =
+                    self.namespaced_crate_names.get(module_name.as_str())
+                else {
+                    return Err(err);
+                };
+
+                debug!(?namespaced_crate_names);
+                for namespaced_crate_name in namespaced_crate_names {
+                    if namespaced_crate_name
+                        .split("::")
+                        .nth(1)
+                        .expect("namespaced crate name has wrong form")
+                        == ident.as_str()
+                    {
+                        return self.resolve_namespaced_crate(
+                            namespaced_crate_name,
+                            finalize,
+                            module.parent.map(|m| ParentScope::module(m, self)),
+                        );
+                    }
+                }
+
+                return Err(err);
+            } else {
+                return Err(err);
+            }
+        }
+
+        result
+    }
+
+    /// Attempts to resolve `ident` in namespaces `ns` of `module`.
+    /// Invariant: if `finalize` is `Some`, expansion and import resolution must be complete.
+    #[instrument(level = "debug", skip(self))]
+    fn try_resolve_ident_in_module_unadjusted(
+        &mut self,
+        module: Module<'ra>,
+        ident: Ident,
+        ns: Namespace,
+        parent_scope: &ParentScope<'ra>,
+        shadowing: Shadowing,
+        finalize: Option<Finalize>,
+        // This binding should be ignored during in-module resolution, so that we don't get
+        // "self-confirming" import resolutions during import validation and checking.
+        ignore_binding: Option<NameBinding<'ra>>,
+        ignore_import: Option<Import<'ra>>,
+    ) -> Result<NameBinding<'ra>, (Determinacy, Weak)> {
         let key = BindingKey::new(ident, ns);
-        let resolution =
-            self.resolution(module, key).try_borrow_mut().map_err(|_| (Determined, Weak::No))?; // This happens when there is a cycle of imports.
+        let resolution = self.resolution(module, key).try_borrow_mut().map_err(|_| {
+            debug!("got resolution error for module {:?}", module);
+            (Determined, Weak::No)
+        })?; // This happens when there is a cycle of imports.
 
         // If the primary binding is unusable, search further and return the shadowed glob
         // binding if it exists. What we really want here is having two separate scopes in
@@ -1034,6 +1108,7 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
             }
         }
 
+        debug!("no resolution");
         // --- From now on we have no resolution. ---
 
         // Now we are in situation when new item/import can appear only from a glob or a macro
@@ -1100,6 +1175,34 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
         Err((Determined, Weak::No))
     }
 
+    // Resolves a namespaced crate.
+    // In general namespaced crates, i.e. those with names of the form `foo::bar`,
+    //  are resolved in the following way. When intializing the `Resolver`, we
+    // collect information on them in the field `namespaced_crate_names`,
+    // which maps the base crate name to a list of namespaced crates with that
+    // base. Suppose we have a base crate named `my_api` and two namespaced
+    // crates named `my_api::utils` and `my_api::core`. When building the reduced
+    // graph for the base crate `my_api` (in `build_reduced_graph_for_external`),
+    // we check whether any namespaced crates exist for it. If any do, we use
+    // `build_reduced_graph_for_namespaced_crate` to build a graph for them.
+    // In the case in which the base crate isn't a dependency and we try to resolve
+    // `my_api` in the path `my_api::utils`, then we resolve the base name to a virtual
+    // module `ModuleKind::NamespaceCrate`. `resolve_ident_in_module`
+    #[instrument(skip(self, finalize))]
+    fn resolve_namespaced_crate(
+        &mut self,
+        namespaced_crate_name: &str,
+        finalize: Option<Finalize>,
+        parent_scope: Option<ParentScope<'ra>>,
+    ) -> Result<NameBinding<'ra>, (Determinacy, Weak)> {
+        self.extern_prelude_get(
+            Ident::from_str(namespaced_crate_name),
+            finalize.is_some(),
+            parent_scope,
+        )
+        .ok_or((Determinacy::Determined, Weak::No))
+    }
+
     /// Validate a local resolution (from ribs).
     #[instrument(level = "debug", skip(self, all_ribs))]
     fn validate_res_from_ribs(
@@ -1550,6 +1653,7 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
                 );
             }
 
+            debug!(?module);
             let binding = if let Some(module) = module {
                 self.resolve_ident_in_module(
                     module,
@@ -1597,6 +1701,7 @@ impl<'ra, 'tcx> Resolver<'ra, 'tcx> {
                 )
             };
 
+            debug!("resolve_path_with_ribs binding: {:?}", binding);
             match binding {
                 Ok(binding) => {
                     if segment_idx == 1 {