feat(compartment-mapper)!: new hooks & type overhaul

packages/compartment-mapper/NEWS.md

@@ -2,6 +2,32 @@ User-visible changes to `@endo/compartment-mapper`:
 
 # Next release
 
+- **Breaking:** `CompartmentMapDescriptor` no longer has a `path` property.
+- **Breaking:** `CompartmentMapDescriptor`'s `label` property is now a
+  _canonical name_ (a string of one or more npm package names separated by `>`).
+- **Breaking:** The `CompartmentMapDescriptor` returned by `captureFromMap()`
+  now uses canonical names as the keys in its `compartments` property.
+- Breaking types: `CompartmentMapDescriptor`, `CompartmentDescriptor`,
+  `ModuleConfiguration` (renamed from `ModuleDescriptor`) and `ModuleSource`
+  have all been narrowed into discrete subtypes.
+- `captureFromMap()`, `loadLocation()` and `importLocation()` now accept a
+  `moduleSourceHook` option. This hook is called when processing each module
+  source, receiving the module source data (location, language, bytes, or error
+  information) and the canonical name of the containing package.
+- `captureFromMap()` now accepts a `packageConnectionsHook` option. This hook is
+  called for each retained compartment with its canonical name and the set of
+  canonical names of compartments it links to (its connections). Useful for
+  analyzing or visualizing the dependency graph.
+- `mapNodeModules()`, `loadLocation()`, `importLocation()`, `makeScript()`,
+  `makeFunctor()`, and `writeScript()` now accept the following hook options:
+  - `unknownCanonicalNameHook`: Called for each canonical name mentioned in
+    policy but not found in the compartment map. Useful for detecting policy
+    misconfigurations.
+  - `packageDependenciesHook`: Called for each package with its set of
+    dependencies. Can return partial updates to modify the dependencies,
+    enabling dependency filtering or injection based on policy.
+  - `packageDataHook`: Called once with data about all packages found while
+    crawling `node_modules`, just prior to creation of a compartment map.
 - When dynamic requires are enabled via configuration, execution now takes
   policy into consideration when no other relationship (for example, a
   dependent/dependee relationship) between two Compartments exists. When policy

packages/compartment-mapper/package.json

@@ -68,6 +68,7 @@
     "ses": "workspace:^"
   },
   "devDependencies": {
+    "@endo/env-options": "workspace:^",
     "ava": "catalog:dev",
     "c8": "catalog:dev",
     "eslint": "catalog:dev",

packages/compartment-mapper/src/archive-lite.js

@@ -1,4 +1,5 @@
-/* Provides functions to create an archive (zip file with a
+/**
+ * Provides functions to create an archive (zip file with a
  * compartment-map.json) from a partially completed compartment map (it must
  * mention all packages/compartments as well as inter-compartment references
  * but does not contain an entry for every module reachable from its entry
@@ -26,6 +27,8 @@
  * In fruition of https://github.com/endojs/endo/issues/400, we will be able to
  * use original source archives on XS and Node.js, but not on the web until the
  * web platform makes further progress on virtual module loaers.
+ *
+ * @module
  */
 
 /* eslint no-shadow: 0 */
@@ -36,8 +39,10 @@
  *   ArchiveResult,
  *   ArchiveWriter,
  *   CaptureSourceLocationHook,
- *   CompartmentMapDescriptor,
+ *   CompartmentsRenameFn,
+ *   FileUrlString,
  *   HashPowers,
+ *   PackageCompartmentMapDescriptor,
  *   ReadFn,
  *   ReadPowers,
  *   Sources,
@@ -55,19 +60,12 @@ import {
 import { unpackReadPowers } from './powers.js';
 import { detectAttenuators } from './policy.js';
 import { digestCompartmentMap } from './digest.js';
+import { stringCompare } from './compartment-map.js';
+import { ATTENUATORS_COMPARTMENT } from './policy-format.js';
 
 const textEncoder = new TextEncoder();
 
-const { assign, create, freeze } = Object;
-
-/**
- * @param {string} rel - a relative URL
- * @param {string} abs - a fully qualified URL
- * @returns {string}
- */
-const resolveLocation = (rel, abs) => new URL(rel, abs).toString();
-
-const { keys } = Object;
+const { assign, create, freeze, keys } = Object;
 
 /**
  * @param {ArchiveWriter} archive
@@ -77,12 +75,10 @@ const addSourcesToArchive = async (archive, sources) => {
   await null;
   for (const compartment of keys(sources).sort()) {
     const modules = sources[compartment];
-    const compartmentLocation = resolveLocation(`${compartment}/`, 'file:///');
     for (const specifier of keys(modules).sort()) {
-      const { bytes, location } = modules[specifier];
-      if (location !== undefined) {
-        const moduleLocation = resolveLocation(location, compartmentLocation);
-        const path = new URL(moduleLocation).pathname.slice(1); // elide initial "/"
+      if ('location' in modules[specifier]) {
+        const { bytes, location } = modules[specifier];
+        const path = `${compartment}/${location}`;
         if (bytes !== undefined) {
           // eslint-disable-next-line no-await-in-loop
           await archive.write(path, bytes);
@@ -100,27 +96,78 @@ const captureSourceLocations = async (sources, captureSourceLocation) => {
   for (const compartmentName of keys(sources).sort()) {
     const modules = sources[compartmentName];
     for (const moduleSpecifier of keys(modules).sort()) {
-      const { sourceLocation } = modules[moduleSpecifier];
-      if (sourceLocation !== undefined) {
+      if ('sourceLocation' in modules[moduleSpecifier]) {
+        const { sourceLocation } = modules[moduleSpecifier];
         captureSourceLocation(compartmentName, moduleSpecifier, sourceLocation);
       }
     }
   }
 };
 
 /**
- * @param {CompartmentMapDescriptor} compartmentMap
+ * @param {PackageCompartmentMapDescriptor} compartmentMap
  * @param {Sources} sources
  * @returns {ArchiveResult}
  */
 export const makeArchiveCompartmentMap = (compartmentMap, sources) => {
+  /** @type {CompartmentsRenameFn<FileUrlString, string>} */
+  const renameCompartments = compartments => {
+    /** @type {Record<FileUrlString, string>} */
+    const compartmentRenames = create(null);
+
+    /**
+     * Get the new name of format `packageName-v${version}` compartments (except
+     * for the attenuators compartment)
+     * @param {string} name
+     * @param {string} version
+     * @returns {string}
+     */
+    const getCompartmentName = (name, version) => {
+      const compartment = compartments[name];
+      return ATTENUATORS_COMPARTMENT === compartment.name
+        ? compartment.name
+        : `${compartment.name}-v${version}`;
+    };
+
+    // The sort below combines two comparators to avoid depending on sort
+    // stability, which became standard as recently as 2019.
+    // If that date seems quaint, please accept my regards from the distant past.
+    // We are very proud of you.
+    const compartmentsByLabel =
+      /** @type {Array<{name: FileUrlString, packageName: string, compartmentName: string}>} */ (
+        Object.entries(compartments)
+          .map(([name, compartment]) => ({
+            name,
+            packageName: compartments[name].name,
+            compartmentName: getCompartmentName(name, compartment.version),
+          }))
+          .sort((a, b) => stringCompare(a.compartmentName, b.compartmentName))
+      );
+
+    /** @type {string|undefined} */
+    let prev;
+    let index = 1;
+    for (const { name, packageName, compartmentName } of compartmentsByLabel) {
+      if (packageName === prev) {
+        compartmentRenames[name] = `${compartmentName}-n${index}`; // Added numeric suffix for duplicates
+        index += 1;
+      } else {
+        compartmentRenames[name] = compartmentName;
+        prev = packageName;
+        index = 1;
+      }
+    }
+    return compartmentRenames;
+  };
+
   const {
     compartmentMap: archiveCompartmentMap,
     sources: archiveSources,
     oldToNewCompartmentNames,
     newToOldCompartmentNames,
     compartmentRenames,
-  } = digestCompartmentMap(compartmentMap, sources);
+  } = digestCompartmentMap(compartmentMap, sources, { renameCompartments });
+
   return {
     archiveCompartmentMap,
     archiveSources,
@@ -130,9 +177,11 @@ export const makeArchiveCompartmentMap = (compartmentMap, sources) => {
   };
 };
 
+const noop = () => {};
+
 /**
  * @param {ReadFn | ReadPowers} powers
- * @param {CompartmentMapDescriptor} compartmentMap
+ * @param {PackageCompartmentMapDescriptor} compartmentMap
  * @param {ArchiveLiteOptions} [options]
  * @returns {Promise<{sources: Sources, compartmentMapBytes: Uint8Array, sha512?: string}>}
  */
@@ -146,6 +195,7 @@ const digestFromMap = async (powers, compartmentMap, options = {}) => {
     policy = undefined,
     sourceMapHook = undefined,
     parserForLanguage: parserForLanguageOption = {},
+    log: _log = noop,
   } = options;
 
   const parserForLanguage = freeze(
@@ -179,6 +229,7 @@ const digestFromMap = async (powers, compartmentMap, options = {}) => {
     importHook: consolidatedExitModuleImportHook,
     sourceMapHook,
   });
+
   // Induce importHook to record all the necessary modules to import the given module specifier.
   const { compartment, attenuatorsCompartment } = link(compartmentMap, {
     resolve,
@@ -229,7 +280,7 @@ const digestFromMap = async (powers, compartmentMap, options = {}) => {
 
 /**
  * @param {ReadFn | ReadPowers} powers
- * @param {CompartmentMapDescriptor} compartmentMap
+ * @param {PackageCompartmentMapDescriptor} compartmentMap
  * @param {ArchiveLiteOptions} [options]
  * @returns {Promise<{bytes: Uint8Array, sha512?: string}>}
  */
@@ -254,7 +305,7 @@ export const makeAndHashArchiveFromMap = async (
 
 /**
  * @param {ReadFn | ReadPowers} powers
- * @param {CompartmentMapDescriptor} compartmentMap
+ * @param {PackageCompartmentMapDescriptor} compartmentMap
  * @param {ArchiveLiteOptions} [options]
  * @returns {Promise<Uint8Array>}
  */
@@ -269,7 +320,7 @@ export const makeArchiveFromMap = async (powers, compartmentMap, options) => {
 
 /**
  * @param {ReadFn | ReadPowers} powers
- * @param {CompartmentMapDescriptor} compartmentMap
+ * @param {PackageCompartmentMapDescriptor} compartmentMap
  * @param {ArchiveLiteOptions} [options]
  * @returns {Promise<Uint8Array>}
  */
@@ -284,7 +335,7 @@ export const mapFromMap = async (powers, compartmentMap, options) => {
 
 /**
  * @param {HashPowers} powers
- * @param {CompartmentMapDescriptor} compartmentMap
+ * @param {PackageCompartmentMapDescriptor} compartmentMap
  * @param {ArchiveLiteOptions} [options]
  * @returns {Promise<string>}
  */
@@ -302,7 +353,7 @@ export const hashFromMap = async (powers, compartmentMap, options) => {
  * @param {WriteFn} write
  * @param {ReadFn | ReadPowers} readPowers
  * @param {string} archiveLocation
- * @param {CompartmentMapDescriptor} compartmentMap
+ * @param {PackageCompartmentMapDescriptor} compartmentMap
  * @param {ArchiveLiteOptions} [options]
  */
 export const writeArchiveFromMap = async (

packages/compartment-mapper/src/archive.js

@@ -24,6 +24,7 @@
  *   ReadPowers,
  *   HashPowers,
  *   WriteFn,
+ *   LogFn,
  * } from './types.js'
  */
 
@@ -169,6 +170,9 @@ export const mapLocation = async (powers, moduleLocation, options = {}) => {
   });
 };
 
+/** @type {LogFn} */
+const noop = () => {};
+
 /**
  * @param {HashPowers} powers
  * @param {string} moduleLocation
@@ -191,10 +195,12 @@ export const hashLocation = async (powers, moduleLocation, options = {}) => {
     workspaceLanguageForExtension,
     workspaceCommonjsLanguageForExtension,
     workspaceModuleLanguageForExtension,
+    log = noop,
     ...otherOptions
   } = assignParserForLanguage(options);
 
   const compartmentMap = await mapNodeModules(powers, moduleLocation, {
+    log,
     dev,
     strict,
     conditions,
@@ -212,6 +218,7 @@ export const hashLocation = async (powers, moduleLocation, options = {}) => {
   return hashFromMap(powers, compartmentMap, {
     parserForLanguage,
     policy,
+    log,
     ...otherOptions,
   });
 };

packages/compartment-mapper/src/bundle-lite.js

@@ -7,9 +7,9 @@
  * } from 'ses'
  * @import {
  *   BundleOptions,
- *   CompartmentDescriptor,
- *   CompartmentMapDescriptor,
  *   CompartmentSources,
+ *   PackageCompartmentDescriptors,
+ *   PackageCompartmentMapDescriptor,
  *   MaybeReadPowers,
  *   ReadFn,
  *   ReadPowers,
@@ -110,6 +110,11 @@ import { defaultParserForLanguage } from './archive-parsers.js';
 import mjsSupport from './bundle-mjs.js';
 import cjsSupport from './bundle-cjs.js';
 import jsonSupport from './bundle-json.js';
+import {
+  isErrorModuleSource,
+  isExitModuleSource,
+  isLocalModuleSource,
+} from './guards.js';
 
 const { quote: q } = assert;
 
@@ -144,7 +149,7 @@ null,
  * The first modules are place-holders for the modules that exit
  * the compartment map to the host's module system.
  *
- * @param {Record<string, CompartmentDescriptor>} compartmentDescriptors
+ * @param {PackageCompartmentDescriptors} compartmentDescriptors
  * @param {Record<string, CompartmentSources>} compartmentSources
  * @param {string} entryCompartmentName
  * @param {string} entryModuleSpecifier
@@ -188,28 +193,18 @@ const sortedModules = (
 
     const source = compartmentSources[compartmentName][moduleSpecifier];
     if (source !== undefined) {
-      const { record, parser, deferredError, bytes, sourceDirname, exit } =
-        source;
-      if (exit !== undefined) {
-        return exit;
-      }
-      assert(
-        bytes !== undefined,
-        `No bytes for ${moduleSpecifier} in ${compartmentName}`,
-      );
-      assert(
-        parser !== undefined,
-        `No parser for ${moduleSpecifier} in ${compartmentName}`,
-      );
-      assert(
-        sourceDirname !== undefined,
-        `No sourceDirname for ${moduleSpecifier} in ${compartmentName}`,
-      );
-      if (deferredError) {
+      if (isErrorModuleSource(source)) {
         throw Error(
-          `Cannot bundle: encountered deferredError ${deferredError}`,
+          `Cannot bundle: encountered deferredError ${source.deferredError}`,
         );
       }
+      if (isExitModuleSource(source)) {
+        return source.exit;
+      }
+      if (!isLocalModuleSource(source)) {
+        throw new TypeError(`Unexpected source type ${JSON.stringify(source)}`);
+      }
+      const { record, parser, bytes, sourceDirname } = source;
       if (record) {
         const { imports = [], reexports = [] } =
           /** @type {PrecompiledStaticModuleInterface} */ (record);
@@ -309,7 +304,7 @@ const getBundlerKitForModule = (module, params) => {
 
 /**
  * @param {ReadFn | ReadPowers | MaybeReadPowers} readPowers
- * @param {CompartmentMapDescriptor} compartmentMap
+ * @param {PackageCompartmentMapDescriptor} compartmentMap
  * @param {BundleOptions} [options]
  * @returns {Promise<string>}
  */
@@ -651,7 +646,7 @@ ${m.bundlerKit.getFunctor()}`,
 
 /**
  * @param {ReadFn | ReadPowers | MaybeReadPowers} readPowers
- * @param {CompartmentMapDescriptor} compartmentMap
+ * @param {PackageCompartmentMapDescriptor} compartmentMap
  * @param {BundleOptions} [options]
  * @returns {Promise<string>}
  */