Cleanup glob handling and do ignore checks in memory using micromatch

package.json

@@ -48,6 +48,7 @@
     "fs-extra": "^11.2.0",
     "globby": "^11.1.0",
     "lodash": "^4.17.20",
+    "micromatch": "^4.0.8",
     "node-object-hash": "^3.0.0",
     "oxc-resolver": "^1.12.0",
     "pkg-dir": "^5.0.0",
@@ -62,6 +63,7 @@
     "@types/find-root": "^1.1.4",
     "@types/fs-extra": "^11.0.4",
     "@types/lodash": "^4.17.13",
+    "@types/micromatch": "^4.0.9",
     "@types/node": "^22.9.3",
     "@types/write-file-atomic": "^4.0.3",
     "@types/yargs": "^15.0.19",

spec/SwcCompiler.test.ts

@@ -41,10 +41,10 @@ test("throws if the file is ignored", async () => {
     }
   }
 
-  expect(error).toBeDefined();
+  expect(error).toBeTruthy();
   expect(error?.ignoredFile).toBeTruthy();
   expect(error?.message).toMatch(
-    /File .+ignored\.ts is imported but not being built because it is explicitly ignored in the wds project config\. It is being ignored by the provided glob pattern '!ignored\.ts', remove this pattern from the project config or don't import this file to fix./
+    /File .+ignored\.ts is imported but not being built because it is explicitly ignored in the wds project config\. It is being ignored by the provided glob pattern 'ignored\.ts', remove this pattern from the project config or don't import this file to fix./
   );
 });
 

src/ProjectConfig.ts

@@ -1,6 +1,7 @@
 import type { Options as SwcOptions } from "@swc/core";
 import fs from "fs-extra";
 import _ from "lodash";
+import micromatch from "micromatch";
 import path from "path";
 import { log } from "./utils.js";
 
@@ -13,7 +14,10 @@ export interface RunOptions {
 }
 
 export interface ProjectConfig {
+  root: string;
   ignore: string[];
+  includeGlob: string;
+  includedMatcher: (filePath: string) => boolean;
   swc?: SwcConfig;
   esm?: boolean;
   extensions: string[];
@@ -23,10 +27,17 @@ export interface ProjectConfig {
 export const projectConfig = async (root: string): Promise<ProjectConfig> => {
   const location = path.join(root, "wds.js");
   const base: ProjectConfig = {
-    ignore: [],
+    root,
+
     extensions: [".ts", ".tsx", ".jsx"],
     cacheDir: path.join(root, "node_modules/.cache/wds"),
     esm: true,
+    /** The list of globby patterns to use when searching for files to build */
+    includeGlob: `**/*`,
+    /** The list of globby patterns to ignore use when searching for files to build */
+    ignore: [],
+    /** A micromatch matcher for userland checking if a file is included */
+    includedMatcher: () => true,
   };
 
   try {
@@ -49,8 +60,10 @@ export const projectConfig = async (root: string): Promise<ProjectConfig> => {
     result.cacheDir = path.resolve(projectRootDir, result.cacheDir);
   }
 
-  // absolutize the ignore paths if not already
-  result.ignore = result.ignore.map((p: string) => (p.startsWith("/") ? p : path.resolve(projectRootDir, p)));
+  // build inclusion glob and matcher
+  result.ignore = _.uniq([`node_modules`, `**/*.d.ts`, `.git/**`, ...result.ignore]);
+  result.includeGlob = `**/*{${result.extensions.join(",")}}`;
+  result.includedMatcher = micromatch.matcher(result.includeGlob, { cwd: result.root, ignore: result.ignore });
 
   return result;
 };

src/SwcCompiler.ts

@@ -99,6 +99,7 @@ export class SwcCompiler implements Compiler {
   private compiledFiles: CompiledFiles;
   private invalidatedFiles: Set<string>;
   private knownCacheEntries = new Set<string>();
+  private projectConfigs: Record<string, ProjectConfig> = {};
 
   static async create(workspaceRoot: string, outDir: string) {
     const compiler = new SwcCompiler(workspaceRoot, outDir);
@@ -169,7 +170,7 @@ export class SwcCompiler implements Compiler {
 
   private async getModule(filename: string) {
     const root = findRoot(path.dirname(filename));
-    const config = await projectConfig(root);
+    const config = await this.getProjectConfig(root);
 
     let swcConfig: Options;
 
@@ -184,11 +185,14 @@ export class SwcCompiler implements Compiler {
       swcConfig = config.swc;
     }
 
-    const globs = [...this.fileGlobPatterns(config), ...this.ignoreFileGlobPatterns(config)];
+    log.debug("searching for filenames", { filename, config });
 
-    log.debug("searching for filenames", { filename, config, root, globs });
-
-    let fileNames = await globby(globs, { cwd: root, absolute: true });
+    let fileNames = await globby(config.includeGlob, {
+      onlyFiles: true,
+      cwd: root,
+      absolute: true,
+      ignore: config.ignore,
+    });
 
     if (process.platform === "win32") {
       fileNames = fileNames.map((fileName) => fileName.replace(/\//g, "\\"));
@@ -276,34 +280,25 @@ export class SwcCompiler implements Compiler {
     }
   }
 
-  /** The list of globby patterns to use when searching for files to build */
-  private fileGlobPatterns(config: ProjectConfig) {
-    return [`**/*{${config.extensions.join(",")}}`];
-  }
-
-  /** The list of globby patterns to ignore use when searching for files to build */
-  private ignoreFileGlobPatterns(config: ProjectConfig) {
-    return [`!node_modules`, `!**/*.d.ts`, ...(config.ignore || []).map((ignore) => `!${ignore}`)];
-  }
-
   /**
    * Detect if a file is being ignored by the ignore glob patterns for a given project
    *
    * Returns false if the file isn't being ignored, or the ignore pattern that is ignoring it if it is.
    */
   private async isFilenameIgnored(filename: string): Promise<string | false> {
     const root = findRoot(filename);
-    const config = await projectConfig(root);
-    const includeGlobs = this.fileGlobPatterns(config);
-    const ignoreGlobs = this.ignoreFileGlobPatterns(config);
-
-    const actual = await globby([...includeGlobs, ...ignoreGlobs], { cwd: root, absolute: true });
-    const all = await globby(includeGlobs, { cwd: root, absolute: true });
-
-    // if the file isn't returned when we use the ignores, but is when we don't use the ignores, it means were ignoring it. Figure out which ignore is causing this
-    if (!actual.includes(filename) && all.includes(filename)) {
-      for (const ignoreGlob of ignoreGlobs) {
-        const withThisIgnore = await globby([...includeGlobs, ignoreGlob], { cwd: root, absolute: true });
+    const config = await this.getProjectConfig(root);
+
+    // check if the file is ignored by any of the ignore patterns using micromatch
+    const included = config.includedMatcher(filename.replace(root + path.sep, ""));
+    if (!included) {
+      // figure out which ignore pattern is causing the file to be ignored for a better error message
+      for (const ignoreGlob of config.ignore) {
+        const withThisIgnore = await globby(config.includeGlob, {
+          cwd: root,
+          absolute: true,
+          ignore: [ignoreGlob],
+        });
         if (!withThisIgnore.includes(filename)) {
           return ignoreGlob;
         }
@@ -326,6 +321,15 @@ export class SwcCompiler implements Compiler {
     );
     return;
   }
+
+  private async getProjectConfig(root: string): Promise<ProjectConfig> {
+    let config = this.projectConfigs[root];
+    if (!config) {
+      config = await projectConfig(root);
+      this.projectConfigs[root] = config;
+    }
+    return config;
+  }
 }
 
 const hashObject = hasher({ sort: true });

src/index.ts

@@ -78,7 +78,7 @@ const startFilesystemWatcher = (project: Project) => {
       if (filePath.endsWith(".DS_Store")) return true;
       if (filePath.endsWith(".tsbuildinfo")) return true;
 
-      return project.config.ignore?.some((ignore) => filePath.startsWith(ignore)) ?? false;
+      return project.config.includedMatcher(filePath);
     }) as any,
   });