Use fs and vm for async chunk loading in Node (vercel/turborepo#8722)

### Description

Use `fs` and `vm` when loading chunks asynchronously in Node, to not
block the thread while reading large chunks

Closes PACK-411

### Testing Instructions

Using the `test/e2e/app-dir/dynamic/` Next.js tests

crates/turbopack-ecmascript-runtime/js/src/nodejs/runtime.ts

@@ -49,6 +49,8 @@ type ModuleFactory = (
 ) => undefined;
 
 const url = require("url");
+const fs = require("fs/promises");
+const vm = require("vm");
 
 const moduleFactories: ModuleFactories = Object.create(null);
 const moduleCache: ModuleCache = Object.create(null);
@@ -118,15 +120,45 @@ async function loadChunkAsync(
  source: SourceInfo,
  chunkData: ChunkData
 ): Promise<any> {
- return new Promise<void>((resolve, reject) => {
- try {
- loadChunk(chunkData, source);
- } catch (err) {
- reject(err);
- return;
+ const chunkPath = typeof chunkData === "string" ? chunkData : chunkData.path;
+ if (!chunkPath.endsWith(".js")) {
+ // We only support loading JS chunks in Node.js.
+ // This branch can be hit when trying to load a CSS chunk.
+ return;
+ }
+
+ const resolved = path.resolve(RUNTIME_ROOT, chunkPath);
+
+ try {
+ const contents = await fs.readFile(resolved, "utf-8");
+
+ const module = {
+ exports: {},
+ };
+ vm.runInThisContext(
+ "(function(module, exports, require, __dirname, __filename) {" +
+ contents +
+ "\n})",
+ resolved
+ )(module, module.exports, require, path.dirname(resolved), resolved);
+
+ const chunkModules: ModuleFactories = module.exports;
+ for (const [moduleId, moduleFactory] of Object.entries(chunkModules)) {
+ if (!moduleFactories[moduleId]) {
+ moduleFactories[moduleId] = moduleFactory;
+ }
  }
- resolve();
- });
+ } catch (e) {
+ let errorMessage = `Failed to load chunk ${chunkPath}`;
+
+ if (source) {
+ errorMessage += ` from ${stringifySourceInfo(source)}`;
+ }
+
+ throw new Error(errorMessage, {
+ cause: e,
+ });
+ }
 }
 
 function loadWebAssembly(chunkPath: ChunkPath, imports: WebAssembly.Imports) {

crates/turbopack-tests/tests/snapshot/runtime/default_build_runtime/output/[turbopack]_runtime.js

@@ -428,6 +428,8 @@ function stringifySourceInfo(source) {
  }
 }
 const url = require("url");
+const fs = require("fs/promises");
+const vm = require("vm");
 const moduleFactories = Object.create(null);
 const moduleCache = Object.create(null);
 /**
@@ -476,15 +478,34 @@ function loadChunkPath(chunkPath, source) {
  }
 }
 async function loadChunkAsync(source, chunkData) {
- return new Promise((resolve, reject)=>{
- try {
- loadChunk(chunkData, source);
- } catch (err) {
- reject(err);
- return;
+ const chunkPath = typeof chunkData === "string" ? chunkData : chunkData.path;
+ if (!chunkPath.endsWith(".js")) {
+ // We only support loading JS chunks in Node.js.
+ // This branch can be hit when trying to load a CSS chunk.
+ return;
+ }
+ const resolved = path.resolve(RUNTIME_ROOT, chunkPath);
+ try {
+ const contents = await fs.readFile(resolved, "utf-8");
+ const module1 = {
+ exports: {}
+ };
+ vm.runInThisContext("(function(module, exports, require, __dirname, __filename) {" + contents + "\n})", resolved)(module1, module1.exports, require, path.dirname(resolved), resolved);
+ const chunkModules = module1.exports;
+ for (const [moduleId, moduleFactory] of Object.entries(chunkModules)){
+ if (!moduleFactories[moduleId]) {
+ moduleFactories[moduleId] = moduleFactory;
+ }
  }
- resolve();
- });
+ } catch (e) {
+ let errorMessage = `Failed to load chunk ${chunkPath}`;
+ if (source) {
+ errorMessage += ` from ${stringifySourceInfo(source)}`;
+ }
+ throw new Error(errorMessage, {
+ cause: e
+ });
+ }
 }
 function loadWebAssembly(chunkPath, imports) {
  const resolved = path.resolve(RUNTIME_ROOT, chunkPath);