test(ses): hermes test runner and multiple tests

Author: naugtur

packages/ses/package.json

@@ -89,7 +89,7 @@
     "prepare": "npm run clean && npm run build",
     "qt": "ava",
     "test": "tsd && ava",
-    "test:hermes": "./scripts/hermes-test.sh",
+    "test:hermes": "node ./scripts/hermes-test.js ./test/hermes/*",
     "test:xs": "xst dist/ses.umd.js test/_lockdown-safe.js && node scripts/generate-test-xs.js && xst tmp/test-xs.js && rm -rf tmp",
     "postpack": "git clean -fX \"*.d.ts*\" \"*.d.cts*\" \"*.d.mts*\" \"*.tsbuildinfo\""
   },

packages/ses/scripts/hermes-test.js

@@ -0,0 +1,159 @@
+#!/usr/bin/env node
+/* global process */
+
+import { existsSync, writeFileSync, mkdtempSync, readFileSync } from 'node:fs';
+import path from 'node:path';
+import { execSync } from 'node:child_process';
+import os from 'node:os';
+
+// Utility function to determine OS-specific directory and create Hermes runner
+function createHermesRunner() {
+  const platform = os.platform();
+  let osDir;
+
+  switch (platform) {
+    case 'linux':
+      osDir = 'linux64-bin';
+      break;
+    case 'darwin':
+      osDir = 'osx-bin';
+      break;
+    case 'win32':
+      osDir = 'win64-bin';
+      break;
+    default:
+      throw new Error(`Unsupported OS: ${platform}`);
+  }
+
+  const hermesc = path.join(
+    '..',
+    '..',
+    'node_modules',
+    'hermes-engine-cli',
+    osDir,
+    'hermesc',
+  );
+  const hermes = path.join(
+    '..',
+    '..',
+    'node_modules',
+    'hermes-engine-cli',
+    osDir,
+    'hermes',
+  );
+
+  // Add .exe extension for Windows
+  const hermescPath = platform === 'win32' ? `${hermesc}.exe` : hermesc;
+  const hermesPath = platform === 'win32' ? `${hermes}.exe` : hermes;
+
+  return {
+    version: () => {
+      execSync(`"${hermesPath}" --version`, {
+        stdio: 'inherit',
+      });
+    },
+    run: jsFile => {
+      const hbcFile = jsFile.replace(/\.js$/, '.hbc');
+
+      // Compile
+      try {
+        execSync(
+          `"${hermescPath}" "${jsFile}" -emit-binary -out "${hbcFile}"`,
+          {
+            stdio: 'pipe', // Capture output to detect compilation errors
+          },
+        );
+      } catch (error) {
+        throw new Error(`Compilation error: ${error.message}`);
+      }
+
+      // Execute
+      execSync(`"${hermesPath}" -b "${hbcFile}"`, {
+        stdio: 'inherit',
+      });
+    },
+  };
+}
+
+const hermesPrefixSES = readFileSync(
+  path.join(import.meta.dirname, '..', 'dist', 'ses-hermes.cjs'),
+  'utf8',
+);
+const tempDir = mkdtempSync(path.join(os.tmpdir(), 'hermes-test-'));
+
+const prepareTestFile = testFile => {
+  const testName = path.basename(testFile);
+  const distFile = path.join(tempDir, `${testName}`);
+
+  const testContent = readFileSync(testFile, 'utf8');
+  // Combine the code in a way that line numbers are preserved and therefore helpful
+  const combined = `${testContent}
+;;;;
+${hermesPrefixSES}
+;;;;
+
+var TEST;
+function test(name, fn) {
+  TEST={fn, name}
+}
+try {
+  TEST.fn()
+} catch(e) {
+  print(e.message)
+  print('  ✗ '+TEST.name)
+  throw e
+}
+print('  ✓ '+TEST.name)
+
+;;;; 
+`;
+  writeFileSync(distFile, combined);
+  return distFile;
+};
+
+function run() {
+  const testFiles = process.argv.slice(2);
+
+  if (testFiles.length === 0) {
+    console.error('Usage: hermes-test.js <test-file1> <test-file2> ...');
+    process.exit(2);
+  }
+
+  // Validate that all test files exist
+  testFiles.forEach(file => {
+    if (!existsSync(file)) {
+      console.error(`Error: Test file not found: ${file}`);
+      process.exit(3);
+    }
+  });
+
+  const hermes = createHermesRunner();
+  let passed = 0;
+  let failed = 0;
+
+  hermes.version();
+
+  // Run each test in a separate clean environment
+  testFiles.forEach(testFile => {
+    console.log(`\n[RUN] ${testFile}`);
+
+    try {
+      hermes.run(prepareTestFile(testFile));
+      passed += 1;
+    } catch (error) {
+      failed += 1;
+    }
+  });
+
+  // Summary
+  console.log(`  -
+
+  ${passed} tests passed`);
+
+  if (failed > 0) {
+    console.log(`  ${failed} tests failed`);
+    process.exit(1);
+  }
+}
+
+run();

packages/ses/scripts/hermes-test.sh

@@ -0,0 +1,23 @@
+/* eslint-disable no-eval */
+/* global test */
+
+test('smoke: lockdown works', () => {
+  lockdown({ evalTaming: 'unsafe-eval' });
+
+  assert(typeof eval === 'function', 'eval is not a function');
+  assert(
+    eval.toString() === 'function eval() { [native code] }',
+    'eval is not a native code function',
+  );
+  // @ts-expect-error expects string
+  assert(eval(42) === 42, 'eval is not functional');
+  assert(
+    eval('42') === 42,
+    'eval called with string argument is not functional',
+  );
+  assert(
+    // eslint-disable-next-line no-new-func
+    Function('return 42')() === 42,
+    'Function constructor is not functional',
+  );
+});

packages/ses/test/_hermes-smoke.js

@@ -0,0 +1,22 @@
+/* global test */
+test('error taming unsafe-debug', () => {
+  lockdown({
+    evalTaming: 'unsafe-eval',
+    errorTaming: 'unsafe-debug',
+    overrideTaming: 'severe',
+    reporting: 'none',
+  });
+
+  try {
+    throw Error('test');
+  } catch (e) {
+    assert(
+      !e.stack.includes('at [object CallSite]'),
+      `stack censorship broken: 
+----
+ ${e.stack}
+----
+`,
+    );
+  }
+});

packages/ses/test/hermes/1smoke.js

@@ -0,0 +1,25 @@
+// tip: use this to trip up platform detection
+// Error.captureStackTrace = null
+
+/* global test */
+test('error taming unsafe', () => {
+  lockdown({
+    evalTaming: 'unsafe-eval',
+    errorTaming: 'unsafe',
+    overrideTaming: 'severe',
+    reporting: 'none',
+  });
+
+  try {
+    throw Error('test');
+  } catch (e) {
+    assert(
+      !e.stack.includes('at [object CallSite]'),
+      `stack censorship broken: 
+----
+ ${e.stack}
+----
+`,
+    );
+  }
+});