feat: handle hanging processes

Author: gajus

cspell.yaml

@@ -19,4 +19,6 @@ words:
   - turborepo
   - turbowatch
   - vitest
-  - wholename
+  - wholename
+  - pidtree
+  - pids

package-lock.json

@@ -10,9 +10,11 @@
   "dependencies": {
     "chalk": "^4.1.2",
     "chokidar": "^3.5.3",
+    "find-process": "^1.4.7",
     "glob": "^9.3.1",
     "jiti": "^1.18.2",
     "micromatch": "^4.0.5",
+    "pidtree": "^0.6.0",
     "randomcolor": "^0.6.2",
     "roarr": "^7.15.0",
     "semver": "^7.3.8",

package.json

@@ -0,0 +1,8 @@
+/* eslint-disable no-console */
+
+const { spawn } = require('node:child_process');
+const { resolve } = require('node:path');
+
+spawn('node', [resolve(__dirname, 'b.js')], {
+  stdio: 'inherit',
+});

src/__fixtures__/killPsTree/badTree/a.js

@@ -0,0 +1,9 @@
+/* eslint-disable no-console */
+
+setInterval(() => {
+  console.log('b');
+}, 1_000);
+
+process.on('SIGTERM', () => {
+  console.log('b: SIGTERM');
+});

src/__fixtures__/killPsTree/badTree/b.js

@@ -0,0 +1,10 @@
+/* eslint-disable no-console */
+
+const { spawn } = require('node:child_process');
+const { resolve } = require('node:path');
+
+const b = spawn('node', [resolve(__dirname, 'b.js')]);
+
+b.stdout.on('data', (data) => {
+  console.log(data.toString());
+});

src/__fixtures__/killPsTree/goodTree/a.js

@@ -0,0 +1,5 @@
+/* eslint-disable no-console */
+
+setInterval(() => {
+  console.log('b');
+}, 1_000);

src/__fixtures__/killPsTree/goodTree/b.js

@@ -2,6 +2,7 @@
 
 import { AbortError, UnexpectedError } from './errors';
 import { findNearestDirectory } from './findNearestDirectory';
+import { killPsTree } from './killPsTree';
 import { Logger } from './Logger';
 import { type Throttle } from './types';
 import chalk from 'chalk';
@@ -117,12 +118,21 @@ export const createSpawn = (
 
     if (abortSignal) {
       const kill = () => {
+        const pid = processPromise.child?.pid;
+
+        if (!pid) {
+          log.warn('no process to kill');
+
+          return;
+        }
+
+        // TODO make this configurable
         // eslint-disable-next-line promise/prefer-await-to-then
-        processPromise.kill().finally(() => {
+        killPsTree(pid, 5_000).then(() => {
           log.debug('task %s was killed', taskId);
 
-          // processPromise.stdout.off('data', onStdout);
-          // processPromise.stderr.off('data', onStderr);
+          processPromise.stdout.off('data', onStdout);
+          processPromise.stderr.off('data', onStderr);
         });
       };
 

src/createSpawn.ts

@@ -0,0 +1,33 @@
+import { killPsTree } from './killPsTree';
+import { exec } from 'node:child_process';
+import { join } from 'node:path';
+import { setTimeout } from 'node:timers/promises';
+import { test } from 'vitest';
+
+test('kills a good process tree', async () => {
+  const childProcess = exec(
+    `node ${join(__dirname, '__fixtures__/killPsTree/goodTree/a.js')}`,
+  );
+
+  if (!childProcess.pid) {
+    throw new Error('Expected child process to have a pid');
+  }
+
+  await setTimeout(500);
+
+  await killPsTree(childProcess.pid);
+});
+
+test('kills a bad process tree', async () => {
+  const childProcess = exec(
+    `node ${join(__dirname, '__fixtures__/killPsTree/badTree/a.js')}`,
+  );
+
+  if (!childProcess.pid) {
+    throw new Error('Expected child process to have a pid');
+  }
+
+  await setTimeout(500);
+
+  await killPsTree(childProcess.pid, 1_000);
+});

src/killPsTree.test.ts

@@ -0,0 +1,66 @@
+import { Logger } from './Logger';
+import findProcess from 'find-process';
+import pidTree from 'pidtree';
+
+const log = Logger.child({
+  namespace: 'killPsTree',
+});
+
+export const killPsTree = async (
+  rootPid: number,
+  gracefulTimeout: number = 30_000,
+) => {
+  const childPids = await pidTree(rootPid);
+
+  const pids = [rootPid, ...childPids];
+
+  for (const pid of pids) {
+    process.kill(pid, 'SIGTERM');
+  }
+
+  let hangingPids = [...pids];
+
+  let hitTimeout = false;
+
+  const timeoutId = setTimeout(() => {
+    hitTimeout = true;
+
+    log.debug({ hangingPids }, 'sending SIGKILL to processes...');
+
+    for (const pid of hangingPids) {
+      process.kill(pid, 'SIGKILL');
+    }
+  }, gracefulTimeout);
+
+  await Promise.all(
+    hangingPids.map((pid) => {
+      return new Promise((resolve) => {
+        const interval = setInterval(async () => {
+          if (hitTimeout) {
+            clearInterval(interval);
+
+            resolve(false);
+
+            return;
+          }
+
+          const processes = await findProcess('pid', pid);
+
+          if (processes.length === 0) {
+            hangingPids = hangingPids.filter(
+              (hangingPid) => hangingPid !== pid,
+            );
+
+            clearInterval(interval);
+
+            resolve(true);
+          }
+        }, 100);
+      });
+    }),
+  );
+
+  clearTimeout(timeoutId);
+
+  log.debug('all processes terminated');
+};