Fix shutdown on Cntrl+c for ci/scripts/run_tests.py (#1886)

According to the docs `ProcessPoolExecutor.shutdown` :
> Any futures that are completed or running won’t be cancelled, regardless of the value of cancel_futures 
ref: https://docs.python.org/3.13/library/concurrent.futures.html#concurrent.futures.Executor.shutdown
Which causes long-running pytest invocation to continue running.

This PR switches to using `multiprocessing.pool.Pool` instead.

## By Submitting this PR I confirm:
- I am familiar with the [Contributing Guidelines](https://github.com/NVIDIA/NeMo-Agent-Toolkit/blob/develop/docs/source/resources/contributing/index.md).
- We require that all contributors "sign-off" on their commits. This certifies that the contribution is your original work, or you have rights to submit it under the same license, or a compatible license.
  - Any contribution which contains commits that are not Signed-Off will not be accepted.
- When the PR is ready for review, new or existing tests cover these changes.
- When the PR is ready for review, the documentation is up to date with these changes.



## Summary by CodeRabbit

* **Refactor**
  * Improved internal test runner to process tasks in submission order for more predictable results.
  * Simplified shutdown behavior to terminate and join worker processes, improving reliability on interrupt.
  * Ensured early-exit stops remaining work when configured to exit first.
  * Enhanced shutdown messaging to provide clearer exit information and restore prior interrupt handling.

Authors:
  - David Gardner (https://github.com/dagardner-nv)

Approvers:
  - Will Killian (https://github.com/willkill07)

URL: #1886

Author: dagardner-nv

ci/scripts/run_tests.py

@@ -21,8 +21,7 @@
 import signal
 import subprocess
 import sys
-from concurrent.futures import ProcessPoolExecutor
-from concurrent.futures import as_completed
+from multiprocessing.pool import Pool
 from pathlib import Path
 
 from dotenv import dotenv_values
@@ -245,42 +244,58 @@ def main(junit_xml: str | None,
 
     failures = 0
 
-    with ProcessPoolExecutor(max_workers=jobs) as executor:
-        ex = executor
+    orig_handler = signal.getsignal(signal.SIGINT)
 
-        def shutdown_executor(_signum, _frame, wait: bool = False):
+    def _restore_handler():
+        if orig_handler is not None:
+            signal.signal(signal.SIGINT, orig_handler)
+
+    with Pool(processes=jobs) as pool:
+        ex = pool
+
+        def shutdown_pool(_signum, _frame):
             nonlocal ex
+
+            shutdown_msg = "Exiting"
             if ex is not None:
-                print("Shutting down executor...")
-                ex.shutdown(wait=wait, cancel_futures=True)
+                print("Shutting down pool...")
+                ex.terminate()
+                ex.join()
+                if _signum is not None:
+                    shutdown_msg = f"Received signal {_signum}, exiting"
+
             else:
-                print("Executor not found")
+                print("Pool not found")
+
+            _restore_handler()
+            raise SystemExit(shutdown_msg)
 
-        signal.signal(signal.SIGINT, shutdown_executor)
+        signal.signal(signal.SIGINT, shutdown_pool)
         futs = [
-            ex.submit(run_one,
-                      p,
-                      enable_coverage=cov_xml is not None,
-                      enable_junit=junit_xml is not None,
-                      run_slow=run_slow,
-                      run_integration=run_integration,
-                      exitfirst=exitfirst,
-                      is_verbose=has_verbose_flag,
-                      extra_flags=extra_flags,
-                      no_tests=no_tests) for p in projects
+            pool.apply_async(run_one,
+                             args=(p, ),
+                             kwds=dict(enable_coverage=cov_xml is not None,
+                                       enable_junit=junit_xml is not None,
+                                       run_slow=run_slow,
+                                       run_integration=run_integration,
+                                       exitfirst=exitfirst,
+                                       is_verbose=has_verbose_flag,
+                                       extra_flags=extra_flags,
+                                       no_tests=no_tests)) for p in projects
         ]
         try:
-            for fut in as_completed(futs):
-                if fut.result() != 0:
+            for fut in futs:
+                if fut.get() != 0:
                     failures += 1
                     if exitfirst:
                         raise TestFailure("Exiting on first failure as requested.")
 
         except TestFailure:
             print("Cancelling remaining tests...")
-            shutdown_executor(None, None, wait=True)
+            shutdown_pool(None, None)
         finally:
             ex = None
+            _restore_handler()
             for p in projects:
                 sh(["rm", "-rf", str(p / ".venv")])