Make GHA tests parallel by using xdist (spack#32361)

* Add two no-op jobs named "all-prechecks" and "all"

These are a suggestion from @tgamblin, they are stable named markers we
can use from gitlab and possibly for required checks to make CI more
resilient to refactors changing the names of specific checks.

* Enable parallel testing using xdist for unit testing in CI

* Normalize tmp paths to deal with macos

* add -u flag compatibility to spack python

As of now, it is accepted and ignored.  The usage with xdist, where it
is invoked specifically by `python -u spack python` which is then passed
`-u` by xdist is the entire reason for doing this.  It should never be
used without explicitly passing -u to the executing python interpreter.

* use spack python in xdist to support python 2

When running on python2, spack has many import cycles unless started
through main.  To allow that, this uses `spack python` as the
interpreter, leveraging the `-u` support so xdist doesn't error out when
it unconditionally requests unbuffered binary IO.

* Use shutil.move to account for tmpdir being in a separate filesystem sometimes

Author: trws

.flake8

@@ -29,6 +29,7 @@ max-line-length = 99
 #
 per-file-ignores =
   var/spack/repos/*/package.py:F403,F405,F821
+  *-ci-package.py:F403,F405,F821
 
 # exclude things we usually do not want linting for.
 # These still get linted when passed explicitly, as when spack flake8 passes

.github/workflows/audit.yaml

@@ -0,0 +1,44 @@
+name: audit
+
+on:
+  workflow_call:
+    inputs:
+      with_coverage:
+        required: true
+        type: string
+      python_version:
+        required: true
+        type: string
+
+concurrency:
+  group: audit-${{inputs.python_version}}-${{github.ref}}-${{github.event.pull_request.number || github.run_number}}
+  cancel-in-progress: true
+
+jobs:
+  # Run audits on all the packages in the built-in repository
+  package-audits:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # @v2
+    - uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5 # @v2
+      with:
+        python-version: ${{inputs.python_version}}
+    - name: Install Python packages
+      run: |
+        pip install --upgrade pip six setuptools pytest codecov 'coverage[toml]<=6.2'
+    - name: Package audits (with coverage)
+      if: ${{ inputs.with_coverage == 'true' }}
+      run: |
+          . share/spack/setup-env.sh
+          coverage run $(which spack) audit packages
+          coverage combine
+          coverage xml
+    - name: Package audits (without coverage)
+      if: ${{ inputs.with_coverage == 'false' }}
+      run: |
+          . share/spack/setup-env.sh
+          $(which spack) audit packages
+    - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # @v2.1.0
+      if: ${{ inputs.with_coverage == 'true' }}
+      with:
+        flags: unittests,linux,audits

.github/workflows/bootstrap.yml

@@ -9,7 +9,7 @@ on:
     - cron: '16 2 * * *'
 
 concurrency:
-  group: bootstrap-${{ github.workflow }}-${{ github.event.pull_request.number || github.run_number }}
+  group: bootstrap-${{github.ref}}-${{github.event.pull_request.number || github.run_number}}
   cancel-in-progress: true
 
 jobs:

.github/workflows/build-containers.yml

@@ -20,7 +20,7 @@ on:
     types: [published]
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_number }}
+  group: build_containers-${{github.ref}}-${{github.event.pull_request.number || github.run_number}}
   cancel-in-progress: true
 
 jobs:

.github/workflows/ci.yaml

@@ -11,7 +11,7 @@ on:
       - releases/**
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_number }}
+  group: ci-${{github.ref}}-${{github.event.pull_request.number || github.run_number}}
   cancel-in-progress: true
 
 jobs:
@@ -20,6 +20,18 @@ jobs:
     uses: ./.github/workflows/valid-style.yml
     with:
       with_coverage: ${{ needs.changes.outputs.with_coverage }}
+  audit-ancient-python:
+    uses: ./.github/workflows/audit.yaml
+    needs: [ changes ]
+    with:
+      with_coverage: ${{ needs.changes.outputs.with_coverage }}
+      python_version: 2.7
+  all-prechecks:
+    needs: [ prechecks ]
+    runs-on: ubuntu-latest
+    steps:
+    - name: Success
+      run: "true"
   # Check which files have been updated by the PR
   changes:
     runs-on: ubuntu-latest
@@ -53,6 +65,7 @@ jobs:
             - 'share/spack/**'
             - '.github/workflows/bootstrap.yml'
             core:
+            - '!lib/spack/docs/**'
             - './!(var/**)/**'
             packages:
             - 'var/**'
@@ -79,15 +92,21 @@ jobs:
     needs: [ prechecks, changes ]
     uses: ./.github/workflows/bootstrap.yml
   unit-tests:
-    if: ${{ github.repository == 'spack/spack' }}
+    if: ${{ github.repository == 'spack/spack' && needs.changes.outputs.core == 'true' }}
     needs: [ prechecks, changes ]
     uses: ./.github/workflows/unit_tests.yaml
     with:
       core: ${{ needs.changes.outputs.core }}
       packages: ${{ needs.changes.outputs.packages }}
       with_coverage: ${{ needs.changes.outputs.with_coverage }}
   windows:
-    if: ${{ github.repository == 'spack/spack' }}
+    if: ${{ github.repository == 'spack/spack' && needs.changes.outputs.core == 'true' }}
     needs: [ prechecks ]
     uses: ./.github/workflows/windows_python.yml
+  all:
+    needs: [ windows, unit-tests, bootstrap, audit-ancient-python ]
+    runs-on: ubuntu-latest
+    steps:
+    - name: Success
+      run: "true"
 

.github/workflows/unit_tests.yaml

@@ -1,6 +1,7 @@
 name: unit tests
 
 on:
+  workflow_dispatch:
   workflow_call:
     inputs:
       core:
@@ -14,7 +15,7 @@ on:
         type: string
 
 concurrency:
-  group: unit_tests-${{ github.workflow }}-${{ github.event.pull_request.number || github.run_number }}
+  group: unit_tests-${{github.ref}}-${{github.event.pull_request.number || github.run_number}}
   cancel-in-progress: true
 
 jobs:
@@ -25,11 +26,26 @@ jobs:
       matrix:
         python-version: ['2.7', '3.6', '3.7', '3.8', '3.9', '3.10']
         concretizer: ['clingo']
+        on_develop:
+        - ${{ github.ref == 'refs/heads/develop' }}
         include:
         - python-version: 2.7
           concretizer: original
-        - python-version: 3.9
+          on_develop: false
+        - python-version: '3.10'
           concretizer: original
+          on_develop: false
+        exclude:
+        - python-version: '3.7'
+          concretizer: 'clingo'
+          on_develop: false
+        - python-version: '3.8'
+          concretizer: 'clingo'
+          on_develop: false
+        - python-version: '3.9'
+          concretizer: 'clingo'
+          on_develop: false
+
     steps:
     - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # @v2
       with:
@@ -46,7 +62,7 @@ jobs:
               patchelf cmake bison libbison-dev kcov
     - name: Install Python packages
       run: |
-          pip install --upgrade pip six setuptools pytest codecov "coverage[toml]<=6.2"
+          pip install --upgrade pip six setuptools pytest codecov "coverage[toml]<=6.2" pytest-xdist
           # ensure style checks are not skipped in unit tests for python >= 3.6
           # note that true/false (i.e., 1/0) are opposite in conditions in python and bash
           if python -c 'import sys; sys.exit(not sys.version_info >= (3, 6))'; then
@@ -108,7 +124,7 @@ jobs:
           sudo apt-get install -y coreutils kcov csh zsh tcsh fish dash bash
     - name: Install Python packages
       run: |
-          pip install --upgrade pip six setuptools pytest codecov coverage[toml]==6.2
+          pip install --upgrade pip six setuptools pytest codecov coverage[toml]==6.2 pytest-xdist
     - name: Setup git configuration
       run: |
           # Need this for the git tests to succeed.
@@ -174,7 +190,7 @@ jobs:
               patchelf kcov
     - name: Install Python packages
       run: |
-          pip install --upgrade pip six setuptools pytest codecov coverage[toml]==6.2 clingo
+          pip install --upgrade pip six setuptools pytest codecov coverage[toml]==6.2 clingo pytest-xdist
     - name: Setup git configuration
       run: |
           # Need this for the git tests to succeed.
@@ -216,7 +232,7 @@ jobs:
     - name: Install Python packages
       run: |
           pip install --upgrade pip six setuptools
-          pip install --upgrade pytest codecov coverage[toml]==6.2
+          pip install --upgrade pytest codecov coverage[toml]==6.2 pytest-xdist
     - name: Setup Homebrew packages
       run: |
         brew install dash fish gcc gnupg2 kcov
@@ -229,17 +245,18 @@ jobs:
         . share/spack/setup-env.sh
         $(which spack) bootstrap untrust spack-install
         $(which spack) solve zlib
+        common_args=(--dist loadfile --tx '4*popen//python=./bin/spack-tmpconfig python -u ./bin/spack python' -x)
         if [ "${{ inputs.with_coverage }}" == "true" ]
         then
-          coverage run $(which spack) unit-test -x
+          coverage run $(which spack) unit-test "${common_args[@]}"
           coverage combine
           coverage xml
           # Delete the symlink going from ./lib/spack/docs/_spack_root back to
           # the initial directory, since it causes ELOOP errors with codecov/actions@2
           rm lib/spack/docs/_spack_root
         else
           echo "ONLY PACKAGE RECIPES CHANGED [skipping coverage]"
-          $(which spack) unit-test -x -m "not maybeslow" -k "test_all_virtual_packages_have_default_providers"
+          $(which spack) unit-test "${common_args[@]}" -m "not maybeslow" -k "test_all_virtual_packages_have_default_providers"
         fi
     - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # @v2.1.0
       if: ${{ inputs.with_coverage == 'true' }}

.github/workflows/valid-style.yml

@@ -8,7 +8,7 @@ on:
         type: string
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.run_number }}
+  group: style-${{github.ref}}-${{github.event.pull_request.number || github.run_number}}
   cancel-in-progress: true
 
 
@@ -53,30 +53,8 @@ jobs:
     - name: Run style tests
       run: |
           share/spack/qa/run-style-tests
-  # Run audits on all the packages in the built-in repository
-  package-audits:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@2541b1294d2704b0964813337f33b291d3f8596b # @v2
-    - uses: actions/setup-python@b55428b1882923874294fa556849718a1d7f2ca5 # @v2
-      with:
-        python-version: '3.10'
-    - name: Install Python packages
-      run: |
-        pip install --upgrade pip six setuptools pytest codecov coverage[toml]==6.2
-    - name: Package audits (with coverage)
-      if: ${{ inputs.with_coverage == 'true' }}
-      run: |
-          . share/spack/setup-env.sh
-          coverage run $(which spack) audit packages
-          coverage combine
-          coverage xml
-    - name: Package audits (without coverage)
-      if: ${{ inputs.with_coverage == 'false' }}
-      run: |
-          . share/spack/setup-env.sh
-          $(which spack) audit packages
-    - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # @v2.1.0
-      if: ${{ inputs.with_coverage == 'true' }}
-      with:
-        flags: unittests,linux,audits
+  audit:
+    uses: ./.github/workflows/audit.yaml
+    with:
+      with_coverage: ${{ inputs.with_coverage }}
+      python_version: '3.10'

.github/workflows/windows_python.yml

@@ -4,7 +4,7 @@ on:
   workflow_call:
 
 concurrency:
-  group: windows-${{ github.workflow }}-${{ github.event.pull_request.number || github.run_number }}
+  group: windows-${{github.ref}}-${{github.event.pull_request.number || github.run_number}}
   cancel-in-progress: true
 
 defaults:

bin/spack-tmpconfig

@@ -7,7 +7,6 @@ export TMPDIR="${XDG_RUNTIME_DIR}"
 export TMP_DIR="$(mktemp -d -t spack-test-XXXXX)"
 clean_up() {
     [[ -n "$TMPCONFIG_DEBUG" ]] && printf "cleaning up: $TMP_DIR\n"
-    [[ -n "$TMPCONFIG_DEBUG" ]] && tree "$TMP_DIR"
     rm -rf "$TMP_DIR"
 }
 trap clean_up EXIT

lib/spack/llnl/util/tty/pty.py

@@ -228,8 +228,8 @@ def __init__(self, controller_function, minion_function):
         self.minion_function = minion_function
 
         # these can be optionally set to change defaults
-        self.controller_timeout = 1
-        self.sleep_time = 0
+        self.controller_timeout = 3
+        self.sleep_time = 0.1
 
     def start(self, **kwargs):
         """Start the controller and minion processes.

lib/spack/spack/cmd/python.py

@@ -30,6 +30,12 @@ def setup_parser(subparser):
         help="print the Python version number and exit",
     )
     subparser.add_argument("-c", dest="python_command", help="command to execute")
+    subparser.add_argument(
+        "-u",
+        dest="unbuffered",
+        action="store_true",
+        help="for compatibility with xdist, do not use without adding -u to the interpreter",
+    )
     subparser.add_argument(
         "-i",
         dest="python_interpreter",