Req-changes: Use dynamic matrix, removed .test_durations JSON

.github/workflows/pyatlan-codeql.yaml

@@ -0,0 +1,31 @@
+name: Pyatlan Code Scanning Build
+
+on: [pull_request, workflow_dispatch]
+
+jobs:
+  CodeQL-Build:
+    runs-on: ubuntu-latest
+
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: ['python']
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+          languages: ${{matrix.language}}
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v3
+        with:
+          category: "/language:${{matrix.language}}"

.github/workflows/merge.yaml → .github/workflows/pyatlan-docs.yaml

@@ -1,6 +1,6 @@
 # SPDX-License-Identifier: Apache-2.0
 # Copyright 2023 Atlan Pte. Ltd.
-name: "Merge"
+name: Pyatlan Sphinx Docs Build
 
 on:
   push:

.github/workflows/pyatlan-pr.yaml

@@ -1,14 +1,18 @@
-name: Pyatlan pull request build
+name: Pyatlan Pull Request Build
 
 on:
   pull_request:
   workflow_dispatch:
 
 jobs:
-  pyatlan-qa-checks-and-unit-tests:
+  qa-checks-and-unit-tests:
     runs-on: ubuntu-latest
+    outputs:
+      files: ${{ steps.distribute-integration-test-files.outputs.files }}
     steps:
-      - uses: actions/checkout@v4
+      - name: Checkout code
+        uses: actions/checkout@v4
+
       - name: Set up Python 3.9
         uses: actions/setup-python@v4
         with:
@@ -32,45 +36,32 @@ jobs:
           MARK_BASE_URL: https://mark.atlan.com
         run: pytest tests/unit
 
-  pyatlan-integration-tests:
+      - name: Prepare integration tests distribution
+        id: distribute-integration-test-files
+        run: |
+          test_ignore_file=tests/integration/.testignore.txt
+          files=$(ls tests/integration/test_*.py tests/integration/*_test.py | grep -v -f $test_ignore_file | tr '\n' ' ')
+          json_files=$(echo "${files[@]}" | jq -R -c 'split(" ")[:-1]')
+          echo "files=$json_files" >> $GITHUB_OUTPUT
+
+  integration-tests:
+    needs: [qa-checks-and-unit-tests]
     runs-on: ubuntu-latest
-    needs: [pyatlan-qa-checks-and-unit-tests]
     strategy:
       fail-fast: false
       matrix:
-        job_id:
-          [
-            "job1",
-            "job2",
-            "job3",
-            "job4",
-            "job5",
-            "job6",
-            "job7",
-            "job8",
-            "job9",
-            "job10",
-            "job11",
-            "job12",
-            "job13",
-            "job14",
-            "job15",
-            "job16",
-            "job17",
-            "job18",
-            "job19",
-            "job20",
-          ]
+        test_file: ${{fromJson(needs.qa-checks-and-unit-tests.outputs.files)}}
     steps:
-      - uses: actions/checkout@v4
+      - name: Checkout code
+        uses: actions/checkout@v4
+
       - name: Set up Python 3.9
         uses: actions/setup-python@v4
         with:
           python-version: 3.9
 
       - name: Install dependencies
         run: |
-          sudo apt-get install -y jq
           python -m pip install --upgrade pip
           if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
           if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
@@ -81,8 +72,4 @@ jobs:
           ATLAN_BASE_URL: ${{ secrets.ATLAN_BASE_URL }}
           MARK_API_KEY: ${{ secrets.MARK_ATLAN_API_KEY }}
           MARK_BASE_URL: https://mark.atlan.com
-        run: |
-          DISTRIBUTED_TESTS=$(python3 distribute_integration_tests.py)
-          TEST_FILES=$(echo "${DISTRIBUTED_TESTS}" | jq -r --arg targetJob "${{ matrix.job_id }}" '.[$targetJob].test_files[]' | tr '\n' ' ')
-          echo "Running ${TEST_FILES}" 
-          pytest ${TEST_FILES}
+        run: pytest ${{ matrix.test_file }}

.github/workflows/python-publish.yaml → .github/workflows/pyatlan-publish.yaml

@@ -6,7 +6,7 @@
 # separate terms of service, privacy policy, and support
 # documentation.
 
-name: Upload Python Package
+name: Publish Pyatlan Package
 
 on:
   release:
@@ -49,6 +49,7 @@ jobs:
         with:
           name: lambda-layer
           path: ./python
+
   publish-images:
     runs-on: ubuntu-latest
     needs: deploy

.github/workflows/pyatlan-cron.yaml → .github/workflows/pyatlan-test-cron.yaml

@@ -1,4 +1,4 @@
-name: Pyatlan cron job build
+name: Pyatlan Test Cron Job Build
 
 on:
   workflow_dispatch: