Req-changes: Use dynamic matrix, removed .test_durations JSON

atlanhq · Jan 23, 2024 · ee0f227 · ee0f227
1 parent bd6a2ba
commit ee0f227
Show file tree

Hide file tree

Showing 4 changed files with 24 additions and 607 deletions.
diff --git a/.github/workflows/pyatlan-cron.yaml b/.github/workflows/pyatlan-cron.yaml
@@ -1,6 +1,8 @@
 name: Pyatlan cron job build
 
 on:
+  push:
+    branches: [main]
   workflow_dispatch:
   schedule:
     - cron: '0 1 * * *' # At 01:00 Daily

diff --git a/.github/workflows/pyatlan-pr.yaml b/.github/workflows/pyatlan-pr.yaml
@@ -5,12 +5,16 @@ on:
   workflow_dispatch:
 
 jobs:
-  pyatlan-qa-checks-and-unit-tests:
+  qa-checks-and-unit-tests:
     runs-on: ubuntu-latest
+    outputs:
+      files: ${{ steps.distribute-integration-test-files.outputs.files }}
     steps:
-      - uses: actions/checkout@v4
+      - name: Checkout code
+        uses: actions/checkout@v2
+
       - name: Set up Python 3.9
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v2
         with:
           python-version: 3.9
 
@@ -32,39 +36,24 @@ jobs:
           MARK_BASE_URL: https://mark.atlan.com
         run: pytest tests/unit
 
-  pyatlan-integration-tests:
+      - name: Prepare integration tests distribution
+        id: distribute-integration-test-files
+        run: |
+          files=$(ls tests/integration/test_*.py tests/integration/*_test.py | grep -v "data_mesh_test.py" | tr '\n' ' ')
+          echo "::set-output name=files::$files"
+
+  integration-tests:
+    needs: [qa-checks-and-unit-tests]
     runs-on: ubuntu-latest
-    needs: [pyatlan-qa-checks-and-unit-tests]
     strategy:
-      fail-fast: false
       matrix:
-        job_id:
-          [
-            "job1",
-            "job2",
-            "job3",
-            "job4",
-            "job5",
-            "job6",
-            "job7",
-            "job8",
-            "job9",
-            "job10",
-            "job11",
-            "job12",
-            "job13",
-            "job14",
-            "job15",
-            "job16",
-            "job17",
-            "job18",
-            "job19",
-            "job20",
-          ]
+        test_file: ${{fromJson(needs.qa-checks-and-unit-tests.outputs.files)}}
     steps:
-      - uses: actions/checkout@v4
+      - name: Checkout code
+        uses: actions/checkout@v2
+
       - name: Set up Python 3.9
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v2
         with:
           python-version: 3.9
 
@@ -82,7 +71,5 @@ jobs:
           MARK_API_KEY: ${{ secrets.MARK_ATLAN_API_KEY }}
           MARK_BASE_URL: https://mark.atlan.com
         run: |
-          DISTRIBUTED_TESTS=$(python3 distribute_integration_tests.py)
-          TEST_FILES=$(echo "${DISTRIBUTED_TESTS}" | jq -r --arg targetJob "${{ matrix.job_id }}" '.[$targetJob].test_files[]' | tr '\n' ' ')
-          echo "Running ${TEST_FILES}" 
-          pytest ${TEST_FILES}
+          echo "Running test 🧪 → ${{ matrix.test_file }}"
+          pytest ${{ matrix.test_file }}