DVX-182: Run integration tests in parallel

Author: Aryamanz29

.github/workflows/pytest-parallel.yaml

@@ -0,0 +1,37 @@
+name: Run tests parallel
+
+on:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        job_id: ["job1", "job2"]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get install -y jq
+          python -m pip install --upgrade pip
+          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+          if [ -f requirements-dev.txt ]; then pip install -r requirements-dev.txt; fi
+
+      - name: Run pytest in parallel
+        env: # Test tenant environment variables
+          ATLAN_API_KEY: ${{ secrets.ATLAN_API_KEY }}
+          ATLAN_BASE_URL: ${{ secrets.ATLAN_BASE_URL }}
+          MARK_API_KEY: ${{ secrets.MARK_ATLAN_API_KEY }}
+          MARK_BASE_URL: https://mark.atlan.com
+        run: |
+          DISTRIBUTED_TESTS=$(python3 st.py)
+          TEST_FILES=$(echo "${DISTRIBUTED_TESTS}" | jq -r --arg targetJob "${{ matrix.job_id }}" '.[$targetJob].test_files[]' | tr '\n' ' ')
+          pytest ${TEST_FILES}

.sample

@@ -0,0 +1,27 @@
+{
+    "tests/integration/connection_test.py::test_invalid_connection": 0.0010759579999999325,
+    "tests/integration/connection_test.py::test_invalid_connection_admin_group": 1.8205267080000018,
+    "tests/integration/connection_test.py::test_invalid_connection_admin_role": 3.566417791,
+    "tests/integration/connection_test.py::test_invalid_connection_admin_user": 13.501227625000002,
+    "tests/integration/purpose_test.py::test_add_policies_to_purpose": 6.234852457000002,
+    "tests/integration/purpose_test.py::test_find_purpose_by_name": 8.90930079200001,
+    "tests/integration/purpose_test.py::test_purpose": 51.40183633299999,
+    "tests/integration/purpose_test.py::test_retrieve_purpose": 49.65388283300001,
+    "tests/integration/purpose_test.py::test_update_purpose": 7.971122498999989,
+    "tests/integration/s3_asset_test.py::test_bucket": 83.91328645800002,
+    "tests/integration/s3_asset_test.py::test_bucket_with_name": 2.5233026249999853,
+    "tests/integration/s3_asset_test.py::test_delete_object": 21.078623375000006,
+    "tests/integration/s3_asset_test.py::test_delete_object_with_name": 15.819571706999966,
+    "tests/integration/s3_asset_test.py::test_object": 3.6064143319999573,
+    "tests/integration/s3_asset_test.py::test_object_with_name": 8.101769499,
+    "tests/integration/s3_asset_test.py::test_read_deleted_object": 1.4334719589999736,
+    "tests/integration/s3_asset_test.py::test_read_deleted_object_with_name": 15.56822116699999,
+    "tests/integration/s3_asset_test.py::test_restore_object": 9.909107290999998,
+    "tests/integration/s3_asset_test.py::test_restore_object_with_name": 50.608775542000046,
+    "tests/integration/s3_asset_test.py::test_retrieve_bucket": 1.5732482919999597,
+    "tests/integration/s3_asset_test.py::test_retrieve_bucket_with_name": 3.184895584000003,
+    "tests/integration/s3_asset_test.py::test_update_bucket": 25.88223066699996,
+    "tests/integration/s3_asset_test.py::test_update_bucket_again": 20.06530891600005,
+    "tests/integration/s3_asset_test.py::test_update_bucket_with_name": 16.880974166999977,
+    "tests/integration/s3_asset_test.py::test_update_bucket_with_name_again": 29.425950040999908
+}

.sample-test

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# Read the distributed test files from the JSON file
+DISTRIBUTED_TESTS=$(python3 st.py)
+
+TEST_FILES=$(echo $DISTRIBUTED_TESTS | jq -r --arg targetJob "job2" '.[$targetJob].test_files[]' | tr '\n' ' ')
+
+echo "pytest $TEST_FILES"

requirements-dev.txt

@@ -4,6 +4,7 @@ black==23.7.0
 types-requests==2.31.0.2
 pytest==7.4.0
 pytest-order==1.1.0
+pytest-split==0.8.1
 retry==0.9.2
 pre-commit==2.20.0
 deepdiff==6.2.1

st.py

@@ -0,0 +1,32 @@
+import json
+
+# Read the test execution times from the JSON file
+with open('.sample', 'r') as f:
+    test_durations = json.load(f)
+
+# Accumulate durations for each test file
+accumulated_durations = {}
+for test, duration in test_durations.items():
+    test_file = test.split('::')[0]
+    accumulated_durations[test_file] = accumulated_durations.get(test_file, 0) + duration
+
+# Sort test files by accumulated duration in descending order
+sorted_test_files = sorted(accumulated_durations.items(), key=lambda x: x[1], reverse=True)
+
+# Distribute test files across multiple parallel jobs while balancing execution time
+num_jobs = 2  # Adjust the number of parallel jobs as needed
+jobs = {f'job{i+1}': {'test_files': [], 'total_time': 0} for i in range(num_jobs)}
+
+for test_file, duration in sorted_test_files:
+    # Find the job with the least total execution time
+    min_job = min(jobs, key=lambda j: jobs[j]['total_time'])
+    
+    # Add the test file to the selected job
+    jobs[min_job]['test_files'].append(test_file)
+    
+    # Update the total execution time for the selected job
+    jobs[min_job]['total_time'] += duration
+
+# Output the distributed test files for each job with total time in seconds and minutes
+result = {f'{job}': {'test_files': jobs[job]['test_files'], 'total_time_seconds': jobs[job]['total_time'], 'total_time_minutes': jobs[job]['total_time'] / 60} for job in jobs}
+print(json.dumps(result, indent=2))