Fix scheduled CI tests (#4742)

* Actually filter to Mesos tests Also run Mesos tests of we touch Mesos. It looks like #4646 added a bunch of Mesos test run steps but didn't include tests= so they just run all tests, even if the dependencies aren't there. * Don't import boto when it may not be installed * Stop pinning very old setuptools and pyyaml This basically reverts 60096d8 and should let us use a setuptools that is new enough for the Python versions we are using. * Run all tests on -fix-ci branches * Put Mesos AWS tests in the Mesos step * Improve WDL documentation (#4732) * Fix code block boundary * Make the CWL quickstart the main one * Talk about Python workflows instead of user scripts * Chase away all the Sphinx warnings so we know the docs should look right * Fail the docs build if the docstrings don't parse cleanly * Encourage installing with cwl and wdl extras * Qualify Python development * Reorganize docs to plug the workflow languages more * Talk a bit about WDL * Add conformance test and install info * Stop trying to draw inheritance diagrams since RtD doesn't give us a dot anyway --------- Co-authored-by: Lon Blauvelt <[email protected]> * Indent docstring to fix doc build failure --------- Co-authored-by: Lon Blauvelt <[email protected]>
DataBiosphere · Jan 11, 2024 · b31e3d5 · b31e3d5
1 parent 1d6e534
commit b31e3d5
Show file tree

Hide file tree

Showing 5 changed files with 41 additions and 24 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -109,6 +109,7 @@ py38_appliance_build:
     - if: $CI_PIPELINE_SOURCE == "schedule"
     - if: $CI_COMMIT_TAG
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
   stage: basic_tests
   script:
     - pwd
@@ -122,6 +123,7 @@ py39_appliance_build:
     - if: $CI_PIPELINE_SOURCE == "schedule"
     - if: $CI_COMMIT_TAG
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
   stage: basic_tests
   script:
     - pwd
@@ -135,6 +137,7 @@ py310_appliance_build:
     - if: $CI_PIPELINE_SOURCE == "schedule"
     - if: $CI_COMMIT_TAG
     - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
   stage: basic_tests
   script:
     - pwd
@@ -174,6 +177,7 @@ slurm_test:
   rules:
     - if: $CI_PIPELINE_SOURCE == "schedule"
     - if: $CI_COMMIT_TAG
+    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
   stage: integration
   script:
     - pwd
@@ -185,6 +189,7 @@ cwl_v1.2:
   rules:
     - if: $CI_PIPELINE_SOURCE == "schedule"
     - if: $CI_COMMIT_TAG
+    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
     - if: $CI_COMMIT_BRANCH
       changes:
         compare_to: 'refs/heads/master'
@@ -212,6 +217,7 @@ cwl_on_arm:
   rules:
     - if: $CI_PIPELINE_SOURCE == "schedule"
     - if: $CI_COMMIT_TAG
+    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
     - if: $CI_COMMIT_BRANCH
       changes:
         compare_to: 'refs/heads/master'
@@ -239,6 +245,7 @@ cwl_misc:
   rules:
     - if: $CI_PIPELINE_SOURCE != "schedule"
     - if: $CI_COMMIT_TAG
+    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
   stage: main_tests
   script:
     - pwd
@@ -302,6 +309,7 @@ jobstore_integration:
   rules:
     - if: $CI_PIPELINE_SOURCE == "schedule"
     - if: $CI_COMMIT_TAG
+    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
   stage: integration
   script:
     - pwd
@@ -318,6 +326,7 @@ server_integration:
   rules:
     - if: $CI_PIPELINE_SOURCE == "schedule"
     - if: $CI_COMMIT_TAG
+    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
   stage: integration
   script:
     - pwd
@@ -335,6 +344,7 @@ provisioner_integration:
   rules:
     - if: $CI_PIPELINE_SOURCE == "schedule"
     - if: $CI_COMMIT_TAG
+    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
   stage: integration
   script:
     - pwd
@@ -346,13 +356,14 @@ provisioner_integration:
     - export TOIL_TEST_INTEGRATIVE=True; export TOIL_AWS_KEYNAME=id_rsa; export TOIL_AWS_ZONE=us-west-2a
     # This reads GITLAB_SECRET_FILE_SSH_KEYS
     - python setup_gitlab_ssh.py
-    - make test threads="${TEST_THREADS}" tests="src/toil/test/sort/sortTest.py src/toil/test/provisioners/clusterScalerTest.py src/toil/test/utils/utilsTest.py::UtilsTest::testAWSProvisionerUtils src/toil/test/provisioners/aws/awsProvisionerTest.py"
+    - make test threads="${TEST_THREADS}" tests="src/toil/test/sort/sortTest.py src/toil/test/provisioners/clusterScalerTest.py src/toil/test/utils/utilsTest.py::UtilsTest::testAWSProvisionerUtils src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSProvisionerBenchTest src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSManagedAutoscaleTest"
 #    - make test tests=src/toil/test/provisioners/gceProvisionerTest.py  # needs env vars set to run
 
 google_jobstore:
   rules:
     - if: $CI_PIPELINE_SOURCE == "schedule"
     - if: $CI_COMMIT_TAG
+    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
   stage: integration
   script:
     - pwd
@@ -370,10 +381,17 @@ mesos:
   rules:
     - if: $CI_PIPELINE_SOURCE == "schedule"
     - if: $CI_COMMIT_TAG
+    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
+    - if: $CI_COMMIT_BRANCH
+      changes:
+        compare_to: 'refs/heads/master'
+        paths:
+          - 'src/toil/test/mesos/*'
+          - 'src/toil/batchSystems/mesos/*'
   stage: integration
   script:
     - pwd
-    - python3.10 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[mesos]
+    - python3.10 -m virtualenv venv && . venv/bin/activate && pip install -U pip wheel && make prepare && make develop extras=[mesos,aws]
     - python setup_gitlab_ssh.py && chmod 400 /root/.ssh/id_rsa
     - echo $'Host *\n    AddressFamily inet' > /root/.ssh/config
     - export LIBPROCESS_IP=127.0.0.1
@@ -383,20 +401,22 @@ mesos:
     - export GOOGLE_APPLICATION_CREDENTIALS=$GOOGLE_CREDENTIALS
     - export TOIL_GOOGLE_KEYNAME=id_rsa
     - export TOIL_GOOGLE_PROJECTID=toil-dev
-    - make test threads="${TEST_THREADS}" src/toil/test/mesos/MesosDataStructuresTest.py::DataStructuresTest
-    - make test threads="${TEST_THREADS}" src/toil/test/batchSystems/batchSystemTest.py::MesosBatchSystemTest
-    - make test threads="${TEST_THREADS}" src/toil/test/sort/sortTest.py::SortTest::testAwsMesos
-    - make test threads="${TEST_THREADS}" src/toil/test/sort/sortTest.py::SortTest::testFileMesos
-    - make test threads="${TEST_THREADS}" src/toil/test/sort/sortTest.py::SortTest::testGoogleMesos
-    - make test threads="${TEST_THREADS}" src/toil/test/cwl/cwlTest.py::CWLv10Test::test_mesos_cwl_conformance
-    - make test threads="${TEST_THREADS}" src/toil/test/cwl/cwlTest.py::CWLv10Test::test_mesos_cwl_conformance_with_caching
-    - make test threads="${TEST_THREADS}" src/toil/test/src/promisedRequirementTest.py::MesosPromisedRequirementsTest
+    - make test threads="${TEST_THREADS}" tests=src/toil/test/mesos/MesosDataStructuresTest.py::DataStructuresTest
+    - make test threads="${TEST_THREADS}" tests=src/toil/test/batchSystems/batchSystemTest.py::MesosBatchSystemTest
+    - make test threads="${TEST_THREADS}" tests=src/toil/test/sort/sortTest.py::SortTest::testAwsMesos
+    - make test threads="${TEST_THREADS}" tests=src/toil/test/sort/sortTest.py::SortTest::testFileMesos
+    - make test threads="${TEST_THREADS}" tests=src/toil/test/sort/sortTest.py::SortTest::testGoogleMesos
+    - make test threads="${TEST_THREADS}" tests=src/toil/test/cwl/cwlTest.py::CWLv10Test::test_mesos_cwl_conformance
+    - make test threads="${TEST_THREADS}" tests=src/toil/test/cwl/cwlTest.py::CWLv10Test::test_mesos_cwl_conformance_with_caching
+    - make test threads="${TEST_THREADS}" tests=src/toil/test/src/promisedRequirementTest.py::MesosPromisedRequirementsTest
+    - make test threads="${TEST_THREADS}" tests="src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSAutoscaleTest src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSStaticAutoscaleTest src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSAutoscaleTestMultipleNodeTypes src/toil/test/provisioners/aws/awsProvisionerTest.py::AWSRestartTest::testAutoScaledCluster"
 
 # Cactus-on-Kubernetes integration (as a script and not a pytest test)
 cactus_integration:
   rules:
     - if: $CI_PIPELINE_SOURCE == "schedule"
     - if: $CI_COMMIT_TAG
+    - if: $CI_COMMIT_BRANCH =~ /.*-fix-ci/
     - if: $CI_COMMIT_BRANCH
       changes:
         compare_to: 'refs/heads/master'

diff --git a/src/toil/test/provisioners/aws/awsProvisionerTest.py b/src/toil/test/provisioners/aws/awsProvisionerTest.py
@@ -29,6 +29,7 @@
                        integrative,
                        needs_aws_ec2,
                        needs_fetchable_appliance,
+                       needs_mesos,
                        slow,
                        timeLimit)
 from toil.test.provisioners.clusterTest import AbstractClusterTest
@@ -171,9 +172,6 @@ def _test(self, preemptibleJobs=False):
         venv_command = ['virtualenv', '--system-site-packages', '--python', exactPython, '--never-download', self.venvDir]
         self.sshUtil(venv_command)
 
-        upgrade_command = [self.pip(), 'install', 'setuptools==28.7.1', 'pyyaml==3.12']
-        self.sshUtil(upgrade_command)
-
         log.info('Set up script...')
         self._getScript()
 
@@ -214,6 +212,7 @@ def _test(self, preemptibleJobs=False):
 
 
 @integrative
+@needs_mesos
 @pytest.mark.timeout(1800)
 class AWSAutoscaleTest(AbstractAWSAutoscaleTest):
     def __init__(self, name):
@@ -282,6 +281,7 @@ def testSpotAutoScaleBalancingTypes(self):
 
 
 @integrative
+@needs_mesos
 @pytest.mark.timeout(2400)
 class AWSStaticAutoscaleTest(AWSAutoscaleTest):
     """Runs the tests on a statically provisioned cluster with autoscaling enabled."""
@@ -357,6 +357,7 @@ def _runScript(self, toilOptions):
 
 
 @integrative
+@needs_mesos
 @pytest.mark.timeout(1200)
 class AWSAutoscaleTestMultipleNodeTypes(AbstractAWSAutoscaleTest):
     def __init__(self, name):
@@ -396,6 +397,7 @@ def testAutoScale(self):
 
 
 @integrative
+@needs_mesos
 @pytest.mark.timeout(1200)
 class AWSRestartTest(AbstractAWSAutoscaleTest):
     """This test insures autoscaling works on a restarted Toil run."""
@@ -459,6 +461,7 @@ def testAutoScaledCluster(self):
 
 
 @integrative
+@needs_mesos
 @pytest.mark.timeout(1200)
 class PreemptibleDeficitCompensationTest(AbstractAWSAutoscaleTest):
     def __init__(self, name):

diff --git a/src/toil/test/provisioners/clusterTest.py b/src/toil/test/provisioners/clusterTest.py
@@ -18,8 +18,6 @@
 import time
 from uuid import uuid4
 
-import boto.ec2
-
 from toil.lib.aws import zone_to_region
 from toil.lib.retry import retry
 from toil.provisioners.aws import get_best_aws_zone
@@ -38,7 +36,9 @@ def __init__(self, methodName):
         self.clusterType = 'mesos'
         self.zone = get_best_aws_zone()
         assert self.zone is not None, "Could not determine AWS availability zone to test in; is TOIL_AWS_ZONE set?"
-        # We need a boto2 connection to EC2 to check on the cluster
+        # We need a boto2 connection to EC2 to check on the cluster.
+        # Since we are protected by needs_aws_ec2 we can import from boto.
+        import boto.ec2
         self.boto2_ec2 = boto.ec2.connect_to_region(zone_to_region(self.zone))
         # Where should we put our virtualenv?
         self.venvDir = '/tmp/venv'

diff --git a/src/toil/test/provisioners/gceProvisionerTest.py b/src/toil/test/provisioners/gceProvisionerTest.py
@@ -136,12 +136,6 @@ def _test(self, preemptibleJobs=False):
                         '--python', exactPython, '/home/venv']
         self.sshUtil(venv_command)
 
-        upgrade_command = ['/home/venv/bin/pip', 'install', 'setuptools==28.7.1']
-        self.sshUtil(upgrade_command)
-
-        yaml_command = ['/home/venv/bin/pip', 'install', 'pyyaml==3.12']
-        self.sshUtil(yaml_command)
-
         self._getScript()
 
         toilOptions = [self.jobStore,

diff --git a/src/toil/utils/toilDebugFile.py b/src/toil/utils/toilDebugFile.py
@@ -63,8 +63,8 @@ def printContentsOfJobStore(job_store: FileJobStore, job_id: Optional[str] = Non
 
     :param job_store: Job store to ask for files from.
     :param job_id: Default is None, which prints out all files in the jobStore.
-    If specified, it will print all jobStore files that have been written to the
-    jobStore by that job.
+        If specified, it will print all jobStore files that have been written
+        to the jobStore by that job.
     """
 
     # TODO: Implement the necessary methods for job stores other than