New AWS jobstore.

.gitlab-ci.yml

@@ -389,7 +389,6 @@ py313_main:
     TOIL_SKIP_DOCKER=true make test threads="${TEST_THREADS}" tests="src/toil/test/lib"
     echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"
 
-
 slurm_test:
   rules:
     - if: $CI_PIPELINE_SOURCE == "schedule"
@@ -464,7 +463,7 @@ cwl_badge:
     rm -rf badges1.2
     echo -e "\e[0Ksection_end:`date +%s`:download_tests\r\e[0K"
     echo -e "\e[0Ksection_start:`date +%s`:test\r\e[0KRun the CWL v1.2 conformance tests."
-    make test tests="src/toil/test/cwl/spec_v12/conformance_tests.cwltest.yaml" threads="${TEST_THREADS}" pytest_args="--cwl-runner-verbose --cwl-badgedir=badges1.2 --junit-xml=in-place-update-conformance-1.2.junit.xml --cwl-args='--relax-path-checks --clean=always --logDebug --statusWait=10 --retryCount=2 --eval-timeout=600 --bypass-file-store'"
+    make test tests="src/toil/test/cwl/spec_v12/conformance_tests.cwltest.yaml" threads="${TEST_THREADS}" pytest_args="--randomly-dont-reorganize --cwl-runner-verbose --cwl-badgedir=badges1.2 --junit-xml=in-place-update-conformance-1.2.junit.xml --cwl-args='--relax-path-checks --clean=always --logDebug --statusWait=10 --retryCount=2 --eval-timeout=600 --bypass-file-store'"
     echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"
   allow_failure: true
   artifacts:
@@ -497,7 +496,7 @@ cwl_badge_release:
     rm -rf badges1.2
     echo -e "\e[0Ksection_end:`date +%s`:download_tests\r\e[0K"
     echo -e "\e[0Ksection_start:`date +%s`:test\r\e[0KRun the CWL v1.2 conformance tests."
-    make test tests="src/toil/test/cwl/spec_v12/conformance_tests.cwltest.yaml" threads="${TEST_THREADS}" pytest_args="--cwl-runner-verbose --cwl-badgedir=badges1.2 --junit-xml=in-place-update-conformance-1.2.junit.xml --cwl-args='--relax-path-checks --clean=always --logDebug --statusWait=10 --retryCount=2 --eval-timeout=600 --bypass-file-store'"
+    make test tests="src/toil/test/cwl/spec_v12/conformance_tests.cwltest.yaml" threads="${TEST_THREADS}" pytest_args="--randomly-dont-reorganize --cwl-runner-verbose --cwl-badgedir=badges1.2 --junit-xml=in-place-update-conformance-1.2.junit.xml --cwl-args='--relax-path-checks --clean=always --logDebug --statusWait=10 --retryCount=2 --eval-timeout=600 --bypass-file-store'"
     echo -e "\e[0Ksection_end:`date +%s`:test\r\e[0K"
   allow_failure: true
   artifacts:

Makefile

@@ -110,7 +110,7 @@ marker=""
 threads:="auto"
 
 dist:="worksteal"
-pytest_args:=""
+pytest_args:="--randomly-dont-reorganize"
 
 # Only pass the threading options if running parallel tests. Otherwise we lose
 # live logging. See <https://stackoverflow.com/q/62533239>
@@ -199,7 +199,7 @@ test_offline: check_venv check_build_reqs
 test_1min: check_venv check_build_reqs
 	TOIL_SKIP_DOCKER=False \
 	TOIL_HISTORY=0 \
-	    python -m pytest $(verbose) $(durations) $(threadopts) -m "$(marker)" $(logging) --timeout=30  --maxfail=1 $(pytest_args) src/toil/test/batchSystems/batchSystemTest.py::SingleMachineBatchSystemTest::test_run_jobs src/toil/test/batchSystems/batchSystemTest.py::KubernetesBatchSystemBenchTest src/toil/test/server/serverTest.py::ToilWESServerBenchTest::test_get_service_info src/toil/test/cwl/cwlTest.py::TestCWLWorkflow::test_run_colon_output src/toil/test/jobStores/jobStoreTest.py::FileJobStoreTest::testUpdateBehavior
+	    python -m pytest $(verbose) $(durations) $(threadopts) -m "$(marker)" $(logging) --timeout=30 --maxfail=1 $(pytest_args) src/toil/test/batchSystems/batchSystemTest.py::SingleMachineBatchSystemTest::test_run_jobs src/toil/test/batchSystems/batchSystemTest.py::KubernetesBatchSystemBenchTest src/toil/test/server/serverTest.py::ToilWESServerBenchTest::test_get_service_info src/toil/test/cwl/cwlTest.py::TestCWLWorkflow::test_run_colon_output src/toil/test/jobStores/jobStoreTest.py::FileJobStoreTest::testUpdateBehavior
 
 ifdef TOIL_DOCKER_REGISTRY
 

contrib/admin/cleanup_aws_resources.py

@@ -70,8 +70,11 @@ def contains_toil_test_patterns(string):
 
 
 def matches(resource_name):
-    if (resource_name.endswith('--files') or resource_name.endswith('--jobs') or resource_name.endswith('_toil')
-            or resource_name.endswith('--internal') or resource_name.startswith('toil-s3test-')):
+    if (
+        resource_name.endswith('--files') or resource_name.endswith('--jobs')
+        or resource_name.endswith('--toil') or resource_name.endswith('_toil')
+        or resource_name.endswith('--internal') or resource_name.startswith('toil-s3test-')
+    ):
         if contains_toil_test_patterns(resource_name):
             return resource_name
 

contrib/admin/mypy-with-ignore.py

@@ -22,7 +22,8 @@ def main():
 
     # TODO: Remove these paths as typing is added and mypy conflicts are addressed.
     # These are handled as path prefixes.
-    ignore_paths = [os.path.abspath(f) for f in [
+
+    ignore_paths = [os.path.join(pkg_root, f) for f in [
         'docs/_build',
         'docker/Dockerfile.py',
         'docs/conf.py',
@@ -61,12 +62,8 @@ def main():
         'src/toil/jobStores/fileJobStore.py',
         'src/toil/jobStores/__init__.py',
         'src/toil/jobStores/googleJobStore.py',
-        'src/toil/jobStores/aws/utils.py',
-        'src/toil/jobStores/aws/jobStore.py',
-        'src/toil/jobStores/aws/__init__.py',
         'src/toil/utils/__init__.py',
         'src/toil/lib/throttle.py',
-        'src/toil/lib/iterables.py',
         'src/toil/lib/bioio.py',
         'src/toil/lib/ec2.py',
         'src/toil/lib/expando.py',
@@ -87,7 +84,6 @@ def main():
         'src/toil/test/cactus',
         'src/toil/test/docs',
         'src/toil/test/jobStores',
-        'src/toil/test/lib/aws',
         'src/toil/test/lib/test_conversions.py',
         'src/toil/test/lib/test_ec2.py',
         'src/toil/test/lib/test_misc.py',

docs/appendices/aws_min_permissions.rst

@@ -18,7 +18,6 @@ with the ``--awsEc2ProfileArn`` parameter.
                 "Action": [
                     "ec2:*",
                     "s3:*",
-                    "sdb:*",
                     "iam:PassRole"
                 ],
                 "Resource": "*"

requirements-dev.txt

@@ -2,7 +2,8 @@ pytest>=6.2.1,<9
 pytest-cov>=2.12.1,<7
 pytest-timeout>=1.4.2,<3
 pytest_httpserver>=1.1.3,<2
-pytest-subtests
+pytest-subtests>=0.14.2,<0.15
+pytest-randomly>=3.16.0,<4
 stubserver>=1.1,<2
 setuptools>=65.5.1,<81
 sphinx>=7,<9

src/toil/fileStores/__init__.py

@@ -28,7 +28,7 @@ class FileID(str):
     the job store if unavailable in the ID.
     """
 
-    def __new__(cls, fileStoreID: str, *args: Any) -> "FileID":
+    def __new__(cls, fileStoreID: str, *args: Any, **kwargs: dict[str, Any]) -> "FileID":
         return super().__new__(cls, fileStoreID)
 
     def __init__(self, fileStoreID: str, size: int, executable: bool = False) -> None:

src/toil/fileStores/cachingFileStore.py

@@ -1207,7 +1207,7 @@ def open(self, job: Job) -> Generator[None, None, None]:
             # its temp dir and database entry.
             self._deallocateSpaceForJob()
 
-    def writeGlobalFile(self, localFileName, cleanup=False, executable=False):
+    def writeGlobalFile(self, localFileName, cleanup=False):
         """
         Creates a file in the jobstore and returns a FileID reference.
         """

src/toil/job.py

@@ -813,7 +813,6 @@ class JobDescription(Requirer):
     Subclassed into variants for checkpoint jobs and service jobs that have
     their specific parameters.
     """
-
     def __init__(
         self,
         requirements: Mapping[str, Union[int, str, float, bool, list]],

src/toil/jobStores/abstractJobStore.py

@@ -90,7 +90,7 @@ def __init__(self, url: ParseResult) -> None:
 class NoSuchJobException(Exception):
     """Indicates that the specified job does not exist."""
 
-    def __init__(self, jobStoreID: FileID):
+    def __init__(self, jobStoreID: Union[FileID, str]):
         """
         :param str jobStoreID: the jobStoreID that was mistakenly assumed to exist
         """
@@ -100,7 +100,7 @@ def __init__(self, jobStoreID: FileID):
 class ConcurrentFileModificationException(Exception):
     """Indicates that the file was attempted to be modified by multiple processes at once."""
 
-    def __init__(self, jobStoreFileID: FileID):
+    def __init__(self, jobStoreFileID: Union[FileID, str]):
         """
         :param jobStoreFileID: the ID of the file that was modified by multiple workers
                or processes concurrently
@@ -112,7 +112,7 @@ class NoSuchFileException(Exception):
     """Indicates that the specified file does not exist."""
 
     def __init__(
-        self, jobStoreFileID: FileID, customName: Optional[str] = None, *extra: Any
+        self, jobStoreFileID: Union[FileID, str], customName: Optional[str] = None, *extra: Any
     ):
         """
         :param jobStoreFileID: the ID of the file that was mistakenly assumed to exist
@@ -140,11 +140,12 @@ class NoSuchJobStoreException(LocatorException):
     def __init__(self, locator: str, prefix: str):
         """
         :param str locator: The location of the job store
+        :param str prefix: The type of job store
         """
         super().__init__(
             "The job store '%s' does not exist, so there is nothing to restart.",
             locator,
-            prefix,
+            prefix
         )
 
 
@@ -159,7 +160,7 @@ def __init__(self, locator: str, prefix: str):
             "The job store '%s' already exists. Use --restart to resume the workflow, or remove "
             "the job store with 'toil clean' to start the workflow from scratch.",
             locator,
-            prefix,
+            prefix
         )
 
 
@@ -240,7 +241,12 @@ def resume(self) -> None:
 
     @property
     def config(self) -> Config:
-        """Return the Toil configuration associated with this job store."""
+        """
+        Return the Toil configuration associated with this job store.
+
+        :raises AttributeError: if the config has not yet been assigned (i.e.
+            during :meth:`resume`).
+        """
         return self.__config
 
     @property
@@ -561,7 +567,6 @@ def _default_export_file(
                 executable = jobStoreFileID.executable
             otherCls._write_to_url(readable, url, executable)
 
-
     @abstractmethod
     def destroy(self) -> None:
         """
@@ -930,15 +935,6 @@ def assign_job_id(self, job_description: JobDescription) -> None:
         """
         raise NotImplementedError()
 
-    @contextmanager
-    def batch(self) -> Iterator[None]:
-        """
-        If supported by the batch system, calls to create() with this context
-        manager active will be performed in a batch after the context manager
-        is released.
-        """
-        yield
-
     @deprecated(new_function_name="create_job")
     def create(self, jobDescription: JobDescription) -> JobDescription:
         return self.create_job(jobDescription)
@@ -1036,6 +1032,15 @@ def load_job(self, job_id: str) -> JobDescription:
     def update(self, jobDescription: JobDescription) -> None:
         return self.update_job(jobDescription)
 
+    @contextmanager
+    def batch(self) -> Iterator[None]:
+        """
+        If supported by the batch system, calls to create() with this context
+        manager active will be performed in a batch after the context manager
+        is released.
+        """
+        yield
+
     @abstractmethod
     def update_job(self, job_description: JobDescription) -> None:
         """
@@ -1206,14 +1211,14 @@ def get_empty_file_store_id(
         Creates an empty file in the job store and returns its ID.
         Call to fileExists(getEmptyFileStoreID(jobStoreID)) will return True.
 
-        :param str job_id: the id of a job, or None. If specified, the may be associated
+        :param job_id: the id of a job, or None. If specified, the may be associated
                with that job in a job-store-specific way. This may influence the returned ID.
 
-        :param bool cleanup: Whether to attempt to delete the file when the job
+        :param cleanup: Whether to attempt to delete the file when the job
                whose jobStoreID was given as jobStoreID is deleted with
                jobStore.delete(job). If jobStoreID was not given, does nothing.
 
-        :param str basename: If supported by the implementation, use the given
+        :param basename: If supported by the implementation, use the given
                file basename so that when searching the job store with a query
                matching that basename, the file will be detected.