From 0f256016823e4c84c81537b643f7a24c28f9ffa9 Mon Sep 17 00:00:00 2001 From: Adrian Fraiha Date: Wed, 27 Jan 2021 16:15:35 -0500 Subject: [PATCH 1/6] batch job status calls --- toil_orchestrator/serializers.py | 11 +++++++++++ toil_orchestrator/views/job_view.py | 15 +++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/toil_orchestrator/serializers.py b/toil_orchestrator/serializers.py index a79e45a6..5c726778 100644 --- a/toil_orchestrator/serializers.py +++ b/toil_orchestrator/serializers.py @@ -90,6 +90,12 @@ class Meta: model = CommandLineToolJob fields = '__all__' +class JobIdsSerializer(serializers.Serializer): + job_ids = serializers.ListField( + allow_empty=True, + child=serializers.UUIDField(required=True), + ) + class JobSerializer(serializers.ModelSerializer): status = serializers.SerializerMethodField() @@ -105,6 +111,11 @@ class Meta: message = MessageField(required=False) commandlinetooljob_set = CommandLineToolJobSerializer(many=True, required=False) + +class JobStatusSerializer(serializers.Serializer): + jobs = serializers.DictField(child=JobSerializer()) + + class JobSubmitSerializer(JobSerializer): diff --git a/toil_orchestrator/views/job_view.py b/toil_orchestrator/views/job_view.py index a68f43e9..29a14f0c 100644 --- a/toil_orchestrator/views/job_view.py +++ b/toil_orchestrator/views/job_view.py @@ -1,5 +1,5 @@ from toil_orchestrator.models import Job, Status -from toil_orchestrator.serializers import JobSerializer, JobSubmitSerializer, JobResumeSerializer +from toil_orchestrator.serializers import JobSerializer, JobSubmitSerializer, JobResumeSerializer, JobIdsSerializer, JobStatusSerializer from toil_orchestrator.tasks import submit_jobs_to_lsf, abort_job from rest_framework import mixins from rest_framework import status @@ -46,6 +46,17 @@ def resume(self, request, pk=None, *args, **kwargs): except Job.DoesNotExist: return Response("Could not find the indicated job to resume", status=status.HTTP_404_NOT_FOUND) + @swagger_auto_schema(request_body=JobIdsSerializer(), responses={status.HTTP_200_OK: JobStatusSerializer}) + @action(detail=False, methods=['post']) + def statuses(self, request): + serializer = JobIdsSerializer(data=request.data) + if not serializer.is_valid(): + return Response(serializer.errors, + status=status.HTTP_400_BAD_REQUEST) + + resp_serializer = JobSerializer(self.queryset.filter(id__in=serializer.data.get("job_ids")), many=True) + return Response(resp_serializer.data) + @swagger_auto_schema(responses={status.HTTP_200_OK: JobSerializer}) @action(detail=True, methods=['get']) def abort(self, request, pk=None, *args, **kwargs): @@ -56,7 +67,7 @@ def abort(self, request, pk=None, *args, **kwargs): abort_job.delay(str(pk)) return Response("Job aborted", status=status.HTTP_200_OK) - @swagger_auto_schema(request_body=JobSubmitSerializer, responses={201: JobSerializer}) + @swagger_auto_schema(request_body=JobSubmitSerializer, responses={status.HTTP_201_CREATED: JobSerializer}) def create(self, request, *args, **kwargs): return self.validate_and_save(request.data) From ae06c7da4bce7e8473047242be9c052be76e9f0c Mon Sep 17 00:00:00 2001 From: Adrian Fraiha Date: Wed, 27 Jan 2021 16:46:20 -0500 Subject: [PATCH 2/6] fix response --- toil_orchestrator/views/job_view.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/toil_orchestrator/views/job_view.py b/toil_orchestrator/views/job_view.py index 29a14f0c..a6c26f8f 100644 --- a/toil_orchestrator/views/job_view.py +++ b/toil_orchestrator/views/job_view.py @@ -54,8 +54,14 @@ def statuses(self, request): return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) - resp_serializer = JobSerializer(self.queryset.filter(id__in=serializer.data.get("job_ids")), many=True) - return Response(resp_serializer.data) + resp_serializer = JobStatusSerializer(data={ + "jobs": {str(j["id"]): j for j in self.queryset.filter(id__in=serializer.validated_data.get("job_ids")).values()} + }) + if resp_serializer.is_valid(): + return Response(resp_serializer.validated_data) + else: + return Response(resp_serializer.errors, + status=status.HTTP_500_INTERNAL_SERVER_ERROR) @swagger_auto_schema(responses={status.HTTP_200_OK: JobSerializer}) @action(detail=True, methods=['get']) From a29a7d5471f9cfecdeb7b9576606dc6f3ab24d83 Mon Sep 17 00:00:00 2001 From: Nikhil Kumar Date: Wed, 3 Feb 2021 11:51:31 -0500 Subject: [PATCH 3/6] Fixed psycopg2 dependency build error --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 25c8ce78..28c798d2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ Django==2.2.13 -psycopg2==2.7.4 +psycopg2==2.8.6 djangorestframework==3.9.4 markdown==3.1.1 django-filter==2.1.0 @@ -12,6 +12,6 @@ celery==4.4.7 cwltool==2.0.20200122124526 ruamel.yaml<=0.16.5 schema-salad==5.0.20200122085940 -psycopg2-binary==2.8.4 +psycopg2-binary==2.8.6 mock==4.0.2 dj-static==0.0.6 From 3fd5bff4441cefaff362e2e9da43baac01d28f4d Mon Sep 17 00:00:00 2001 From: Nikhil Date: Wed, 3 Feb 2021 16:52:18 -0500 Subject: [PATCH 4/6] Refactored job statuses to add commandline job status --- toil_orchestrator/serializers.py | 2 +- toil_orchestrator/views/job_view.py | 14 +++++++++----- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/toil_orchestrator/serializers.py b/toil_orchestrator/serializers.py index 5c726778..8c0ab8db 100644 --- a/toil_orchestrator/serializers.py +++ b/toil_orchestrator/serializers.py @@ -113,7 +113,7 @@ class Meta: class JobStatusSerializer(serializers.Serializer): - jobs = serializers.DictField(child=JobSerializer()) + jobs = serializers.DictField() class JobSubmitSerializer(JobSerializer): diff --git a/toil_orchestrator/views/job_view.py b/toil_orchestrator/views/job_view.py index a6c26f8f..a50748d8 100644 --- a/toil_orchestrator/views/job_view.py +++ b/toil_orchestrator/views/job_view.py @@ -53,12 +53,16 @@ def statuses(self, request): if not serializer.is_valid(): return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) - - resp_serializer = JobStatusSerializer(data={ - "jobs": {str(j["id"]): j for j in self.queryset.filter(id__in=serializer.validated_data.get("job_ids")).values()} - }) + job_ids = serializer.validated_data.get("job_ids") + job_status_data = {} + for single_job in self.queryset.filter(id__in=job_ids): + job_obj = JobSerializer(single_job) + job_data = job_obj.data + job_id = job_data["id"] + job_status_data[job_id] = job_data + resp_serializer = JobStatusSerializer(data={'jobs':job_status_data}) if resp_serializer.is_valid(): - return Response(resp_serializer.validated_data) + return Response(resp_serializer.data) else: return Response(resp_serializer.errors, status=status.HTTP_500_INTERNAL_SERVER_ERROR) From efa3d79bf0d6e93bbe175c4b7b5ce793b0bdc74a Mon Sep 17 00:00:00 2001 From: Sinisa Ivkovic Date: Fri, 12 Feb 2021 14:50:29 -0500 Subject: [PATCH 5/6] Fix updating log path --- toil_orchestrator/tasks.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/toil_orchestrator/tasks.py b/toil_orchestrator/tasks.py index 04001af9..f9a98249 100644 --- a/toil_orchestrator/tasks.py +++ b/toil_orchestrator/tasks.py @@ -42,7 +42,7 @@ def get_message(job_obj): def set_message(job_obj, message_obj): message_str = json.dumps(message_obj, sort_keys=True, indent=1, cls=DjangoJSONEncoder) job_obj.message = message_str - job_obj.save() + job_obj.save(update_fields=['message']) def update_message_by_key(job_obj, key, value): @@ -98,8 +98,12 @@ def submit_jobs_to_lsf(self, job_id): job.output_directory = job_output_dir job.status = Status.PENDING log_path = os.path.join(job_work_dir, 'lsf.log') - update_message_by_key(job,'log',log_path) - job.save() + update_message_by_key(job, 'log', log_path) + job.save(update_fields=['external_id', + 'job_store_location', + 'working_dir', + 'output_directory', + 'status']) @shared_task(bind=True, max_retries=10, retry_jitter=True, retry_backoff=60) From 9771f75d6d2dd6b7dbdb6b4149c50a0e7d92c48b Mon Sep 17 00:00:00 2001 From: Nikhil Kumar Date: Tue, 16 Feb 2021 14:24:38 -0500 Subject: [PATCH 6/6] Bumped version --- ridgeback/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ridgeback/__init__.py b/ridgeback/__init__.py index 8d6f94b6..a3174951 100644 --- a/ridgeback/__init__.py +++ b/ridgeback/__init__.py @@ -1 +1 @@ -__version__="1.5.0" +__version__="1.7.0"