Skip to content

Commit

Permalink
Merge pull request #122 from mskcc/develop
Browse files Browse the repository at this point in the history
Develop, for release 1.6.0
  • Loading branch information
allanbolipata authored Dec 28, 2020
2 parents 6e21b05 + e84874e commit 3537fa1
Show file tree
Hide file tree
Showing 12 changed files with 169 additions and 73 deletions.
18 changes: 18 additions & 0 deletions batch_systems/lsf_client/lsf_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,24 @@ def submit(self, command, job_args, stdout):
universal_newlines=True, env=current_env)
return self._parse_procid(process.stdout)

def abort(self, external_job_id):
'''
Kill LSF job
Args:
external_job_id (str): external_job_id
Returns:
bool: successful
'''
bkill_command = ['bkill', external_job_id]
process = subprocess.run(
bkill_command, check=True, stdout=subprocess.PIPE,
universal_newlines=True)
if process.returncode == 0:
return True
return False

def parse_bjobs(self, bjobs_output_str):
"""
Parse the output of bjobs into a descriptive dict
Expand Down
4 changes: 2 additions & 2 deletions container/celery_services.def
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@ Includecmd: no
-Q ${RIDGEBACK_DEFAULT_QUEUE} \
-f ${CELERY_LOG_PATH}/ridgeback_worker.log \
--pidfile ${CELERY_PID_PATH}/${CELERY_EVENT_QUEUE_PREFIX}.ridgeback_worker.pid \
-n ${CELERY_EVENT_QUEUE_PREFIX}.ridgeback_worker
-n ${CELERY_EVENT_QUEUE_PREFIX}.ridgeback_worker &


%post
Expand All @@ -108,4 +108,4 @@ Includecmd: no
&& git clone https://github.com/mskcc/ridgeback --branch $RIDGEBACK_BRANCH
cd /usr/bin/ridgeback \
&& python3 -m pip install python-ldap \
&& pip3 install -r requirements.txt
&& pip3 install -r requirements.txt
2 changes: 1 addition & 1 deletion ridgeback/settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
SECRET_KEY = '3gpghwoqas_6ei_efvb%)5s&lwgs#o99c9(ovmi=1od*e6ezvw'

# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
DEBUG = False

ALLOWED_HOSTS = os.environ.get('RIDGEBACK_ALLOWED_HOSTS', 'localhost').split(',')

Expand Down
3 changes: 3 additions & 0 deletions submitter/jobsubmitter.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,9 @@ def submit(self):
def status(self, external_id):
return self.lsf_client.status(external_id)

def abort(self, external_id):
return self.lsf_client.abort(external_id)

def get_outputs(self):
error_message = None
result_json = None
Expand Down
50 changes: 24 additions & 26 deletions tests/test_migrations.py
Original file line number Diff line number Diff line change
@@ -1,82 +1,80 @@
import os
from django.apps import apps
from django.test import TestCase
from django.db.migrations.executor import MigrationExecutor
from django.db import connection
from toil_orchestrator import migrations
from django_test_migrations.contrib.unittest_case import MigratorTestCase
from toil_orchestrator.models import Status
import json

current_app = 'toil_orchestrator'


class TestMigration(MigratorTestCase):

def setUp(self,migrate_from,migrate_to):
def setUp(self, migrate_from, migrate_to):
self.migrate_from = (current_app, self.get_migration_file_name(migrate_from))
self.migrate_to = (current_app, self.get_migration_file_name(migrate_to))
super().setUp()

def get_migration_file_name(self,number):
def get_migration_file_name(self, number):
migration_path = migrations.__path__[0]
migration_files = os.listdir(migration_path)
for single_file in migration_files:
single_file_split = single_file.split('_')
if str(number) in single_file_split[0]:
return os.path.splitext(single_file)[0]


class TestMessageMigration(TestMigration):

def setUp(self):
super().setUp(13,17)
super().setUp(13, 17)

def prepare(self):
job = self.old_state.apps.get_model('toil_orchestrator','job')
job = self.old_state.apps.get_model('toil_orchestrator', 'job')
job.objects.create(
app={
"github": {
"github": {
"entrypoint": "tempo.cwl",
"repository": "https://github.com/mskcc-cwl/tempo"
}
},
}
},
message="sample_info_string"

)
)

def test_migration(self):
job = self.new_state.apps.get_model('toil_orchestrator','job')
job = self.new_state.apps.get_model('toil_orchestrator', 'job')
current_job = job.objects.all().first()

def test_message(self):
job = self.new_state.apps.get_model('toil_orchestrator','job')
job = self.new_state.apps.get_model('toil_orchestrator', 'job')
current_job = job.objects.all().first()
new_message_obj = json.loads(current_job.message)
self.assertEqual(new_message_obj['info'],"sample_info_string")
self.assertEqual(new_message_obj['info'], "sample_info_string")


class TestDateMigration(TestMigration):

def setUp(self):
super().setUp(12,13)
super().setUp(12, 13)

def prepare(self):
job = self.old_state.apps.get_model('toil_orchestrator','job')
job = self.old_state.apps.get_model('toil_orchestrator', 'job')
job.objects.create(
app={
"github": {
"github": {
"entrypoint": "tempo.cwl",
"repository": "https://github.com/mskcc-cwl/tempo"
}
},
}
},
status=Status.COMPLETED,
finished=None,
started=None,
submitted=None
)
)

def test_date_change(self):
job = self.new_state.apps.get_model('toil_orchestrator','job')
job = self.new_state.apps.get_model('toil_orchestrator', 'job')
current_job = job.objects.all().first()
self.assertNotEqual(current_job.started,None)
self.assertNotEqual(current_job.submitted,None)
self.assertNotEqual(current_job.finished,None)

self.assertNotEqual(current_job.started, None)
self.assertNotEqual(current_job.submitted, None)
self.assertNotEqual(current_job.finished, None)
29 changes: 14 additions & 15 deletions tests/test_tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,21 +6,21 @@

class TestTasks(TestCase):
fixtures = [
"toil_orchestrator.job.json"
"toil_orchestrator.job.json"
]

def setUp(self):
self.current_job = Job.objects.first()

def test_failure_to_submit(self):
on_failure_to_submit(None,None,None,[self.current_job.id],None,None)
on_failure_to_submit(None, None, None, [self.current_job.id], None, None)
self.current_job.refresh_from_db()
self.assertEqual(self.current_job.status,Status.FAILED)
self.assertNotEqual(self.current_job.finished,None)
self.assertEqual(self.current_job.status, Status.FAILED)
self.assertNotEqual(self.current_job.finished, None)
info_message = get_message(self.current_job)['info']
log_path = get_message(self.current_job)['log']
self.assertEqual(info_message,'Failed to submit job')
self.assertNotEqual(log_path,None)
self.assertEqual(info_message, 'Failed to submit job')
self.assertNotEqual(log_path, None)

@patch('submitter.jobsubmitter.JobSubmitter.__init__')
@patch('submitter.jobsubmitter.JobSubmitter.submit')
Expand All @@ -31,8 +31,8 @@ def test_submit(self, save_job_info, submit, init):
submit.return_value = self.current_job.external_id, self.current_job.job_store_location, self.current_job.working_dir, self.current_job.output_directory
submit_jobs_to_lsf(self.current_job.id)
self.current_job.refresh_from_db()
self.assertEqual(self.current_job.status,Status.PENDING)
self.assertEqual(self.current_job.finished,None)
self.assertEqual(self.current_job.status, Status.PENDING)
self.assertEqual(self.current_job.finished, None)

@patch('toil_orchestrator.tasks.get_job_info_path')
@patch('submitter.jobsubmitter.JobSubmitter.__init__')
Expand Down Expand Up @@ -67,15 +67,15 @@ def test_fail(self, status, init, get_job_info_path):
failed_jobs = get_message(self.current_job)['failed_jobs']
unknown_jobs = get_message(self.current_job)['unknown_jobs']
expected_failed_jobs = {
'failed_job_1': ['failed_job_1_id'],
'failed_job_2': ['failed_job_2_id']
'failed_job_1': ['failed_job_1_id'],
'failed_job_2': ['failed_job_2_id']
}
expected_unknown_jobs = {
'unknown_job': ['unknown_job_id_1', 'unknown_job_id_2']
'unknown_job': ['unknown_job_id_1', 'unknown_job_id_2']
}
self.assertEqual(info_message,'submitter reason')
self.assertEqual(info_message, 'submitter reason')
self.assertEqual(failed_jobs, expected_failed_jobs)
self.assertEqual(unknown_jobs,expected_unknown_jobs)
self.assertEqual(unknown_jobs, expected_unknown_jobs)

@patch('toil_orchestrator.tasks.get_job_info_path')
@patch('submitter.jobsubmitter.JobSubmitter.__init__')
Expand Down Expand Up @@ -107,5 +107,4 @@ def test_fail_not_submitted(self):
expected_unknown_jobs = {}
self.assertTrue('External id not provided' in info_message)
self.assertEqual(failed_jobs, expected_failed_jobs)
self.assertEqual(unknown_jobs,expected_unknown_jobs)

self.assertEqual(unknown_jobs, expected_unknown_jobs)
4 changes: 3 additions & 1 deletion toil_orchestrator/celery.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,9 @@
# }

app.conf.task_routes = {'toil_orchestrator.tasks.submit_jobs_to_lsf': {'queue': settings.RIDGEBACK_DEFAULT_QUEUE},
'toil_orchestrator.tasks.cleanup_folder': {'queue': settings.RIDGEBACK_DEFAULT_QUEUE}}
'toil_orchestrator.tasks.cleanup_folder': {'queue': settings.RIDGEBACK_DEFAULT_QUEUE},
'toil_orchestrator.tasks.abort_job': {'queue': settings.RIDGEBACK_DEFAULT_QUEUE},
}
#
# app.conf.task_queues = (
# Queue('toil', routing_key='submit'),
Expand Down
4 changes: 2 additions & 2 deletions toil_orchestrator/fixtures/toil_orchestrator.job.json
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@
"pk": "316a4495-ad98-4799-9b73-b4990d3cc259",
"fields": {
"root": "1cdbb924-9078-400f-b4b9-4bef1ea5ce1a",
"status": 5,
"status": 6,
"started": null,
"submitted": null,
"finished": null,
Expand All @@ -159,7 +159,7 @@
"pk": "ca899e87-0f7c-4e81-9eb2-0ffd47d0935b",
"fields": {
"root": "1cdbb924-9078-400f-b4b9-4bef1ea5ce1a",
"status": 5,
"status": 6,
"started": null,
"submitted": null,
"finished": null,
Expand Down
34 changes: 34 additions & 0 deletions toil_orchestrator/migrations/0018_auto_20201221_1300.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
# Generated by Django 2.2.10 on 2020-12-21 18:00

from django.db import migrations, models
import toil_orchestrator.models


def migrate_statuses(apps, schema_editor):
Job = apps.get_model('toil_orchestrator', 'Job')
jobs = Job.objects.all()
for job in jobs:
if job.status == 5:
job.status = 6
job.save()


class Migration(migrations.Migration):

dependencies = [
('toil_orchestrator', '0017_auto_20200624_1635'),
]

operations = [
migrations.AlterField(
model_name='commandlinetooljob',
name='status',
field=models.IntegerField(choices=[(0, 'CREATED'), (1, 'PENDING'), (2, 'RUNNING'), (3, 'COMPLETED'), (4, 'FAILED'), (5, 'ABORTED'), (6, 'UNKNOWN')], default=0),
),
migrations.AlterField(
model_name='job',
name='status',
field=models.IntegerField(choices=[(0, 'CREATED'), (1, 'PENDING'), (2, 'RUNNING'), (3, 'COMPLETED'), (4, 'FAILED'), (5, 'ABORTED'), (6, 'UNKNOWN')], default=toil_orchestrator.models.Status(0)),
),
migrations.RunPython(migrate_statuses)
]
3 changes: 2 additions & 1 deletion toil_orchestrator/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,8 @@ class Status(IntEnum):
RUNNING = 2
COMPLETED = 3
FAILED = 4
UNKNOWN = 5
ABORTED = 5
UNKNOWN = 6


class BaseModel(models.Model):
Expand Down
Loading

0 comments on commit 3537fa1

Please sign in to comment.