diff --git a/orchestrator/models.py b/orchestrator/models.py old mode 100644 new mode 100755 diff --git a/submitter/app/app.py b/submitter/app/app.py old mode 100644 new mode 100755 index 20bab91..0c73389 --- a/submitter/app/app.py +++ b/submitter/app/app.py @@ -11,7 +11,11 @@ def factory(app): repo = app["github"]["repository"] entrypoint = app["github"]["entrypoint"] version = app["github"].get("version", "master") - return GithubApp(repo, entrypoint, version) + if app["github"].get("nfcore_template"): + nfcore_template = app["github"]["nfcore_template"] + else: + nfcore_template = None + return GithubApp(repo, entrypoint, nfcore_template, version) elif app.get("base64"): raise Exception("Base64 app not implemented yet") elif app.get("app"): @@ -32,11 +36,12 @@ class GithubApp(App): type = "github" logger = logging.getLogger(__name__) - def __init__(self, github, entrypoint, version="master"): + def __init__(self, github, entrypoint, nfcore_template, version="master"): super().__init__() self.github = github self.entrypoint = entrypoint self.version = version + self.nfcore_template = nfcore_template def resolve(self, location): dirname = os.path.join(location, self._extract_dirname_from_github_link()) diff --git a/submitter/nextflow_submitter/nextflow_jobsubmitter.py b/submitter/nextflow_submitter/nextflow_jobsubmitter.py old mode 100644 new mode 100755 index 7f31c8a..749d581 --- a/submitter/nextflow_submitter/nextflow_jobsubmitter.py +++ b/submitter/nextflow_submitter/nextflow_jobsubmitter.py @@ -1,6 +1,7 @@ import os import shutil import hashlib +import json from django.conf import settings from submitter import JobSubmitter @@ -47,6 +48,10 @@ def __init__( JobSubmitter.__init__(self, job_id, app, inputs, walltime, tool_walltime, memlimit, log_dir, app_name) self.resume_jobstore = resume_jobstore dir_config = settings.PIPELINE_CONFIG.get(self.app_name) + if self.app.nfcore_template: + self.cli_output_name = "--outdir" + else: + self.cli_output_name = "--outDir" if not dir_config: dir_config = settings.PIPELINE_CONFIG["NA"] if resume_jobstore: @@ -117,34 +122,53 @@ def _size(self, path): except Exception: return 0 + def _output_construct(self, path): + location = self._location(path) + basename = self._basename(path) + checksum = self._checksum(path) + size = self._size(path) + nameroot = self._nameroot(path) + nameext = self._nameext(path) + file_obj = { + "location": location, + "basename": basename, + "checksum": checksum, + "size": size, + "nameroot": nameroot, + "nameext": nameext, + "class": "File", + } + return file_obj + def get_outputs(self): - result = list() error_message = None - try: - with open(self.inputs["outputs"]) as f: - files = f.readlines() - for f in files: - path = f.strip() - location = self._location(path) - basename = self._basename(path) - checksum = self._checksum(path) - size = self._size(path) - nameroot = self._nameroot(path) - nameext = self._nameext(path) - file_obj = { - "location": location, - "basename": basename, - "checksum": checksum, - "size": size, - "nameroot": nameroot, - "nameext": nameext, - "class": "File", - } - result.append(file_obj) - except FileNotFoundError: - error_message = "Could not find %s" % self.inputs["outputs"] - except Exception: - error_message = "Could not parse %s" % self.inputs["outputs"] + result = list() + prov_file = os.path.join(self.job_outputs_dir, "manifest.json") + if os.path.exists(prov_file): + try: + with open(prov_file, "r") as f: + prov_json = json.loads(f.read()) + published_json = prov_json["published"] + for f in published_json: + path = f["target"] + file_obj = self._output_construct(path) + result.append(file_obj) + except (IndexError, ValueError): + error_message = "Could not parse json from %s" % prov_file + except FileNotFoundError: + error_message = "Could not find %s" % prov_file + else: + try: + with open(self.inputs["outputs"]) as f: + files = f.readlines() + for f in files: + path = f.strip() + file_obj = self._output_construct(path) + result.append(file_obj) + except FileNotFoundError: + error_message = "Could not find %s" % self.inputs["outputs"] + except Exception: + error_message = "Could not parse %s" % self.inputs["outputs"] result_json = {"outputs": result} return result_json, error_message @@ -172,8 +196,8 @@ def _dump_app_inputs(self): inputs = self.inputs.get("inputs", []) for i in inputs: input_map[i["name"]] = self._dump_input(i["name"], i["content"], self.job_work_dir) - if self.log_dir: - input_map[i["name"]] = self._dump_input(i["name"], i["content"], self.log_dir) + # if self.log_dir: + # input_map[i["name"]] = self._dump_input(i["name"], i["content"], self.log_dir) config = self.inputs.get("config") if config: self._dump_config(config) @@ -220,8 +244,8 @@ def _command_line(self): "-profile", profile, "-w", - self.job_store_dir, - "--outDir", + self.job_work_dir, + self.cli_output_name, self.job_outputs_dir, ] for k, v in self.inputs_location.items():