feat: split output files into subdirectories

.github/workflows/main.yml

@@ -69,7 +69,7 @@ jobs:
 
       - name: Free Disk Space (Ubuntu)
         if: ${{ matrix.case.cleanup }}
-        uses: jlumbroso/[email protected].0
+        uses: jlumbroso/[email protected].1
         with:
           # this might remove tools that are actually needed,
           # if set to "true" but frees about 6 GB

workflow/rules/annotation.smk

@@ -1,22 +1,22 @@
 rule annotate_candidate_variants:
     input:
-        calls="results/candidate-calls/{group}.{caller}.{scatteritem}.bcf",
+        calls="results/calls/candidates/{caller}/{group}/{group}.{scatteritem}.bcf",
         cache=access.random("resources/vep/cache"),
         plugins=access.random("resources/vep/plugins"),
         fasta=access.random(genome),
         fai=genome_fai,
     output:
-        calls="results/candidate-calls/{group}.{caller}.{scatteritem}.annotated.bcf",
-        stats="results/candidate-calls/{group}.{caller}.{scatteritem}.stats.html",
+        calls="results/calls/candidates/{caller}/{group}/{group}.{scatteritem}.annotated.bcf",
+        stats="results/calls/candidates/{caller}/{group}/{group}.{scatteritem}.stats.html",
     params:
         plugins=config["annotations"]["vep"]["candidate_calls"]["plugins"],
         extra="{} --vcf_info_field ANN ".format(
             config["annotations"]["vep"]["candidate_calls"]["params"]
         ),
     log:
-        "logs/vep/{group}.{caller}.{scatteritem}.annotate_candidates.log",
+        "logs/vep/{group}/{group}.{caller}.{scatteritem}.annotate_candidates.log",
     benchmark:
-        "benchmarks/vep/{group}.{caller}.{scatteritem}.annotate_candidates.tsv"
+        "benchmarks/vep/{group}/{group}.{caller}.{scatteritem}.annotate_candidates.tsv"
     threads: 4
     group:
         "candidate-annotation"
@@ -26,16 +26,16 @@ rule annotate_candidate_variants:
 
 rule annotate_variants:
     input:
-        calls="results/calls/{group}.{calling_type}.{scatteritem}.bcf",
+        calls="results/calls/varlociraptor/{group}/{group}.{calling_type}.{scatteritem}.bcf",
         cache=access.random("resources/vep/cache"),
         plugins=access.random("resources/vep/plugins"),
         revel=lambda wc: get_plugin_aux("REVEL"),
         revel_tbi=lambda wc: get_plugin_aux("REVEL", True),
         fasta=access.random(genome),
         fai=genome_fai,
     output:
-        calls="results/calls/{group}.{calling_type}.{scatteritem}.annotated.bcf",
-        stats="results/calls/{group}.{calling_type}.{scatteritem}.stats.html",
+        calls="results/calls/vep_annotated/{group}/{group}.{calling_type}.{scatteritem}.bcf",
+        stats="results/calls/vep_annotated/{group}/{group}.{calling_type}.{scatteritem}.stats.html",
     params:
         # Pass a list of plugins to use, see https://www.ensembl.org/info/docs/tools/vep/script/vep_plugins.html
         # Plugin args can be added as well, e.g. via an entry "MyPlugin,1,FOO", see docs.
@@ -55,11 +55,11 @@ rule annotate_variants:
 # TODO What about multiple ID Fields?
 rule annotate_vcfs:
     input:
-        bcf="results/calls/{prefix}.bcf",
+        bcf="results/calls/vep_annotated/{prefix}.bcf",
         annotations=get_annotation_vcfs(),
         idx=get_annotation_vcfs(idx=True),
     output:
-        "results/calls/{prefix}.db-annotated.bcf",
+        "results/calls/db_annotated/{prefix}.bcf",
     log:
         "logs/annotate-vcfs/{prefix}.log",
     params:
@@ -76,11 +76,11 @@ rule annotate_vcfs:
 
 rule annotate_dgidb:
     input:
-        "results/calls/{prefix}.bcf",
+        get_annotate_dgidb_input,
     params:
         datasources=get_dgidb_datasources(),
     output:
-        "results/calls/{prefix}.dgidb.bcf",
+        "results/calls/dgidb_annotated/{prefix}.bcf",
     log:
         "logs/annotate-dgidb/{prefix}.log",
     conda:
@@ -106,9 +106,9 @@ rule gather_annotated_calls:
         calls=get_gather_annotated_calls_input(),
         idx=get_gather_annotated_calls_input(ext="bcf.csi"),
     output:
-        "results/final-calls/{group}.{calling_type}.annotated.bcf",
+        "results/final-calls/{group}/{group}.{calling_type}.annotated.bcf",
     log:
-        "logs/gather-annotated-calls/{group}.{calling_type}.log",
+        "logs/gather-annotated-calls/{group}/{group}.{calling_type}.log",
     params:
         extra="-a",
     group:

workflow/rules/calling.smk

@@ -47,7 +47,7 @@ rule varlociraptor_preprocess:
         bai="results/recal/{sample}.bai",
         alignment_props=get_alignment_props,
     output:
-        "results/observations/{group}/{sample}.{caller}.{scatteritem}.bcf",
+        "results/observations/{caller}/{group}/{sample}.{scatteritem}.bcf",
     params:
         extra=lambda wc: get_varlociraptor_params(
             wc, config["params"]["varlociraptor"]["preprocess"]
@@ -69,9 +69,9 @@ rule varlociraptor_call:
         obs=get_group_observations,
         scenario="results/scenarios/{group}.yaml",
     output:
-        temp("results/calls/{group}.{caller}.{scatteritem}.unsorted.bcf"),
+        temp("results/calls/varlociraptor/{caller}/{group}/{group}.{scatteritem}.unsorted.bcf"),
     log:
-        "logs/varlociraptor/call/{group}.{caller}.{scatteritem}.log",
+        "logs/varlociraptor/call/{caller}/{group}.{scatteritem}.log",
     params:
         obs=get_varlociraptor_obs_args,
         # Add -o as workaround to separate info field entries from subcommand
@@ -87,24 +87,24 @@ rule varlociraptor_call:
     conda:
         "../envs/varlociraptor.yaml"
     benchmark:
-        "benchmarks/varlociraptor/call/{group}.{caller}.{scatteritem}.tsv"
+        "benchmarks/varlociraptor/call/{group}/{group}.{caller}.{scatteritem}.tsv"
     shell:
         "(varlociraptor call variants {params.extra} generic --obs {params.obs}"
         " --scenario {input.scenario} {params.postprocess} {output}) 2> {log}"
 
 
 rule sort_calls:
     input:
-        "results/calls/{group}.{caller}.{scatteritem}.unsorted.bcf",
+        "results/calls/varlociraptor/{caller}/{group}/{group}.{scatteritem}.unsorted.bcf",
     output:
-        temp("results/calls/{group}.{caller}.{scatteritem}.bcf"),
+        temp("results/calls/varlociraptor/{caller}/{group}/{group}.{scatteritem}.bcf"),
     params:
         # Set to True, in case you want uncompressed BCF output
         uncompressed_bcf=False,
         # Extra arguments
         extras="",
     log:
-        "logs/bcf-sort/{group}.{caller}.{scatteritem}.log",
+        "logs/bcf-sort/{group}/{group}.{caller}.{scatteritem}.log",
     resources:
         mem_mb=8000,
     wrapper:
@@ -116,9 +116,9 @@ rule bcftools_concat:
         calls=get_scattered_calls(),
         indexes=get_scattered_calls(ext="bcf.csi"),
     output:
-        "results/calls/{group}.{calling_type}.{scatteritem}.bcf",
+        "results/calls/varlociraptor/{group}/{group}.{calling_type}.{scatteritem}.bcf",
     log:
-        "logs/concat-calls/{group}.{calling_type}.{scatteritem}.log",
+        "logs/concat-calls/{group}/{group}.{calling_type}.{scatteritem}.log",
     params:
         extra="-a",  # TODO Check this
     wrapper:

workflow/rules/candidate_calling.smk

@@ -7,11 +7,11 @@ rule freebayes:
         alns=access.random(lambda w: get_group_bams(w)),
         idxs=lambda w: get_group_bams(w, bai=True),
     output:
-        "results/candidate-calls/{group}.freebayes.bcf",
+        "results/calls/candidates/freebayes/{group}/{group}.bcf",
     log:
         "logs/freebayes/{group}.log",
     params:
-        # genotyping is performed by varlociraptor, hence we deactivate it in freebayes by 
+        # genotyping is performed by varlociraptor, hence we deactivate it in freebayes by
         # always setting --pooled-continuous
         extra="--pooled-continuous --min-alternate-count {} --min-alternate-fraction {} {}".format(
             1 if is_activated("calc_consensus_reads") else 2,
@@ -31,7 +31,7 @@ rule delly:
         index=lambda w: get_group_bams(w, bai=True),
         exclude=get_delly_excluded_regions(),
     output:
-        "results/candidate-calls/{group}.delly.bcf",
+        "results/calls/candidates/delly/{group}/{group}.bcf",
     log:
         "logs/delly/{group}.log",
     params:
@@ -44,9 +44,9 @@ rule delly:
 # Delly breakends lead to invalid BCFs after VEP annotation (invalid RLEN). Therefore we exclude them for now.
 rule fix_delly_calls:
     input:
-        "results/candidate-calls/{group}.delly.bcf",
+        "results/calls/candidates/delly/{group}/{group}.bcf",
     output:
-        "results/candidate-calls/{group}.delly.no_bnds.bcf",
+        "results/calls/candidates/delly/{group}/{group}.no_bnds.bcf",
     log:
         "logs/fix_delly_calls/{group}.log",
     conda:
@@ -61,26 +61,26 @@ rule filter_offtarget_variants:
         index=get_fixed_candidate_calls("bcf.csi"),
         regions="resources/target_regions/target_regions.bed",
     output:
-        "results/candidate-calls/{group}.{caller}.filtered.bcf",
+        "results/calls/candidates/{caller}/filtered/{group}/{group}.bcf",
     params:
         extra="",
     log:
-        "logs/filter_offtarget_variants/{group}.{caller}.log",
+        "logs/filter_offtarget_variants/{group}/{group}.{caller}.log",
     wrapper:
         "v2.3.2/bio/bcftools/filter"
 
 
 rule scatter_candidates:
     input:
-        "results/candidate-calls/{group}.{caller}.filtered.bcf"
+        "results/calls/candidates/{caller}/filtered/{group}/{group}.bcf"
         if config.get("target_regions", None)
         else get_fixed_candidate_calls("bcf"),
     output:
         scatter.calling(
-            "results/candidate-calls/{{group}}.{{caller}}.{scatteritem}.bcf"
+            "results/calls/candidates/{{caller}}/{{group}}/{{group}}.{scatteritem}.bcf"
         ),
     log:
-        "logs/scatter-candidates/{group}.{caller}.log",
+        "logs/scatter-candidates/{caller}/{group}/{group}.log",
     conda:
         "../envs/rbt.yaml"
     shell: