Restructure for nextflow pipeline

Author: grst

.editorconfig

@@ -0,0 +1,36 @@
+# http://editorconfig.org
+root = true
+
+[*]
+indent_style = space
+indent_size = 4
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+# Use 4 spaces for the Python files
+[*.py]
+indent_size = 4
+max_line_length = 80
+
+# The JSON files contain newlines inconsistently
+[*.json]
+insert_final_newline = ignore
+
+# Minified JavaScript files shouldn't be changed
+[**.min.js]
+indent_style = ignore
+insert_final_newline = ignore
+
+# Makefiles always use tabs for indentation
+[Makefile]
+indent_style = tab
+
+# Batch files use tabs for indentation
+[*.bat]
+indent_style = tab
+
+[*.md]
+trim_trailing_whitespace = false
+

.gitattributes

@@ -0,0 +1 @@
+* text=auto eol=lf

.gitmodules

@@ -1,6 +1,3 @@
-[submodule "single-cell-analysis-nf"]
-	path = single-cell-analysis-nf
-	url = [email protected]:icbi-lab/single-cell-analysis-nf.git
 [submodule "lib/scanpy_helper_submodule"]
 	path = lib/scanpy_helper_submodule
 	url = [email protected]:icbi-lab/tools/scanpy_helpers.git

analyses/10_integrate_scrnaseq_data/11_merge_all.ipynb renamed to analyses/20_integrate_scrnaseq_data/21_merge_all.ipynb

@@ -2,13 +2,23 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": 2,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "The autoreload extension is already loaded. To reload it, use:\n",
+      "  %reload_ext autoreload\n"
+     ]
+    }
+   ],
    "source": [
     "%load_ext autoreload\n",
     "%autoreload 2\n",
     "\n",
+    "from nxfvars import nxfvars\n",
     "import scanpy as sc\n",
     "import numpy as np\n",
     "import itertools\n",
@@ -37,6 +47,15 @@
     "import re"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "out_dir = nxfvars.get(\"outdir\", \"/tmp\")"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": 2,
@@ -54,7 +73,7 @@
     }
    ],
    "source": [
-    "threadpool_limits(8)"
+    "threadpool_limits(int(nxfvars.get(\"cpus\", \"8\")))"
    ]
   },
   {
@@ -72,7 +91,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "dataset_table = pd.read_csv(\"../../tables/samplesheet_scrnaseq_preprocessing.csv\")"
+    "dataset_table = pd.read_csv(nxfvars.get(\"samplesheet\", \"../../tables/samplesheet_scrnaseq_preprocessing.csv\"))"
    ]
   },
   {
@@ -478,9 +497,10 @@
     }
    ],
    "source": [
+    "dataset_path = nxfvars.get(\"dataset_path\", \"../../data/20_qc_norm_scrnaseq/01_qc_and_filtering\")\n",
     "datasets = {\n",
     "    dataset_id: sc.read_h5ad(\n",
-    "        f\"../../data/20_qc_norm_scrnaseq/01_qc_and_filtering/{dataset_id}/{dataset_id}.qc.h5ad\"\n",
+    "        f\"{dataset_id}.qc.h5ad\" if dataset_path == \".\" else f\"{dataset_path}/{dataset_id}/{dataset_id}.qc.h5ad\"\n",
     "    )\n",
     "    for dataset_id in tqdm(dataset_table[\"id\"])\n",
     "}"
@@ -2621,7 +2641,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "obs_all.to_excel(\"../../data/50_integrate_scrnaseq_data/51_merge_all/obs_all.xlsx\")"
+    "obs_all.to_excel(f\"{out_dir}/obs_all.xlsx\")"
    ]
   },
   {
@@ -2921,64 +2941,9 @@
    ],
    "source": [
     "merged_all.write_h5ad(\n",
-    "    \"../../data/50_integrate_scrnaseq_data/51_merge_all/merged_all.h5ad\"\n",
+    "    \"{out_dir}/merged_all.h5ad\"\n",
     ")"
    ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "## Export for NSCLC heterogeneity\n",
-    " * only tumor samples (no controls, no metastases)\n",
-    " * all NSCLC subtypes"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 33,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# datasets_nsclc_heterogeneity = dict()\n",
-    "# for dataset_id, dataset in datasets.items():\n",
-    "#     if \"tumor_primary\" in dataset.obs[\"origin\"].values:\n",
-    "#         datasets_nsclc_heterogeneity[dataset_id] = dataset[\n",
-    "#             dataset.obs[\"origin\"] == \"tumor_primary\", :\n",
-    "#         ].copy()\n",
-    "# del datasets_nsclc_heterogeneity[\"Pircher_batch1_NSCLC\"]"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 34,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# merged_nsclc_heterogeneity = merge_datasets(\n",
-    "#     datasets_nsclc_heterogeneity.values(), symbol_in_n_datasets=5\n",
-    "# )"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 35,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# merged_nsclc_heterogeneity.shape"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 36,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# merged_nsclc_heterogeneity.write_h5ad(\n",
-    "#     \"../../data/50_integrate_scrnaseq_data/51_merge_all/merged_nsclc_heterogeneity.h5ad\"\n",
-    "# )"
-   ]
   }
  ],
  "metadata": {

analyses/10_integrate_scrnaseq_data/11_merge_all.py renamed to analyses/20_integrate_scrnaseq_data/21_merge_all.py

@@ -16,6 +16,7 @@
 # %load_ext autoreload
 # %autoreload 2
 
+from nxfvars import nxfvars
 import scanpy as sc
 import numpy as np
 import itertools
@@ -44,21 +45,25 @@
 import re
 
 # %%
-threadpool_limits(8)
+out_dir = nxfvars.get("outdir", "/tmp")
+
+# %%
+threadpool_limits(int(nxfvars.get("cpus", "8")))
 
 # %%
 sc.set_figure_params(figsize=(5, 5))
 
 # %%
-dataset_table = pd.read_csv("../../tables/samplesheet_scrnaseq_preprocessing.csv")
+dataset_table = pd.read_csv(nxfvars.get("samplesheet", "../../tables/samplesheet_scrnaseq_preprocessing.csv"))
 
 # %%
 dataset_table
 
 # %%
+dataset_path = nxfvars.get("dataset_path", "../../data/20_qc_norm_scrnaseq/01_qc_and_filtering")
 datasets = {
     dataset_id: sc.read_h5ad(
-        f"../../data/20_qc_norm_scrnaseq/01_qc_and_filtering/{dataset_id}/{dataset_id}.qc.h5ad"
+        f"{dataset_id}.qc.h5ad" if dataset_path == "." else f"{dataset_path}/{dataset_id}/{dataset_id}.qc.h5ad"
     )
     for dataset_id in tqdm(dataset_table["id"])
 }
@@ -288,7 +293,7 @@
 )
 
 # %%
-obs_all.to_excel("../../data/50_integrate_scrnaseq_data/51_merge_all/obs_all.xlsx")
+obs_all.to_excel(f"{out_dir}/obs_all.xlsx")
 
 # %%
 merged_all = merge_datasets(datasets.values(), symbol_in_n_datasets=17)
@@ -301,32 +306,5 @@
 
 # %%
 merged_all.write_h5ad(
-    "../../data/50_integrate_scrnaseq_data/51_merge_all/merged_all.h5ad"
+    "{out_dir}/merged_all.h5ad"
 )
-
-# %% [markdown]
-# ## Export for NSCLC heterogeneity
-#  * only tumor samples (no controls, no metastases)
-#  * all NSCLC subtypes
-
-# %%
-# datasets_nsclc_heterogeneity = dict()
-# for dataset_id, dataset in datasets.items():
-#     if "tumor_primary" in dataset.obs["origin"].values:
-#         datasets_nsclc_heterogeneity[dataset_id] = dataset[
-#             dataset.obs["origin"] == "tumor_primary", :
-#         ].copy()
-# del datasets_nsclc_heterogeneity["Pircher_batch1_NSCLC"]
-
-# %%
-# merged_nsclc_heterogeneity = merge_datasets(
-#     datasets_nsclc_heterogeneity.values(), symbol_in_n_datasets=5
-# )
-
-# %%
-# merged_nsclc_heterogeneity.shape
-
-# %%
-# merged_nsclc_heterogeneity.write_h5ad(
-#     "../../data/50_integrate_scrnaseq_data/51_merge_all/merged_nsclc_heterogeneity.h5ad"
-# )

analyses/20_annotate_and_infercnv/61_infercnvpy-all.ipynb renamed to analyses/20_integrate_scrnaseq_data/61_infercnvpy-all.ipynb

@@ -20,8 +20,17 @@
  params {
     modules {
         'SCQC' {
-            publish_dir   = '01_qc_and_filtering'
+            publish_dir = '20_qc_and_filtering'
             publish_by_id = true
         }
+        'SCQC_MERGE_STATS' {
+            publish_dir = '20_qc_and_filtering'
+        }
+        'P11_MERGE_ALL' {
+            publish_dir = '21_merge_all'
+        }
+        'SCVI' {
+            publish_dir = '21_merge_all'
+        }
     }
-}
+}

analyses/20_annotate_and_infercnv/61_infercnvpy-all.py renamed to analyses/20_integrate_scrnaseq_data/61_infercnvpy-all.py

@@ -0,0 +1,9 @@
+includeConfig 'conf/modules.config'
+
+
+params {
+    input = "tables/samplesheet_scrnaseq_preprocessing.csv"
+    outdir = "data/20_integrate_scrnaseq_data"
+    publish_dir_mode = "link"
+}
+

conf/modules.config

@@ -0,0 +1,32 @@
+#!/usr/bin/env nextflow
+
+nextflow.enable.dsl = 2
+
+def modules = params.modules.clone()
+assert params.input: "Input samplesheet not specified!"
+
+include { check_samplesheet }  from './modules/local/check_samplesheet' params(params)
+
+include { SCQC } from "./modules/local/scqc/main.nf" addParams(
+    options: modules['SCQC']
+)
+include { SCQC_MERGE_STATS } from "./modules/local/scqc_merge_stats/main.nf" addParams(
+    options: modules['SCQC_MERGE_STATS']
+)
+include { P11_MERGE_ALL } from "./modules/local/analyses/20_integrate_scrnaseq_data.nf" addParams (
+    options: modules["P11_MERGE_ALL"]
+)
+
+
+workflow {
+    ch_samples = Channel.from(check_samplesheet(params.input))
+
+    SCQC(ch_samples)
+    SCQC_MERGE_STATS(SCQC.out.qc_stats.collect())
+
+    P11_MERGE_ALL(
+        Channel.fromPath(params.input),
+        SCQC.out.adata.flatMap{ meta, adata -> adata }
+    )
+}
+