Enhancement snakemake analysis

.github/workflows/tests.yml

@@ -15,8 +15,12 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: 3.9
+
+      - name: Set PYTHONPATH
+        run: echo "PYTHONPATH=." >> $GITHUB_ENV
+
       - name: Install packages
-        run: pip install -U pip; pip install cryptography; pip install jupyter_client==6.1.12; pip install ipython_genutils==0.2.0; pip install nbconvert==5.6.1; pip install rpy2; pip install ipykernel; pip install pandas==1.2.3; pip install pillow; pip install -U Jinja2==3.0
+        run: pip install -U pip; pip install cryptography; pip install jupyter_client==6.1.12; pip install ipython_genutils==0.2.0; pip install nbconvert==5.6.1; pip install rpy2; pip install ipykernel; pip install pandas==1.2.3; pip install pillow; pip install -U Jinja2==3.0; pip install coverage;
 
       - name: install pandoc
         run: sudo apt-get install pandoc;
@@ -26,25 +30,28 @@ jobs:
       - name: Mysql Setup
         run: mysql -u root -proot < setup.sql
       - name: Server data setup
-        run: mkdir /home/runner/mmeds_server_data; mkdir /home/runner/conda_envs; mkdir -p /home/runner/conda_env_files/pheniqs; mkdir -p /home/runner/.modules/modulefiles; cp -r ./test_files /home/runner/mmeds_server_data/.; cp ./mmeds/resources/mmeds_stats.yaml /home/runner/mmeds_server_data/.; cp -r ./modules/* /home/runner/.modules/modulefiles/.; cp -r ./conda_env_files/* /home/runner/conda_env_files/.;
-
+        run: mkdir /home/runner/mmeds_server_data; mkdir /home/runner/conda_envs; mkdir -p /home/runner/conda_env_files/pheniqs; mkdir -p /home/runner/.modules/modulefiles; cp -r ./test_files /home/runner/mmeds_server_data/.; cp ./mmeds/resources/mmeds_stats.yaml /home/runner/mmeds_server_data/.; cp -r ./modules/* /home/runner/.modules/modulefiles/.; cp -r ./conda_env_files/* /home/runner/conda_env_files/.; mkdir /home/runner/mmeds_server_data/taxonomic_databases; touch /home/runner/mmeds_server_data/taxonomic_databases/dummy_classifier.qza;
+    
       - name: Python install
         run: sudo python setup.py install --verbose; pip install wheel; pip install git+https://github.com/clemente-lab/mmeds-meta@$GITHUB_REF_NAME;
 
-      - name: install jupyter
-        run: conda env create -f ./conda_env_files/jupyter_env-github.yaml -p /home/runner/conda_envs/jupyter; 
+        #- name: install jupyter
+        #run: conda env create -f ./conda_env_files/jupyter_env-github.yaml -p /home/runner/conda_envs/jupyter; 
+
 
       - name: add bioconda channel for qiime
         run: conda config --append channels conda-forge --append channels bioconda --append channels defaults
 
       - name: install qiime
         run: conda env create -f ./conda_env_files/qiime2-2020.8_env.yaml -p /home/runner/conda_envs/qiime2-2020.8
 
-      - name: install R
-        run: sudo apt install dirmngr gnupg apt-transport-https ca-certificates software-properties-common; sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9; sudo add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu focal-cran40/'; sudo apt install r-base;
+
+    #- name: install R
+        #run: sudo apt install dirmngr gnupg apt-transport-https ca-certificates software-properties-common; sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9; sudo add-apt-repository 'deb https://cloud.r-project.org/bin/linux/ubuntu focal-cran40/'; sudo apt install r-base;
 
-      - name: install R dependencies
-        run: sudo Rscript -e 'install.packages("GGally",dependencies=TRUE)'; sudo Rscript -e 'install.packages("ggrepel",dependencies=TRUE)'; sudo Rscript -e 'install.packages("RColorBrewer",dependencies=TRUE)'; sudo Rscript -e 'install.packages("ggplot2",dependencies=TRUE)';
+        #- name: install R dependencies
+        #run: sudo Rscript -e 'install.packages("GGally",dependencies=TRUE)'; sudo Rscript -e 'install.packages("ggrepel",dependencies=TRUE)'; sudo Rscript -e 'install.packages("RColorBrewer",dependencies=TRUE)'; sudo Rscript -e 'install.packages("ggplot2",dependencies=TRUE)';
+
 
       - name: install pheniqs
         run: conda env create -f ./conda_env_files/pheniqs_env.yaml -p /home/runner/conda_envs/pheniqs
@@ -54,7 +61,6 @@ jobs:
 
       - name: Unit Tests
         run: |
-         coverage run --parallel-mode --concurrency=multiprocessing ./mmeds/host/manager.py&
          coverage run --parallel-mode --concurrency=multiprocessing ./mmeds/tests/unit/test.py analysis demultiplex;
 
       - uses: actions/upload-artifact@v3
@@ -63,7 +69,9 @@ jobs:
             path: /home/runner/mmeds_server_data/test_files/test_study/Qiime2_0/summary/[email protected]
 
       - name: Upload Unit coverage
-        uses: codecov/codecov-action@v2
+        uses: codecov/codecov-action@v4
+        with:
+            token: ${{ secrets.CODECOV_TOKEN }}
 
   run-tests-simple:
     runs-on: ubuntu-20.04
@@ -74,7 +82,10 @@ jobs:
         with:
           python-version: 3.9
       - name: Install packages
-        run: pip install -U pip; sudo apt-get install tidy environment-modules -y; pip install cryptography;
+        run: pip install -U pip; sudo apt-get install tidy environment-modules -y; pip install cryptography; pip install coverage;
+
+      - name: Set PYTHONPATH
+        run: echo "PYTHONPATH=." >> $GITHUB_ENV
 
       - name: Start Services
         run: sudo systemctl start mysql.service; sudo systemctl start mongod.service;
@@ -83,21 +94,25 @@ jobs:
         run: mysql -u root -proot < setup.sql
 
       - name: Server data setup
-        run: mkdir /home/runner/mmeds_server_data; mkdir /home/runner/conda_envs; mkdir -p /home/runner/conda_env_files/pheniqs; mkdir -p /home/runner/.modules/modulefiles; cp -r ./test_files /home/runner/mmeds_server_data/.; cp ./mmeds/resources/mmeds_stats.yaml /home/runner/mmeds_server_data/.; cp -r ./modules/* /home/runner/.modules/modulefiles/.; cp -r ./conda_env_files/* /home/runner/conda_env_files/.;
+        run: mkdir /home/runner/mmeds_server_data; mkdir /home/runner/conda_envs; mkdir -p /home/runner/conda_env_files/pheniqs; mkdir -p /home/runner/.modules/modulefiles; cp -r ./test_files /home/runner/mmeds_server_data/.; cp ./mmeds/resources/mmeds_stats.yaml /home/runner/mmeds_server_data/.; cp -r ./modules/* /home/runner/.modules/modulefiles/.; cp -r ./conda_env_files/* /home/runner/conda_env_files/.; mkdir /home/runner/mmeds_server_data/taxonomic_databases; touch /home/runner/mmeds_server_data/taxonomic_databases/dummy_classifier.qza;
 
       - name: Python install
         run: sudo python setup.py install --verbose; pip install wheel; pip install git+https://github.com/clemente-lab/mmeds-meta@$GITHUB_REF_NAME;
 
+      - name: Install GraphViz
+        run: sudo apt-get install graphviz
+
       - name: install pheniqs
         run: conda env create -f ./conda_env_files/pheniqs_env.yaml -p /home/runner/conda_envs/pheniqs
 
       - name: Unit Tests
         run: |
-         coverage run --parallel-mode --concurrency=multiprocessing ./mmeds/host/manager.py&
-         coverage run --parallel-mode --concurrency=multiprocessing ./mmeds/tests/unit/test.py adder authentication database documents error formatter spawn tool tools uploader util validate;
+            coverage run --parallel-mode --concurrency=multiprocessing ./mmeds/tests/unit/test.py adder authentication database documents error formatter tools uploader util validate spawn snakemake;
 
       - name: Upload Unit coverage
-        uses: codecov/codecov-action@v2
+        uses: codecov/codecov-action@v4
+        with:
+            token: ${{ secrets.CODECOV_TOKEN }}
 
       - name: Server Tests
         run: |
@@ -106,8 +121,20 @@ jobs:
           pytest --cov=mmeds -W ignore::DeprecationWarning -W ignore::FutureWarning -s ./mmeds/tests/server/test_server.py -x --durations=0;
 
       - name: Upload Server Coverage
-        uses: codecov/codecov-action@v2
+        uses: codecov/codecov-action@v4
+        with:
+            token: ${{ secrets.CODECOV_TOKEN }}
 
       - name: Metadata Tests
         run: pytest --cov=mmeds -W ignore::DeprecationWarning -W ignore::FutureWarning -s ./mmeds/tests/metadata/test_error_metadata.py;
 
+      - name: Load File Action
+        if: failure()
+        id: read_file
+        uses: guibranco/github-file-reader-action-v2@latest
+        with:
+            path: "/home/runner/mmeds_server_data/MMEDS_log.txt"
+
+      - name: Print MMEDS Logs
+        if: failure()
+        run: echo "${{ steps.read_file.outputs.contents }}"

.gitignore

@@ -43,3 +43,7 @@ htmlcov/
 
 # My Git tags file
 tags
+
+# Snakemake work files
+**/.snakemake
+**/.snakemake/*

MANIFEST.in

@@ -2,3 +2,7 @@ include mmeds/html/*
 include mmeds/CSS/*
 include mmeds/resources/*
 include mmeds/test_files/*
+include mmeds/tools/*
+include mmeds/snakemake/*
+include mmeds/snakemake/rules/*
+include mmeds/snakemake/workflows/*

conda_env_files/jupyter_env-github.yaml

@@ -1,21 +1,23 @@
 name: jupyter
 channels:
   - conda-forge
+  - bioconda
 dependencies:
+  - python=3.9
   - jupyter_client=6.1.12
   - nbconvert=5.6.1
   - rpy2
   - ipykernel
-  - pandas=1.2.3
+  - pandas=2.2.2
   - pillow
-  - ipython_genutils=0.2.0
-  - Jinja2=3.0
+  - ipython_genutils
+  - Jinja2
   - r-base=4.1
   - r-ggrepel
   - r-GGally
   - pip
   - pip:
-    - git+https://github.com/clemente-lab/mmeds-meta@Enhancement-PreviousChanges
+    - git+https://github.com/clemente-lab/mmeds-meta@Enhancement-SnakemakeAnalysis
 # at the moment, I don't have a better solution other than changing this branch for each pull request
 
 

conda_env_files/qiime2-2020.8_env.yaml

@@ -356,8 +356,8 @@ dependencies:
   - scikit-bio=0.5.5
   - scikit-learn=0.23.1
   - scipy=1.5.2
-  - seaborn=0.10.1
-  - seaborn-base=0.10.1
+  - seaborn=0.11.1
+  - seaborn-base=0.11.1
   - send2trash=1.5.0
   - sepp=4.3.10
   - setuptools=49.6.0

mmeds/config.py

@@ -8,6 +8,7 @@
 import mmeds.secrets as sec
 import mmeds.html as html
 import mmeds.resources as resources
+import mmeds.snakemake as snakemake
 import mmeds
 import hashlib
 import re
@@ -20,16 +21,18 @@
 IS_PRODUCTION = 'web01' in getfqdn().split('.')
 
 # While this is false, users cannot be added, cannot upload, and cannot query from webpage
-LIVE_PROD_ACCESS = False
+LIVE_PROD_ACCESS = True
 
 # Every MMEDs script imports config.py and thus tries to connect to the database even if it doesn't need to
 # this flag is just to disable that behavior as necessary
 DB_INSTALLED = True
 
+ROOT = Path(mmeds.__file__).parent.resolve()
+HTML_DIR = Path(html.__file__).parent.resolve()
+STORAGE_DIR = Path(resources.__file__).parent.resolve()
+SNAKEMAKE_DIR = Path(snakemake.__file__).parent.resolve()
+
 if TESTING:
-    ROOT = Path(mmeds.__file__).parent.resolve()
-    HTML_DIR = Path(html.__file__).parent.resolve()
-    STORAGE_DIR = Path(resources.__file__).parent.resolve()
     # If apache is trying to access apache's home directory for `mmeds_server_data` rather than your user account's home
     # you may need to hardcode the path here
     # TODO: Add mmeds configuration options for DATABASE_DIR
@@ -44,21 +47,13 @@
     SERVER_PATH = 'http://localhost/myapp/'
     CSS_DIR = 'http://localhost/CSS/'
     IMAGE_PATH = str(CSS_DIR) + '/'
+
 else:
     # We're on web01 and using MMEDs out of if it's project diredctory
     # OR, we're in the folder /sc/arion/projects/MMEDS
-    # TODO: this isn't great, means we have to be careful not naming folders, files as MMEDS
-    # However, it's a solution that doesn't add any dependancies, and I think we should rework config.py anyway
-    if 'matt' in Path.cwd().parts and 'clemej05a' in Path.cwd().parts:
-        DATABASE_DIR = Path('/hpc/users/stapym01') / 'mmeds_server_data'
-        ROOT = Path('/sc/arion/projects/clemej05a/matt/mmeds-meta/')
-    else:
-        DATABASE_DIR = Path('/sc/arion/projects/MMEDS/mmeds_server_data')
-        ROOT = Path('/hpc/users/mmedsadmin/www/mmeds-meta/')
+    DATABASE_DIR = Path('/sc/arion/projects/MMEDS/mmeds_server_data')
 
-    HTML_DIR = ROOT / 'mmeds/html'
-    CSS_DIR = ROOT / 'mmeds/CSS'
-    STORAGE_DIR = ROOT / 'mmeds/resources'
+    CSS_DIR = ROOT / 'CSS'
     WWW_ROOT = "https://mmedsadmin.dmz.hpc.mssm.edu/"
     SERVER_ROOT = WWW_ROOT + "mmeds_app/"
     # Replace the old version
@@ -67,9 +62,13 @@
     IMAGE_PATH = WWW_ROOT + 'mmeds/CSS/'
 
 
+TOOLS_DIR = ROOT / 'tools'
 STUDIES_DIR = DATABASE_DIR / 'studies'
 SEQUENCING_DIR = DATABASE_DIR / 'sequencing_runs'
 SESSION_PATH = DATABASE_DIR / 'CherryPySessions'
+TAXONOMIC_DATABASE_DIR = DATABASE_DIR / "taxonomic_databases"
+SNAKEMAKE_WORKFLOWS_DIR = SNAKEMAKE_DIR / "workflows"
+SNAKEMAKE_RULES_DIR = SNAKEMAKE_DIR / "rules"
 
 ############################
 # CONFIGURE SERVER GLOBALS #
@@ -263,23 +262,42 @@
     '#': '#',
 }
 
-##########################
-# CONFIGURE TOOL GLOBALS #
-###########################
-
-
-TOOL_FILES = {
-    'child_analysis': ['otu_table'],
-    'qiime1': ['data', 'for_reads', 'rev_reads', 'barcodes', 'for_barcodes', 'rev_barcodes', 'metadata'],
-    'qiime2': ['data', 'for_reads', 'rev_reads', 'barcodes', 'for_barcodes', 'rev_barcodes', 'metadata'],
-    'sparcc': ['otu_table'],
-    'cutie': ['otu_table'],
-    'lefse': ['lefse_table'],
-    'picrust1': ['otu_table'],
-    'picrust2': ['otu_table'],
-    'test': []
+##############################
+# CONFIGURE ANALYSIS GLOBALS #
+##############################
+
+
+WORKFLOWS = {
+    "core_pipeline_taxonomic": {
+        "parameters": [
+            'sampling_depth',
+            'metadata',
+            'alpha_metrics',
+            'beta_metrics',
+            'taxonomic_database',
+            'sequencing_runs',
+            'taxa_levels'
+        ]
+    },
+    "lefse": {
+        "parameters": [
+            "tables",
+            "classes",
+            "subclasses"
+        ]
+    }
 }
 
+TAXONOMIC_DATABASES = {
+    "greengenes": TAXONOMIC_DATABASE_DIR / "gg-13-8-99-nb-2020-8-0.qza",
+    "greengenes2": TAXONOMIC_DATABASE_DIR / "greengenes2.2020-10.nb-classifier.qza",
+    "silva": TAXONOMIC_DATABASE_DIR / "silva-138-99-nb-classifier.qza"
+}
+
+if TESTING:
+    TAXONOMIC_DATABASES["test"] = TAXONOMIC_DATABASE_DIR / "dummy_classifier.qza"
+
+
 UPLOADED_FP = 'uploaded_file'
 ERROR_FP = 'error_log.tsv'
 
@@ -332,17 +350,13 @@
 
 CURRENT_PROCESSES = DATABASE_DIR / 'current_processes.yaml'
 CONFIG_PARAMETERS = {
-    'qiime2': [
+    'core_pipeline_taxonomic': [
         'sampling_depth',
         'metadata',
-        'taxa_levels',
-        'abundance_threshold',
-        'font_size',
-        'sub_analysis',
-        'additional_analysis',
-        'iterations',
-        'permutations',
-        'type'],
+        'alpha_metrics',
+        'beta_metrics',
+        'sequencing_runs',
+        'taxa_levels'],
     'cutie': [
         'statistic',
         'feature_table',
@@ -360,6 +374,18 @@
         'permutations'],
     'test': []
 }
+
+CONFIG_LISTS = [
+    'metadata',
+    'taxa_levels',
+    'alpha_metrics',
+    'beta_metrics',
+    'sequencing_runs',
+    'tables',
+    'classes',
+    'subclasses'
+]
+
 CONTACT_EMAIL = '[email protected]'
 MMEDS_EMAIL = '[email protected]'
 TEST_EMAIL = '[email protected]'
@@ -400,6 +426,7 @@
 TEST_CODE_SHORT = 'singlereadsshort'
 TEST_CODE_PAIRED = 'pairedreads'
 TEST_CODE_DEMUX = 'demuxedreads'
+TEST_CODE_MIXED = 'mixedstudy'
 TEST_CODE_OTU = 'otutable'
 TEST_CODE_LEFSE = 'lefsetable'
 TEST_MIXS = str(TEST_PATH / 'test_MIxS.tsv')
@@ -422,6 +449,7 @@
     str(TEST_PATH / 'test_stripped_1'),
     str(TEST_PATH / 'test_stripped_2')
 ]
+TEST_SNAKEMAKE_DIR = str(TEST_PATH / 'test_snakemake')
 TEST_PHENIQS_MAPPING = str(TEST_PATH / 'test_pheniqs_mapping_file.tsv')
 TEST_STUDY = str(TEST_PATH / 'test_study')
 TEST_SPECIMEN = str(TEST_PATH / 'test_specimen.tsv')
@@ -443,6 +471,7 @@
 TEST_ADD_SUBJECT = str(TEST_PATH / 'test_add_subject.tsv')
 TEST_ANIMAL_SUBJECT = str(TEST_PATH / 'test_animal_subject.tsv')
 TEST_MIXED_SUBJECT = str(TEST_PATH / 'test_mixed_subject.tsv')
+TEST_MIXED_SPECIMEN = str(TEST_PATH / 'test_mixed_specimen.tsv')
 TEST_SUBJECT_ERROR = str(TEST_PATH / 'validation_files/test_subject_error.tsv')
 TEST_SUBJECT_WARN = str(TEST_PATH / 'validation_files/test_subject_warn.tsv')
 TEST_SUBJECT_ALT = str(TEST_PATH / 'test_subject_alt.tsv')
@@ -478,6 +507,8 @@
 SPECIMEN_TEMPLATE = str(TEST_PATH / 'specimen_template.tsv')
 CONFIG_EXAMPLE = str(TEST_PATH / 'config_example.yaml')
 
+TEST_SEQUENCING_NAME = "TEST_RUN"
+
 
 # Defaults
 QIIME_SAMPLE_ID_CATS = ('#SampleID', '#q2:types')