Merge pull request #207 from MatthewMaher/master

pheweb/conf.py

@@ -164,6 +164,7 @@ def get_top_hits_pval_cutoff() -> float: return _get_config_float('top_hits_pval
 
 ## Pheno correlation config
 def should_show_correlations() -> bool: return _get_config_bool('show_correlations', False)
+def pval_is_neglog10() -> bool: return _get_config_bool('pval_is_neglog10', False)
 def get_pheno_correlations_pvalue_threshold() -> float: return _get_config_float('pheno_correlations_pvalue_threshold', 0.05)
 
 

pheweb/load/cluster.py

@@ -9,20 +9,26 @@
 from boltons.iterutils import chunked
 from typing import List,Dict,Any
 
-N_AT_A_TIME = 5
-
 header_template = {
     'slurm': '''\
 #!/bin/bash
-#SBATCH --array=0-{n_jobs}
+#SBATCH --array=0-{n_jobs_m1}
 #SBATCH --mem=4G
 #SBATCH --time=5-0:0
 #SBATCH --output={tmp_path}/slurm-%j.out
 #SBATCH --error={tmp_path}/slurm-%j.out
 ''',
     'sge': '''\
 #!/bin/bash
-#$ -t 0-{n_jobs}
+#$ -t 0-{n_jobs_m1}
+#$ -l h_vmem=4G
+#$ -l h_rt=120:00:00
+#$ -o {tmp_path}
+#$ -e {tmp_path}
+''',
+    'uge': '''\
+#!/bin/bash
+#$ -t 1-{n_jobs}
 #$ -l h_vmem=4G
 #$ -l h_rt=120:00:00
 #$ -o {tmp_path}
@@ -32,20 +38,24 @@
 array_id_variable = {
     'slurm': 'SLURM_ARRAY_TASK_ID',
     'sge': 'SGE_TASK_ID',
+    'uge': '(($SGE_TASK_ID - 1))',
 }
 submit_command = {
     'slurm': 'sbatch',
     'sge': 'qsub',
+    'uge': 'qsub',
 }
 monitor_command = {
     'slurm': 'squeue --long --array --job',
     'sge': 'qstat -j',
+    'uge': 'qstat -j',
 }
 
 def run(argv:List[str]) -> None:
     parser = argparse.ArgumentParser()
-    parser.add_argument('--engine', choices=['slurm', 'sge'], required=True)
+    parser.add_argument('--engine', choices=['slurm', 'sge', 'uge'], required=True)
     parser.add_argument('--step', choices=['parse', 'augment-phenos', 'manhattan', 'qq'], required=True)
+    parser.add_argument('--N_per_job', default=5)
     args = parser.parse_args(argv)
 
     def should_process(pheno:Dict[str,Any]) -> bool:
@@ -77,12 +87,12 @@ def should_process(pheno:Dict[str,Any]) -> bool:
         print('All phenos are up-to-date!')
         exit(0)
 
-    jobs = chunked(idxs, N_AT_A_TIME)
+    jobs = chunked(idxs, args.N_per_job)
     batch_filepath = get_dated_tmp_path('{}-{}'.format(args.engine, args.step)) + '.sh'
     tmp_path = get_tmp_path(args.step)
     mkdir_p(tmp_path)
     with open(batch_filepath, 'w') as f:
-        f.write(header_template[args.engine].format(n_jobs = len(jobs)-1, tmp_path=tmp_path))
+        f.write(header_template[args.engine].format(n_jobs_m1 = len(jobs)-1, n_jobs = len(jobs), tmp_path=tmp_path))
         f.write('\n\njobs=(\n')
         for job in jobs:
             f.write(','.join(map(str,job)) + '\n')

pheweb/parse_utils.py

@@ -1,6 +1,7 @@
 from . import utils
 from .utils import PheWebError
 from . import conf
+#import sys
 
 import itertools
 from collections import OrderedDict,Counter
@@ -78,6 +79,7 @@ def scientific_int(s:str) -> int:
         'required': True,
         'type': float,
         'nullable': True,
+        'could_be_neglog10': True,
         'range': [0, 1],
         'sigfigs': 2,
         'tooltip_lztemplate': {
@@ -200,6 +202,11 @@ def parse(self, value):
             return ''
         # type
         x = self._d['type'](value)
+
+        # It's POSSIBLE that conversion of -log10 representation is required
+        if 'could_be_neglog10' in self._d and self._d['could_be_neglog10'] and conf.pval_is_neglog10():
+            x = 10**-x
+
         # range
         if 'range' in self._d:
             assert self._d['range'][0] is None or x >= self._d['range'][0]