3_munge_sumstats.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import os\n",
    "import glob\n",
    "import time\n",
    "import subprocess as sp\n",
    "import multiprocessing as mp\n",
    "import psutil\n",
    "\n",
    "from basic_tools import *\n",
    "\n",
    "\n",
    "debug=False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# functions used for loading gwas result files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def save_sumstats(gwas_result,pheno_code):\n",
    "    dest=premunge_path.format(pheno_code)\n",
    "    if os.path.exists(dest) or os.path.exists(sumstats_path.format(pheno_code)+'.sumstats.gz'):\n",
    "        print(\"Congratulations!. premunged sumstat of\",pheno_code,\"exists. passed.\")\n",
    "        return\n",
    "    elif os.path.exists(sumstats_path.format(pheno_code)+'.sumstats.gz'):\n",
    "        print(\"Congratulations!. munged sumstat of\",pheno_code,\"exists. passed.\")\n",
    "        return       \n",
    "    print(\"Saving sumstats:\",dest)\n",
    "    tosave=gwas_result\n",
    "    tosave.to_csv(dest,sep=' ',columns=['SNP','A1','A2','N','BETA','P','MAF','INFO'],index=False)\n",
    "    print(\"Finished saving sumstats:\",dest)\n",
    "\n",
    "def munge_sumstats(pheno_code):\n",
    "    if os.path.exists(sumstats_path.format(pheno_code)+'.sumstats.gz'):\n",
    "        print(\"Congratulations!. munged sumstats of\",pheno_code,\"exists. passed.\")\n",
    "        return\n",
    "    script=['munge_sumstats.py','--sumstats',premunge_path.format(pheno_code),'--merge-alleles',snplist_path,'--out',sumstats_path.format(pheno_code)]\n",
    "    print('Started:',' '.join(script))\n",
    "    sp.call(script) \n",
    "    print('Finished:',' '.join(script))\n",
    "    #sp.call(['munge_sumstats.py','--sumstats',premunge_path.format(pheno_code),'--merge-alleles',snplist_path,'--out',sumstats_path.format(pheno_code)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "if debug:\n",
    "    gwas_result=read_gwas2('23115_irnt')\n",
    "    gwas_result.to_pickle('23115_irnt.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "start=int(sys.argv[1])\n",
    "end=int(sys.argv[2])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "```\n",
    "jupyter nbconvert 3_munge_sumstats.ipynb --to script && mkdir ~/premunge &&python 3_munge_sumstats.py 0 50\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2968, 24)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "phenotypes_filtered.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for idx,row in phenotypes_filtered.iloc[start:end].iterrows():\n",
    "    pheno_code=idx\n",
    "    if os.path.exists(sumstats_path.format(pheno_code)+'.sumstats.gz'):\n",
    "        print(\"Congratulations!. passed\",pheno_code,\"exists. passed.\")\n",
    "        continue\n",
    "    print(pheno_code)\n",
    "    gwas_result=read_gwas2(pheno_code)\n",
    "    save_sumstats(gwas_result,pheno_code)\n",
    "    munge_sumstats(pheno_code)\n",
    "    \n",
    "    run_command('rm {}'.format(premunge_path.format(pheno_code)))\n",
    "    \n",
    "#    print(pheno_code)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5855_2\n",
      "20426\n",
      "4233_raw\n",
      "30510_raw\n"
     ]
    }
   ],
   "source": [
    "idx_list=[]\n",
    "for idx,row in phenotypes_filtered.iterrows():\n",
    "    pheno_code=idx\n",
    "    if os.path.exists(sumstats_path.format(pheno_code)+'.sumstats.gz'):\n",
    "        #print(\"Congratulations!. passed\",row['phenotype'],\"exists. passed.\")\n",
    "        continue\n",
    "    idx_list.append(idx)\n",
    "    print(idx)\n",
    "#    print(pheno_code)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1631, 1637, 1638, 1639, 1640, 1641, 1642, 1643]\n",
      "[1644, 1645, 1646, 1647, 1648, 1649, 1650, 1651]\n",
      "[1652, 1653, 1654, 1655, 1656, 1657, 1658, 1659]\n",
      "[1660, 1661, 1662, 1663, 1664, 1665, 1666, 1667]\n",
      "[1668, 1669, 1670, 1671, 1672, 1673, 1674, 1675]\n",
      "[1676, 1677, 1678, 1679, 1680, 1681, 1682, 1683]\n",
      "[1684, 1685, 1686, 1687, 1688, 1689, 1690, 1691]\n",
      "[1692, 1693, 1694, 1695, 1696, 1697, 1698, 1699]\n",
      "[1700, 1701, 1702, 1703, 1704, 1705, 1706, 1707]\n",
      "[1708, 1709, 1710, 1711, 1712, 1713, 1714, 1715]\n",
      "[1716, 1717, 1718, 1719, 1720, 1721, 1722, 1723]\n",
      "[1724, 1725, 1726, 1727, 1728, 1729, 1730, 1731]\n",
      "[1732, 1733, 1734, 1735, 1736, 1737, 1738, 1739]\n",
      "[1740, 1741, 1742, 1743, 1744, 1745, 1746, 1747]\n",
      "[1748, 1749, 1750, 1751, 1752, 1753, 1754, 1755]\n",
      "[1756, 1757, 1758, 1759, 1760, 1761, 1762, 1763]\n",
      "[1764, 1765, 1766, 1767, 1768, 1769, 1770, 1771]\n",
      "[1772, 1773, 1774, 1775, 1776, 1777, 1778, 1779]\n",
      "[1780, 1781, 1782, 1783, 1784, 1785, 1786, 1787]\n",
      "[1788, 1789, 1790, 1791, 1792, 1793, 1794, 1795]\n",
      "[1796, 1797, 1798, 1799, 1800, 1801, 1802, 1803]\n",
      "[1804, 1805, 1806, 1807, 1808, 1809, 1810, 1811]\n",
      "[1812, 1813, 1814, 1815, 1816, 1817, 1818, 1819]\n",
      "[1820, 1821, 1822, 1823, 1824, 1825, 1826, 1827]\n",
      "[1828, 1829, 1830, 1831, 1832, 1833, 1834, 1835]\n",
      "[1836, 1837, 1838, 1839, 1840, 1841, 1842, 1843]\n",
      "[1844, 1845, 2415, 2446, 2454, 2485, 2492, 2516]\n",
      "[2522, 2599, 2638, 2641, 2642, 2679, 2687]\n"
     ]
    }
   ],
   "source": [
    "for i in range(0, len(idx_list), int(len(idx_list)/25)):\n",
    "    print(idx_list[i:i + int(len(idx_list)/25)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "a=pd.read_csv('/home/ch6845/premunge/20426.premunge',sep=' ')\n",
    "a.shape,a[np.logical_not(a['BETA']=='Infinity')].shape,a[np.logical_not(a['BETA']=='-Infinity')].shape\n",
    "a[np.logical_not(a['BETA']=='Infinity')].to_csv('/home/ch6845/premunge/20426.premunge',sep=' ',index=None)\n",
    "\"\"\""
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}