Meta_Analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "Bad key \"text.kerning_factor\" on line 4 in\n",
      "/pylon5/ac5616p/yuke/anaconda3/envs/R3Kernel/lib/python3.7/site-packages/matplotlib/mpl-data/stylelib/_classic_test_patch.mplstyle.\n",
      "You probably need to get an updated matplotlibrc file from\n",
      "https://github.com/matplotlib/matplotlib/blob/v3.1.3/matplotlibrc.template\n",
      "or from the matplotlib source distribution\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import string\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import pydicom"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "r3_path = '/pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R3'\n",
    "r3_notes_path = '/pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R3'\n",
    "r3_img_path = '/pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R3/20_Batmanghelich'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# R3 Demographics Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "r3_demo_path = '/pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R3/20_BATMANGHELICH_NOTES_DATA_2020_04_03/R3_20_BATMANGHELICH_DEMOGRAPHICS_2020_04_02.xlsx'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "r3_demo_df = pd.read_excel(r3_demo_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PATIENT_STUDY_ID</th>\n",
       "      <th>BIRTH_DATE</th>\n",
       "      <th>DEATH_DATE</th>\n",
       "      <th>GENDER</th>\n",
       "      <th>RACE</th>\n",
       "      <th>ETHNICITY</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>111111139</td>\n",
       "      <td>04/20/1938</td>\n",
       "      <td>NaN</td>\n",
       "      <td>MALE</td>\n",
       "      <td>WHITE</td>\n",
       "      <td>NOT HISPANIC OR LATINO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>111111670</td>\n",
       "      <td>12/01/1950</td>\n",
       "      <td>04/23/2018</td>\n",
       "      <td>MALE</td>\n",
       "      <td>WHITE</td>\n",
       "      <td>NOT HISPANIC OR LATINO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>111111263</td>\n",
       "      <td>05/28/1950</td>\n",
       "      <td>NaN</td>\n",
       "      <td>MALE</td>\n",
       "      <td>WHITE</td>\n",
       "      <td>NOT HISPANIC OR LATINO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>111110014</td>\n",
       "      <td>02/25/1935</td>\n",
       "      <td>07/12/2018</td>\n",
       "      <td>MALE</td>\n",
       "      <td>WHITE</td>\n",
       "      <td>NOT HISPANIC OR LATINO</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>111110002</td>\n",
       "      <td>08/27/1942</td>\n",
       "      <td>NaN</td>\n",
       "      <td>FEMALE</td>\n",
       "      <td>BLACK</td>\n",
       "      <td>NOT HISPANIC OR LATINO</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   PATIENT_STUDY_ID  BIRTH_DATE  DEATH_DATE  GENDER   RACE  \\\n",
       "0         111111139  04/20/1938         NaN    MALE  WHITE   \n",
       "1         111111670  12/01/1950  04/23/2018    MALE  WHITE   \n",
       "2         111111263  05/28/1950         NaN    MALE  WHITE   \n",
       "3         111110014  02/25/1935  07/12/2018    MALE  WHITE   \n",
       "4         111110002  08/27/1942         NaN  FEMALE  BLACK   \n",
       "\n",
       "                ETHNICITY  \n",
       "0  NOT HISPANIC OR LATINO  \n",
       "1  NOT HISPANIC OR LATINO  \n",
       "2  NOT HISPANIC OR LATINO  \n",
       "3  NOT HISPANIC OR LATINO  \n",
       "4  NOT HISPANIC OR LATINO  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r3_demo_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total distinct patients:10174\n"
     ]
    }
   ],
   "source": [
    "# total number of patients\n",
    "print('Total distinct patients:{0}'.format(r3_demo_df['PATIENT_STUDY_ID'].nunique()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GENDER\n",
       "FEMALE    5177\n",
       "MALE      4997\n",
       "Name: PATIENT_STUDY_ID, dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# counts by gender\n",
    "r3_demo_df.groupby('GENDER')['PATIENT_STUDY_ID'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RACE\n",
       "ALASKA NATIVE                1\n",
       "AMERICAN INDIAN             12\n",
       "BLACK                      838\n",
       "CHINESE                     22\n",
       "DECLINED                   117\n",
       "FILIPINO                     4\n",
       "GUAM/CHAMORRO                3\n",
       "HAWAIIAN                     2\n",
       "INDIAN (ASIAN)              15\n",
       "JAPANESE                     3\n",
       "KOREAN                       3\n",
       "NOT SPECIFIED               92\n",
       "OTHER ASIAN                 34\n",
       "OTHER PACIFIC ISLANDER       4\n",
       "SAMOAN                       1\n",
       "VIETNAMESE                   4\n",
       "WHITE                     9006\n",
       "Name: PATIENT_STUDY_ID, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# counts by race\n",
    "r3_demo_df.groupby('RACE')['PATIENT_STUDY_ID'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ETHNICITY\n",
       "DECLINED                   270\n",
       "HISPANIC OR LATINO          61\n",
       "NOT HISPANIC OR LATINO    9432\n",
       "NOT SPECIFIED              253\n",
       "Name: PATIENT_STUDY_ID, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# counts by ethinicity\n",
    "r3_demo_df.groupby('ETHNICITY')['PATIENT_STUDY_ID'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2060"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# number of deaths (about 20%)\n",
    "sum(-r3_demo_df['DEATH_DATE'].isnull())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# R3 Image"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## data profiling preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = '/pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R3/20_Batmanghelich/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "root_lst = []\n",
    "series_lst = []\n",
    "sid_lst = []\n",
    "pid_lst = []\n",
    "files_lst = []\n",
    "meta_lst = []\n",
    "patient_age_lst = []\n",
    "patient_sex_lst = []\n",
    "scan_modality_lst = []\n",
    "scan_date_lst = []\n",
    "scan_time_lst = []\n",
    "scan_body_part_lst = []\n",
    "scan_option_lst = []\n",
    "scan_slicethickness_lst = []\n",
    "scan_spacing_lst = []\n",
    "scan_pixel_spacing_1_lst = []\n",
    "scan_pixel_spacing_2_lst = []\n",
    "scan_rows_lst = []\n",
    "scan_cols_lst = []\n",
    "scan_kvp_lst = []\n",
    "scan_xraycurrent_lst = []\n",
    "scan_exposuretime_lst = []\n",
    "\n",
    "for root, dirs, files in os.walk(path, topdown=True):\n",
    "    if root.count('/') == 11: # if it is a dicom series folder\n",
    "        # use path as the key\n",
    "        root_lst.append(root)\n",
    "        # extract patient id from the path\n",
    "        sub_path = root.split('/')[9]\n",
    "        pid_lst.append(sub_path.split('_')[-1])\n",
    "        # extract study id from the path\n",
    "        sub_path = root.split('/')[10]\n",
    "        sid_lst.append(sub_path.split('_')[-1])\n",
    "        # extract series folder name\n",
    "        series_lst.append(root.split('/')[-1])\n",
    "        # count number of dicom images\n",
    "        files_lst.append(len(files))\n",
    "        \n",
    "        # read the first dicom file in current path\n",
    "        try:\n",
    "            dicom_file_path = os.path.join(root, files[0])\n",
    "            # meta elements in the dicom file\n",
    "            ds = pydicom.filereader.dcmread(dicom_file_path)\n",
    "        except:\n",
    "            ds = None\n",
    "        try:\n",
    "            patient_age_lst.append(ds.PatientAge) # age\n",
    "        except:\n",
    "            patient_age_lst.append(None)\n",
    "        try:\n",
    "            patient_sex_lst.append(ds.PatientSex) # sex\n",
    "        except:\n",
    "            patient_sex_lst.append(None)\n",
    "        try:\n",
    "            scan_modality_lst.append(ds.Modality) # scan modality\n",
    "        except:\n",
    "            scan_modality_lst.append(None)\n",
    "        try:\n",
    "            scan_date_lst.append(ds.AcquisitionDate) # scan date\n",
    "        except:\n",
    "            scan_date_lst.append(None)\n",
    "        try:\n",
    "            scan_time_lst.append(ds.AcquisitionTime) # scan time\n",
    "        except:\n",
    "            scan_time_lst.append(None)\n",
    "        try:\n",
    "            scan_body_part_lst.append(ds.BodyPartExamined) # scan body part\n",
    "        except:\n",
    "            scan_body_part_lst.append(None)\n",
    "        try:\n",
    "            scan_slicethickness_lst.append(ds.SliceThickness) # slice thickness\n",
    "        except:\n",
    "            scan_slicethickness_lst.append(None)\n",
    "        try:\n",
    "            scan_spacing_lst.append(ds.SpacingBetweenSlices) # slice spacing\n",
    "        except:\n",
    "            scan_spacing_lst.append(None)\n",
    "        try:\n",
    "            scan_pixel_spacing_1_lst.append(ds.PixelSpacing[0]) # pixel spacing direction 1\n",
    "        except:\n",
    "            scan_pixel_spacing_1_lst.append(None)\n",
    "        try:\n",
    "            scan_pixel_spacing_2_lst.append(ds.PixelSpacing[1]) # pixel spacing direction 2\n",
    "        except:\n",
    "            scan_pixel_spacing_2_lst.append(None)\n",
    "        try:\n",
    "            scan_rows_lst.append(ds.Rows) # number of pixels per row\n",
    "        except:\n",
    "            scan_rows_lst.append(None)\n",
    "        try:\n",
    "            scan_cols_lst.append(ds.Columns) # number of pixels per column\n",
    "        except:\n",
    "            scan_cols_lst.append(None)\n",
    "        try:\n",
    "            scan_kvp_lst.append(ds.KVP) # kilovoltage applied to the x-ray tube\n",
    "        except:\n",
    "            scan_kvp_lst.append(None)\n",
    "        try:\n",
    "            scan_xraycurrent_lst.append(ds.XRayTubeCurrent) # x-ray current\n",
    "        except:\n",
    "            scan_xraycurrent_lst.append(None)\n",
    "        try:\n",
    "            scan_exposuretime_lst.append(ds.ExposureTime) # exposure time\n",
    "        except:\n",
    "            scan_exposuretime_lst.append(None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "dicom_profile = {'path':root_lst,\n",
    "                'pid':pid_lst,\n",
    "                'sid':sid_lst,\n",
    "                'series':series_lst,\n",
    "                'files':files_lst,\n",
    "                'age':patient_age_lst,\n",
    "                'sex':patient_sex_lst,\n",
    "                'modality':scan_modality_lst,\n",
    "                'date': scan_date_lst,\n",
    "                'time': scan_time_lst,\n",
    "                'body_part': scan_body_part_lst,\n",
    "                'slice_thickness': scan_slicethickness_lst,\n",
    "                'slice_spacing': scan_spacing_lst,\n",
    "                'pixcel_spacing_1': scan_pixel_spacing_1_lst,\n",
    "                'pixcel_spacing_2': scan_pixel_spacing_2_lst,\n",
    "                'rows':scan_rows_lst,\n",
    "                'columns':scan_cols_lst,\n",
    "                'kvp': scan_kvp_lst,\n",
    "                'x_ray_current': scan_xraycurrent_lst,\n",
    "                'exposure_time': scan_exposuretime_lst\n",
    "                }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "dicom_profile_df = pd.DataFrame.from_dict(dicom_profile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>path</th>\n",
       "      <th>pid</th>\n",
       "      <th>sid</th>\n",
       "      <th>series</th>\n",
       "      <th>files</th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>modality</th>\n",
       "      <th>date</th>\n",
       "      <th>time</th>\n",
       "      <th>body_part</th>\n",
       "      <th>slice_thickness</th>\n",
       "      <th>slice_spacing</th>\n",
       "      <th>pixcel_spacing_1</th>\n",
       "      <th>pixcel_spacing_2</th>\n",
       "      <th>rows</th>\n",
       "      <th>columns</th>\n",
       "      <th>kvp</th>\n",
       "      <th>x_ray_current</th>\n",
       "      <th>exposure_time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>/pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...</td>\n",
       "      <td>0111259663</td>\n",
       "      <td>41209592</td>\n",
       "      <td>Series_302_LUNG_2.5</td>\n",
       "      <td>129</td>\n",
       "      <td>076Y</td>\n",
       "      <td>M</td>\n",
       "      <td>CT</td>\n",
       "      <td>20170101</td>\n",
       "      <td>084817.737107</td>\n",
       "      <td>CHEST</td>\n",
       "      <td>2.500</td>\n",
       "      <td>-2.500</td>\n",
       "      <td>0.710938</td>\n",
       "      <td>0.710938</td>\n",
       "      <td>512.0</td>\n",
       "      <td>512.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>185.0</td>\n",
       "      <td>349.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>/pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...</td>\n",
       "      <td>0111259663</td>\n",
       "      <td>41209592</td>\n",
       "      <td>Series_1_Scout</td>\n",
       "      <td>2</td>\n",
       "      <td>076Y</td>\n",
       "      <td>M</td>\n",
       "      <td>CT</td>\n",
       "      <td>20170101</td>\n",
       "      <td>084729.320407</td>\n",
       "      <td>CHEST</td>\n",
       "      <td>370.000</td>\n",
       "      <td>10.000</td>\n",
       "      <td>0.636364</td>\n",
       "      <td>0.646341</td>\n",
       "      <td>582.0</td>\n",
       "      <td>820.0</td>\n",
       "      <td>80.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>3782.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>/pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...</td>\n",
       "      <td>0111259663</td>\n",
       "      <td>41209592</td>\n",
       "      <td>Series_2_CHEST_WITHOUT</td>\n",
       "      <td>513</td>\n",
       "      <td>076Y</td>\n",
       "      <td>M</td>\n",
       "      <td>CT</td>\n",
       "      <td>20170101</td>\n",
       "      <td>084817.737107</td>\n",
       "      <td>CHEST</td>\n",
       "      <td>0.625</td>\n",
       "      <td>-0.625</td>\n",
       "      <td>0.710938</td>\n",
       "      <td>0.710938</td>\n",
       "      <td>512.0</td>\n",
       "      <td>512.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>231.0</td>\n",
       "      <td>349.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>/pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...</td>\n",
       "      <td>0111259663</td>\n",
       "      <td>41209592</td>\n",
       "      <td>Series_301_SOFT_2.5</td>\n",
       "      <td>129</td>\n",
       "      <td>076Y</td>\n",
       "      <td>M</td>\n",
       "      <td>CT</td>\n",
       "      <td>20170101</td>\n",
       "      <td>084817.737107</td>\n",
       "      <td>CHEST</td>\n",
       "      <td>2.500</td>\n",
       "      <td>-2.500</td>\n",
       "      <td>0.710938</td>\n",
       "      <td>0.710938</td>\n",
       "      <td>512.0</td>\n",
       "      <td>512.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>185.0</td>\n",
       "      <td>349.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>/pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...</td>\n",
       "      <td>0111225336</td>\n",
       "      <td>43814074</td>\n",
       "      <td>Series_5_CHEST_WO</td>\n",
       "      <td>436</td>\n",
       "      <td>081Y</td>\n",
       "      <td>F</td>\n",
       "      <td>CT</td>\n",
       "      <td>20180101</td>\n",
       "      <td>151917.602455</td>\n",
       "      <td>None</td>\n",
       "      <td>0.625</td>\n",
       "      <td>0.625</td>\n",
       "      <td>0.689453</td>\n",
       "      <td>0.689453</td>\n",
       "      <td>512.0</td>\n",
       "      <td>512.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>647.0</td>\n",
       "      <td>400.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                path         pid       sid  \\\n",
       "0  /pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...  0111259663  41209592   \n",
       "1  /pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...  0111259663  41209592   \n",
       "2  /pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...  0111259663  41209592   \n",
       "3  /pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...  0111259663  41209592   \n",
       "4  /pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...  0111225336  43814074   \n",
       "\n",
       "                   series  files   age sex modality      date           time  \\\n",
       "0     Series_302_LUNG_2.5    129  076Y   M       CT  20170101  084817.737107   \n",
       "1          Series_1_Scout      2  076Y   M       CT  20170101  084729.320407   \n",
       "2  Series_2_CHEST_WITHOUT    513  076Y   M       CT  20170101  084817.737107   \n",
       "3     Series_301_SOFT_2.5    129  076Y   M       CT  20170101  084817.737107   \n",
       "4       Series_5_CHEST_WO    436  081Y   F       CT  20180101  151917.602455   \n",
       "\n",
       "  body_part  slice_thickness  slice_spacing  pixcel_spacing_1  \\\n",
       "0     CHEST            2.500         -2.500          0.710938   \n",
       "1     CHEST          370.000         10.000          0.636364   \n",
       "2     CHEST            0.625         -0.625          0.710938   \n",
       "3     CHEST            2.500         -2.500          0.710938   \n",
       "4      None            0.625          0.625          0.689453   \n",
       "\n",
       "   pixcel_spacing_2   rows  columns    kvp  x_ray_current  exposure_time  \n",
       "0          0.710938  512.0    512.0  120.0          185.0          349.0  \n",
       "1          0.646341  582.0    820.0   80.0           20.0         3782.0  \n",
       "2          0.710938  512.0    512.0  120.0          231.0          349.0  \n",
       "3          0.710938  512.0    512.0  120.0          185.0          349.0  \n",
       "4          0.689453  512.0    512.0  120.0          647.0          400.0  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dicom_profile_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "dicom_profile_df.to_csv('R3_dicom_profile.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## series profiling analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "dicom_profile_df = pd.read_csv('R3_dicom_profile.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>path</th>\n",
       "      <th>pid</th>\n",
       "      <th>sid</th>\n",
       "      <th>series</th>\n",
       "      <th>files</th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>modality</th>\n",
       "      <th>date</th>\n",
       "      <th>time</th>\n",
       "      <th>body_part</th>\n",
       "      <th>slice_thickness</th>\n",
       "      <th>slice_spacing</th>\n",
       "      <th>rows</th>\n",
       "      <th>columns</th>\n",
       "      <th>kvp</th>\n",
       "      <th>x_ray_current</th>\n",
       "      <th>exposure_time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>/pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...</td>\n",
       "      <td>0111259663</td>\n",
       "      <td>41209592</td>\n",
       "      <td>Series_302_LUNG_2.5</td>\n",
       "      <td>129</td>\n",
       "      <td>076Y</td>\n",
       "      <td>M</td>\n",
       "      <td>CT</td>\n",
       "      <td>20170101.0</td>\n",
       "      <td>84817.737107</td>\n",
       "      <td>CHEST</td>\n",
       "      <td>2.500</td>\n",
       "      <td>-2.500</td>\n",
       "      <td>512.0</td>\n",
       "      <td>512.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>185.0</td>\n",
       "      <td>349.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>/pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...</td>\n",
       "      <td>0111259663</td>\n",
       "      <td>41209592</td>\n",
       "      <td>Series_1_Scout</td>\n",
       "      <td>2</td>\n",
       "      <td>076Y</td>\n",
       "      <td>M</td>\n",
       "      <td>CT</td>\n",
       "      <td>20170101.0</td>\n",
       "      <td>84729.320407</td>\n",
       "      <td>CHEST</td>\n",
       "      <td>370.000</td>\n",
       "      <td>10.000</td>\n",
       "      <td>582.0</td>\n",
       "      <td>820.0</td>\n",
       "      <td>80.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>3782.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>/pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...</td>\n",
       "      <td>0111259663</td>\n",
       "      <td>41209592</td>\n",
       "      <td>Series_2_CHEST_WITHOUT</td>\n",
       "      <td>513</td>\n",
       "      <td>076Y</td>\n",
       "      <td>M</td>\n",
       "      <td>CT</td>\n",
       "      <td>20170101.0</td>\n",
       "      <td>84817.737107</td>\n",
       "      <td>CHEST</td>\n",
       "      <td>0.625</td>\n",
       "      <td>-0.625</td>\n",
       "      <td>512.0</td>\n",
       "      <td>512.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>231.0</td>\n",
       "      <td>349.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>/pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...</td>\n",
       "      <td>0111259663</td>\n",
       "      <td>41209592</td>\n",
       "      <td>Series_301_SOFT_2.5</td>\n",
       "      <td>129</td>\n",
       "      <td>076Y</td>\n",
       "      <td>M</td>\n",
       "      <td>CT</td>\n",
       "      <td>20170101.0</td>\n",
       "      <td>84817.737107</td>\n",
       "      <td>CHEST</td>\n",
       "      <td>2.500</td>\n",
       "      <td>-2.500</td>\n",
       "      <td>512.0</td>\n",
       "      <td>512.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>185.0</td>\n",
       "      <td>349.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>/pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...</td>\n",
       "      <td>0111225336</td>\n",
       "      <td>43814074</td>\n",
       "      <td>Series_5_CHEST_WO</td>\n",
       "      <td>436</td>\n",
       "      <td>081Y</td>\n",
       "      <td>F</td>\n",
       "      <td>CT</td>\n",
       "      <td>20180101.0</td>\n",
       "      <td>151917.602455</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.625</td>\n",
       "      <td>0.625</td>\n",
       "      <td>512.0</td>\n",
       "      <td>512.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>647.0</td>\n",
       "      <td>400.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                path         pid       sid  \\\n",
       "0  /pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...  0111259663  41209592   \n",
       "1  /pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...  0111259663  41209592   \n",
       "2  /pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...  0111259663  41209592   \n",
       "3  /pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...  0111259663  41209592   \n",
       "4  /pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...  0111225336  43814074   \n",
       "\n",
       "                   series  files   age sex modality        date  \\\n",
       "0     Series_302_LUNG_2.5    129  076Y   M       CT  20170101.0   \n",
       "1          Series_1_Scout      2  076Y   M       CT  20170101.0   \n",
       "2  Series_2_CHEST_WITHOUT    513  076Y   M       CT  20170101.0   \n",
       "3     Series_301_SOFT_2.5    129  076Y   M       CT  20170101.0   \n",
       "4       Series_5_CHEST_WO    436  081Y   F       CT  20180101.0   \n",
       "\n",
       "            time body_part  slice_thickness  slice_spacing   rows  columns  \\\n",
       "0   84817.737107     CHEST            2.500         -2.500  512.0    512.0   \n",
       "1   84729.320407     CHEST          370.000         10.000  582.0    820.0   \n",
       "2   84817.737107     CHEST            0.625         -0.625  512.0    512.0   \n",
       "3   84817.737107     CHEST            2.500         -2.500  512.0    512.0   \n",
       "4  151917.602455       NaN            0.625          0.625  512.0    512.0   \n",
       "\n",
       "     kvp  x_ray_current  exposure_time  \n",
       "0  120.0          185.0          349.0  \n",
       "1   80.0           20.0         3782.0  \n",
       "2  120.0          231.0          349.0  \n",
       "3  120.0          185.0          349.0  \n",
       "4  120.0          647.0          400.0  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# show samples\n",
    "dicom_profile_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of dicom series: 49933\n"
     ]
    }
   ],
   "source": [
    "# total number of dicom series\n",
    "print('Total number of dicom series:', dicom_profile_df.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    49933.000000\n",
       "mean       176.207037\n",
       "std        171.725817\n",
       "min          0.000000\n",
       "25%         65.000000\n",
       "50%        126.000000\n",
       "75%        266.000000\n",
       "max       2560.000000\n",
       "Name: files, dtype: float64"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# distribution of number of files\n",
    "dicom_profile_df['files'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10117"
      ]
     },
     "execution_count": 143,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# number of patients (roughly match with the demographics data)\n",
    "dicom_profile_df['pid'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9872"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# number of patients who have at lest one seriers > 266 dicom images\n",
    "# (most of the patients, or 97.6% patients, have at least one series containing 266 or more images)\n",
    "ind = dicom_profile_df['files'] >= 266\n",
    "dicom_profile_df[ind]['pid'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "# is there any naming pattern associated with the number of images?\n",
    "s1 = dicom_profile_df.groupby('series')['files'].mean()\n",
    "s2 = dicom_profile_df.groupby('series')['files'].count()\n",
    "series_df = pd.concat([s1,s2],axis=1).reset_index()\n",
    "series_df.columns = ['series', 'avg_files', 'count']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>series</th>\n",
       "      <th>avg_files</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>121</th>\n",
       "      <td>Series_1_SCOUT</td>\n",
       "      <td>1.989225</td>\n",
       "      <td>6682</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>811</th>\n",
       "      <td>Series_4_LUNG_2.5</td>\n",
       "      <td>119.510458</td>\n",
       "      <td>2247</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>683</th>\n",
       "      <td>Series_3_SOFT_2.5</td>\n",
       "      <td>119.014027</td>\n",
       "      <td>2210</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>241</th>\n",
       "      <td>Series_2_DR_30</td>\n",
       "      <td>474.576617</td>\n",
       "      <td>1762</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>805</th>\n",
       "      <td>Series_4_LUNG</td>\n",
       "      <td>132.627890</td>\n",
       "      <td>1384</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>740</th>\n",
       "      <td>Series_401_Reformatted</td>\n",
       "      <td>142.569421</td>\n",
       "      <td>1210</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>738</th>\n",
       "      <td>Series_400_Reformatted</td>\n",
       "      <td>235.137759</td>\n",
       "      <td>1205</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>127</th>\n",
       "      <td>Series_1_Scout</td>\n",
       "      <td>2.001735</td>\n",
       "      <td>1153</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>134</th>\n",
       "      <td>Series_1_scout</td>\n",
       "      <td>2.003413</td>\n",
       "      <td>879</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>112</th>\n",
       "      <td>Series_1_AP_LAT_SCOUT</td>\n",
       "      <td>1.998792</td>\n",
       "      <td>828</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>762</th>\n",
       "      <td>Series_4_2.5MM_LUNG</td>\n",
       "      <td>120.600543</td>\n",
       "      <td>736</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>743</th>\n",
       "      <td>Series_402_Reformatted</td>\n",
       "      <td>141.975172</td>\n",
       "      <td>725</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>745</th>\n",
       "      <td>Series_403_Reformatted</td>\n",
       "      <td>105.071749</td>\n",
       "      <td>669</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1073</th>\n",
       "      <td>Series_601_8_X_1_CHEST</td>\n",
       "      <td>295.814024</td>\n",
       "      <td>656</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1107</th>\n",
       "      <td>Series_605_SPINE</td>\n",
       "      <td>129.401840</td>\n",
       "      <td>652</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>593</th>\n",
       "      <td>Series_3_2.5_MM</td>\n",
       "      <td>132.986090</td>\n",
       "      <td>647</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>630</th>\n",
       "      <td>Series_3_CHEST_WO</td>\n",
       "      <td>124.329173</td>\n",
       "      <td>641</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>980</th>\n",
       "      <td>Series_5_CHEST_WO</td>\n",
       "      <td>493.903276</td>\n",
       "      <td>641</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>728</th>\n",
       "      <td>Series_3_lung</td>\n",
       "      <td>121.307210</td>\n",
       "      <td>638</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>196</th>\n",
       "      <td>Series_2_2.5_CHEST</td>\n",
       "      <td>120.201908</td>\n",
       "      <td>629</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      series   avg_files  count\n",
       "121           Series_1_SCOUT    1.989225   6682\n",
       "811        Series_4_LUNG_2.5  119.510458   2247\n",
       "683        Series_3_SOFT_2.5  119.014027   2210\n",
       "241           Series_2_DR_30  474.576617   1762\n",
       "805            Series_4_LUNG  132.627890   1384\n",
       "740   Series_401_Reformatted  142.569421   1210\n",
       "738   Series_400_Reformatted  235.137759   1205\n",
       "127           Series_1_Scout    2.001735   1153\n",
       "134           Series_1_scout    2.003413    879\n",
       "112    Series_1_AP_LAT_SCOUT    1.998792    828\n",
       "762      Series_4_2.5MM_LUNG  120.600543    736\n",
       "743   Series_402_Reformatted  141.975172    725\n",
       "745   Series_403_Reformatted  105.071749    669\n",
       "1073  Series_601_8_X_1_CHEST  295.814024    656\n",
       "1107        Series_605_SPINE  129.401840    652\n",
       "593          Series_3_2.5_MM  132.986090    647\n",
       "630        Series_3_CHEST_WO  124.329173    641\n",
       "980        Series_5_CHEST_WO  493.903276    641\n",
       "728            Series_3_lung  121.307210    638\n",
       "196       Series_2_2.5_CHEST  120.201908    629"
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "series_df.sort_values(by='count', ascending=False).head(20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## dicom meta data analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "dicom_profile_df = pd.read_csv('R3_dicom_profile.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### meta samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>path</th>\n",
       "      <th>pid</th>\n",
       "      <th>sid</th>\n",
       "      <th>series</th>\n",
       "      <th>files</th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>modality</th>\n",
       "      <th>date</th>\n",
       "      <th>time</th>\n",
       "      <th>body_part</th>\n",
       "      <th>slice_thickness</th>\n",
       "      <th>slice_spacing</th>\n",
       "      <th>rows</th>\n",
       "      <th>columns</th>\n",
       "      <th>kvp</th>\n",
       "      <th>x_ray_current</th>\n",
       "      <th>exposure_time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>/pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...</td>\n",
       "      <td>0111259663</td>\n",
       "      <td>41209592</td>\n",
       "      <td>Series_302_LUNG_2.5</td>\n",
       "      <td>129</td>\n",
       "      <td>076Y</td>\n",
       "      <td>M</td>\n",
       "      <td>CT</td>\n",
       "      <td>20170101.0</td>\n",
       "      <td>84817.737107</td>\n",
       "      <td>CHEST</td>\n",
       "      <td>2.500000</td>\n",
       "      <td>-2.500</td>\n",
       "      <td>512.0</td>\n",
       "      <td>512.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>185.0</td>\n",
       "      <td>349.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>/pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...</td>\n",
       "      <td>0111259663</td>\n",
       "      <td>41209592</td>\n",
       "      <td>Series_1_Scout</td>\n",
       "      <td>2</td>\n",
       "      <td>076Y</td>\n",
       "      <td>M</td>\n",
       "      <td>CT</td>\n",
       "      <td>20170101.0</td>\n",
       "      <td>84729.320407</td>\n",
       "      <td>CHEST</td>\n",
       "      <td>370.000000</td>\n",
       "      <td>10.000</td>\n",
       "      <td>582.0</td>\n",
       "      <td>820.0</td>\n",
       "      <td>80.0</td>\n",
       "      <td>20.0</td>\n",
       "      <td>3782.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>/pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...</td>\n",
       "      <td>0111259663</td>\n",
       "      <td>41209592</td>\n",
       "      <td>Series_2_CHEST_WITHOUT</td>\n",
       "      <td>513</td>\n",
       "      <td>076Y</td>\n",
       "      <td>M</td>\n",
       "      <td>CT</td>\n",
       "      <td>20170101.0</td>\n",
       "      <td>84817.737107</td>\n",
       "      <td>CHEST</td>\n",
       "      <td>0.625000</td>\n",
       "      <td>-0.625</td>\n",
       "      <td>512.0</td>\n",
       "      <td>512.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>231.0</td>\n",
       "      <td>349.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>/pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...</td>\n",
       "      <td>0111259663</td>\n",
       "      <td>41209592</td>\n",
       "      <td>Series_301_SOFT_2.5</td>\n",
       "      <td>129</td>\n",
       "      <td>076Y</td>\n",
       "      <td>M</td>\n",
       "      <td>CT</td>\n",
       "      <td>20170101.0</td>\n",
       "      <td>84817.737107</td>\n",
       "      <td>CHEST</td>\n",
       "      <td>2.500000</td>\n",
       "      <td>-2.500</td>\n",
       "      <td>512.0</td>\n",
       "      <td>512.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>185.0</td>\n",
       "      <td>349.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>/pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...</td>\n",
       "      <td>0111225336</td>\n",
       "      <td>43814074</td>\n",
       "      <td>Series_5_CHEST_WO</td>\n",
       "      <td>436</td>\n",
       "      <td>081Y</td>\n",
       "      <td>F</td>\n",
       "      <td>CT</td>\n",
       "      <td>20180101.0</td>\n",
       "      <td>151917.602455</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.625000</td>\n",
       "      <td>0.625</td>\n",
       "      <td>512.0</td>\n",
       "      <td>512.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>647.0</td>\n",
       "      <td>400.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>/pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...</td>\n",
       "      <td>0111225336</td>\n",
       "      <td>43814074</td>\n",
       "      <td>Series_605_SPINE</td>\n",
       "      <td>136</td>\n",
       "      <td>081Y</td>\n",
       "      <td>F</td>\n",
       "      <td>CT</td>\n",
       "      <td>20180101.0</td>\n",
       "      <td>151917.602455</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>512.0</td>\n",
       "      <td>512.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>400.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>/pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...</td>\n",
       "      <td>0111225336</td>\n",
       "      <td>43814074</td>\n",
       "      <td>Series_601_8_X_1_CHEST</td>\n",
       "      <td>286</td>\n",
       "      <td>081Y</td>\n",
       "      <td>F</td>\n",
       "      <td>CT</td>\n",
       "      <td>20180101.0</td>\n",
       "      <td>151917.602455</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8.000001</td>\n",
       "      <td>NaN</td>\n",
       "      <td>512.0</td>\n",
       "      <td>512.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>400.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>/pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...</td>\n",
       "      <td>0111225336</td>\n",
       "      <td>43814074</td>\n",
       "      <td>Series_1_SCOUT</td>\n",
       "      <td>2</td>\n",
       "      <td>081Y</td>\n",
       "      <td>F</td>\n",
       "      <td>CT</td>\n",
       "      <td>20180101.0</td>\n",
       "      <td>151719.738681</td>\n",
       "      <td>NaN</td>\n",
       "      <td>355.545457</td>\n",
       "      <td>10.000</td>\n",
       "      <td>651.0</td>\n",
       "      <td>888.0</td>\n",
       "      <td>80.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>3626.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>/pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...</td>\n",
       "      <td>0111225336</td>\n",
       "      <td>43814074</td>\n",
       "      <td>Series_3_CHEST_WO</td>\n",
       "      <td>109</td>\n",
       "      <td>081Y</td>\n",
       "      <td>F</td>\n",
       "      <td>CT</td>\n",
       "      <td>20180101.0</td>\n",
       "      <td>151917.602455</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.500000</td>\n",
       "      <td>0.625</td>\n",
       "      <td>512.0</td>\n",
       "      <td>512.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>647.0</td>\n",
       "      <td>400.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>/pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...</td>\n",
       "      <td>0111225336</td>\n",
       "      <td>43814074</td>\n",
       "      <td>Series_4_LUNG</td>\n",
       "      <td>109</td>\n",
       "      <td>081Y</td>\n",
       "      <td>F</td>\n",
       "      <td>CT</td>\n",
       "      <td>20180101.0</td>\n",
       "      <td>151917.602455</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.500000</td>\n",
       "      <td>0.625</td>\n",
       "      <td>512.0</td>\n",
       "      <td>512.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>647.0</td>\n",
       "      <td>400.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>/pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...</td>\n",
       "      <td>0110950981</td>\n",
       "      <td>42086009</td>\n",
       "      <td>Series_2_DR_30</td>\n",
       "      <td>519</td>\n",
       "      <td>050Y</td>\n",
       "      <td>M</td>\n",
       "      <td>CT</td>\n",
       "      <td>20170101.0</td>\n",
       "      <td>224711.918638</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.625000</td>\n",
       "      <td>0.625</td>\n",
       "      <td>512.0</td>\n",
       "      <td>512.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>696.0</td>\n",
       "      <td>400.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>/pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...</td>\n",
       "      <td>0110950981</td>\n",
       "      <td>42086009</td>\n",
       "      <td>Series_1_SCOUT</td>\n",
       "      <td>2</td>\n",
       "      <td>050Y</td>\n",
       "      <td>M</td>\n",
       "      <td>CT</td>\n",
       "      <td>20170101.0</td>\n",
       "      <td>224607.176761</td>\n",
       "      <td>NaN</td>\n",
       "      <td>400.545457</td>\n",
       "      <td>10.000</td>\n",
       "      <td>734.0</td>\n",
       "      <td>888.0</td>\n",
       "      <td>80.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>4076.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>/pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...</td>\n",
       "      <td>0110950981</td>\n",
       "      <td>42086009</td>\n",
       "      <td>Series_3_SOFT_2.5</td>\n",
       "      <td>131</td>\n",
       "      <td>050Y</td>\n",
       "      <td>M</td>\n",
       "      <td>CT</td>\n",
       "      <td>20170101.0</td>\n",
       "      <td>224711.918638</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.500000</td>\n",
       "      <td>0.625</td>\n",
       "      <td>512.0</td>\n",
       "      <td>512.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>695.0</td>\n",
       "      <td>400.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>/pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...</td>\n",
       "      <td>0110950981</td>\n",
       "      <td>42086009</td>\n",
       "      <td>Series_603_MIP_1_X_8</td>\n",
       "      <td>310</td>\n",
       "      <td>050Y</td>\n",
       "      <td>M</td>\n",
       "      <td>CT</td>\n",
       "      <td>20170101.0</td>\n",
       "      <td>224711.918638</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>512.0</td>\n",
       "      <td>512.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>400.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>/pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...</td>\n",
       "      <td>0110950981</td>\n",
       "      <td>42086009</td>\n",
       "      <td>Series_4_LUNG_2.5</td>\n",
       "      <td>131</td>\n",
       "      <td>050Y</td>\n",
       "      <td>M</td>\n",
       "      <td>CT</td>\n",
       "      <td>20170101.0</td>\n",
       "      <td>224711.918638</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.500000</td>\n",
       "      <td>0.625</td>\n",
       "      <td>512.0</td>\n",
       "      <td>512.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>696.0</td>\n",
       "      <td>400.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>/pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...</td>\n",
       "      <td>0119239036</td>\n",
       "      <td>40732683</td>\n",
       "      <td>Series_6_HI_RES_PRONE-INSPIRATION</td>\n",
       "      <td>35</td>\n",
       "      <td>070Y</td>\n",
       "      <td>M</td>\n",
       "      <td>CT</td>\n",
       "      <td>20170101.0</td>\n",
       "      <td>193601.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.250000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>512.0</td>\n",
       "      <td>512.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>311.0</td>\n",
       "      <td>1000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>/pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...</td>\n",
       "      <td>0119239036</td>\n",
       "      <td>40732683</td>\n",
       "      <td>Series_4_HI_RES_SUPINE-_EXPIRATION</td>\n",
       "      <td>36</td>\n",
       "      <td>070Y</td>\n",
       "      <td>M</td>\n",
       "      <td>CT</td>\n",
       "      <td>20170101.0</td>\n",
       "      <td>191920.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.250000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>512.0</td>\n",
       "      <td>512.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>340.0</td>\n",
       "      <td>1000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>/pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...</td>\n",
       "      <td>0119239036</td>\n",
       "      <td>40732683</td>\n",
       "      <td>Series_301_Reformatted</td>\n",
       "      <td>152</td>\n",
       "      <td>070Y</td>\n",
       "      <td>M</td>\n",
       "      <td>CT</td>\n",
       "      <td>20170101.0</td>\n",
       "      <td>191517.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>512.0</td>\n",
       "      <td>512.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>397.0</td>\n",
       "      <td>808.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>/pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...</td>\n",
       "      <td>0119239036</td>\n",
       "      <td>40732683</td>\n",
       "      <td>Series_1_SCOUTS</td>\n",
       "      <td>2</td>\n",
       "      <td>070Y</td>\n",
       "      <td>M</td>\n",
       "      <td>CT</td>\n",
       "      <td>20170101.0</td>\n",
       "      <td>191402.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>360.181824</td>\n",
       "      <td>NaN</td>\n",
       "      <td>660.0</td>\n",
       "      <td>888.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>10.0</td>\n",
       "      <td>3672.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>/pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...</td>\n",
       "      <td>0119239036</td>\n",
       "      <td>40732683</td>\n",
       "      <td>Series_300_Reformatted</td>\n",
       "      <td>184</td>\n",
       "      <td>070Y</td>\n",
       "      <td>M</td>\n",
       "      <td>CT</td>\n",
       "      <td>20170101.0</td>\n",
       "      <td>191517.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>512.0</td>\n",
       "      <td>512.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>397.0</td>\n",
       "      <td>808.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 path         pid       sid  \\\n",
       "0   /pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...  0111259663  41209592   \n",
       "1   /pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...  0111259663  41209592   \n",
       "2   /pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...  0111259663  41209592   \n",
       "3   /pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...  0111259663  41209592   \n",
       "4   /pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...  0111225336  43814074   \n",
       "5   /pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...  0111225336  43814074   \n",
       "6   /pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...  0111225336  43814074   \n",
       "7   /pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...  0111225336  43814074   \n",
       "8   /pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...  0111225336  43814074   \n",
       "9   /pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...  0111225336  43814074   \n",
       "10  /pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...  0110950981  42086009   \n",
       "11  /pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...  0110950981  42086009   \n",
       "12  /pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...  0110950981  42086009   \n",
       "13  /pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...  0110950981  42086009   \n",
       "14  /pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...  0110950981  42086009   \n",
       "15  /pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...  0119239036  40732683   \n",
       "16  /pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...  0119239036  40732683   \n",
       "17  /pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...  0119239036  40732683   \n",
       "18  /pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...  0119239036  40732683   \n",
       "19  /pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...  0119239036  40732683   \n",
       "\n",
       "                                series  files   age sex modality        date  \\\n",
       "0                  Series_302_LUNG_2.5    129  076Y   M       CT  20170101.0   \n",
       "1                       Series_1_Scout      2  076Y   M       CT  20170101.0   \n",
       "2               Series_2_CHEST_WITHOUT    513  076Y   M       CT  20170101.0   \n",
       "3                  Series_301_SOFT_2.5    129  076Y   M       CT  20170101.0   \n",
       "4                    Series_5_CHEST_WO    436  081Y   F       CT  20180101.0   \n",
       "5                     Series_605_SPINE    136  081Y   F       CT  20180101.0   \n",
       "6               Series_601_8_X_1_CHEST    286  081Y   F       CT  20180101.0   \n",
       "7                       Series_1_SCOUT      2  081Y   F       CT  20180101.0   \n",
       "8                    Series_3_CHEST_WO    109  081Y   F       CT  20180101.0   \n",
       "9                        Series_4_LUNG    109  081Y   F       CT  20180101.0   \n",
       "10                      Series_2_DR_30    519  050Y   M       CT  20170101.0   \n",
       "11                      Series_1_SCOUT      2  050Y   M       CT  20170101.0   \n",
       "12                   Series_3_SOFT_2.5    131  050Y   M       CT  20170101.0   \n",
       "13                Series_603_MIP_1_X_8    310  050Y   M       CT  20170101.0   \n",
       "14                   Series_4_LUNG_2.5    131  050Y   M       CT  20170101.0   \n",
       "15   Series_6_HI_RES_PRONE-INSPIRATION     35  070Y   M       CT  20170101.0   \n",
       "16  Series_4_HI_RES_SUPINE-_EXPIRATION     36  070Y   M       CT  20170101.0   \n",
       "17              Series_301_Reformatted    152  070Y   M       CT  20170101.0   \n",
       "18                     Series_1_SCOUTS      2  070Y   M       CT  20170101.0   \n",
       "19              Series_300_Reformatted    184  070Y   M       CT  20170101.0   \n",
       "\n",
       "             time body_part  slice_thickness  slice_spacing   rows  columns  \\\n",
       "0    84817.737107     CHEST         2.500000         -2.500  512.0    512.0   \n",
       "1    84729.320407     CHEST       370.000000         10.000  582.0    820.0   \n",
       "2    84817.737107     CHEST         0.625000         -0.625  512.0    512.0   \n",
       "3    84817.737107     CHEST         2.500000         -2.500  512.0    512.0   \n",
       "4   151917.602455       NaN         0.625000          0.625  512.0    512.0   \n",
       "5   151917.602455       NaN         2.000000            NaN  512.0    512.0   \n",
       "6   151917.602455       NaN         8.000001            NaN  512.0    512.0   \n",
       "7   151719.738681       NaN       355.545457         10.000  651.0    888.0   \n",
       "8   151917.602455       NaN         2.500000          0.625  512.0    512.0   \n",
       "9   151917.602455       NaN         2.500000          0.625  512.0    512.0   \n",
       "10  224711.918638       NaN         0.625000          0.625  512.0    512.0   \n",
       "11  224607.176761       NaN       400.545457         10.000  734.0    888.0   \n",
       "12  224711.918638       NaN         2.500000          0.625  512.0    512.0   \n",
       "13  224711.918638       NaN         8.000000            NaN  512.0    512.0   \n",
       "14  224711.918638       NaN         2.500000          0.625  512.0    512.0   \n",
       "15  193601.000000       NaN         1.250000            NaN  512.0    512.0   \n",
       "16  191920.000000       NaN         1.250000            NaN  512.0    512.0   \n",
       "17  191517.000000       NaN         2.000000            NaN  512.0    512.0   \n",
       "18  191402.000000       NaN       360.181824            NaN  660.0    888.0   \n",
       "19  191517.000000       NaN         2.000000            NaN  512.0    512.0   \n",
       "\n",
       "      kvp  x_ray_current  exposure_time  \n",
       "0   120.0          185.0          349.0  \n",
       "1    80.0           20.0         3782.0  \n",
       "2   120.0          231.0          349.0  \n",
       "3   120.0          185.0          349.0  \n",
       "4   120.0          647.0          400.0  \n",
       "5   120.0            NaN          400.0  \n",
       "6   120.0            NaN          400.0  \n",
       "7    80.0           40.0         3626.0  \n",
       "8   120.0          647.0          400.0  \n",
       "9   120.0          647.0          400.0  \n",
       "10  120.0          696.0          400.0  \n",
       "11   80.0           40.0         4076.0  \n",
       "12  120.0          695.0          400.0  \n",
       "13  120.0            NaN          400.0  \n",
       "14  120.0          696.0          400.0  \n",
       "15  120.0          311.0         1000.0  \n",
       "16  120.0          340.0         1000.0  \n",
       "17  120.0          397.0          808.0  \n",
       "18  120.0           10.0         3672.0  \n",
       "19  120.0          397.0          808.0  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# show samples\n",
    "dicom_profile_df.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R3/20_Batmanghelich/10/Patient_0111259663/Study_CT_CHEST_WITHOUT_CONTRAST_41209592/Series_2_CHEST_WITHOUT/1.173628060384414930913011079890345343151.dcm\n",
      "(0008, 0005) Specific Character Set              CS: 'ISO_IR 100'\n",
      "(0008, 0008) Image Type                          CS: ['ORIGINAL', 'PRIMARY', 'AXIAL']\n",
      "(0008, 0012) Instance Creation Date              DA: '20170101'\n",
      "(0008, 0013) Instance Creation Time              TM: '084830'\n",
      "(0008, 0016) SOP Class UID                       UI: CT Image Storage\n",
      "(0008, 0018) SOP Instance UID                    UI: 1.173628060384414930913011079890345343151\n",
      "(0008, 0020) Study Date                          DA: '20170101'\n",
      "(0008, 0021) Series Date                         DA: '20170101'\n",
      "(0008, 0022) Acquisition Date                    DA: '20170101'\n",
      "(0008, 0023) Content Date                        DA: '20170101'\n",
      "(0008, 0030) Study Time                          TM: '084635'\n",
      "(0008, 0031) Series Time                         TM: '084819'\n",
      "(0008, 0032) Acquisition Time                    TM: '084817.737107'\n",
      "(0008, 0033) Content Time                        TM: '084830'\n",
      "(0008, 0050) Accession Number                    SH: '41209592'\n",
      "(0008, 0060) Modality                            CS: 'CT'\n",
      "(0008, 0070) Manufacturer                        LO: 'GE MEDICAL SYSTEMS'\n",
      "(0008, 0080) Institution Name                    LO: ''\n",
      "(0008, 0090) Referring Physician's Name          PN: ' '\n",
      "(0008, 1030) Study Description                   LO: 'CT CHEST WITHOUT CONTRAST'\n",
      "(0008, 1032)  Procedure Code Sequence   1 item(s) ---- \n",
      "   (0008, 0100) Code Value                          SH: 'CTCHESTX'\n",
      "   (0008, 0104) Code Meaning                        LO: 'CT CHEST WITHOUT CONTRAST'\n",
      "   ---------\n",
      "(0008, 103e) Series Description                  LO: 'CHEST WITHOUT'\n",
      "(0008, 1090) Manufacturer's Model Name           LO: 'Revolution CT'\n",
      "(0008, 3010) Irradiation Event UID               UI: 1.195801622828940421564270430252158880138\n",
      "(0010, 0010) Patient's Name                      PN: ' '\n",
      "(0010, 0020) Patient ID                          LO: '0111259663'\n",
      "(0010, 0030) Patient's Birth Date                DA: '19410101'\n",
      "(0010, 0040) Patient's Sex                       CS: 'M'\n",
      "(0010, 1010) Patient's Age                       AS: '076Y'\n",
      "(0012, 0062) Patient Identity Removed            CS: 'YES'\n",
      "(0012, 0063) De-identification Method            LO: 'CTP Default:  based on DICOM PS3.15 AnnexE. Details in 0012,0064'\n",
      "(0012, 0064)  De-identification Method Code Sequence   5 item(s) ---- \n",
      "   (0008, 0100) Code Value                          SH: '113100'\n",
      "   (0008, 0102) Coding Scheme Designator            SH: 'DCM'\n",
      "   (0008, 0104) Code Meaning                        LO: 'Basic Application Confidentiality Profile'\n",
      "   ---------\n",
      "   (0008, 0100) Code Value                          SH: '113105'\n",
      "   (0008, 0102) Coding Scheme Designator            SH: 'DCM'\n",
      "   (0008, 0104) Code Meaning                        LO: 'Clean Descriptors Option'\n",
      "   ---------\n",
      "   (0008, 0100) Code Value                          SH: '113107'\n",
      "   (0008, 0102) Coding Scheme Designator            SH: 'DCM'\n",
      "   (0008, 0104) Code Meaning                        LO: 'Retain Longitudinal Temporal Information Modified Dates Option'\n",
      "   ---------\n",
      "   (0008, 0100) Code Value                          SH: '113108'\n",
      "   (0008, 0102) Coding Scheme Designator            SH: 'DCM'\n",
      "   (0008, 0104) Code Meaning                        LO: 'Retain Patient Characteristics Option'\n",
      "   ---------\n",
      "   (0008, 0100) Code Value                          SH: '113109'\n",
      "   (0008, 0102) Coding Scheme Designator            SH: 'DCM'\n",
      "   (0008, 0104) Code Meaning                        LO: 'Retain Device Identity Option'\n",
      "   ---------\n",
      "(0018, 0015) Body Part Examined                  CS: 'CHEST'\n",
      "(0018, 0022) Scan Options                        CS: 'HELICAL MODE'\n",
      "(0018, 0050) Slice Thickness                     DS: \"0.625\"\n",
      "(0018, 0060) KVP                                 DS: \"120.0\"\n",
      "(0018, 0088) Spacing Between Slices              DS: \"-0.625\"\n",
      "(0018, 0090) Data Collection Diameter            DS: \"500.0\"\n",
      "(0018, 1000) Device Serial Number                LO: 'UNKNOWN'\n",
      "(0018, 1020) Software Versions                   LO: 'revo_1.5_m3b_xt.53'\n",
      "(0018, 1100) Reconstruction Diameter             DS: \"364.0\"\n",
      "(0018, 1110) Distance Source to Detector         DS: \"1097.61\"\n",
      "(0018, 1111) Distance Source to Patient          DS: \"625.61\"\n",
      "(0018, 1120) Gantry/Detector Tilt                DS: \"0.0\"\n",
      "(0018, 1130) Table Height                        DS: \"174.5\"\n",
      "(0018, 1140) Rotation Direction                  CS: 'CW'\n",
      "(0018, 1150) Exposure Time                       IS: \"349\"\n",
      "(0018, 1151) X-Ray Tube Current                  IS: \"231\"\n",
      "(0018, 1152) Exposure                            IS: \"0\"\n",
      "(0018, 1160) Filter Type                         SH: 'BODY FILTER'\n",
      "(0018, 1170) Generator Power                     IS: \"42000\"\n",
      "(0018, 1190) Focal Spot(s)                       DS: \"1.2\"\n",
      "(0018, 1210) Convolution Kernel                  SH: 'STANDARD'\n",
      "(0018, 5100) Patient Position                    CS: 'FFS'\n",
      "(0018, 9305) Revolution Time                     FD: 0.35\n",
      "(0018, 9306) Single Collimation Width            FD: 0.625\n",
      "(0018, 9307) Total Collimation Width             FD: 80.0\n",
      "(0018, 9309) Table Speed                         FD: 226.7856924874442\n",
      "(0018, 9310) Table Feed per Rotation             FD: 79.37499237060547\n",
      "(0018, 9311) Spiral Pitch Factor                 FD: 0.9921874046325684\n",
      "(0020, 000d) Study Instance UID                  UI: 1.206195360372899552434858802707494844338\n",
      "(0020, 000e) Series Instance UID                 UI: 1.189291886103607707359098180926816002252\n",
      "(0020, 0010) Study ID                            SH: '1.96701558009142014686363983942228233978'\n",
      "(0020, 0011) Series Number                       IS: \"2\"\n",
      "(0020, 0012) Acquisition Number                  IS: \"1\"\n",
      "(0020, 0013) Instance Number                     IS: \"164\"\n",
      "(0020, 0032) Image Position (Patient)            DS: [-184.926, -173.867, -132.249]\n",
      "(0020, 0037) Image Orientation (Patient)         DS: [1.000000, 0.000000, 0.000000, 0.000000, 1.000000, 0.000000]\n",
      "(0020, 0052) Frame of Reference UID              UI: 1.46173189128964008911985340719396327420\n",
      "(0020, 1040) Position Reference Indicator        LO: 'SN'\n",
      "(0020, 1041) Slice Location                      DS: \"-132.249\"\n",
      "(0028, 0002) Samples per Pixel                   US: 1\n",
      "(0028, 0004) Photometric Interpretation          CS: 'MONOCHROME2'\n",
      "(0028, 0010) Rows                                US: 512\n",
      "(0028, 0011) Columns                             US: 512\n",
      "(0028, 0030) Pixel Spacing                       DS: [0.710938, 0.710938]\n",
      "(0028, 0100) Bits Allocated                      US: 16\n",
      "(0028, 0101) Bits Stored                         US: 16\n",
      "(0028, 0102) High Bit                            US: 15\n",
      "(0028, 0103) Pixel Representation                US: 1\n",
      "(0028, 0120) Pixel Padding Value                 SS: -2000\n",
      "(0028, 0303) Longitudinal Temporal Information M CS: 'MODIFIED'\n",
      "(0028, 1050) Window Center                       DS: \"40.0\"\n",
      "(0028, 1051) Window Width                        DS: \"400.0\"\n",
      "(0028, 1052) Rescale Intercept                   DS: \"-1024.0\"\n",
      "(0028, 1053) Rescale Slope                       DS: \"1.0\"\n",
      "(0028, 1054) Rescale Type                        LO: 'HU'\n",
      "(0040, 0244) Performed Procedure Step Start Date DA: '20170101'\n",
      "(0040, 0245) Performed Procedure Step Start Time TM: '084750'\n",
      "(7fe0, 0010) Pixel Data                          OW: Array of 524288 elements\n",
      "\n",
      "\n",
      "/pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R3/20_Batmanghelich/10/Patient_0111259663/Study_CT_CHEST_WITHOUT_CONTRAST_41209592/Series_2_CHEST_WITHOUT/1.260626262539811748548983709090876033134.dcm\n",
      "(0008, 0005) Specific Character Set              CS: 'ISO_IR 100'\n",
      "(0008, 0008) Image Type                          CS: ['ORIGINAL', 'PRIMARY', 'AXIAL']\n",
      "(0008, 0012) Instance Creation Date              DA: '20170101'\n",
      "(0008, 0013) Instance Creation Time              TM: '084831'\n",
      "(0008, 0016) SOP Class UID                       UI: CT Image Storage\n",
      "(0008, 0018) SOP Instance UID                    UI: 1.260626262539811748548983709090876033134\n",
      "(0008, 0020) Study Date                          DA: '20170101'\n",
      "(0008, 0021) Series Date                         DA: '20170101'\n",
      "(0008, 0022) Acquisition Date                    DA: '20170101'\n",
      "(0008, 0023) Content Date                        DA: '20170101'\n",
      "(0008, 0030) Study Time                          TM: '084635'\n",
      "(0008, 0031) Series Time                         TM: '084819'\n",
      "(0008, 0032) Acquisition Time                    TM: '084817.737107'\n",
      "(0008, 0033) Content Time                        TM: '084831'\n",
      "(0008, 0050) Accession Number                    SH: '41209592'\n",
      "(0008, 0060) Modality                            CS: 'CT'\n",
      "(0008, 0070) Manufacturer                        LO: 'GE MEDICAL SYSTEMS'\n",
      "(0008, 0080) Institution Name                    LO: ''\n",
      "(0008, 0090) Referring Physician's Name          PN: ' '\n",
      "(0008, 1030) Study Description                   LO: 'CT CHEST WITHOUT CONTRAST'\n",
      "(0008, 1032)  Procedure Code Sequence   1 item(s) ---- \n",
      "   (0008, 0100) Code Value                          SH: 'CTCHESTX'\n",
      "   (0008, 0104) Code Meaning                        LO: 'CT CHEST WITHOUT CONTRAST'\n",
      "   ---------\n",
      "(0008, 103e) Series Description                  LO: 'CHEST WITHOUT'\n",
      "(0008, 1090) Manufacturer's Model Name           LO: 'Revolution CT'\n",
      "(0008, 3010) Irradiation Event UID               UI: 1.195801622828940421564270430252158880138\n",
      "(0010, 0010) Patient's Name                      PN: ' '\n",
      "(0010, 0020) Patient ID                          LO: '0111259663'\n",
      "(0010, 0030) Patient's Birth Date                DA: '19410101'\n",
      "(0010, 0040) Patient's Sex                       CS: 'M'\n",
      "(0010, 1010) Patient's Age                       AS: '076Y'\n",
      "(0012, 0062) Patient Identity Removed            CS: 'YES'\n",
      "(0012, 0063) De-identification Method            LO: 'CTP Default:  based on DICOM PS3.15 AnnexE. Details in 0012,0064'\n",
      "(0012, 0064)  De-identification Method Code Sequence   5 item(s) ---- \n",
      "   (0008, 0100) Code Value                          SH: '113100'\n",
      "   (0008, 0102) Coding Scheme Designator            SH: 'DCM'\n",
      "   (0008, 0104) Code Meaning                        LO: 'Basic Application Confidentiality Profile'\n",
      "   ---------\n",
      "   (0008, 0100) Code Value                          SH: '113105'\n",
      "   (0008, 0102) Coding Scheme Designator            SH: 'DCM'\n",
      "   (0008, 0104) Code Meaning                        LO: 'Clean Descriptors Option'\n",
      "   ---------\n",
      "   (0008, 0100) Code Value                          SH: '113107'\n",
      "   (0008, 0102) Coding Scheme Designator            SH: 'DCM'\n",
      "   (0008, 0104) Code Meaning                        LO: 'Retain Longitudinal Temporal Information Modified Dates Option'\n",
      "   ---------\n",
      "   (0008, 0100) Code Value                          SH: '113108'\n",
      "   (0008, 0102) Coding Scheme Designator            SH: 'DCM'\n",
      "   (0008, 0104) Code Meaning                        LO: 'Retain Patient Characteristics Option'\n",
      "   ---------\n",
      "   (0008, 0100) Code Value                          SH: '113109'\n",
      "   (0008, 0102) Coding Scheme Designator            SH: 'DCM'\n",
      "   (0008, 0104) Code Meaning                        LO: 'Retain Device Identity Option'\n",
      "   ---------\n",
      "(0018, 0015) Body Part Examined                  CS: 'CHEST'\n",
      "(0018, 0022) Scan Options                        CS: 'HELICAL MODE'\n",
      "(0018, 0050) Slice Thickness                     DS: \"0.625\"\n",
      "(0018, 0060) KVP                                 DS: \"120.0\"\n",
      "(0018, 0088) Spacing Between Slices              DS: \"-0.625\"\n",
      "(0018, 0090) Data Collection Diameter            DS: \"500.0\"\n",
      "(0018, 1000) Device Serial Number                LO: 'UNKNOWN'\n",
      "(0018, 1020) Software Versions                   LO: 'revo_1.5_m3b_xt.53'\n",
      "(0018, 1100) Reconstruction Diameter             DS: \"364.0\"\n",
      "(0018, 1110) Distance Source to Detector         DS: \"1097.61\"\n",
      "(0018, 1111) Distance Source to Patient          DS: \"625.61\"\n",
      "(0018, 1120) Gantry/Detector Tilt                DS: \"0.0\"\n",
      "(0018, 1130) Table Height                        DS: \"174.5\"\n",
      "(0018, 1140) Rotation Direction                  CS: 'CW'\n",
      "(0018, 1150) Exposure Time                       IS: \"349\"\n",
      "(0018, 1151) X-Ray Tube Current                  IS: \"197\"\n",
      "(0018, 1152) Exposure                            IS: \"0\"\n",
      "(0018, 1160) Filter Type                         SH: 'BODY FILTER'\n",
      "(0018, 1170) Generator Power                     IS: \"42000\"\n",
      "(0018, 1190) Focal Spot(s)                       DS: \"1.2\"\n",
      "(0018, 1210) Convolution Kernel                  SH: 'STANDARD'\n",
      "(0018, 5100) Patient Position                    CS: 'FFS'\n",
      "(0018, 9305) Revolution Time                     FD: 0.35\n",
      "(0018, 9306) Single Collimation Width            FD: 0.625\n",
      "(0018, 9307) Total Collimation Width             FD: 80.0\n",
      "(0018, 9309) Table Speed                         FD: 226.7856924874442\n",
      "(0018, 9310) Table Feed per Rotation             FD: 79.37499237060547\n",
      "(0018, 9311) Spiral Pitch Factor                 FD: 0.9921874046325684\n",
      "(0020, 000d) Study Instance UID                  UI: 1.206195360372899552434858802707494844338\n",
      "(0020, 000e) Series Instance UID                 UI: 1.189291886103607707359098180926816002252\n",
      "(0020, 0010) Study ID                            SH: '1.96701558009142014686363983942228233978'\n",
      "(0020, 0011) Series Number                       IS: \"2\"\n",
      "(0020, 0012) Acquisition Number                  IS: \"1\"\n",
      "(0020, 0013) Instance Number                     IS: \"200\"\n",
      "(0020, 0032) Image Position (Patient)            DS: [-184.926, -173.867, -154.749]\n",
      "(0020, 0037) Image Orientation (Patient)         DS: [1.000000, 0.000000, 0.000000, 0.000000, 1.000000, 0.000000]\n",
      "(0020, 0052) Frame of Reference UID              UI: 1.46173189128964008911985340719396327420\n",
      "(0020, 1040) Position Reference Indicator        LO: 'SN'\n",
      "(0020, 1041) Slice Location                      DS: \"-154.749\"\n",
      "(0028, 0002) Samples per Pixel                   US: 1\n",
      "(0028, 0004) Photometric Interpretation          CS: 'MONOCHROME2'\n",
      "(0028, 0010) Rows                                US: 512\n",
      "(0028, 0011) Columns                             US: 512\n",
      "(0028, 0030) Pixel Spacing                       DS: [0.710938, 0.710938]\n",
      "(0028, 0100) Bits Allocated                      US: 16\n",
      "(0028, 0101) Bits Stored                         US: 16\n",
      "(0028, 0102) High Bit                            US: 15\n",
      "(0028, 0103) Pixel Representation                US: 1\n",
      "(0028, 0120) Pixel Padding Value                 SS: -2000\n",
      "(0028, 0303) Longitudinal Temporal Information M CS: 'MODIFIED'\n",
      "(0028, 1050) Window Center                       DS: \"40.0\"\n",
      "(0028, 1051) Window Width                        DS: \"400.0\"\n",
      "(0028, 1052) Rescale Intercept                   DS: \"-1024.0\"\n",
      "(0028, 1053) Rescale Slope                       DS: \"1.0\"\n",
      "(0028, 1054) Rescale Type                        LO: 'HU'\n",
      "(0040, 0244) Performed Procedure Step Start Date DA: '20170101'\n",
      "(0040, 0245) Performed Procedure Step Start Time TM: '084750'\n",
      "(7fe0, 0010) Pixel Data                          OW: Array of 524288 elements\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# show 2 dicom meta data samples\n",
    "sample_path = dicom_profile_df.iloc[2,:]\n",
    "dir_path = sample_path.path\n",
    "\n",
    "i = 0\n",
    "for _,_,files in os.walk(dir_path):\n",
    "    for f in files:\n",
    "        file_path = os.path.join(dir_path, f)\n",
    "        print(file_path)\n",
    "        ds = pydicom.filereader.dcmread(file_path)\n",
    "        print(ds)\n",
    "        print('\\n')\n",
    "        i+=1\n",
    "        if(i>=2):\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.710938, 0.710938]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds.PixelSpacing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### meta statistics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "# reduce series level to patient level\n",
    "# select series with most images per patient\n",
    "idx = dicom_profile_df.groupby(['pid'])['files'].transform(max) == dicom_profile_df['files']\n",
    "pt_dicom_profile_df = dicom_profile_df[idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>path</th>\n",
       "      <th>pid</th>\n",
       "      <th>sid</th>\n",
       "      <th>series</th>\n",
       "      <th>files</th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>modality</th>\n",
       "      <th>date</th>\n",
       "      <th>time</th>\n",
       "      <th>body_part</th>\n",
       "      <th>slice_thickness</th>\n",
       "      <th>slice_spacing</th>\n",
       "      <th>rows</th>\n",
       "      <th>columns</th>\n",
       "      <th>kvp</th>\n",
       "      <th>x_ray_current</th>\n",
       "      <th>exposure_time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>/pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...</td>\n",
       "      <td>0111259663</td>\n",
       "      <td>41209592</td>\n",
       "      <td>Series_2_CHEST_WITHOUT</td>\n",
       "      <td>513</td>\n",
       "      <td>076Y</td>\n",
       "      <td>M</td>\n",
       "      <td>CT</td>\n",
       "      <td>20170101.0</td>\n",
       "      <td>84817.737107</td>\n",
       "      <td>CHEST</td>\n",
       "      <td>0.625</td>\n",
       "      <td>-0.625</td>\n",
       "      <td>512.0</td>\n",
       "      <td>512.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>231.0</td>\n",
       "      <td>349.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>/pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...</td>\n",
       "      <td>0111225336</td>\n",
       "      <td>43814074</td>\n",
       "      <td>Series_5_CHEST_WO</td>\n",
       "      <td>436</td>\n",
       "      <td>081Y</td>\n",
       "      <td>F</td>\n",
       "      <td>CT</td>\n",
       "      <td>20180101.0</td>\n",
       "      <td>151917.602455</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.625</td>\n",
       "      <td>0.625</td>\n",
       "      <td>512.0</td>\n",
       "      <td>512.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>647.0</td>\n",
       "      <td>400.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>/pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...</td>\n",
       "      <td>0110950981</td>\n",
       "      <td>42086009</td>\n",
       "      <td>Series_2_DR_30</td>\n",
       "      <td>519</td>\n",
       "      <td>050Y</td>\n",
       "      <td>M</td>\n",
       "      <td>CT</td>\n",
       "      <td>20170101.0</td>\n",
       "      <td>224711.918638</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.625</td>\n",
       "      <td>0.625</td>\n",
       "      <td>512.0</td>\n",
       "      <td>512.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>696.0</td>\n",
       "      <td>400.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>/pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...</td>\n",
       "      <td>0119239036</td>\n",
       "      <td>40732683</td>\n",
       "      <td>Series_300_Reformatted</td>\n",
       "      <td>184</td>\n",
       "      <td>070Y</td>\n",
       "      <td>M</td>\n",
       "      <td>CT</td>\n",
       "      <td>20170101.0</td>\n",
       "      <td>191517.000000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>512.0</td>\n",
       "      <td>512.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>397.0</td>\n",
       "      <td>808.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>/pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...</td>\n",
       "      <td>0110316446</td>\n",
       "      <td>43970439</td>\n",
       "      <td>Series_4_THINS</td>\n",
       "      <td>588</td>\n",
       "      <td>063Y</td>\n",
       "      <td>M</td>\n",
       "      <td>CT</td>\n",
       "      <td>20180101.0</td>\n",
       "      <td>172511.791334</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.625</td>\n",
       "      <td>2.500</td>\n",
       "      <td>512.0</td>\n",
       "      <td>512.0</td>\n",
       "      <td>120.0</td>\n",
       "      <td>600.0</td>\n",
       "      <td>400.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 path         pid       sid  \\\n",
       "2   /pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...  0111259663  41209592   \n",
       "4   /pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...  0111225336  43814074   \n",
       "10  /pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...  0110950981  42086009   \n",
       "19  /pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...  0119239036  40732683   \n",
       "22  /pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R...  0110316446  43970439   \n",
       "\n",
       "                    series  files   age sex modality        date  \\\n",
       "2   Series_2_CHEST_WITHOUT    513  076Y   M       CT  20170101.0   \n",
       "4        Series_5_CHEST_WO    436  081Y   F       CT  20180101.0   \n",
       "10          Series_2_DR_30    519  050Y   M       CT  20170101.0   \n",
       "19  Series_300_Reformatted    184  070Y   M       CT  20170101.0   \n",
       "22          Series_4_THINS    588  063Y   M       CT  20180101.0   \n",
       "\n",
       "             time body_part  slice_thickness  slice_spacing   rows  columns  \\\n",
       "2    84817.737107     CHEST            0.625         -0.625  512.0    512.0   \n",
       "4   151917.602455       NaN            0.625          0.625  512.0    512.0   \n",
       "10  224711.918638       NaN            0.625          0.625  512.0    512.0   \n",
       "19  191517.000000       NaN            2.000            NaN  512.0    512.0   \n",
       "22  172511.791334       NaN            0.625          2.500  512.0    512.0   \n",
       "\n",
       "      kvp  x_ray_current  exposure_time  \n",
       "2   120.0          231.0          349.0  \n",
       "4   120.0          647.0          400.0  \n",
       "10  120.0          696.0          400.0  \n",
       "19  120.0          397.0          808.0  \n",
       "22  120.0          600.0          400.0  "
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pt_dicom_profile_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### age distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "# age distribution\n",
    "idx = -pt_dicom_profile_df['age'].isna()\n",
    "age = list(pt_dicom_profile_df[idx]['age'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {},
   "outputs": [],
   "source": [
    "age_n = []\n",
    "for x in age:\n",
    "    s= x.strip('0')[:-1]\n",
    "    if s=='':\n",
    "        a=0\n",
    "    else:\n",
    "        a=int(s)\n",
    "    age_n.append(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    10489.000000\n",
       "mean        65.971971\n",
       "std         14.959052\n",
       "min          0.000000\n",
       "25%         58.000000\n",
       "50%         67.000000\n",
       "75%         76.000000\n",
       "max        116.000000\n",
       "dtype: float64"
      ]
     },
     "execution_count": 105,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.Series(age_n).describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAicAAAF0CAYAAAD4opkZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAcEElEQVR4nO3df5Bd513f8fcHy0mMRSObpDtGNpU7KFAnKk7Ysc2k064T8K9kcJjS4IybyCGMYMaG0IoWGaZjSHDHTOOkzQDuCCzitCHCTWKisR2McLykYcbxD2Is/yBYjRUsjWMDdkw2DqZKv/3jPjI3yq726q5299Hu+zVzZ+95znPOfe53zt397Pl1U1VIkiT14tuWewCSJEnDDCeSJKkrhhNJktQVw4kkSeqK4USSJHXFcCJJkroybzhJ8rIk9yT5syQPJ/mV1v6hJI8neaA9zm7tSfLBJHuTPJjkdUPr2pzksfbYvHhvS5IkHa/WjNDnBeANVTWT5ETgs0k+1eb9h6r62GH9LwY2tse5wA3AuUlOBa4BJoEC7k+yq6qeneuFX/GKV9SGDRuO6g2N6mtf+xonn3zyoqx7pbN247N247Fu47N247N24ztS7e6///6/rqpXHmn5ecNJDe7SNtMmT2yPI9257VLgw225u5OsS3IaMAXsrqpnAJLsBi4CPjrXijZs2MB999033xDHMj09zdTU1KKse6WzduOzduOxbuOzduOzduM7Uu2SfGm+5Uc65yTJCUkeAJ5mEDA+12Zd2w7dfCDJS1vbeuCJocX3t7a52iVJkl40ymEdquobwNlJ1gG3JHkNcDXwZeAlwHbgF4D3LHRASbYAWwAmJiaYnp5e6CpnNTMzs2jrXums3fis3Xis2/is3fis3fgWWruRwskhVfWVJHcBF1XV+1rzC0l+B/j5Nn0AOGNosdNb2wEGh3aG26dneY3tDMIOk5OTtVi71NxdNz5rNz5rNx7rNj5rNz5rN76F1m6Uq3Ve2faYkOQk4IeBP2/nkZAkwFuAh9oiu4B3tKt2zgOeq6ongTuAC5KckuQU4ILWJkmS9KJR9pycBtyU5AQGYebmqro1yaeTvBII8ADw063/7cAlwF7geeCdAFX1TJL3Ave2fu85dHKsJEnSIaNcrfMg8NpZ2t8wR/8Crpxj3g5gx1GOUZIkrSLeIVaSJHXFcCJJkrpiOJEkSV0xnEiSpK4YTiRJUlcMJ5IkqSuGE0mS1JWjun29JC2WDdtum3Pe1k0HueII85fLvuvetNxDkFYk95xIkqSuGE4kSVJXDCeSJKkrhhNJktQVw4kkSeqK4USSJHXFcCJJkrpiOJEkSV0xnEiSpK4YTiRJUlcMJ5IkqSuGE0mS1BXDiSRJ6orhRJIkdcVwIkmSumI4kSRJXTGcSJKkrhhOJElSVwwnkiSpK4YTSZLUFcOJJEnqiuFEkiR1xXAiSZK6YjiRJEldMZxIkqSuGE4kSVJXDCeSJKkrhhNJktQVw4kkSerKvOEkycuS3JPkz5I8nORXWvuZST6XZG+S30vyktb+0ja9t83fMLSuq1v7F5JcuFhvSpIkHb/WjNDnBeANVTWT5ETgs0k+Bfx74ANVtTPJfwfeBdzQfj5bVd+T5DLg14AfT3IWcBnwauC7gD9K8qqq+sYivC9JWnQbtt223EOY19ZNB7nisHHuu+5NyzQaaTTz7jmpgZk2eWJ7FPAG4GOt/SbgLe35pW2aNv+NSdLad1bVC1X1OLAXOOeYvAtJkrRijHTOSZITkjwAPA3sBv4P8JWqOti67AfWt+frgScA2vzngO8cbp9lGUmSJGC0wzq0Qy9nJ1kH3AJ832INKMkWYAvAxMQE09PTi/I6MzMzi7bulc7ajc/azW3rpoNzzps46cjzNbfZauc2OBo/r+NbaO1GCieHVNVXktwF/CCwLsmatnfkdOBA63YAOAPYn2QN8HLgb4baDxleZvg1tgPbASYnJ2tqauqo3tCopqenWax1r3TWbnzWbm6HnxcxbOumg1y/56h+XamZrXb7Lp9ansEcZ/y8jm+htRvlap1Xtj0mJDkJ+GHgUeAu4Mdat83AJ9vzXW2aNv/TVVWt/bJ2Nc+ZwEbgnrFHLkmSVqRR/hU5DbgpyQkMwszNVXVrkkeAnUl+Ffg8cGPrfyPwP5LsBZ5hcIUOVfVwkpuBR4CDwJVeqSNJkg43bzipqgeB187S/kVmudqmqv4O+DdzrOta4NqjH6YkSVotvEOsJEnqiuFEkiR1xXAiSZK6YjiRJEldMZxIkqSuGE4kSVJXDCeSJKkrhhNJktQVw4kkSeqK4USSJHXFcCJJkrpiOJEkSV0xnEiSpK4YTiRJUlcMJ5IkqSuGE0mS1BXDiSRJ6orhRJIkdcVwIkmSumI4kSRJXTGcSJKkrhhOJElSVwwnkiSpK4YTSZLUFcOJJEnqiuFEkiR1xXAiSZK6YjiRJEldMZxIkqSuGE4kSVJXDCeSJKkrhhNJktQVw4kkSeqK4USSJHXFcCJJkrpiOJEkSV0xnEiSpK7MG06SnJHkriSPJHk4ybtb+y8nOZDkgfa4ZGiZq5PsTfKFJBcOtV/U2vYm2bY4b0mSJB3P1ozQ5yCwtar+NMl3APcn2d3mfaCq3jfcOclZwGXAq4HvAv4oyava7N8AfhjYD9ybZFdVPXIs3ogkSVoZ5g0nVfUk8GR7/tUkjwLrj7DIpcDOqnoBeDzJXuCcNm9vVX0RIMnO1tdwIkmSXnRU55wk2QC8Fvhca7oqyYNJdiQ5pbWtB54YWmx/a5urXZIk6UWpqtE6JmuBPwaurapPJJkA/hoo4L3AaVX1E0l+Hbi7qv5nW+5G4FNtNRdV1U+29rcD51bVVYe9zhZgC8DExMQP7Ny5c6HvcVYzMzOsXbt2Uda90lm78Vm7ue058Nyc8yZOgqe+voSDWUFmq92m9S9fnsEcZ/y8ju9ItTv//PPvr6rJIy0/yjknJDkR+Djwkar6BEBVPTU0/7eAW9vkAeCMocVPb20cof1FVbUd2A4wOTlZU1NTowzxqE1PT7NY617prN34rN3crth225zztm46yPV7Rvp1pcPMVrt9l08tz2COM35ex7fQ2o1ytU6AG4FHq+r9Q+2nDXX7UeCh9nwXcFmSlyY5E9gI3APcC2xMcmaSlzA4aXbX2COXJEkr0ij/irweeDuwJ8kDre0XgbclOZvBYZ19wE8BVNXDSW5mcKLrQeDKqvoGQJKrgDuAE4AdVfXwMXwvkiRpBRjlap3PApll1u1HWOZa4NpZ2m8/0nKSJEneIVaSJHXFcCJJkrpiOJEkSV0xnEiSpK4YTiRJUlcMJ5IkqSuGE0mS1BXDiSRJ6orhRJIkdcVwIkmSumI4kSRJXTGcSJKkrhhOJElSVwwnkiSpK4YTSZLUFcOJJEnqiuFEkiR1xXAiSZK6YjiRJEldMZxIkqSuGE4kSVJXDCeSJKkrhhNJktQVw4kkSeqK4USSJHXFcCJJkrpiOJEkSV0xnEiSpK4YTiRJUlcMJ5IkqSuGE0mS1BXDiSRJ6orhRJIkdcVwIkmSumI4kSRJXTGcSJKkrhhOJElSVwwnkiSpK/OGkyRnJLkrySNJHk7y7tZ+apLdSR5rP09p7UnywSR7kzyY5HVD69rc+j+WZPPivS1JknS8GmXPyUFga1WdBZwHXJnkLGAbcGdVbQTubNMAFwMb22MLcAMMwgxwDXAucA5wzaFAI0mSdMi84aSqnqyqP23Pvwo8CqwHLgVuat1uAt7Snl8KfLgG7gbWJTkNuBDYXVXPVNWzwG7gomP6biRJ0nEvVTV652QD8BngNcBfVtW61h7g2apal+RW4Lqq+mybdyfwC8AU8LKq+tXW/p+Ar1fV+w57jS0M9rgwMTHxAzt37lzI+5vTzMwMa9euXZR1r3TWbnzWbm57Djw357yJk+Cpry/hYFaQ2Wq3af3Ll2cwxxk/r+M7Uu3OP//8+6tq8kjLrxn1hZKsBT4O/FxV/e0gjwxUVSUZPeUcQVVtB7YDTE5O1tTU1LFY7beYnp5msda90lm78Vm7uV2x7bY5523ddJDr94z860pDZqvdvsunlmcwxxk/r+NbaO1GulonyYkMgslHquoTrfmpdriG9vPp1n4AOGNo8dNb21ztkiRJLxrlap0ANwKPVtX7h2btAg5dcbMZ+ORQ+zvaVTvnAc9V1ZPAHcAFSU5pJ8Je0NokSZJeNMp+0tcDbwf2JHmgtf0icB1wc5J3AV8C3trm3Q5cAuwFngfeCVBVzyR5L3Bv6/eeqnrmmLwLSZK0YswbTtqJrZlj9htn6V/AlXOsawew42gGKEmSVhfvECtJkrpiOJEkSV0xnEiSpK4YTiRJUlcMJ5IkqSuGE0mS1BXDiSRJ6orhRJIkdcVwIkmSumI4kSRJXTGcSJKkrhhOJElSVwwnkiSpK4YTSZLUFcOJJEnqiuFEkiR1xXAiSZK6YjiRJEldMZxIkqSuGE4kSVJXDCeSJKkrhhNJktQVw4kkSeqK4USSJHXFcCJJkrpiOJEkSV0xnEiSpK4YTiRJUlcMJ5IkqSuGE0mS1BXDiSRJ6orhRJIkdcVwIkmSumI4kSRJXTGcSJKkrhhOJElSVwwnkiSpK/OGkyQ7kjyd5KGhtl9OciDJA+1xydC8q5PsTfKFJBcOtV/U2vYm2Xbs34okSVoJRtlz8iHgolnaP1BVZ7fH7QBJzgIuA17dlvnNJCckOQH4DeBi4Czgba2vJEnSN1kzX4eq+kySDSOu71JgZ1W9ADyeZC9wTpu3t6q+CJBkZ+v7yFGPWNIRbdh223IPQZIWZN5wcgRXJXkHcB+wtaqeBdYDdw/12d/aAJ44rP3c2VaaZAuwBWBiYoLp6ekFDHFuMzMzi7bulc7ajW8pard108FFXf9ymDhpZb6vpTBb7fz8jsbfdeNbaO3GDSc3AO8Fqv28HviJsUcxpKq2A9sBJicna2pq6lis9ltMT0+zWOte6azd+JaidleswD0nWzcd5Po9C/lfavWarXb7Lp9ansEcZ/xdN76F1m6sT3tVPXXoeZLfAm5tkweAM4a6nt7aOEK7JEnSi8a6lDjJaUOTPwocupJnF3BZkpcmORPYCNwD3AtsTHJmkpcwOGl21/jDliRJK9W8e06SfBSYAl6RZD9wDTCV5GwGh3X2AT8FUFUPJ7mZwYmuB4Erq+obbT1XAXcAJwA7qurhY/5uJEnScW+Uq3XeNkvzjUfofy1w7SzttwO3H9XoJEnSquMdYiVJUlcMJ5IkqSuGE0mS1BXDiSRJ6orhRJIkdcVwIkmSumI4kSRJXfHLKiRplTlev7l633VvWu4haIm450SSJHXFcCJJkrpiOJEkSV0xnEiSpK4YTiRJUlcMJ5IkqSuGE0mS1BXDiSRJ6orhRJIkdcVwIkmSumI4kSRJXTGcSJKkrhhOJElSVwwnkiSpK4YTSZLUFcOJJEnqiuFEkiR1xXAiSZK6YjiRJEldMZxIkqSuGE4kSVJXDCeSJKkrhhNJktQVw4kkSeqK4USSJHXFcCJJkrpiOJEkSV0xnEiSpK4YTiRJUlfmDSdJdiR5OslDQ22nJtmd5LH285TWniQfTLI3yYNJXje0zObW/7Ekmxfn7UiSpOPdKHtOPgRcdFjbNuDOqtoI3NmmAS4GNrbHFuAGGIQZ4BrgXOAc4JpDgUaSJGnYvOGkqj4DPHNY86XATe35TcBbhto/XAN3A+uSnAZcCOyuqmeq6llgN98aeCRJksY+52Siqp5sz78MTLTn64Enhvrtb21ztUuSJH2TNQtdQVVVkjoWgwFIsoXBISEmJiaYnp4+Vqv+JjMzM4u27pXO2o1vKWq3ddPBRV3/cpg4aWW+r6Wwkmq31L93/F03voXWbtxw8lSS06rqyXbY5unWfgA4Y6jf6a3tADB1WPv0bCuuqu3AdoDJycmampqarduCTU9Ps1jrXums3fiWonZXbLttUde/HLZuOsj1exb8v9SqtJJqt+/yqSV9PX/XjW+htRv3sM4u4NAVN5uBTw61v6NdtXMe8Fw7/HMHcEGSU9qJsBe0NkmSpG8yb5xO8lEGez1ekWQ/g6turgNuTvIu4EvAW1v324FLgL3A88A7AarqmSTvBe5t/d5TVYefZCtJkjR/OKmqt80x642z9C3gyjnWswPYcVSjkyRJq453iJUkSV0xnEiSpK4YTiRJUlcMJ5IkqSuGE0mS1BXDiSRJ6orhRJIkdcVwIkmSumI4kSRJXTGcSJKkrhhOJElSVwwnkiSpK4YTSZLUFcOJJEnqiuFEkiR1xXAiSZK6YjiRJEldMZxIkqSuGE4kSVJXDCeSJKkrhhNJktQVw4kkSeqK4USSJHXFcCJJkrpiOJEkSV0xnEiSpK4YTiRJUlcMJ5IkqStrlnsAUq82bLvtmK9z66aDXLEI65WklcQ9J5IkqSuGE0mS1BXDiSRJ6orhRJIkdcVwIkmSumI4kSRJXTGcSJKkrhhOJElSVxYUTpLsS7InyQNJ7mttpybZneSx9vOU1p4kH0yyN8mDSV53LN6AJElaWY7FnpPzq+rsqpps09uAO6tqI3Bnmwa4GNjYHluAG47Ba0uSpBVmMQ7rXArc1J7fBLxlqP3DNXA3sC7JaYvw+pIk6Ti20HBSwB8muT/JltY2UVVPtudfBiba8/XAE0PL7m9tkiRJL0pVjb9wsr6qDiT5x8Bu4GeAXVW1bqjPs1V1SpJbgeuq6rOt/U7gF6rqvsPWuYXBYR8mJiZ+YOfOnWOP70hmZmZYu3btoqx7pVsttdtz4Lljvs6Jk+Cprx/z1a541m18K6l2m9a/fElfb7X8rlsMR6rd+eeff//QqSCzWtC3ElfVgfbz6SS3AOcATyU5raqebIdtnm7dDwBnDC1+ems7fJ3bge0Ak5OTNTU1tZAhzml6eprFWvdKt1pqtxjfHrx100Gu3+OXgR8t6za+lVS7fZdPLenrrZbfdYthobUb+7BOkpOTfMeh58AFwEPALmBz67YZ+GR7vgt4R7tq5zzguaHDP5IkScDC9pxMALckObSe362qP0hyL3BzkncBXwLe2vrfDlwC7AWeB965gNeWJEkr1NjhpKq+CHz/LO1/A7xxlvYCrhz39SRJ0urgHWIlSVJXDCeSJKkrhhNJktQVw4kkSeqK4USSJHXFcCJJkrpiOJEkSV0xnEiSpK4YTiRJUlcMJ5IkqSsr46sq1b0Ni/ANv5Kklck9J5IkqSuGE0mS1BXDiSRJ6orhRJIkdcVwIkmSumI4kSRJXTGcSJKkrhhOJElSVwwnkiSpK4YTSZLUFcOJJEnqiuFEkiR1xXAiSZK6YjiRJEldMZxIkqSuGE4kSVJX1iz3ACRJGsWGbbct6ett3XSQKxb4mvuue9MxGs3q4p4TSZLUFcOJJEnqiuFEkiR1xXAiSZK6YjiRJEld8Wqd48xSn60+m2NxBrskSXNxz4kkSeqK4USSJHXFcCJJkrqy5OEkyUVJvpBkb5JtS/36kiSpb0saTpKcAPwGcDFwFvC2JGct5RgkSVLflnrPyTnA3qr6YlX9PbATuHSJxyBJkjq21JcSrweeGJreD5y7xGMAYM+B57wcVpKkDqWqlu7Fkh8DLqqqn2zTbwfOraqrhvpsAba0ye8FvrBIw3kF8NeLtO6VztqNz9qNx7qNz9qNz9qN70i1+ydV9cojLbzUe04OAGcMTZ/e2l5UVduB7Ys9kCT3VdXkYr/OSmTtxmftxmPdxmftxmftxrfQ2i31OSf3AhuTnJnkJcBlwK4lHoMkSerYku45qaqDSa4C7gBOAHZU1cNLOQZJktS3Jf9unaq6Hbh9qV93Fot+6GgFs3bjs3bjsW7js3bjs3bjW1DtlvSEWEmSpPl4+3pJktSVVRlOvIX+aJKckeSuJI8keTjJu1v7qUl2J3ms/TxlucfaqyQnJPl8klvb9JlJPte2vd9rJ4brMEnWJflYkj9P8miSH3S7G02Sf9c+rw8l+WiSl7ndzS7JjiRPJ3loqG3W7SwDH2w1fDDJ65Zv5Mtrjrr9l/Z5fTDJLUnWDc27utXtC0kuHOU1Vl048Rb6R+UgsLWqzgLOA65stdoG3FlVG4E727Rm927g0aHpXwM+UFXfAzwLvGtZRtW//wb8QVV9H/D9DGrodjePJOuBnwUmq+o1DC48uAy3u7l8CLjosLa5trOLgY3tsQW4YYnG2KMP8a112w28pqr+OfAXwNUA7W/GZcCr2zK/2f4OH9GqCyd4C/2RVdWTVfWn7flXGfyBWM+gXje1bjcBb1meEfYtyenAm4DfbtMB3gB8rHWxdrNI8nLgXwI3AlTV31fVV3C7G9Ua4KQka4BvB57E7W5WVfUZ4JnDmufazi4FPlwDdwPrkpy2NCPty2x1q6o/rKqDbfJuBvcxg0HddlbVC1X1OLCXwd/hI1qN4WS2W+ivX6axHDeSbABeC3wOmKiqJ9usLwMTyzSs3v1X4D8C/69NfyfwlaEPsNve7M4E/gr4nXZI7LeTnIzb3byq6gDwPuAvGYSS54D7cbs7GnNtZ/7tGN1PAJ9qz8eq22oMJzpKSdYCHwd+rqr+dnheDS738pKvwyR5M/B0Vd2/3GM5Dq0BXgfcUFWvBb7GYYdw3O5m186PuJRBwPsu4GS+dfe7RuR2dvSS/BKDUwI+spD1rMZwMu8t9PUPkpzIIJh8pKo+0ZqfOrQ7s/18ernG17HXAz+SZB+DQ4dvYHAexbq2ux3c9uayH9hfVZ9r0x9jEFbc7ub3Q8DjVfVXVfV/gU8w2Bbd7kY313bm3455JLkCeDNwef3DfUrGqttqDCfeQn9E7RyJG4FHq+r9Q7N2AZvb883AJ5d6bL2rqqur6vSq2sBgG/t0VV0O3AX8WOtm7WZRVV8Gnkjyva3pjcAjuN2N4i+B85J8e/v8Hqqd293o5trOdgHvaFftnAc8N3T4Z9VLchGDw9g/UlXPD83aBVyW5KVJzmRwQvE9865vNd6ELcklDM4HOHQL/WuXeUhdSvIvgP8N7OEfzpv4RQbnndwMfDfwJeCtVXX4SWVqkkwBP19Vb07yTxnsSTkV+Dzwb6vqheUcX4+SnM3gROKXAF8E3sngnym3u3kk+RXgxxnsWv888JMMjvG73R0myUeBKQbfoPsUcA3w+8yynbWw9+sMDpM9D7yzqu5bjnEvtznqdjXwUuBvWre7q+qnW/9fYnAeykEGpwd86vB1fstrrMZwIkmS+rUaD+tIkqSOGU4kSVJXDCeSJKkrhhNJktQVw4kkSeqK4USSJHXFcCJJkrpiOJG0JJL8fpL7kzycZEtre1eSv0hyT5LfSvLrrf2VST6e5N72eP3yjl7SUvImbJKWRJJT2502T2LwNRIXAn/C4Htzvgp8Gvizqroqye8Cv1lVn03y3cAdVfXPlm3wkpbUmvm7SNIx8bNJfrQ9PwN4O/DHh25Bn+R/Aa9q838IOGtwx3AA/lGStVU1s5QDlrQ8DCeSFl37fqEfAn6wqp5PMg38OTDX3pBvA86rqr9bmhFK6onnnEhaCi8Hnm3B5PuA84CTgX+V5JQka4B/PdT/D4GfOTTRvghQ0iphOJG0FP4AWJPkUeA64G7gAPCfGXx9+p8A+4DnWv+fBSaTPJjkEeCnl3zEkpaNJ8RKWjaHziNpe05uAXZU1S3LPS5Jy8s9J5KW0y8neQB4CHgc+P1lHo+kDrjnRJIkdcU9J5IkqSuGE0mS1BXDiSRJ6orhRJIkdcVwIkmSumI4kSRJXfn/o2UXGrbKlnIAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 648x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig = plt.figure(figsize=(9,6))\n",
    "ax = pd.Series(age_n).hist()\n",
    "ax.set_xlabel('age')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### slice spacing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    7440.000000\n",
       "mean        1.818149\n",
       "std         5.444075\n",
       "min        -2.500000\n",
       "25%         0.625000\n",
       "50%         0.625000\n",
       "75%         0.800000\n",
       "max        35.000000\n",
       "Name: slice_spacing, dtype: float64"
      ]
     },
     "execution_count": 107,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# age distribution\n",
    "idx = -pt_dicom_profile_df['slice_spacing'].isna()\n",
    "pt_dicom_profile_df[idx]['slice_spacing'].describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiQAAAF3CAYAAACVAVC0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAYVklEQVR4nO3df7DddZ3f8edLoisFFRX3lgJtmMpocbOiExGLnV5AIaAVu/UHO1aDpZNpB7s6E2c3dNux/urg+INd1122mYURXVdkdRmoYdUUuHW0VX4IgoAsKYtDUjSjQTRa2Ube/eN849xmc7034d7zzr3n+Zi5c8/5nO85388n3ITn/Z7vOSdVhSRJUqcndU9AkiTJIJEkSe0MEkmS1M4gkSRJ7QwSSZLUziCRJEntFhQkSR5McleSO5LcOow9K8nWJPcP3585jCfJR5NsS3JnkhfPepz1w/b3J1m/NEuSJEnLzYEcITm9qk6uqrXD9U3ADVV1InDDcB3gHODE4WsDcBmMAgZ4F/BS4BTgXXsjRpIkTbYn8pTNecCVw+UrgdfOGv9EjXwNOCrJMcDZwNaq2lVVjwBbgXVPYP+SJGmFWLXA7Qr4UpIC/ktVbQamqurh4fbvAlPD5WOBh2bdd/swNtf4nI4++uhavXr1Aqe4fPzkJz/hiCOO6J7G2EzaesE1TwrXPBlc8+K67bbbvl9Vz9l3fKFB8vKq2pHkV4GtSb49+8aqqiFWnrAkGxg91cPU1BQf+tCHFuNhDym7d+/myCOP7J7G2EzaesE1TwrXPBlc8+I6/fTTv7O/8QUFSVXtGL7vTHINo3NAvpfkmKp6eHhKZuew+Q7g+Fl3P24Y2wFM7zM+s599bQY2A6xdu7amp6f33WTZm5mZYSWuay6Ttl5wzZPCNU8G1zwe855DkuSIJE/bexk4C/gWcB2w95Uy64Frh8vXAW8ZXm1zKvDo8NTOF4GzkjxzOJn1rGFMkiRNuIUcIZkCrkmyd/s/q6ovJLkFuDrJhcB3gDcM218PnAtsA34KvBWgqnYleS9wy7Dde6pq16KtRJIkLVvzBklVPQC8cD/jPwDO3M94ARfN8VhXAFcc+DQlSdJK5ju1SpKkdgaJJElqZ5BIkqR2BokkSWpnkEiSpHYGiSRJameQSJKkdgaJJElqZ5BIkqR2C/20X0nLxOpNW1r2u3HNHi44yH0/eMmrFnk2kpYbj5BIkqR2BokkSWpnkEiSpHYGiSRJameQSJKkdgaJJElqZ5BIkqR2BokkSWpnkEiSpHYGiSRJameQSJKkdgaJJElqZ5BIkqR2BokkSWpnkEiSpHYGiSRJameQSJKkdgaJJElqZ5BIkqR2BokkSWpnkEiSpHYGiSRJameQSJKkdgaJJElqZ5BIkqR2BokkSWpnkEiSpHYGiSRJameQSJKkdgaJJElqZ5BIkqR2BokkSWpnkEiSpHYGiSRJameQSJKkdgaJJElqZ5BIkqR2BokkSWpnkEiSpHYGiSRJameQSJKkdgaJJElqZ5BIkqR2BokkSWpnkEiSpHYGiSRJarfgIElyWJLbk3x+uH5Ckq8n2ZbkM0meMoz/ynB923D76lmPcfEwfl+Ssxd7MZIkaXk6kCMkbwfunXX9A8ClVfVc4BHgwmH8QuCRYfzSYTuSnAScD7wAWAf8UZLDntj0JUnSSrCgIElyHPAq4E+G6wHOAD47bHIl8Nrh8nnDdYbbzxy2Pw+4qqoeq6q/BrYBpyzGIiRJ0vK20CMkvwf8NvD4cP3ZwA+ras9wfTtw7HD5WOAhgOH2R4ftfzG+n/tIkqQJtmq+DZK8GthZVbclmV7qCSXZAGwAmJqaYmZmZql3OXa7d+9ekeuay6StF3rXvHHNnvk3WgJThx/8vpfrz4c/25PBNY/HvEECnAa8Jsm5wFOBpwO/DxyVZNVwFOQ4YMew/Q7geGB7klXAM4AfzBrfa/Z9fqGqNgObAdauXVvT09MHsaxD28zMDCtxXXOZtPVC75ov2LSlZb8b1+zhw3ct5J+Uv+3BN00v7mTGxJ/tyeCax2Pep2yq6uKqOq6qVjM6KfXGqnoTcBPwumGz9cC1w+XrhusMt99YVTWMnz+8CucE4ETg5kVbiSRJWrYO7teZkd8BrkryPuB24PJh/HLgk0m2AbsYRQxVdXeSq4F7gD3ARVX18yewf0mStEIcUJBU1QwwM1x+gP28Sqaqfga8fo77vx94/4FOUpIkrWy+U6skSWpnkEiSpHYGiSRJameQSJKkdgaJJElqZ5BIkqR2BokkSWpnkEiSpHYGiSRJameQSJKkdgaJJElqZ5BIkqR2BokkSWpnkEiSpHYGiSRJameQSJKkdgaJJElqZ5BIkqR2BokkSWpnkEiSpHYGiSRJameQSJKkdgaJJElqZ5BIkqR2BokkSWpnkEiSpHYGiSRJameQSJKkdgaJJElqZ5BIkqR2BokkSWpnkEiSpHYGiSRJameQSJKkdgaJJElqZ5BIkqR2BokkSWpnkEiSpHYGiSRJameQSJKkdgaJJElqZ5BIkqR2BokkSWpnkEiSpHYGiSRJameQSJKkdgaJJElqZ5BIkqR2BokkSWpnkEiSpHYGiSRJameQSJKkdgaJJElqZ5BIkqR2BokkSWpnkEiSpHbzBkmSpya5Ock3k9yd5N3D+AlJvp5kW5LPJHnKMP4rw/Vtw+2rZz3WxcP4fUnOXqpFSZKk5WUhR0geA86oqhcCJwPrkpwKfAC4tKqeCzwCXDhsfyHwyDB+6bAdSU4CzgdeAKwD/ijJYYu5GEmStDzNGyQ1snu4+uThq4AzgM8O41cCrx0unzdcZ7j9zCQZxq+qqseq6q+BbcApi7IKSZK0rC3oHJIkhyW5A9gJbAX+F/DDqtozbLIdOHa4fCzwEMBw+6PAs2eP7+c+kiRpgq1ayEZV9XPg5CRHAdcAz1+qCSXZAGwAmJqaYmZmZql21Wb37t0rcl1zmbT1Qu+aN67ZM/9GS2Dq8IPf93L9+fBnezK45vFYUJDsVVU/THIT8DLgqCSrhqMgxwE7hs12AMcD25OsAp4B/GDW+F6z7zN7H5uBzQBr166t6enpA1rQcjAzM8NKXNdcJm290LvmCzZtadnvxjV7+PBdB/RPyi88+KbpxZ3MmPizPRlc83gs5FU2zxmOjJDkcOCVwL3ATcDrhs3WA9cOl68brjPcfmNV1TB+/vAqnBOAE4GbF2shkiRp+VrIrzPHAFcOr4h5EnB1VX0+yT3AVUneB9wOXD5sfznwySTbgF2MXllDVd2d5GrgHmAPcNHwVJAkSZpw8wZJVd0JvGg/4w+wn1fJVNXPgNfP8VjvB95/4NOUJEkrme/UKkmS2hkkkiSpnUEiSZLaGSSSJKmdQSJJktoZJJIkqZ1BIkmS2hkkkiSpnUEiSZLaGSSSJKmdQSJJktoZJJIkqZ1BIkmS2hkkkiSpnUEiSZLaGSSSJKmdQSJJktoZJJIkqZ1BIkmS2hkkkiSpnUEiSZLaGSSSJKmdQSJJktoZJJIkqZ1BIkmS2hkkkiSpnUEiSZLaGSSSJKmdQSJJktoZJJIkqZ1BIkmS2hkkkiSpnUEiSZLaGSSSJKmdQSJJktoZJJIkqZ1BIkmS2hkkkiSpnUEiSZLaGSSSJKmdQSJJktoZJJIkqZ1BIkmS2hkkkiSpnUEiSZLaGSSSJKmdQSJJktoZJJIkqZ1BIkmS2hkkkiSpnUEiSZLaGSSSJKmdQSJJktoZJJIkqZ1BIkmS2hkkkiSp3bxBkuT4JDcluSfJ3UnePow/K8nWJPcP3585jCfJR5NsS3JnkhfPeqz1w/b3J1m/dMuSJEnLyUKOkOwBNlbVScCpwEVJTgI2ATdU1YnADcN1gHOAE4evDcBlMAoY4F3AS4FTgHftjRhJkjTZ5g2Sqnq4qr4xXP4xcC9wLHAecOWw2ZXAa4fL5wGfqJGvAUclOQY4G9haVbuq6hFgK7BuUVcjSZKWpQM6hyTJauBFwNeBqap6eLjpu8DUcPlY4KFZd9s+jM01LkmSJtyqhW6Y5Ejgc8A7qupHSX5xW1VVklqMCSXZwOipHqamppiZmVmMhz2k7N69e0Wuay6Ttl7oXfPGNXta9jt1+MHve7n+fPizPRlc83gsKEiSPJlRjHyqqv5iGP5ekmOq6uHhKZmdw/gO4PhZdz9uGNsBTO8zPrPvvqpqM7AZYO3atTU9Pb3vJsvezMwMK3Fdc5m09ULvmi/YtKVlvxvX7OHDdy34d5z/z4Nvml7cyYyJP9uTwTWPx0JeZRPgcuDeqvrIrJuuA/a+UmY9cO2s8bcMr7Y5FXh0eGrni8BZSZ45nMx61jAmSZIm3EJ+nTkNeDNwV5I7hrF/D1wCXJ3kQuA7wBuG264HzgW2AT8F3gpQVbuSvBe4ZdjuPVW1a1FWIUmSlrV5g6SqvgJkjpvP3M/2BVw0x2NdAVxxIBOUJEkrn+/UKkmS2hkkkiSpnUEiSZLaGSSSJKmdQSJJktoZJJIkqZ1BIkmS2hkkkiSpnUEiSZLaGSSSJKmdQSJJktoZJJIkqZ1BIkmS2hkkkiSpnUEiSZLaGSSSJKmdQSJJktoZJJIkqZ1BIkmS2hkkkiSpnUEiSZLaGSSSJKmdQSJJktoZJJIkqZ1BIkmS2hkkkiSpnUEiSZLaGSSSJKmdQSJJktoZJJIkqZ1BIkmS2hkkkiSpnUEiSZLaGSSSJKmdQSJJktoZJJIkqZ1BIkmS2hkkkiSpnUEiSZLaGSSSJKmdQSJJktoZJJIkqZ1BIkmS2hkkkiSpnUEiSZLaGSSSJKmdQSJJktoZJJIkqZ1BIkmS2hkkkiSpnUEiSZLaGSSSJKmdQSJJktoZJJIkqZ1BIkmS2hkkkiSp3bxBkuSKJDuTfGvW2LOSbE1y//D9mcN4knw0ybYkdyZ58az7rB+2vz/J+qVZjiRJWo5WLWCbjwMfAz4xa2wTcENVXZJk03D9d4BzgBOHr5cClwEvTfIs4F3AWqCA25JcV1WPLNZCJGnc7trxKBds2tI9jQP24CWv6p6C9LfMe4Skqr4M7Npn+DzgyuHylcBrZ41/oka+BhyV5BjgbGBrVe0aImQrsG4xFiBJkpa/gz2HZKqqHh4ufxeYGi4fCzw0a7vtw9hc45IkSQt6yuaXqqpKUosxGYAkG4ANAFNTU8zMzCzWQx8ydu/evSLXNZdJWy/0rnnjmj0t+506/OD3vVx/Pp7Imjs9kT9v/z5Pho41H2yQfC/JMVX18PCUzM5hfAdw/KztjhvGdgDT+4zP7O+Bq2ozsBlg7dq1NT09vb/NlrWZmRlW4rrmMmnrhd41d53TsHHNHj5818H9k/Lgm6YXdzJj8gefuvag19zpifx5+/d5MnSs+WD/Jl0HrAcuGb5fO2v8bUmuYnRS66NDtHwR+M97X40DnAVcfPDT1nLiiX+SpPnMGyRJPs3o6MbRSbYzerXMJcDVSS4EvgO8Ydj8euBcYBvwU+CtAFW1K8l7gVuG7d5TVfueKCtJkibUvEFSVb85x01n7mfbAi6a43GuAK44oNlJkqSJ4Du1SpKkdgaJJElqZ5BIkqR2BokkSWpnkEiSpHYGiSRJameQSJKkdgaJJElqZ5BIkqR2BokkSWpnkEiSpHYGiSRJameQSJKkdgaJJElqZ5BIkqR2BokkSWpnkEiSpHYGiSRJameQSJKkdgaJJElqZ5BIkqR2BokkSWpnkEiSpHYGiSRJameQSJKkdgaJJElqZ5BIkqR2BokkSWpnkEiSpHYGiSRJameQSJKkdgaJJElqZ5BIkqR2q7onIEnSSrV605buKRyUj687Yuz79AiJJElqZ5BIkqR2BokkSWpnkEiSpHYGiSRJameQSJKkdgaJJElqZ5BIkqR2BokkSWpnkEiSpHYGiSRJameQSJKkdgaJJElqZ5BIkqR2BokkSWpnkEiSpHYGiSRJameQSJKkdgaJJElqZ5BIkqR2BokkSWpnkEiSpHZjD5Ik65Lcl2Rbkk3j3r8kSTr0rBrnzpIcBvwh8EpgO3BLkuuq6p5xzkPSoWX1pi3dUzgoG9d0z0BaOcZ9hOQUYFtVPVBVfwNcBZw35jlIkqRDzLiD5FjgoVnXtw9jkiRpgqWqxrez5HXAuqr618P1NwMvraq3zdpmA7BhuPo84L6xTXB8jga+3z2JMZq09YJrnhSueTK45sX1D6rqOfsOjvUcEmAHcPys68cNY79QVZuBzeOc1LglubWq1nbPY1wmbb3gmieFa54Mrnk8xv2UzS3AiUlOSPIU4HzgujHPQZIkHWLGeoSkqvYkeRvwReAw4Iqqunucc5AkSYeecT9lQ1VdD1w/7v0eYlb0U1L7MWnrBdc8KVzzZHDNYzDWk1olSZL2x7eOlyRJ7QySBkk+mOTbSe5Mck2So7rntNSSvD7J3UkeT7Kiz1aftI9HSHJFkp1JvtU9l3FJcnySm5LcM/xcv717TkstyVOT3Jzkm8Oa3909p3FIcliS25N8vnsu45DkwSR3Jbkjya3j3LdB0mMr8GtV9evAXwEXN89nHL4F/Abw5e6JLKVZH49wDnAS8JtJTuqd1ZL7OLCuexJjtgfYWFUnAacCF03Af+fHgDOq6oXAycC6JKc2z2kc3g7c2z2JMTu9qk5e6S/7FVBVX6qqPcPVrzF6P5YVraruraqV+CZ3+5q4j0eoqi8Du7rnMU5V9XBVfWO4/GNG/8Na0e86XSO7h6tPHr5W9EmISY4DXgX8SfdcJoFB0u9fAX/ZPQktGj8eYcIkWQ28CPh670yW3vD0xR3ATmBrVa30Nf8e8NvA490TGaMCvpTktuGd08dm7C/7nRRJ/hvwd/dz0+9W1bXDNr/L6NDvp8Y5t6WykDVLK0mSI4HPAe+oqh91z2epVdXPgZOH896uSfJrVbUizx1K8mpgZ1XdlmS6ez5j9PKq2pHkV4GtSb49HAVdcgbJEqmqV/yy25NcALwaOLNWyGuv51vzhJj34xG0MiR5MqMY+VRV/UX3fMapqn6Y5CZG5w6tyCABTgNek+Rc4KnA05P8aVX9y+Z5Lamq2jF835nkGkZPQ48lSHzKpkGSdYwOA76mqn7aPR8tKj8eYQIkCXA5cG9VfaR7PuOQ5Dl7XxGY5HDglcC3e2e1dKrq4qo6rqpWM/p7fONKj5EkRyR52t7LwFmMMTgNkh4fA57G6HDYHUn+uHtCSy3JP0+yHXgZsCXJF7vntBSGk5X3fjzCvcDVK/3jEZJ8GvifwPOSbE9yYfecxuA04M3AGcPf4TuG36RXsmOAm5LcySi8t1bVRLwUdoJMAV9J8k3gZmBLVX1hXDv3nVolSVI7j5BIkqR2BokkSWpnkEiSpHYGiSRJameQSJKkdgaJJElqZ5BI2q8kM0nWDpev3/umWIeyJGuTfLR7HpIOnG8dL2leVbUs3vSrqm4Fbu2eh6QD5xESSXvfMnpLkm8m+VaSN+5z+4NJjh4uvyXJncO2nxzGnpPkc0luGb5O+yX7+qez3t309iRPSzKd5MvDHO5L8sdJnjRsf1mSW5PcneTdsx7nJUn+xzCPm2c9zueH2/9TkiuGIz0PJPmtWff9j8N+vpLk00neubh/opIOlEdIJMHoQ9L+d1W9CiDJM4B/u+9GSV4A/AfgH1fV95M8a7jp94FLq+orSf4+o7fO/0dz7OudwEVV9dXh03J/NoyfApwEfAf4AvAbwGcZfVr0riSHATck+XVGn6HyGeCNVXVLkqcD/2c/+3o+cDqjj2q4L8llwMnAvwBeCDwZ+AZw24L+lCQtGY+QSAK4C3hlkg8k+SdV9egc250B/HlVfR+gqnYN468APpbkDkYfJvj0ITb256vAR4YjFkcNn/8DcHNVPTB8xP2ngZcP429I8g3gduAFjKLlecDDVXXLMI8fzXqc2bZU1WPDfHcy+qyO04Brq+pnVfVj4L/O+6cjacl5hEQSVfVXSV4MnAu8L8kNB/gQTwJOraqfzbdhVV2SZMuwr68mOXvvTftumuQERkdUXlJVjyT5OKOPgl+ox2Zd/jn+mycdsjxCIokkfw/4aVX9KfBB4MVzbHoj8Pokzx7ut/cpmy8B/27W4538S/b1D6vqrqr6AKNPjX3+cNMpSU4Yzh15I/AV4OnAT4BHk0wB5wzb3gcck+Qlw2M+LclCY+OrwD9L8tThKM6rF3g/SUvI3xYkAawBPpjkceD/Mjp/5EP7blRVdyd5P/Dfk/yc0dMoFwC/Bfzh8NH0q4AvA/9mjn29I8npwOPA3cBfAi9jFCcfA54L3ARcU1WPJ7md0TkjDzGKCarqb4YTb/8gyeGMzh95xUIWOpxzch1wJ/A9Rk9XzfUUlaQxSdW+R0klabySTAPvrKqxHK1IcmRV7U7ydxjF04aq+sY49i1p/zxCImkSbU5yEqPzUa40RqR+HiGRtCSSvBV4+z7DX62qizrmI+nQZpBIkqR2vspGkiS1M0gkSVI7g0SSJLUzSCRJUjuDRJIktft/uwfMKjEACngAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 648x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# excluding slice spacing > 10 mm\n",
    "fig = plt.figure(figsize=(9,6))\n",
    "idx = pt_dicom_profile_df['slice_spacing'] < 10\n",
    "ax = pt_dicom_profile_df[idx]['slice_spacing'].hist()\n",
    "ax.set_xlabel('slice_spacing')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# R3 Notes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "r3_notes_dict = '/pghbio/dbmi/batmanlab/Data/UPMC_Lung_Images/R3/20_BATMANGHELICH_NOTES_DATA_2020_04_03/R3_20_BATMANGHELICH_PROCEDURE_NOTES_METADATA_DEID_2020_04_02.xlsx'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "r3_notes_meta_df = pd.read_excel(r3_notes_dict, dtype=str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PATIENT_STUDY_ID</th>\n",
       "      <th>ORDER_DATE</th>\n",
       "      <th>RESULT_TIME</th>\n",
       "      <th>PROCEDURE_CODE</th>\n",
       "      <th>PROCEDURE_NAME</th>\n",
       "      <th>ACCESSION_STUDY_ID</th>\n",
       "      <th>NOTE_STUDY_ID</th>\n",
       "      <th>NOTE_CSN_STUDY_ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0110000087</td>\n",
       "      <td>02/14/2017 15:35:00</td>\n",
       "      <td>02/17/2017 15:22:00</td>\n",
       "      <td>74176</td>\n",
       "      <td>CT ABDOMEN AND PELVIS WITHOUT CONTRAST</td>\n",
       "      <td>49955263</td>\n",
       "      <td>539216381</td>\n",
       "      <td>442362360</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0110000087</td>\n",
       "      <td>03/12/2017 18:40:00</td>\n",
       "      <td>03/12/2017 20:11:00</td>\n",
       "      <td>74176</td>\n",
       "      <td>CT ABDOMEN AND PELVIS WITHOUT CONTRAST</td>\n",
       "      <td>49229914</td>\n",
       "      <td>533526235</td>\n",
       "      <td>457830446</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0110000087</td>\n",
       "      <td>02/08/2018 13:06:00</td>\n",
       "      <td>02/19/2018 10:53:00</td>\n",
       "      <td>71250</td>\n",
       "      <td>CT CHEST WITHOUT CONTRAST</td>\n",
       "      <td>43754750</td>\n",
       "      <td>7603198677</td>\n",
       "      <td>520069287</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0110000087</td>\n",
       "      <td>07/05/2017 00:06:00</td>\n",
       "      <td>07/05/2017 00:24:00</td>\n",
       "      <td>74176</td>\n",
       "      <td>CT ABDOMEN AND PELVIS WITHOUT CONTRAST</td>\n",
       "      <td>40436615</td>\n",
       "      <td>7666701228</td>\n",
       "      <td>570342923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0110000639</td>\n",
       "      <td>09/22/2017 13:18:00</td>\n",
       "      <td>09/29/2017 01:24:00</td>\n",
       "      <td>71250</td>\n",
       "      <td>CT CHEST WITHOUT CONTRAST</td>\n",
       "      <td>41382337</td>\n",
       "      <td>7673928041</td>\n",
       "      <td>598512550</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  PATIENT_STUDY_ID           ORDER_DATE          RESULT_TIME PROCEDURE_CODE  \\\n",
       "0       0110000087  02/14/2017 15:35:00  02/17/2017 15:22:00          74176   \n",
       "1       0110000087  03/12/2017 18:40:00  03/12/2017 20:11:00          74176   \n",
       "2       0110000087  02/08/2018 13:06:00  02/19/2018 10:53:00          71250   \n",
       "3       0110000087  07/05/2017 00:06:00  07/05/2017 00:24:00          74176   \n",
       "4       0110000639  09/22/2017 13:18:00  09/29/2017 01:24:00          71250   \n",
       "\n",
       "                           PROCEDURE_NAME ACCESSION_STUDY_ID NOTE_STUDY_ID  \\\n",
       "0  CT ABDOMEN AND PELVIS WITHOUT CONTRAST           49955263     539216381   \n",
       "1  CT ABDOMEN AND PELVIS WITHOUT CONTRAST           49229914     533526235   \n",
       "2               CT CHEST WITHOUT CONTRAST           43754750    7603198677   \n",
       "3  CT ABDOMEN AND PELVIS WITHOUT CONTRAST           40436615    7666701228   \n",
       "4               CT CHEST WITHOUT CONTRAST           41382337    7673928041   \n",
       "\n",
       "  NOTE_CSN_STUDY_ID  \n",
       "0         442362360  \n",
       "1         457830446  \n",
       "2         520069287  \n",
       "3         570342923  \n",
       "4         598512550  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r3_notes_meta_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PATIENT_STUDY_ID</th>\n",
       "      <th>ORDER_DATE</th>\n",
       "      <th>RESULT_TIME</th>\n",
       "      <th>PROCEDURE_CODE</th>\n",
       "      <th>PROCEDURE_NAME</th>\n",
       "      <th>ACCESSION_STUDY_ID</th>\n",
       "      <th>NOTE_STUDY_ID</th>\n",
       "      <th>NOTE_CSN_STUDY_ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>113</th>\n",
       "      <td>0110006016</td>\n",
       "      <td>11/17/2017 16:18:00</td>\n",
       "      <td>11/18/2017 11:52:00</td>\n",
       "      <td>71250</td>\n",
       "      <td>CT CHEST WITHOUT CONTRAST</td>\n",
       "      <td>42928897</td>\n",
       "      <td>7684776429</td>\n",
       "      <td>500735035</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>114</th>\n",
       "      <td>0110006016</td>\n",
       "      <td>11/17/2017 16:18:00</td>\n",
       "      <td>11/18/2017 11:52:00</td>\n",
       "      <td>71250</td>\n",
       "      <td>CT CHEST WITHOUT CONTRAST</td>\n",
       "      <td>42928897</td>\n",
       "      <td>7684776319</td>\n",
       "      <td>500735913</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115</th>\n",
       "      <td>0110006016</td>\n",
       "      <td>12/01/2017 15:59:00</td>\n",
       "      <td>12/08/2017 16:27:00</td>\n",
       "      <td>71250</td>\n",
       "      <td>CT CHEST WITHOUT CONTRAST</td>\n",
       "      <td>42180042</td>\n",
       "      <td>7698748740</td>\n",
       "      <td>504081756</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116</th>\n",
       "      <td>0110006016</td>\n",
       "      <td>12/01/2017 15:59:00</td>\n",
       "      <td>12/08/2017 16:27:00</td>\n",
       "      <td>71250</td>\n",
       "      <td>CT CHEST WITHOUT CONTRAST</td>\n",
       "      <td>42180042</td>\n",
       "      <td>7698709464</td>\n",
       "      <td>504941771</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117</th>\n",
       "      <td>0110006016</td>\n",
       "      <td>02/26/2018 12:27:00</td>\n",
       "      <td>02/28/2018 16:55:00</td>\n",
       "      <td>71250</td>\n",
       "      <td>CT CHEST WITHOUT CONTRAST</td>\n",
       "      <td>43905578</td>\n",
       "      <td>7605453139</td>\n",
       "      <td>522428661</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118</th>\n",
       "      <td>0110006016</td>\n",
       "      <td>02/26/2018 12:27:00</td>\n",
       "      <td>02/28/2018 16:55:00</td>\n",
       "      <td>71250</td>\n",
       "      <td>CT CHEST WITHOUT CONTRAST</td>\n",
       "      <td>43905578</td>\n",
       "      <td>7605563232</td>\n",
       "      <td>522438302</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    PATIENT_STUDY_ID           ORDER_DATE          RESULT_TIME PROCEDURE_CODE  \\\n",
       "113       0110006016  11/17/2017 16:18:00  11/18/2017 11:52:00          71250   \n",
       "114       0110006016  11/17/2017 16:18:00  11/18/2017 11:52:00          71250   \n",
       "115       0110006016  12/01/2017 15:59:00  12/08/2017 16:27:00          71250   \n",
       "116       0110006016  12/01/2017 15:59:00  12/08/2017 16:27:00          71250   \n",
       "117       0110006016  02/26/2018 12:27:00  02/28/2018 16:55:00          71250   \n",
       "118       0110006016  02/26/2018 12:27:00  02/28/2018 16:55:00          71250   \n",
       "\n",
       "                PROCEDURE_NAME ACCESSION_STUDY_ID NOTE_STUDY_ID  \\\n",
       "113  CT CHEST WITHOUT CONTRAST           42928897    7684776429   \n",
       "114  CT CHEST WITHOUT CONTRAST           42928897    7684776319   \n",
       "115  CT CHEST WITHOUT CONTRAST           42180042    7698748740   \n",
       "116  CT CHEST WITHOUT CONTRAST           42180042    7698709464   \n",
       "117  CT CHEST WITHOUT CONTRAST           43905578    7605453139   \n",
       "118  CT CHEST WITHOUT CONTRAST           43905578    7605563232   \n",
       "\n",
       "    NOTE_CSN_STUDY_ID  \n",
       "113         500735035  \n",
       "114         500735913  \n",
       "115         504081756  \n",
       "116         504941771  \n",
       "117         522428661  \n",
       "118         522438302  "
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pid = '0110006016'\n",
    "sid = '42928897'\n",
    "ind = r3_notes_meta_df['PATIENT_STUDY_ID'] == pid\n",
    "r3_notes_meta_df[ind]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(38459, 8)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r3_notes_meta_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "38459"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r3_notes_meta_df['NOTE_STUDY_ID'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "38459"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r3_notes_meta_df['NOTE_CSN_STUDY_ID'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10102"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r3_notes_meta_df['PATIENT_STUDY_ID'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "25805"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "r3_notes_meta_df['ACCESSION_STUDY_ID'].nunique()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## R3 keywords (defined in chexpert) profiling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "1. cd /pylon5/ac5616p/yuke/PythonProject/chexpert-labeler\n",
    "2. conda activate negbio3.7\n",
    "3. python label.py\n",
    "4. output to /pylon5/ac5616p/yuke/PythonProject/chexpert-labeler/R3_Notes_Terms.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "f_path = '/pylon5/ac5616p/yuke/PythonProject/chexpert-labeler/R3_Notes_Terms.csv'\n",
    "R3_term_df = pd.read_csv(f_path, index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>term</th>\n",
       "      <th>mention</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>emphysema</td>\n",
       "      <td>No Finding</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>nodule</td>\n",
       "      <td>Lung Lesion</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>nodule</td>\n",
       "      <td>Lung Lesion</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>nodule</td>\n",
       "      <td>Lung Lesion</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>atelecta</td>\n",
       "      <td>Atelectasis</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        term      mention\n",
       "0  emphysema   No Finding\n",
       "1     nodule  Lung Lesion\n",
       "2     nodule  Lung Lesion\n",
       "3     nodule  Lung Lesion\n",
       "4   atelecta  Atelectasis"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "R3_term_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "139"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "R3_term_df['term'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "R3_term_df['mention'].nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "term_profile = R3_term_df.groupby(['mention','term']).size().reset_index(name='counts').sort_values('counts',ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>mention</th>\n",
       "      <th>term</th>\n",
       "      <th>counts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>Lung Lesion</td>\n",
       "      <td>nodule</td>\n",
       "      <td>76559</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>64</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>calcifi</td>\n",
       "      <td>51275</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>104</th>\n",
       "      <td>Pleural Effusion</td>\n",
       "      <td>effusion</td>\n",
       "      <td>41204</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>Lung Lesion</td>\n",
       "      <td>mass</td>\n",
       "      <td>28117</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>Lung Opacity</td>\n",
       "      <td>opaci</td>\n",
       "      <td>27386</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Atelectasis</td>\n",
       "      <td>atelecta</td>\n",
       "      <td>25302</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>thickening</td>\n",
       "      <td>23613</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Consolidation</td>\n",
       "      <td>consolidat</td>\n",
       "      <td>19350</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>emphysema</td>\n",
       "      <td>15844</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>atherosclero</td>\n",
       "      <td>15311</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116</th>\n",
       "      <td>Pneumothorax</td>\n",
       "      <td>pneumothorax</td>\n",
       "      <td>14954</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>hernia</td>\n",
       "      <td>14733</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>Fracture</td>\n",
       "      <td>fracture</td>\n",
       "      <td>14294</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Cardiomegaly</td>\n",
       "      <td>the heart</td>\n",
       "      <td>13839</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>inflam</td>\n",
       "      <td>12467</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>Lung Opacity</td>\n",
       "      <td>scar</td>\n",
       "      <td>12333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>71</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>degenera</td>\n",
       "      <td>12295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>114</th>\n",
       "      <td>Pneumonia</td>\n",
       "      <td>pneumonia</td>\n",
       "      <td>12170</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>72</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>density</td>\n",
       "      <td>10490</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50</th>\n",
       "      <td>Lung Opacity</td>\n",
       "      <td>wall thickening</td>\n",
       "      <td>10108</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Cardiomegaly</td>\n",
       "      <td>heart size</td>\n",
       "      <td>9223</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138</th>\n",
       "      <td>Support Devices</td>\n",
       "      <td>tube</td>\n",
       "      <td>9122</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>prominen</td>\n",
       "      <td>9053</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>121</th>\n",
       "      <td>Support Devices</td>\n",
       "      <td>catheter</td>\n",
       "      <td>8179</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>106</th>\n",
       "      <td>Pleural Other</td>\n",
       "      <td>fibrosis</td>\n",
       "      <td>6244</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Edema</td>\n",
       "      <td>edema</td>\n",
       "      <td>6166</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>granuloma</td>\n",
       "      <td>5823</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>Lung Opacity</td>\n",
       "      <td>infiltrate</td>\n",
       "      <td>5757</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Enlarged Cardiomediastinum</td>\n",
       "      <td>mediastinum</td>\n",
       "      <td>5667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Cardiomegaly</td>\n",
       "      <td>cardiomegaly</td>\n",
       "      <td>5633</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>Lung Lesion</td>\n",
       "      <td>carcinoma</td>\n",
       "      <td>5454</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>hematoma</td>\n",
       "      <td>5117</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>76</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>fissure</td>\n",
       "      <td>5056</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>112</th>\n",
       "      <td>Pneumonia</td>\n",
       "      <td>infectious</td>\n",
       "      <td>4611</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>metastasis</td>\n",
       "      <td>4113</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>Lung Lesion</td>\n",
       "      <td>tumor</td>\n",
       "      <td>4019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>aspiration</td>\n",
       "      <td>3892</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>93</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>plaque</td>\n",
       "      <td>3746</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>105</th>\n",
       "      <td>Pleural Effusion</td>\n",
       "      <td>pleural fluid</td>\n",
       "      <td>3714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>Lung Opacity</td>\n",
       "      <td>infiltration</td>\n",
       "      <td>3372</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>139</th>\n",
       "      <td>Support Devices</td>\n",
       "      <td>valve</td>\n",
       "      <td>3180</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Atelectasis</td>\n",
       "      <td>collapse</td>\n",
       "      <td>3009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>Lung Lesion</td>\n",
       "      <td>neoplasm</td>\n",
       "      <td>2977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>reticula</td>\n",
       "      <td>2942</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>111</th>\n",
       "      <td>Pneumonia</td>\n",
       "      <td>infection</td>\n",
       "      <td>2685</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>110</th>\n",
       "      <td>Pleural Other</td>\n",
       "      <td>pleural thickening</td>\n",
       "      <td>2444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Enlarged Cardiomediastinum</td>\n",
       "      <td>contour</td>\n",
       "      <td>2309</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>84</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>hypertension</td>\n",
       "      <td>2301</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>Lung Opacity</td>\n",
       "      <td>airspace disease</td>\n",
       "      <td>2141</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>bronchiolitis</td>\n",
       "      <td>2129</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>135</th>\n",
       "      <td>Support Devices</td>\n",
       "      <td>plate</td>\n",
       "      <td>2120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>Lung Lesion</td>\n",
       "      <td>nodular opacities</td>\n",
       "      <td>2050</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>124</th>\n",
       "      <td>Support Devices</td>\n",
       "      <td>device</td>\n",
       "      <td>1881</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Cardiomegaly</td>\n",
       "      <td>cardiac enlargement</td>\n",
       "      <td>1689</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>Lung Opacity</td>\n",
       "      <td>interstitial lung</td>\n",
       "      <td>1571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>Lung Lesion</td>\n",
       "      <td>nodular density</td>\n",
       "      <td>1434</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>130</th>\n",
       "      <td>Support Devices</td>\n",
       "      <td>marker</td>\n",
       "      <td>1432</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>79</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>hemorrhage</td>\n",
       "      <td>1383</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>deformity</td>\n",
       "      <td>1380</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>125</th>\n",
       "      <td>Support Devices</td>\n",
       "      <td>drain</td>\n",
       "      <td>1373</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>copd</td>\n",
       "      <td>1372</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>tracheostomy</td>\n",
       "      <td>1331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>Lung Lesion</td>\n",
       "      <td>nodular densities</td>\n",
       "      <td>1294</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>Lung Lesion</td>\n",
       "      <td>nodular opacity</td>\n",
       "      <td>1283</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>contusion</td>\n",
       "      <td>1272</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>134</th>\n",
       "      <td>Support Devices</td>\n",
       "      <td>picc</td>\n",
       "      <td>1234</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>Lung Opacity</td>\n",
       "      <td>parenchymal scarring</td>\n",
       "      <td>1217</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>117</th>\n",
       "      <td>Support Devices</td>\n",
       "      <td>line</td>\n",
       "      <td>1197</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>pneumoperitoneum</td>\n",
       "      <td>1128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>60</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>bronchitis</td>\n",
       "      <td>1102</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>129</th>\n",
       "      <td>Support Devices</td>\n",
       "      <td>lines</td>\n",
       "      <td>1033</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Cardiomegaly</td>\n",
       "      <td>cardiac silhouette</td>\n",
       "      <td>1031</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>103</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>volume loss</td>\n",
       "      <td>1004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>pneumomediastinum</td>\n",
       "      <td>998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>Lung Opacity</td>\n",
       "      <td>reticulation</td>\n",
       "      <td>983</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>137</th>\n",
       "      <td>Support Devices</td>\n",
       "      <td>support</td>\n",
       "      <td>955</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>scoliosis</td>\n",
       "      <td>948</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>132</th>\n",
       "      <td>Support Devices</td>\n",
       "      <td>pacemaker</td>\n",
       "      <td>914</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>osteopenia</td>\n",
       "      <td>829</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>elevation</td>\n",
       "      <td>762</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>Lung Lesion</td>\n",
       "      <td>cavitary lesion</td>\n",
       "      <td>748</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>hyperinflat</td>\n",
       "      <td>718</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>127</th>\n",
       "      <td>Support Devices</td>\n",
       "      <td>hardware</td>\n",
       "      <td>698</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Cardiomegaly</td>\n",
       "      <td>cardiac size</td>\n",
       "      <td>683</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>133</th>\n",
       "      <td>Support Devices</td>\n",
       "      <td>pacer</td>\n",
       "      <td>574</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119</th>\n",
       "      <td>Support Devices</td>\n",
       "      <td>arthroplast</td>\n",
       "      <td>537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Edema</td>\n",
       "      <td>vascular congestion</td>\n",
       "      <td>531</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113</th>\n",
       "      <td>Pneumonia</td>\n",
       "      <td>infectious process</td>\n",
       "      <td>510</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>Lung Opacity</td>\n",
       "      <td>peribronchial thickening</td>\n",
       "      <td>449</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>122</th>\n",
       "      <td>Support Devices</td>\n",
       "      <td>coil</td>\n",
       "      <td>424</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>109</th>\n",
       "      <td>Pleural Other</td>\n",
       "      <td>pleural scar</td>\n",
       "      <td>424</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>128</th>\n",
       "      <td>Support Devices</td>\n",
       "      <td>icd</td>\n",
       "      <td>421</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>131</th>\n",
       "      <td>Support Devices</td>\n",
       "      <td>mediport</td>\n",
       "      <td>400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>bullae</td>\n",
       "      <td>397</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>kyphosis</td>\n",
       "      <td>387</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>airways disease</td>\n",
       "      <td>386</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>hyperinflation</td>\n",
       "      <td>368</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Edema</td>\n",
       "      <td>heart failure</td>\n",
       "      <td>352</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>108</th>\n",
       "      <td>Pleural Other</td>\n",
       "      <td>pleural parenchymal scar</td>\n",
       "      <td>341</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>Lung Opacity</td>\n",
       "      <td>increased density</td>\n",
       "      <td>337</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>115</th>\n",
       "      <td>Pneumothorax</td>\n",
       "      <td>pneumothoraces</td>\n",
       "      <td>335</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>120</th>\n",
       "      <td>Support Devices</td>\n",
       "      <td>cannula</td>\n",
       "      <td>326</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>asthma</td>\n",
       "      <td>311</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>136</th>\n",
       "      <td>Support Devices</td>\n",
       "      <td>screw</td>\n",
       "      <td>291</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Edema</td>\n",
       "      <td>chf</td>\n",
       "      <td>269</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>arthropathy</td>\n",
       "      <td>233</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>radiopaque</td>\n",
       "      <td>229</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>Lung Lesion</td>\n",
       "      <td>lump</td>\n",
       "      <td>202</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>Lung Opacity</td>\n",
       "      <td>interstitial marking</td>\n",
       "      <td>190</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>86</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>interstitial markings</td>\n",
       "      <td>190</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>thoracentesis</td>\n",
       "      <td>173</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>defib</td>\n",
       "      <td>110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>123</th>\n",
       "      <td>Support Devices</td>\n",
       "      <td>defib</td>\n",
       "      <td>110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>osteodystrophy</td>\n",
       "      <td>61</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>bullet</td>\n",
       "      <td>60</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>eventration</td>\n",
       "      <td>59</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>Lung Opacity</td>\n",
       "      <td>interstitial pattern</td>\n",
       "      <td>46</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>68</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>cuffing</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>107</th>\n",
       "      <td>Pleural Other</td>\n",
       "      <td>fibrothorax</td>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Cardiomegaly</td>\n",
       "      <td>enlarged heart</td>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>Lung Lesion</td>\n",
       "      <td>nodular opacification</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>osteophytosis</td>\n",
       "      <td>27</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>blunt</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>Lung Opacity</td>\n",
       "      <td>reticular marking</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Edema</td>\n",
       "      <td>indistinctness</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Edema</td>\n",
       "      <td>pulmonary congestion</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>aortic disease</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>Enlarged Cardiomediastinum</td>\n",
       "      <td>mediastinal silhouette</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>118</th>\n",
       "      <td>Support Devices</td>\n",
       "      <td>apparatus</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Edema</td>\n",
       "      <td>vascular prominence</td>\n",
       "      <td>15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>hyperexpansion</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>Lung Opacity</td>\n",
       "      <td>reticular pattern</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>Lung Opacity</td>\n",
       "      <td>air space disease</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>aerophagia</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>61</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>bronchospasm</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>126</th>\n",
       "      <td>Support Devices</td>\n",
       "      <td>equipment</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>Enlarged Cardiomediastinum</td>\n",
       "      <td>mediastinal configuration</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>irregular lucency</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Cardiomegaly</td>\n",
       "      <td>cardiac contour</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65</th>\n",
       "      <td>No Finding</td>\n",
       "      <td>cephalization</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        mention                       term  counts\n",
       "33                  Lung Lesion                     nodule   76559\n",
       "64                   No Finding                    calcifi   51275\n",
       "104            Pleural Effusion                   effusion   41204\n",
       "26                  Lung Lesion                       mass   28117\n",
       "43                 Lung Opacity                      opaci   27386\n",
       "0                   Atelectasis                   atelecta   25302\n",
       "100                  No Finding                 thickening   23613\n",
       "10                Consolidation                 consolidat   19350\n",
       "74                   No Finding                  emphysema   15844\n",
       "57                   No Finding               atherosclero   15311\n",
       "116                Pneumothorax               pneumothorax   14954\n",
       "80                   No Finding                     hernia   14733\n",
       "22                     Fracture                   fracture   14294\n",
       "9                  Cardiomegaly                  the heart   13839\n",
       "85                   No Finding                     inflam   12467\n",
       "49                 Lung Opacity                       scar   12333\n",
       "71                   No Finding                   degenera   12295\n",
       "114                   Pneumonia                  pneumonia   12170\n",
       "72                   No Finding                    density   10490\n",
       "50                 Lung Opacity            wall thickening   10108\n",
       "8                  Cardiomegaly                 heart size    9223\n",
       "138             Support Devices                       tube    9122\n",
       "96                   No Finding                   prominen    9053\n",
       "121             Support Devices                   catheter    8179\n",
       "106               Pleural Other                   fibrosis    6244\n",
       "12                        Edema                      edema    6166\n",
       "77                   No Finding                  granuloma    5823\n",
       "38                 Lung Opacity                 infiltrate    5757\n",
       "18   Enlarged Cardiomediastinum                mediastinum    5667\n",
       "6                  Cardiomegaly               cardiomegaly    5633\n",
       "23                  Lung Lesion                  carcinoma    5454\n",
       "78                   No Finding                   hematoma    5117\n",
       "76                   No Finding                    fissure    5056\n",
       "112                   Pneumonia                 infectious    4611\n",
       "89                   No Finding                 metastasis    4113\n",
       "34                  Lung Lesion                      tumor    4019\n",
       "55                   No Finding                 aspiration    3892\n",
       "93                   No Finding                     plaque    3746\n",
       "105            Pleural Effusion              pleural fluid    3714\n",
       "39                 Lung Opacity               infiltration    3372\n",
       "139             Support Devices                      valve    3180\n",
       "1                   Atelectasis                   collapse    3009\n",
       "27                  Lung Lesion                   neoplasm    2977\n",
       "98                   No Finding                   reticula    2942\n",
       "111                   Pneumonia                  infection    2685\n",
       "110               Pleural Other         pleural thickening    2444\n",
       "19   Enlarged Cardiomediastinum                    contour    2309\n",
       "84                   No Finding               hypertension    2301\n",
       "36                 Lung Opacity           airspace disease    2141\n",
       "59                   No Finding              bronchiolitis    2129\n",
       "135             Support Devices                      plate    2120\n",
       "31                  Lung Lesion          nodular opacities    2050\n",
       "124             Support Devices                     device    1881\n",
       "3                  Cardiomegaly        cardiac enlargement    1689\n",
       "40                 Lung Opacity          interstitial lung    1571\n",
       "29                  Lung Lesion            nodular density    1434\n",
       "130             Support Devices                     marker    1432\n",
       "79                   No Finding                 hemorrhage    1383\n",
       "70                   No Finding                  deformity    1380\n",
       "125             Support Devices                     drain     1373\n",
       "67                   No Finding                       copd    1372\n",
       "102                  No Finding               tracheostomy    1331\n",
       "28                  Lung Lesion          nodular densities    1294\n",
       "32                  Lung Lesion            nodular opacity    1283\n",
       "66                   No Finding                  contusion    1272\n",
       "134             Support Devices                       picc    1234\n",
       "44                 Lung Opacity       parenchymal scarring    1217\n",
       "117             Support Devices                      line     1197\n",
       "95                   No Finding           pneumoperitoneum    1128\n",
       "60                   No Finding                 bronchitis    1102\n",
       "129             Support Devices                      lines    1033\n",
       "4                  Cardiomegaly         cardiac silhouette    1031\n",
       "103                  No Finding                volume loss    1004\n",
       "94                   No Finding          pneumomediastinum     998\n",
       "48                 Lung Opacity               reticulation     983\n",
       "137             Support Devices                    support     955\n",
       "99                   No Finding                  scoliosis     948\n",
       "132             Support Devices                  pacemaker     914\n",
       "91                   No Finding                 osteopenia     829\n",
       "73                   No Finding                  elevation     762\n",
       "24                  Lung Lesion            cavitary lesion     748\n",
       "82                   No Finding                hyperinflat     718\n",
       "127             Support Devices                   hardware     698\n",
       "5                  Cardiomegaly               cardiac size     683\n",
       "133             Support Devices                      pacer     574\n",
       "119             Support Devices                arthroplast     537\n",
       "16                        Edema        vascular congestion     531\n",
       "113                   Pneumonia         infectious process     510\n",
       "45                 Lung Opacity   peribronchial thickening     449\n",
       "122             Support Devices                       coil     424\n",
       "109               Pleural Other               pleural scar     424\n",
       "128             Support Devices                        icd     421\n",
       "131             Support Devices                   mediport     400\n",
       "62                   No Finding                     bullae     397\n",
       "88                   No Finding                   kyphosis     387\n",
       "52                   No Finding            airways disease     386\n",
       "83                   No Finding             hyperinflation     368\n",
       "13                        Edema              heart failure     352\n",
       "108               Pleural Other   pleural parenchymal scar     341\n",
       "37                 Lung Opacity          increased density     337\n",
       "115                Pneumothorax             pneumothoraces     335\n",
       "120             Support Devices                    cannula     326\n",
       "56                   No Finding                     asthma     311\n",
       "136             Support Devices                      screw     291\n",
       "11                        Edema                        chf     269\n",
       "54                   No Finding                arthropathy     233\n",
       "97                   No Finding                 radiopaque     229\n",
       "25                  Lung Lesion                       lump     202\n",
       "41                 Lung Opacity       interstitial marking     190\n",
       "86                   No Finding      interstitial markings     190\n",
       "101                  No Finding              thoracentesis     173\n",
       "69                   No Finding                      defib     110\n",
       "123             Support Devices                      defib     110\n",
       "90                   No Finding             osteodystrophy      61\n",
       "63                   No Finding                     bullet      60\n",
       "75                   No Finding                eventration      59\n",
       "42                 Lung Opacity       interstitial pattern      46\n",
       "68                   No Finding                    cuffing      34\n",
       "107               Pleural Other                fibrothorax      31\n",
       "7                  Cardiomegaly             enlarged heart      31\n",
       "30                  Lung Lesion      nodular opacification      28\n",
       "92                   No Finding              osteophytosis      27\n",
       "58                   No Finding                      blunt      26\n",
       "46                 Lung Opacity          reticular marking      25\n",
       "14                        Edema             indistinctness      22\n",
       "15                        Edema       pulmonary congestion      20\n",
       "53                   No Finding             aortic disease      19\n",
       "21   Enlarged Cardiomediastinum     mediastinal silhouette      18\n",
       "118             Support Devices                  apparatus      15\n",
       "17                        Edema        vascular prominence      15\n",
       "81                   No Finding             hyperexpansion      11\n",
       "47                 Lung Opacity          reticular pattern       9\n",
       "35                 Lung Opacity          air space disease       8\n",
       "51                   No Finding                 aerophagia       4\n",
       "61                   No Finding               bronchospasm       3\n",
       "126             Support Devices                  equipment       2\n",
       "20   Enlarged Cardiomediastinum  mediastinal configuration       2\n",
       "87                   No Finding          irregular lucency       1\n",
       "2                  Cardiomegaly            cardiac contour       1\n",
       "65                   No Finding              cephalization       1"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.set_option('display.max_rows', 500)\n",
    "term_profile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "kong_py35",
   "language": "python",
   "name": "kong_py35"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}