Data Cleaning.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.pyplot import figure"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py:2698: DtypeWarning: Columns (18,19,20,21,22,23,24,33,34,35,37,43,45) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  interactivity=interactivity, compiler=compiler, result=result)\n"
     ]
    }
   ],
   "source": [
    "data = pd.read_csv('data/DC_Properties.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data Cleaning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## remove rows with missing values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# since our many concern is the price of the building, rows without the price have only a small to none value\n",
    "data = data[data[\"PRICE\"].notnull()]\n",
    "\n",
    "# remove rows with strange prices\n",
    "data = data.loc[(data[\"PRICE\"] <= 25000000) & (data[\"PRICE\"] >= 40000)]\n",
    "\n",
    "# unqualifed data is not representative for the market\n",
    "data = data[data.QUALIFIED != \"U\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## drop columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# see above\n",
    "data = data.drop([\"QUALIFIED\"], axis=1)\n",
    "# this column has only one value \"2018-07-22 18:01:43\" => useless\n",
    "data = data.drop([\"GIS_LAST_MOD_DTTM\"], axis=1)\n",
    "# since the unique id is equal to the row number we don't need it\n",
    "data = data.drop([\"Unnamed: 0\"], axis=1)\n",
    "# many many missing values\n",
    "data = data.drop([\"LIVING_GBA\", \"CMPLX_NUM\"], axis=1)\n",
    "# these columns have nothing to say\n",
    "data = data.drop([\"STATE\", \"CITY\", \"SOURCE\", \"BLDG_NUM\"], axis=1)\n",
    "# these columns contain only unique texts which can't be computed\n",
    "data = data.drop([\"NATIONALGRID\", \"FULLADDRESS\"], axis=1)\n",
    "# in our other notebook we proved that there is a realy high (< 0.999) correlation between x, y and longitude and latititude because in both show the longitude and latitdude of a building\n",
    "data = data.drop([\"X\", \"Y\"], axis=1)\n",
    "# this column has way to many unique values => makes the onehot encoded data really big. It correlates strongly with the other location information \n",
    "data = data.drop([\"CENSUS_BLOCK\"], axis=1)\n",
    "# we have to many columns with location information\n",
    "data = data.drop([\"ZIPCODE\", \"ASSESSMENT_NBHD\", \"CENSUS_TRACT\", \"WARD\", \"SQUARE\", \"QUADRANT\"], axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## data insertion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\anaconda3\\lib\\site-packages\\ipykernel_launcher.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n",
      "C:\\anaconda3\\lib\\site-packages\\ipykernel_launcher.py:8: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n",
      "C:\\anaconda3\\lib\\site-packages\\ipykernel_launcher.py:9: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  if __name__ == '__main__':\n",
      "C:\\anaconda3\\lib\\site-packages\\ipykernel_launcher.py:10: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  # Remove the CWD from sys.path while we load stuff.\n",
      "C:\\anaconda3\\lib\\site-packages\\ipykernel_launcher.py:11: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  # This is added back by InteractiveShellApp.init_path()\n",
      "C:\\anaconda3\\lib\\site-packages\\ipykernel_launcher.py:12: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  if sys.path[0] == '':\n",
      "C:\\anaconda3\\lib\\site-packages\\ipykernel_launcher.py:13: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  del sys.path[0]\n",
      "C:\\anaconda3\\lib\\site-packages\\ipykernel_launcher.py:14: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n",
      "C:\\anaconda3\\lib\\site-packages\\ipykernel_launcher.py:15: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  from ipykernel import kernelapp as app\n",
      "C:\\anaconda3\\lib\\site-packages\\ipykernel_launcher.py:16: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  app.launch_new_instance()\n",
      "C:\\anaconda3\\lib\\site-packages\\ipykernel_launcher.py:17: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
     ]
    }
   ],
   "source": [
    "data.loc[data['YR_RMDL'].isnull(), 'YR_RMDL'] = data['EYB']\n",
    "data.loc[data['AYB'].isnull(), 'AYB'] = data['EYB']\n",
    "# set the missing sale date to the year it was build\n",
    "data[\"SALEDATE\"] = np.where(data[\"SALEDATE\"].isnull(), data[\"AYB\"], data[\"SALEDATE\"])\n",
    "# set missing values with the mean, mode\n",
    "data.NUM_UNITS[data.NUM_UNITS.isnull()] =  data.NUM_UNITS.mode().iloc[0]\n",
    "data.STORIES=data.STORIES.round()\n",
    "data.STORIES[data.STORIES.isnull()] =  data.STORIES.mode().iloc[0]\n",
    "data.GBA[data.GBA.isnull()] =  data.GBA.mean()\n",
    "data.STYLE[data.STYLE.isnull()] =  data.STYLE.mode().iloc[0]\n",
    "data.STRUCT[data.STRUCT.isnull()] =  data.STRUCT.mode().iloc[0]\n",
    "data.GRADE[data.GRADE.isnull()] =  data.GRADE.mode().iloc[0]\n",
    "data.CNDTN[data.CNDTN.isnull()] =  data.CNDTN.mode().iloc[0]\n",
    "data.EXTWALL[data.EXTWALL.isnull()] =  data.EXTWALL.mode().iloc[0]\n",
    "data.ROOF[data.ROOF.isnull()] =  data.ROOF.mode().iloc[0]\n",
    "data.INTWALL[data.INTWALL.isnull()] =  data.INTWALL.mode().iloc[0]\n",
    "data.KITCHENS[data.KITCHENS.isnull()] =  data.KITCHENS.mode().iloc[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# these values make no sense\n",
    "data.ROOMS = data.ROOMS.astype(int)\n",
    "data = data[data.ROOMS != 0]\n",
    "data = data[data.AC != \"0\"]\n",
    "data.STORIES = data.STORIES.astype(int)\n",
    "data = data[data.STORIES < 50]\n",
    "data = data[data.FIREPLACES < 20]\n",
    "data = data[data.FIREPLACES < 40]\n",
    "data = data[data.ASSESSMENT_SUBNBHD.notnull()]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## data convertion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#convert saledate to datetime\n",
    "data['SALEDATE'] = pd.to_datetime(data['SALEDATE'])\n",
    "#Calculating the difference in years between Last Sale Date and Year Built\n",
    "data['SalevYB']=data['SALEDATE'].dt.year - data['AYB']\n",
    "#Calculating the difference in years between Last Sale Date and Year Improved\n",
    "data['SalevYI']=data['SALEDATE'].dt.year - data['EYB']\n",
    "\n",
    "data['SalevYR']=data['SALEDATE'].dt.year - data['YR_RMDL']\n",
    "\n",
    "data = data.drop([\"SALEDATE\", \"EYB\", \"AYB\", \"YR_RMDL\"], axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## final results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All Data:\n",
      "Row Count: 59874\n",
      "Col Count: 28\n",
      "\n",
      "BATHRM                0\n",
      "HF_BATHRM             0\n",
      "HEAT                  0\n",
      "AC                    0\n",
      "NUM_UNITS             0\n",
      "ROOMS                 0\n",
      "BEDRM                 0\n",
      "STORIES               0\n",
      "PRICE                 0\n",
      "SALE_NUM              0\n",
      "GBA                   0\n",
      "STYLE                 0\n",
      "STRUCT                0\n",
      "GRADE                 0\n",
      "CNDTN                 0\n",
      "EXTWALL               0\n",
      "ROOF                  0\n",
      "INTWALL               0\n",
      "KITCHENS              0\n",
      "FIREPLACES            0\n",
      "USECODE               0\n",
      "LANDAREA              0\n",
      "LATITUDE              0\n",
      "LONGITUDE             0\n",
      "ASSESSMENT_SUBNBHD    0\n",
      "SalevYB               0\n",
      "SalevYI               0\n",
      "SalevYR               0\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "missing_values = data.isnull().sum()\n",
    "print(\"All Data:\")\n",
    "print(\"Row Count: \" + str(data.shape[0]))\n",
    "print(\"Col Count: \" + str(data.shape[1]))\n",
    "print()\n",
    "print(missing_values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>BATHRM</th>\n",
       "      <th>HF_BATHRM</th>\n",
       "      <th>NUM_UNITS</th>\n",
       "      <th>ROOMS</th>\n",
       "      <th>BEDRM</th>\n",
       "      <th>STORIES</th>\n",
       "      <th>PRICE</th>\n",
       "      <th>SALE_NUM</th>\n",
       "      <th>GBA</th>\n",
       "      <th>KITCHENS</th>\n",
       "      <th>FIREPLACES</th>\n",
       "      <th>USECODE</th>\n",
       "      <th>LANDAREA</th>\n",
       "      <th>LATITUDE</th>\n",
       "      <th>LONGITUDE</th>\n",
       "      <th>SalevYB</th>\n",
       "      <th>SalevYI</th>\n",
       "      <th>SalevYR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>59874.000000</td>\n",
       "      <td>59874.000000</td>\n",
       "      <td>59874.000000</td>\n",
       "      <td>59874.000000</td>\n",
       "      <td>59874.000000</td>\n",
       "      <td>59874.000000</td>\n",
       "      <td>5.987400e+04</td>\n",
       "      <td>59874.000000</td>\n",
       "      <td>59874.000000</td>\n",
       "      <td>59874.000000</td>\n",
       "      <td>59874.000000</td>\n",
       "      <td>59874.000000</td>\n",
       "      <td>59874.000000</td>\n",
       "      <td>59874.000000</td>\n",
       "      <td>59874.000000</td>\n",
       "      <td>59874.000000</td>\n",
       "      <td>59874.000000</td>\n",
       "      <td>59874.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>1.930053</td>\n",
       "      <td>0.481879</td>\n",
       "      <td>1.139977</td>\n",
       "      <td>6.117129</td>\n",
       "      <td>2.731002</td>\n",
       "      <td>2.062197</td>\n",
       "      <td>5.420962e+05</td>\n",
       "      <td>1.936283</td>\n",
       "      <td>1709.249545</td>\n",
       "      <td>1.161055</td>\n",
       "      <td>0.496342</td>\n",
       "      <td>14.369810</td>\n",
       "      <td>2083.298744</td>\n",
       "      <td>38.912689</td>\n",
       "      <td>-77.014304</td>\n",
       "      <td>69.980359</td>\n",
       "      <td>44.854344</td>\n",
       "      <td>16.855530</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.983598</td>\n",
       "      <td>0.575252</td>\n",
       "      <td>0.492488</td>\n",
       "      <td>2.574053</td>\n",
       "      <td>1.354157</td>\n",
       "      <td>0.346256</td>\n",
       "      <td>4.797784e+05</td>\n",
       "      <td>1.469934</td>\n",
       "      <td>633.601846</td>\n",
       "      <td>0.508770</td>\n",
       "      <td>0.818503</td>\n",
       "      <td>3.836785</td>\n",
       "      <td>2483.519796</td>\n",
       "      <td>0.029545</td>\n",
       "      <td>0.039694</td>\n",
       "      <td>36.920528</td>\n",
       "      <td>28.458298</td>\n",
       "      <td>23.318441</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.000000e+04</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>252.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>38.819953</td>\n",
       "      <td>-77.113909</td>\n",
       "      <td>-11.000000</td>\n",
       "      <td>-21.000000</td>\n",
       "      <td>-26.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>2.680000e+05</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1360.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>11.000000</td>\n",
       "      <td>581.000000</td>\n",
       "      <td>38.894615</td>\n",
       "      <td>-77.037668</td>\n",
       "      <td>50.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>2.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>4.300000e+05</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1730.484413</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>13.000000</td>\n",
       "      <td>1463.000000</td>\n",
       "      <td>38.913210</td>\n",
       "      <td>-77.018128</td>\n",
       "      <td>78.000000</td>\n",
       "      <td>44.000000</td>\n",
       "      <td>7.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>2.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>7.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>6.850000e+05</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>1730.484413</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>16.000000</td>\n",
       "      <td>2520.000000</td>\n",
       "      <td>38.931395</td>\n",
       "      <td>-76.988052</td>\n",
       "      <td>97.000000</td>\n",
       "      <td>55.000000</td>\n",
       "      <td>32.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>11.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>2.200000e+07</td>\n",
       "      <td>15.000000</td>\n",
       "      <td>10419.000000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>13.000000</td>\n",
       "      <td>117.000000</td>\n",
       "      <td>155905.000000</td>\n",
       "      <td>38.982532</td>\n",
       "      <td>-76.909758</td>\n",
       "      <td>262.000000</td>\n",
       "      <td>195.000000</td>\n",
       "      <td>1995.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             BATHRM     HF_BATHRM     NUM_UNITS         ROOMS         BEDRM  \\\n",
       "count  59874.000000  59874.000000  59874.000000  59874.000000  59874.000000   \n",
       "mean       1.930053      0.481879      1.139977      6.117129      2.731002   \n",
       "std        0.983598      0.575252      0.492488      2.574053      1.354157   \n",
       "min        0.000000      0.000000      0.000000      1.000000      0.000000   \n",
       "25%        1.000000      0.000000      1.000000      4.000000      2.000000   \n",
       "50%        2.000000      0.000000      1.000000      6.000000      3.000000   \n",
       "75%        2.000000      1.000000      1.000000      7.000000      3.000000   \n",
       "max       11.000000      5.000000      6.000000     25.000000     20.000000   \n",
       "\n",
       "            STORIES         PRICE      SALE_NUM           GBA      KITCHENS  \\\n",
       "count  59874.000000  5.987400e+04  59874.000000  59874.000000  59874.000000   \n",
       "mean       2.062197  5.420962e+05      1.936283   1709.249545      1.161055   \n",
       "std        0.346256  4.797784e+05      1.469934    633.601846      0.508770   \n",
       "min        0.000000  4.000000e+04      1.000000    252.000000      0.000000   \n",
       "25%        2.000000  2.680000e+05      1.000000   1360.000000      1.000000   \n",
       "50%        2.000000  4.300000e+05      1.000000   1730.484413      1.000000   \n",
       "75%        2.000000  6.850000e+05      3.000000   1730.484413      1.000000   \n",
       "max       20.000000  2.200000e+07     15.000000  10419.000000      6.000000   \n",
       "\n",
       "         FIREPLACES       USECODE       LANDAREA      LATITUDE     LONGITUDE  \\\n",
       "count  59874.000000  59874.000000   59874.000000  59874.000000  59874.000000   \n",
       "mean       0.496342     14.369810    2083.298744     38.912689    -77.014304   \n",
       "std        0.818503      3.836785    2483.519796      0.029545      0.039694   \n",
       "min        0.000000     11.000000       0.000000     38.819953    -77.113909   \n",
       "25%        0.000000     11.000000     581.000000     38.894615    -77.037668   \n",
       "50%        0.000000     13.000000    1463.000000     38.913210    -77.018128   \n",
       "75%        1.000000     16.000000    2520.000000     38.931395    -76.988052   \n",
       "max       13.000000    117.000000  155905.000000     38.982532    -76.909758   \n",
       "\n",
       "            SalevYB       SalevYI       SalevYR  \n",
       "count  59874.000000  59874.000000  59874.000000  \n",
       "mean      69.980359     44.854344     16.855530  \n",
       "std       36.920528     28.458298     23.318441  \n",
       "min      -11.000000    -21.000000    -26.000000  \n",
       "25%       50.000000     30.000000      1.000000  \n",
       "50%       78.000000     44.000000      7.000000  \n",
       "75%       97.000000     55.000000     32.000000  \n",
       "max      262.000000    195.000000   1995.000000  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## save to csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "data.to_csv(\"data/data_cleaned.csv\", index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}