ML_Project_TelecomChurnAnalysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.preprocessing import Imputer\n",
    "from sklearn import tree\n",
    "from sklearn import metrics\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "% matplotlib inline\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "import itertools\n",
    "#local_path=\"C:/sandbox/gitrepos/fbproject/\"\n",
    "local_path=\"./\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>State</th>\n",
       "      <th>AccountLength</th>\n",
       "      <th>AreaCode</th>\n",
       "      <th>Phone</th>\n",
       "      <th>InternationalPlan</th>\n",
       "      <th>VMailPlan</th>\n",
       "      <th>VMailMessage</th>\n",
       "      <th>DayMins</th>\n",
       "      <th>DayCalls</th>\n",
       "      <th>DayCharge</th>\n",
       "      <th>...</th>\n",
       "      <th>EveCalls</th>\n",
       "      <th>EveCharge</th>\n",
       "      <th>NightMins</th>\n",
       "      <th>NightCalls</th>\n",
       "      <th>NightCharge</th>\n",
       "      <th>InternationalMins</th>\n",
       "      <th>InternationalCalls</th>\n",
       "      <th>InternationalCharge</th>\n",
       "      <th>CustServCalls</th>\n",
       "      <th>Churn</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>KS</td>\n",
       "      <td>128</td>\n",
       "      <td>415</td>\n",
       "      <td>382-4657</td>\n",
       "      <td>no</td>\n",
       "      <td>yes</td>\n",
       "      <td>25</td>\n",
       "      <td>265.1</td>\n",
       "      <td>110</td>\n",
       "      <td>45.07</td>\n",
       "      <td>...</td>\n",
       "      <td>99</td>\n",
       "      <td>16.78</td>\n",
       "      <td>244.7</td>\n",
       "      <td>91</td>\n",
       "      <td>11.01</td>\n",
       "      <td>10.0</td>\n",
       "      <td>3</td>\n",
       "      <td>2.70</td>\n",
       "      <td>1</td>\n",
       "      <td>False.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>OH</td>\n",
       "      <td>107</td>\n",
       "      <td>415</td>\n",
       "      <td>371-7191</td>\n",
       "      <td>no</td>\n",
       "      <td>yes</td>\n",
       "      <td>26</td>\n",
       "      <td>161.6</td>\n",
       "      <td>123</td>\n",
       "      <td>27.47</td>\n",
       "      <td>...</td>\n",
       "      <td>103</td>\n",
       "      <td>16.62</td>\n",
       "      <td>254.4</td>\n",
       "      <td>103</td>\n",
       "      <td>11.45</td>\n",
       "      <td>13.7</td>\n",
       "      <td>3</td>\n",
       "      <td>3.70</td>\n",
       "      <td>1</td>\n",
       "      <td>False.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NJ</td>\n",
       "      <td>137</td>\n",
       "      <td>415</td>\n",
       "      <td>358-1921</td>\n",
       "      <td>no</td>\n",
       "      <td>no</td>\n",
       "      <td>0</td>\n",
       "      <td>243.4</td>\n",
       "      <td>114</td>\n",
       "      <td>41.38</td>\n",
       "      <td>...</td>\n",
       "      <td>110</td>\n",
       "      <td>10.30</td>\n",
       "      <td>162.6</td>\n",
       "      <td>104</td>\n",
       "      <td>7.32</td>\n",
       "      <td>12.2</td>\n",
       "      <td>5</td>\n",
       "      <td>3.29</td>\n",
       "      <td>0</td>\n",
       "      <td>False.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>OH</td>\n",
       "      <td>84</td>\n",
       "      <td>408</td>\n",
       "      <td>375-9999</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "      <td>0</td>\n",
       "      <td>299.4</td>\n",
       "      <td>71</td>\n",
       "      <td>50.90</td>\n",
       "      <td>...</td>\n",
       "      <td>88</td>\n",
       "      <td>5.26</td>\n",
       "      <td>196.9</td>\n",
       "      <td>89</td>\n",
       "      <td>8.86</td>\n",
       "      <td>6.6</td>\n",
       "      <td>7</td>\n",
       "      <td>1.78</td>\n",
       "      <td>2</td>\n",
       "      <td>False.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>OK</td>\n",
       "      <td>75</td>\n",
       "      <td>415</td>\n",
       "      <td>330-6626</td>\n",
       "      <td>yes</td>\n",
       "      <td>no</td>\n",
       "      <td>0</td>\n",
       "      <td>166.7</td>\n",
       "      <td>113</td>\n",
       "      <td>28.34</td>\n",
       "      <td>...</td>\n",
       "      <td>122</td>\n",
       "      <td>12.61</td>\n",
       "      <td>186.9</td>\n",
       "      <td>121</td>\n",
       "      <td>8.41</td>\n",
       "      <td>10.1</td>\n",
       "      <td>3</td>\n",
       "      <td>2.73</td>\n",
       "      <td>3</td>\n",
       "      <td>False.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  State  AccountLength  AreaCode      Phone InternationalPlan VMailPlan  \\\n",
       "0    KS            128       415   382-4657                no       yes   \n",
       "1    OH            107       415   371-7191                no       yes   \n",
       "2    NJ            137       415   358-1921                no        no   \n",
       "3    OH             84       408   375-9999               yes        no   \n",
       "4    OK             75       415   330-6626               yes        no   \n",
       "\n",
       "   VMailMessage  DayMins  DayCalls  DayCharge   ...     EveCalls  EveCharge  \\\n",
       "0            25    265.1       110      45.07   ...           99      16.78   \n",
       "1            26    161.6       123      27.47   ...          103      16.62   \n",
       "2             0    243.4       114      41.38   ...          110      10.30   \n",
       "3             0    299.4        71      50.90   ...           88       5.26   \n",
       "4             0    166.7       113      28.34   ...          122      12.61   \n",
       "\n",
       "   NightMins  NightCalls  NightCharge  InternationalMins  InternationalCalls  \\\n",
       "0      244.7          91        11.01               10.0                   3   \n",
       "1      254.4         103        11.45               13.7                   3   \n",
       "2      162.6         104         7.32               12.2                   5   \n",
       "3      196.9          89         8.86                6.6                   7   \n",
       "4      186.9         121         8.41               10.1                   3   \n",
       "\n",
       "   InternationalCharge  CustServCalls    Churn  \n",
       "0                 2.70              1   False.  \n",
       "1                 3.70              1   False.  \n",
       "2                 3.29              0   False.  \n",
       "3                 1.78              2   False.  \n",
       "4                 2.73              3   False.  \n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "customer_all_df = pd.read_csv(local_path+\"telecom_churn_data.txt\"\n",
    "                              , names=[\"State\",\n",
    "                                        \"AccountLength\", \"AreaCode\",\n",
    "                                        \"Phone\", \"InternationalPlan\", \n",
    "                                        \"VMailPlan\", \"VMailMessage\",\n",
    "                                        \"DayMins\", \"DayCalls\", \"DayCharge\",\n",
    "                                        \"EveMins\", \"EveCalls\", \"EveCharge\",\n",
    "                                        \"NightMins\", \"NightCalls\", \"NightCharge\", \n",
    "                                        \"InternationalMins\", \"InternationalCalls\", \"InternationalCharge\", \n",
    "                                        \"CustServCalls\", \"Churn\"])\n",
    "customer_all_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AccountLength</th>\n",
       "      <th>AreaCode</th>\n",
       "      <th>VMailMessage</th>\n",
       "      <th>DayMins</th>\n",
       "      <th>DayCalls</th>\n",
       "      <th>DayCharge</th>\n",
       "      <th>EveMins</th>\n",
       "      <th>EveCalls</th>\n",
       "      <th>EveCharge</th>\n",
       "      <th>NightMins</th>\n",
       "      <th>NightCalls</th>\n",
       "      <th>NightCharge</th>\n",
       "      <th>InternationalMins</th>\n",
       "      <th>InternationalCalls</th>\n",
       "      <th>InternationalCharge</th>\n",
       "      <th>CustServCalls</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>4617.000000</td>\n",
       "      <td>4617.000000</td>\n",
       "      <td>4617.000000</td>\n",
       "      <td>4617.000000</td>\n",
       "      <td>4617.000000</td>\n",
       "      <td>4617.000000</td>\n",
       "      <td>4617.000000</td>\n",
       "      <td>4617.000000</td>\n",
       "      <td>4617.000000</td>\n",
       "      <td>4617.000000</td>\n",
       "      <td>4617.000000</td>\n",
       "      <td>4617.000000</td>\n",
       "      <td>4617.000000</td>\n",
       "      <td>4617.000000</td>\n",
       "      <td>4617.000000</td>\n",
       "      <td>4617.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>100.645224</td>\n",
       "      <td>437.046350</td>\n",
       "      <td>7.849903</td>\n",
       "      <td>180.447152</td>\n",
       "      <td>100.054364</td>\n",
       "      <td>30.676576</td>\n",
       "      <td>200.429088</td>\n",
       "      <td>100.179770</td>\n",
       "      <td>17.036703</td>\n",
       "      <td>200.623933</td>\n",
       "      <td>99.944120</td>\n",
       "      <td>9.028185</td>\n",
       "      <td>10.279294</td>\n",
       "      <td>4.433831</td>\n",
       "      <td>2.775926</td>\n",
       "      <td>1.567035</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>39.597194</td>\n",
       "      <td>42.288212</td>\n",
       "      <td>13.592333</td>\n",
       "      <td>53.983540</td>\n",
       "      <td>19.883027</td>\n",
       "      <td>9.177145</td>\n",
       "      <td>50.557001</td>\n",
       "      <td>19.821314</td>\n",
       "      <td>4.297332</td>\n",
       "      <td>50.543616</td>\n",
       "      <td>19.935053</td>\n",
       "      <td>2.274488</td>\n",
       "      <td>2.757361</td>\n",
       "      <td>2.457615</td>\n",
       "      <td>0.744413</td>\n",
       "      <td>1.307019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>408.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>23.200000</td>\n",
       "      <td>12.000000</td>\n",
       "      <td>1.040000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>74.000000</td>\n",
       "      <td>408.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>143.700000</td>\n",
       "      <td>87.000000</td>\n",
       "      <td>24.430000</td>\n",
       "      <td>165.900000</td>\n",
       "      <td>87.000000</td>\n",
       "      <td>14.100000</td>\n",
       "      <td>167.100000</td>\n",
       "      <td>87.000000</td>\n",
       "      <td>7.520000</td>\n",
       "      <td>8.600000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>2.320000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>100.000000</td>\n",
       "      <td>415.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>180.000000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>30.600000</td>\n",
       "      <td>200.800000</td>\n",
       "      <td>101.000000</td>\n",
       "      <td>17.070000</td>\n",
       "      <td>200.800000</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>9.040000</td>\n",
       "      <td>10.300000</td>\n",
       "      <td>4.000000</td>\n",
       "      <td>2.780000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>127.000000</td>\n",
       "      <td>510.000000</td>\n",
       "      <td>17.000000</td>\n",
       "      <td>216.800000</td>\n",
       "      <td>113.000000</td>\n",
       "      <td>36.860000</td>\n",
       "      <td>234.000000</td>\n",
       "      <td>114.000000</td>\n",
       "      <td>19.890000</td>\n",
       "      <td>234.900000</td>\n",
       "      <td>113.000000</td>\n",
       "      <td>10.570000</td>\n",
       "      <td>12.100000</td>\n",
       "      <td>6.000000</td>\n",
       "      <td>3.270000</td>\n",
       "      <td>2.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>243.000000</td>\n",
       "      <td>510.000000</td>\n",
       "      <td>51.000000</td>\n",
       "      <td>351.500000</td>\n",
       "      <td>165.000000</td>\n",
       "      <td>59.760000</td>\n",
       "      <td>363.700000</td>\n",
       "      <td>170.000000</td>\n",
       "      <td>30.910000</td>\n",
       "      <td>395.000000</td>\n",
       "      <td>175.000000</td>\n",
       "      <td>17.770000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>20.000000</td>\n",
       "      <td>5.400000</td>\n",
       "      <td>9.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       AccountLength     AreaCode  VMailMessage      DayMins     DayCalls  \\\n",
       "count    4617.000000  4617.000000   4617.000000  4617.000000  4617.000000   \n",
       "mean      100.645224   437.046350      7.849903   180.447152   100.054364   \n",
       "std        39.597194    42.288212     13.592333    53.983540    19.883027   \n",
       "min         1.000000   408.000000      0.000000     0.000000     0.000000   \n",
       "25%        74.000000   408.000000      0.000000   143.700000    87.000000   \n",
       "50%       100.000000   415.000000      0.000000   180.000000   100.000000   \n",
       "75%       127.000000   510.000000     17.000000   216.800000   113.000000   \n",
       "max       243.000000   510.000000     51.000000   351.500000   165.000000   \n",
       "\n",
       "         DayCharge      EveMins     EveCalls    EveCharge    NightMins  \\\n",
       "count  4617.000000  4617.000000  4617.000000  4617.000000  4617.000000   \n",
       "mean     30.676576   200.429088   100.179770    17.036703   200.623933   \n",
       "std       9.177145    50.557001    19.821314     4.297332    50.543616   \n",
       "min       0.000000     0.000000     0.000000     0.000000    23.200000   \n",
       "25%      24.430000   165.900000    87.000000    14.100000   167.100000   \n",
       "50%      30.600000   200.800000   101.000000    17.070000   200.800000   \n",
       "75%      36.860000   234.000000   114.000000    19.890000   234.900000   \n",
       "max      59.760000   363.700000   170.000000    30.910000   395.000000   \n",
       "\n",
       "        NightCalls  NightCharge  InternationalMins  InternationalCalls  \\\n",
       "count  4617.000000  4617.000000        4617.000000         4617.000000   \n",
       "mean     99.944120     9.028185          10.279294            4.433831   \n",
       "std      19.935053     2.274488           2.757361            2.457615   \n",
       "min      12.000000     1.040000           0.000000            0.000000   \n",
       "25%      87.000000     7.520000           8.600000            3.000000   \n",
       "50%     100.000000     9.040000          10.300000            4.000000   \n",
       "75%     113.000000    10.570000          12.100000            6.000000   \n",
       "max     175.000000    17.770000          20.000000           20.000000   \n",
       "\n",
       "       InternationalCharge  CustServCalls  \n",
       "count          4617.000000    4617.000000  \n",
       "mean              2.775926       1.567035  \n",
       "std               0.744413       1.307019  \n",
       "min               0.000000       0.000000  \n",
       "25%               2.320000       1.000000  \n",
       "50%               2.780000       1.000000  \n",
       "75%               3.270000       2.000000  \n",
       "max               5.400000       9.000000  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "customer_all_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "customer_all_df['Churn'] = customer_all_df['Churn'].apply(lambda x: 1 if x.strip() == \"True.\" else 0)\n",
    "customer_all_df[\"InternationalPlan\"] = customer_all_df[\"InternationalPlan\"].apply(lambda y: 1 if y.strip() == \"yes\" else 0)\n",
    "customer_all_df[\"VMailPlan\"] = customer_all_df[\"VMailPlan\"].apply(lambda z: 1 if z.strip() == \"yes\" else 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style>\n",
       "    .dataframe thead tr:only-child th {\n",
       "        text-align: right;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>State</th>\n",
       "      <th>AccountLength</th>\n",
       "      <th>AreaCode</th>\n",
       "      <th>Phone</th>\n",
       "      <th>InternationalPlan</th>\n",
       "      <th>VMailPlan</th>\n",
       "      <th>VMailMessage</th>\n",
       "      <th>DayMins</th>\n",
       "      <th>DayCalls</th>\n",
       "      <th>DayCharge</th>\n",
       "      <th>...</th>\n",
       "      <th>EveCalls</th>\n",
       "      <th>EveCharge</th>\n",
       "      <th>NightMins</th>\n",
       "      <th>NightCalls</th>\n",
       "      <th>NightCharge</th>\n",
       "      <th>InternationalMins</th>\n",
       "      <th>InternationalCalls</th>\n",
       "      <th>InternationalCharge</th>\n",
       "      <th>CustServCalls</th>\n",
       "      <th>Churn</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>KS</td>\n",
       "      <td>128</td>\n",
       "      <td>415</td>\n",
       "      <td>382-4657</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>25</td>\n",
       "      <td>265.1</td>\n",
       "      <td>110</td>\n",
       "      <td>45.07</td>\n",
       "      <td>...</td>\n",
       "      <td>99</td>\n",
       "      <td>16.78</td>\n",
       "      <td>244.7</td>\n",
       "      <td>91</td>\n",
       "      <td>11.01</td>\n",
       "      <td>10.0</td>\n",
       "      <td>3</td>\n",
       "      <td>2.70</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>OH</td>\n",
       "      <td>107</td>\n",
       "      <td>415</td>\n",
       "      <td>371-7191</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>26</td>\n",
       "      <td>161.6</td>\n",
       "      <td>123</td>\n",
       "      <td>27.47</td>\n",
       "      <td>...</td>\n",
       "      <td>103</td>\n",
       "      <td>16.62</td>\n",
       "      <td>254.4</td>\n",
       "      <td>103</td>\n",
       "      <td>11.45</td>\n",
       "      <td>13.7</td>\n",
       "      <td>3</td>\n",
       "      <td>3.70</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NJ</td>\n",
       "      <td>137</td>\n",
       "      <td>415</td>\n",
       "      <td>358-1921</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>243.4</td>\n",
       "      <td>114</td>\n",
       "      <td>41.38</td>\n",
       "      <td>...</td>\n",
       "      <td>110</td>\n",
       "      <td>10.30</td>\n",
       "      <td>162.6</td>\n",
       "      <td>104</td>\n",
       "      <td>7.32</td>\n",
       "      <td>12.2</td>\n",
       "      <td>5</td>\n",
       "      <td>3.29</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>OH</td>\n",
       "      <td>84</td>\n",
       "      <td>408</td>\n",
       "      <td>375-9999</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>299.4</td>\n",
       "      <td>71</td>\n",
       "      <td>50.90</td>\n",
       "      <td>...</td>\n",
       "      <td>88</td>\n",
       "      <td>5.26</td>\n",
       "      <td>196.9</td>\n",
       "      <td>89</td>\n",
       "      <td>8.86</td>\n",
       "      <td>6.6</td>\n",
       "      <td>7</td>\n",
       "      <td>1.78</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>OK</td>\n",
       "      <td>75</td>\n",
       "      <td>415</td>\n",
       "      <td>330-6626</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>166.7</td>\n",
       "      <td>113</td>\n",
       "      <td>28.34</td>\n",
       "      <td>...</td>\n",
       "      <td>122</td>\n",
       "      <td>12.61</td>\n",
       "      <td>186.9</td>\n",
       "      <td>121</td>\n",
       "      <td>8.41</td>\n",
       "      <td>10.1</td>\n",
       "      <td>3</td>\n",
       "      <td>2.73</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  State  AccountLength  AreaCode      Phone  InternationalPlan  VMailPlan  \\\n",
       "0    KS            128       415   382-4657                  0          1   \n",
       "1    OH            107       415   371-7191                  0          1   \n",
       "2    NJ            137       415   358-1921                  0          0   \n",
       "3    OH             84       408   375-9999                  1          0   \n",
       "4    OK             75       415   330-6626                  1          0   \n",
       "\n",
       "   VMailMessage  DayMins  DayCalls  DayCharge  ...    EveCalls  EveCharge  \\\n",
       "0            25    265.1       110      45.07  ...          99      16.78   \n",
       "1            26    161.6       123      27.47  ...         103      16.62   \n",
       "2             0    243.4       114      41.38  ...         110      10.30   \n",
       "3             0    299.4        71      50.90  ...          88       5.26   \n",
       "4             0    166.7       113      28.34  ...         122      12.61   \n",
       "\n",
       "   NightMins  NightCalls  NightCharge  InternationalMins  InternationalCalls  \\\n",
       "0      244.7          91        11.01               10.0                   3   \n",
       "1      254.4         103        11.45               13.7                   3   \n",
       "2      162.6         104         7.32               12.2                   5   \n",
       "3      196.9          89         8.86                6.6                   7   \n",
       "4      186.9         121         8.41               10.1                   3   \n",
       "\n",
       "   InternationalCharge  CustServCalls  Churn  \n",
       "0                 2.70              1      0  \n",
       "1                 3.70              1      0  \n",
       "2                 3.29              0      0  \n",
       "3                 1.78              2      0  \n",
       "4                 2.73              3      0  \n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "customer_all_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def testFeaturesWithDecisionTree(features, labels):\n",
    "    features_train, features_test, labels_train, labels_test = train_test_split(features, labels, random_state=42)\n",
    "\n",
    "    churn_tree = tree.DecisionTreeClassifier(criterion=\"entropy\", max_depth=6)\n",
    "    #churn_tree = tree.DecisionTreeClassifier(criterion=\"gini\", max_depth=5)\n",
    "    \n",
    "    #churn_tree = churn_tree.fit(features, labels)\n",
    "    churn_tree = churn_tree.fit(features_train, labels_train)\n",
    "\n",
    "    print(\"Decision Tree\")\n",
    "    print(\"=============\")\n",
    "    print(\"Feature Importance : \")\n",
    "    print(churn_tree.feature_importances_)\n",
    "    \n",
    "    print(\"Tree Score : \")\n",
    "    #print(churn_tree.score(features, labels))\n",
    "    print(churn_tree.score(features_train, labels_train))\n",
    "\n",
    "    print(\"Predicts Probability : \")\n",
    "    prediction = churn_tree.predict_proba(features_test)[:,1]\n",
    "    print(prediction)\n",
    "    print(\"Test Features (last 3 rows) : \")\n",
    "    print(features_test[-3:])\n",
    "    print(\"Test Labels (last 3 rows) : \")\n",
    "    print(labels_test[-3:])\n",
    "    #print(len(features_test))\n",
    "    #print(prediction.size)\n",
    "    \n",
    "    print(\"Accuracy (ROC AUC Score) : \")\n",
    "    print(roc_auc_score(labels_test, prediction))\n",
    "    \n",
    "    print(\"\")\n",
    "    \n",
    "    return churn_tree"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def testFeaturesWithRandomForest(features, labels):\n",
    "    features_train, features_test, labels_train, labels_test = train_test_split(features, labels, random_state=42)\n",
    "\n",
    "    ran_forest = RandomForestClassifier(criterion=\"entropy\", max_depth = 10, min_samples_split=2, n_estimators = 100, random_state = 1)\n",
    "    #ran_forest = RandomForestClassifier(criterion=\"gini\", max_depth = 10, min_samples_split=2, n_estimators = 100, random_state = 1)\n",
    "\n",
    "    #ran_forest = churn_tree.fit(features, labels)\n",
    "    ran_forest = ran_forest.fit(features_train, labels_train)\n",
    "\n",
    "    print(\"Random Forest\")\n",
    "    print(\"=============\")\n",
    "    print(\"Feature Importance : \")\n",
    "    print(ran_forest.feature_importances_)\n",
    "    \n",
    "    print(\"Tree Score : \")\n",
    "    #print(churn_tree.score(features, labels))\n",
    "    print(ran_forest.score(features_train, labels_train))\n",
    "\n",
    "    print(\"Predict Probability : \")\n",
    "    prediction = ran_forest.predict_proba(features_test)[:,1]\n",
    "    print(prediction)\n",
    "    print(\"Test Features (last 3 rows) : \")\n",
    "    print(features_test[-3:])\n",
    "    print(\"Test Labels (last 3 rows) : \")\n",
    "    print(labels_test[-3:])\n",
    "    #print(len(features_test))\n",
    "    #print(prediction.size)\n",
    "    \n",
    "    print(\"Accuracy (ROC AUC Score) : \")\n",
    "    print(roc_auc_score(labels_test, prediction))\n",
    "    \n",
    "    #Churn score/rate\n",
    "    test_count = len(features_test)\n",
    "    print(\"Number of test customer: \" + str(test_count))\n",
    "    churn_count =  sum(list(labels_test))\n",
    "    print(\"Number of predicted Churn customers: \" + str(churn_count))\n",
    "    global churn_score\n",
    "    churn_score = round(churn_count/test_count*100,3)\n",
    "    print(\"Estimated churn score: \" + str(churn_score))\n",
    "    \n",
    "    print(\"\")\n",
    "\n",
    "    return ran_forest"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['AccountLength', 'AreaCode', 'InternationalPlan', 'VMailPlan', 'VMailMessage', 'DayMins', 'EveMins', 'NightMins', 'InternationalMins', 'DayCalls', 'EveCalls', 'NightCalls', 'InternationalCalls', 'DayCharge', 'EveCharge', 'NightCharge', 'InternationalCharge', 'CustServCalls']\n",
      "Random Forest\n",
      "=============\n",
      "Feature Importance : \n",
      "[ 0.02455573  0.00626417  0.10003628  0.02482651  0.02895192  0.15492081\n",
      "  0.06538043  0.03647185  0.04136246  0.02319451  0.02079514  0.02605463\n",
      "  0.04757159  0.13839118  0.05946713  0.03868534  0.03996788  0.12310245]\n",
      "Tree Score : \n",
      "0.978047371462\n",
      "Predict Probability : \n",
      "[ 0.03258336  0.06471487  0.06800621 ...,  0.03933377  0.0872315\n",
      "  0.63097451]\n",
      "Test Features (last 3 rows) : \n",
      "[[  19.    408.      0.      0.      0.    186.1   254.3   214.     14.6\n",
      "    98.     57.    127.      7.     31.64   21.62    9.63    3.94    2.  ]\n",
      " [ 135.    415.      1.      0.      0.    177.2   209.3   236.3    11.1\n",
      "   151.     83.    114.      5.     30.12   17.79   10.63    3.      2.  ]\n",
      " [  73.    415.      0.      1.     31.     82.3   256.1   229.6    11.8\n",
      "   105.     91.     98.      2.     13.99   21.77   10.33    3.19    6.  ]]\n",
      "Test Labels (last 3 rows) : \n",
      "[0 0 1]\n",
      "Accuracy (ROC AUC Score) : \n",
      "0.935207995609\n",
      "Number of test customer: 1155\n",
      "Number of predicted Churn customers: 157\n",
      "Estimated churn score: 13.593\n",
      "\n"
     ]
    }
   ],
   "source": [
    "feature_columns = list([\n",
    "    #\"State\", \n",
    "    \"AccountLength\", \n",
    "    \"AreaCode\", \n",
    "    #\"Phone\", \n",
    "    \"InternationalPlan\", \n",
    "    \"VMailPlan\", \n",
    "    \"VMailMessage\",\n",
    "    \"DayMins\", \"EveMins\", \n",
    "    \"NightMins\", \n",
    "    \"InternationalMins\",\n",
    "    \"DayCalls\", \n",
    "    \"EveCalls\", \n",
    "    \"NightCalls\", \n",
    "    \"InternationalCalls\",\n",
    "    \"DayCharge\", \"EveCharge\", \"NightCharge\", \n",
    "    \"InternationalCharge\", \n",
    "    \"CustServCalls\"\n",
    "])\n",
    "label_columns = \"Churn\"\n",
    "features = customer_all_df[feature_columns].values\n",
    "labels = customer_all_df[label_columns].values\n",
    "\n",
    "print(feature_columns)\n",
    "#churn_tree = testFeaturesWithDecisionTree(features, labels)\n",
    "ran_forest = testFeaturesWithRandomForest(features, labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['InternationalPlan', 'DayMins', 'EveMins', 'NightMins', 'InternationalCalls', 'CustServCalls']\n",
      "Random Forest\n",
      "=============\n",
      "Feature Importance : \n",
      "[ 0.11225104  0.35912139  0.16488464  0.11456806  0.08417336  0.16500151]\n",
      "Tree Score : \n",
      "0.96591565569\n",
      "Predict Probability : \n",
      "[ 0.02800203  0.04747157  0.02243403 ...,  0.02251604  0.14301965\n",
      "  0.81763068]\n",
      "Test Features (last 3 rows) : \n",
      "[[   0.   186.1  254.3  214.     7.     2. ]\n",
      " [   1.   177.2  209.3  236.3    5.     2. ]\n",
      " [   0.    82.3  256.1  229.6    2.     6. ]]\n",
      "Test Labels (last 3 rows) : \n",
      "[0 0 1]\n",
      "Accuracy (ROC AUC Score) : \n",
      "0.906500899889\n",
      "Number of test customer: 1155\n",
      "Number of predicted Churn customers: 157\n",
      "Estimated churn score: 13.593\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x119a090b8>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAFWCAYAAABuJfanAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xu8HWV59vHfRThTwAMRSgIkaJTiAbQBUamKFgpajNgq\noIJFMKWvFNTair7vq1b6Vmm1tSKaRsXWE3jERomggKcWD0kQw0HoGwM2SdUknBEwBK7+MbNlstk7\ne3ay9pq1Zq7v57M/e80zM3vda+/kXs+655nnkW0iIqI7tmk6gIiI6K8k/oiIjknij4jomCT+iIiO\nSeKPiOiYJP6IiI5J4o+I6Jgk/thqkm6RdJ+keypfe2/lz3y+pNW9irHmc/6LpL/p53OOR9I7JX2q\n6TiinZL4o1eOtf1bla//bjIYSds2+fxbY5hjj+GQxB9TStJhkq6SdIekH0t6fmXfKZJ+IuluSSsl\n/WnZvgvwNWDv6ieI0T3y0Z8Kyk8eb5G0HPiVpG3L874oaZ2kmyWdWTPuWZJcxrhK0u2STpd0iKTl\n5ev5YOX4P5H0H5I+KOlOSTdKemFl/96SFkm6TdIKSa+r7HunpC9I+pSku4DTgbcBx5ev/ceb+31V\nfxeS/kLSWkk/l3RKZf9Okt4n6WdlfP8uaaeJ/kbRTulZxJSRNAO4BDgJuBR4IfBFSQfYXgesBf4Q\nWAk8F/iapCW2r5Z0DPAp2zMrP6/O054IvBhYDzwEfAX4t7J9JnC5pJtsX1bzZTwTmFPGt6h8Hb8P\nbAf8SNLnbX+7cuwXgD2AlwFfkjTb9m3ARcB1wN7AAcA3JP3U9pXlufOAlwMnAzuUP+MJtl9diWXc\n31e5fy9gd2AGcCTwBUlftn078F7gycCzgV+UsT5U428ULZQef/TKl8se4x2Svly2vRpYbHux7Yds\nfwNYCrwIwPYltn/qwreBrwO/t5VxfMD2Ktv3AYcA022/y/YG2yuBjwAnTOLnnWP7fttfB34FXGh7\nre01wHeBp1eOXQu83/YDtj8L3AS8WNI+wHOAt5Q/6xrgoxRJfsT3bH+5/D3dN1YgNX5fDwDvKp9/\nMXAP8CRJ2wCvBc6yvcb2g7avsv1rJvgbRTulxx+98lLbl49q2w94uaRjK23bAd8EKHv17wCeSNEJ\n2Rm4divjWDXq+feWdEelbRpFwq7rl5XH942x/VuV7TXedNbDn1H08PcGbrN996h9c8eJe0w1fl+3\n2t5Y2b63jG8PYEfgp2P82M3+jaKdkvhjKq0CPmn7daN3SNoB+CJFr/ffbD9QflIYqeeMNW3sryiS\n3Yi9xjimet4q4Gbbc7Yk+C0wQ5IqyX9fivLQfwOPkbRrJfnvC6ypnDv69W6yXeP3tTnrgfuBxwM/\nHrVv3L9RtFdKPTGVPgUcK+kPJE2TtGN5EXImsD1FLXsdsLHszR5VOfeXwGMl7V5puwZ4kaTHSNoL\neMMEz/9D4O7ygu9OZQxPkXRIz17hph4HnClpO0kvB36HooyyCrgKeHf5O3gacCrF72c8vwRmlWUa\nmPj3NS7bDwEXAP9QXmSeJulZ5ZvJ5v5G0VJJ/DFlyoQ3j2KEyjqK3uVfAtuUPd8zgc8BtwOvpOgd\nj5x7I3AhsLK8brA38EmKHustFPXtz07w/A9SXAw9GLiZouf7UYoLoFPhBxQXgtcD/w/4Y9u3lvtO\nBGZR9P4vBt4xRmms6vPl91slXT3R76uGN1OUhZYAtwHnUvwdxv0bTeJnx5BRFmKJ2HqS/gQ4zfbh\nTccSMZG8q0dEdEwSf0REx6TUExHRMenxR0R0zECO499jjz08a9aspsOIiBgay5YtW297ep1jBzLx\nz5o1i6VLlzYdRkTE0JD0s7rHptQTEdExSfwRER2TxB8R0TFJ/BERHZPEHxHRMUn8EREdk8QfEdEx\nSfwRER2TxB8R0TEDeefulpp19iV9e65b3vPivj1XREQvpccfEdExtRK/pKMl3SRphaSzx9g/T9Jy\nSddIWirp8Mq+WyRdO7Kvl8FHRMTkTVjqkTQNOB84ElgNLJG0yPYNlcOuABbZdrmQ9OeAAyr7j7C9\nvodxR0TEFqrT4z8UWGF7pe0NwEUUizP/hu17/PCKLrsAWd0lImJA1Un8M4BVle3VZdsmJB0n6Ubg\nEuC1lV0GLpe0TNL88Z5E0vyyTLR03bp19aKPiIhJ69nFXdsX2z4AeClwTmXX4bYPBo4BXi/pueOc\nv9D2XNtzp0+vtZZARERsgTqJfw2wT2V7Ztk2JtvfAfaXtEe5vab8vha4mKJ0FBERDamT+JcAcyTN\nlrQ9cAKwqHqApCdIUvn4GcAOwK2SdpG0a9m+C3AUcF0vX0BEREzOhKN6bG+UdAZwGTANuMD29ZJO\nL/cvAP4IOFnSA8B9wPHlCJ89gYvL94Rtgc/YvnSKXktERNRQ685d24uBxaPaFlQenwucO8Z5K4GD\ntjLGiIjoody5GxHRMUn8EREdk8QfEdExSfwRER2TxB8R0TFJ/BERHZPEHxHRMUn8EREdk8QfEdEx\nSfwRER2TxB8R0TFJ/BERHZPEHxHRMUn8EREdk8QfEdExSfwRER2TxB8R0TFJ/BERHZPEHxHRMUn8\nEREdUyvxSzpa0k2SVkg6e4z98yQtl3SNpKWSDq97bkRE9NeEiV/SNOB84BjgQOBESQeOOuwK4CDb\nBwOvBT46iXMjIqKP6vT4DwVW2F5pewNwETCveoDte2y73NwFcN1zIyKiv+ok/hnAqsr26rJtE5KO\nk3QjcAlFr7/2ueX588sy0dJ169bViT0iIrZAzy7u2r7Y9gHAS4FztuD8hbbn2p47ffr0XoUVERGj\n1En8a4B9Ktszy7Yx2f4OsL+kPSZ7bkRETL06iX8JMEfSbEnbAycAi6oHSHqCJJWPnwHsANxa59yI\niOivbSc6wPZGSWcAlwHTgAtsXy/p9HL/AuCPgJMlPQDcBxxfXuwd89wpei0REVHDhIkfwPZiYPGo\ntgWVx+cC59Y9NyIimpM7dyMiOiaJPyKiY5L4IyI6Jok/IqJjkvgjIjomiT8iomOS+CMiOiaJPyKi\nY5L4IyI6Jok/IqJjkvgjIjomiT8iomOS+CMiOiaJPyKiY5L4IyI6Jok/IqJjkvgjIjomiT8iomOS\n+CMiOqZW4pd0tKSbJK2QdPYY+18labmkayVdJemgyr5byvZrJC3tZfARETF5Ey62LmkacD5wJLAa\nWCJpke0bKofdDDzP9u2SjgEWAs+s7D/C9voexh0REVuoTo//UGCF7ZW2NwAXAfOqB9i+yvbt5eb3\ngZm9DTMiInqlTuKfAayqbK8u28ZzKvC1yraByyUtkzR/8iFGREQvTVjqmQxJR1Ak/sMrzYfbXiPp\nccA3JN1o+ztjnDsfmA+w77779jKsiIioqNPjXwPsU9meWbZtQtLTgI8C82zfOtJue035fS1wMUXp\n6BFsL7Q91/bc6dOn138FERExKXUS/xJgjqTZkrYHTgAWVQ+QtC/wJeAk2/9Zad9F0q4jj4GjgOt6\nFXxEREzehKUe2xslnQFcBkwDLrB9vaTTy/0LgLcDjwU+JAlgo+25wJ7AxWXbtsBnbF86Ja8kIiJq\nqVXjt70YWDyqbUHl8WnAaWOctxI4aHR7REQ0J3fuRkR0TBJ/RETHJPFHRHRMEn9ERMck8UdEdEwS\nf0RExyTxR0R0TBJ/RETHJPFHRHRMEn9ERMck8UdEdEwSf0RExyTxR0R0TBJ/RETHJPFHRHRMEn9E\nRMck8UdEdEwSf0RExyTxR0R0TBJ/RETH1Er8ko6WdJOkFZLOHmP/qyQtl3StpKskHVT33IiI6K8J\nE7+kacD5wDHAgcCJkg4cddjNwPNsPxU4B1g4iXMjIqKP6vT4DwVW2F5pewNwETCveoDtq2zfXm5+\nH5hZ99yIiOivOol/BrCqsr26bBvPqcDXJnuupPmSlkpaum7duhphRUTElujpxV1JR1Ak/rdM9lzb\nC23PtT13+vTpvQwrIiIqtq1xzBpgn8r2zLJtE5KeBnwUOMb2rZM5NyIi+qdOj38JMEfSbEnbAycA\ni6oHSNoX+BJwku3/nMy5ERHRXxP2+G1vlHQGcBkwDbjA9vWSTi/3LwDeDjwW+JAkgI1l2WbMc6fo\ntURERA11Sj3YXgwsHtW2oPL4NOC0uudGRERzcuduRETHJPFHRHRMEn9ERMck8UdEdEwSf0RExyTx\nR0R0TBJ/RETHJPFHRHRMEn9ERMck8UdEdEwSf0RExyTxR0R0TBJ/RETHJPFHRHRMEn9ERMfUmo8/\nmjfr7Ev6+ny3vOfFfX2+tr++iEGSHn9ERMck8UdEdEwSf0RExyTxR0R0TK3EL+loSTdJWiHp7DH2\nHyDpe5J+LenNo/bdIulaSddIWtqrwCMiYstMOKpH0jTgfOBIYDWwRNIi2zdUDrsNOBN46Tg/5gjb\n67c22IiI2Hp1evyHAitsr7S9AbgImFc9wPZa20uAB6YgxoiI6KE6iX8GsKqyvbpsq8vA5ZKWSZo/\n3kGS5ktaKmnpunXrJvHjIyJiMvpxcfdw2wcDxwCvl/TcsQ6yvdD2XNtzp0+f3oewIiK6qU7iXwPs\nU9meWbbVYntN+X0tcDFF6SgiIhpSJ/EvAeZImi1pe+AEYFGdHy5pF0m7jjwGjgKu29JgIyJi6004\nqsf2RklnAJcB04ALbF8v6fRy/wJJewFLgd2AhyS9ATgQ2AO4WNLIc33G9qVT81IiIqKOWpO02V4M\nLB7VtqDy+BcUJaDR7gIO2poAIyKit3LnbkRExyTxR0R0TBJ/RETHJPFHRHRMEn9ERMck8UdEdEwS\nf0RExyTxR0R0TBJ/RETHJPFHRHRMEn9ERMck8UdEdEwSf0RExyTxR0R0TBJ/RETH1JqPPyK2zqyz\nL+nr893ynhf39fliuCTxR8RWyxvbcEmpJyKiY5L4IyI6Jok/IqJjaiV+SUdLuknSCklnj7H/AEnf\nk/RrSW+ezLkREdFfEyZ+SdOA84FjgAOBEyUdOOqw24AzgfduwbkREdFHdXr8hwIrbK+0vQG4CJhX\nPcD2WttLgAcme25ERPRXncQ/A1hV2V5dttVR+1xJ8yUtlbR03bp1NX98RERM1sBc3LW90PZc23On\nT5/edDgREa1VJ/GvAfapbM8s2+rYmnMjImIK1En8S4A5kmZL2h44AVhU8+dvzbkRETEFJpyywfZG\nSWcAlwHTgAtsXy/p9HL/Akl7AUuB3YCHJL0BOND2XWOdO1UvJiIiJlZrrh7bi4HFo9oWVB7/gqKM\nU+vciIhozsBc3I2IiP5I4o+I6Jgk/oiIjsl8/BERE+jnegP9WGsgPf6IiI5J4o+I6Jgk/oiIjkni\nj4jomCT+iIiOSeKPiOiYJP6IiI5J4o+I6Jgk/oiIjknij4jomCT+iIiOSeKPiOiYJP6IiI5J4o+I\n6Jgk/oiIjknij4jomFqJX9LRkm6StELS2WPsl6QPlPuXS3pGZd8tkq6VdI2kpb0MPiIiJm/CFbgk\nTQPOB44EVgNLJC2yfUPlsGOAOeXXM4EPl99HHGF7fc+ijoiILVanx38osML2StsbgIuAeaOOmQd8\nwoXvA4+S9Ns9jjUiInqgTuKfAayqbK8u2+oeY+ByScskzR/vSSTNl7RU0tJ169bVCCsiIrZEPy7u\nHm77YIpy0OslPXesg2wvtD3X9tzp06f3IayIiG6qk/jXAPtUtmeWbbWOsT3yfS1wMUXpKCIiGlIn\n8S8B5kiaLWl74ARg0ahjFgEnl6N7DgPutP1zSbtI2hVA0i7AUcB1PYw/IiImacJRPbY3SjoDuAyY\nBlxg+3pJp5f7FwCLgRcBK4B7gVPK0/cELpY08lyfsX1pz19FRETUNmHiB7C9mCK5V9sWVB4beP0Y\n560EDtrKGCMioody525ERMck8UdEdEwSf0RExyTxR0R0TBJ/RETHJPFHRHRMEn9ERMck8UdEdEwS\nf0RExyTxR0R0TBJ/RETHJPFHRHRMEn9ERMck8UdEdEwSf0RExyTxR0R0TBJ/RETHJPFHRHRMEn9E\nRMck8UdEdEytxC/paEk3SVoh6ewx9kvSB8r9yyU9o+65ERHRXxMmfknTgPOBY4ADgRMlHTjqsGOA\nOeXXfODDkzg3IiL6qE6P/1Bghe2VtjcAFwHzRh0zD/iEC98HHiXpt2ueGxERfbRtjWNmAKsq26uB\nZ9Y4ZkbNcwGQNJ/i0wLAPZJuqhFbL+wBrJ/sSTp3CiKZGnl9Y8jrGxhtfn39fm371T2wTuLvC9sL\ngYX9fl5JS23P7ffz9kte33DL6xteg/za6iT+NcA+le2ZZVudY7arcW5ERPRRnRr/EmCOpNmStgdO\nABaNOmYRcHI5uucw4E7bP695bkRE9NGEPX7bGyWdAVwGTAMusH29pNPL/QuAxcCLgBXAvcApmzt3\nSl7Jlut7eanP8vqGW17f8BrY1ybbTccQERF9lDt3IyI6Jok/IqJjkvgjIjomib/lJD1a0tOajiMi\nBkcnE7+kXSRtUz5+oqSXSNqu6bh6RdK3JO0m6THA1cBHJP1D03H1iqSzytcnSR+TdLWko5qOq1ck\nPUfSLuXjV0v6B0m178ocdJJ2kPRKSW+T9PaRr6bj6gVJj5e0Q/n4+ZLOlPSopuMarZOJH/gOsKOk\nGcDXgZOAf2k0ot7a3fZdwMso5lB6JvD7DcfUS68tX99RwKMp/n7vaTaknvowcK+kg4C/AH4KfKLZ\nkHrq3yjm7NoI/Kry1QZfBB6U9ASK4Zz7AJ9pNqRHGpgpG/pMtu+VdCrwIdt/J+mapoPqoW3LSfJe\nAfzvpoOZAiq/vwj4ZHlfiTZ3wpDZaNuS5gEftP2x8t9qW8y0fXTTQUyRh8r7l44DzrN9nqQfNR3U\naF3t8UvSs4BXAZeUbdMajKfX3kVx09wK20sk7Q/8/4Zj6qVlkr5Okfgvk7Qr8FDDMfXS3ZLeCrwa\nuKQsS7amFAlcJempTQcxRR6QdCLwGuCrZdvA/e06eQOXpOcCbwb+w/a5ZWJ8g+0zGw4taigT4cHA\nStt3SHosMMP28oZD6wlJewGvBJbY/q6kfYHn225FuUfSDcATgJuBX1N8grPtoR+EUK43cjrwPdsX\nSpoNvML2QM0n2snE31aSzgPG/YO26Y2tvD6zH5Vype3vNBdR1DXehWrbP+t3LF3VyRq/pCdS9Phn\nsWnieEFTMfXI0qYD6AdJ5wLHAzcAD5bNprhoP/QkvQw4F3gcRW94pEe8W6OB9chIgpf0OGDHhsPp\nCUnXsvlO10B9mulkj1/Sj4EFwDIeThzYXtZYUFFbuUjP02z/uulYpoKkFcCxtn/SdCxTQdJLgPcB\newNrKT65/cT2kxsNbCtMNNx20D7NdLLHTzFq4sNNB9Frkr7C5nsdL+ljOFNpJcUFs1YmfuCXbU36\npXOAw4DLbT9d0hEUF7KH1qAl9ol0NfF/RdL/Ai6mkjxs39ZcSD3x3qYD6JN7gWskXcGmf7+2XMNY\nKumzwJfZ9PV9qbmQeuoB27dK2kbSNra/Ken9TQe1NSTdzdidroEs03U18b+m/P6XlTYD+zcQS8/Y\n/nbTMfTJItq9oM9uFG9u1buRDbQl8d8h6bcorsl8WtJahvwGLtu7Nh3DZHSyxt92kuYA7wYOpHLx\nzPZQv7FFO5TTUdxP0Rt+FbA78GnbtzYaWA+NvnBt+78aDOcROtXjl/QC21eWoyYeoUUfpT8OvAP4\nR+AIihXRhv5mPUmfs/2K8UZQDNrIicmS9FflXeRjDsttSynLdrV3/6+NBTIFxrtwDQzUhetOJX7g\necCVwLFj7GvTR+mdbF8hSeVFp3dKWgYM+0RYZ5Xf/7DRKKbOyAXdVg7LrdTBR6bXGHlzG8g6+BYa\nigvXKfW0kKSrgMOBL1C80a0B3mP7SY0GFtFykpbanlsOGX+67Yck/dj2QU3HVtWpHr+kN21uv+22\nTF18FrAzcCZFD+QIHr6gPbTGGDkhHu5BDn2PUdJmL1gP+3BcSTtSTGfwBGA5cIHtjc1G1XNDceG6\nUz1+SQ8B1wBf4+E5Qn7D9l83EVevlP+xdrW9blT744C7bN/fTGS9IenLwF4UJbmLBu2C2daStA5Y\nBVwI/IBH/vsc6lFb5RDVB4DvAscAP7N91ubPGi7lhev7KK6pDeyF664l/oOAE4GjKe7avRC4wi35\nJUhaCFw6+iJ1OUXsUbb/rJnIekfS7hTrDJxAMWrisxRvAsN+DwaSpgFHUvwbfRrFzLEX2r6+0cB6\nRNK1tp9aPt4W+KHtZzQcVk+U8+/vafs/RrUfDvzc9k+biWxsQz/SYzJs/9j22bYPBj5GsRjEDeWV\n+Db43bFGJtm+GHhuA/H0nO07bX+cosf4zxRTUP9Jo0H1iO0HbV9q+zUUFwhXAN+SdEbDofXKAyMP\nWljieT9w1xjtd5b7BkqnavwjJE0Hng48FVhNMeyqDXbezL5WvMlLejZFj/j3gH8HjrP93Waj6p1y\n2b4XU7zGWcAHKO4wb4ODJI0kRwE7ldttuEazp+1rRzfavlbSrP6Hs3mdSvySXkuxKtWOFCNeXmG7\nLUkfYK2kQ23/sNoo6RBg3TjnDA1JtwB3ABcB8ymW7kPSMwBsX91YcD0g6RPAU4DFwF/bvq7hkHrK\ndpsWOxptc+vq7tS3KGrqWo3/IeA6YGRCpU1efAtGTRwKfI5i/eCRmUbnAicDJ9j+QUOh9YSkbzH+\nJHQe9mm1y3+fIyNAHjF6ach7xEh6zOb2D/N1GkkXAlfa/sio9tOAI20f30xkY+ta4n/e5vYP+6gJ\n+M0IntdT9BwBrqdYt7VNn2xiCEm6mU1v4KryME8pImlPipLcBjbtdG1PUY78RVOxjaVTiX+EpGOB\nS2y3aZ3W3yinpLikxfPV7wy8CdjX9vxybqIn2f7qBKcOBUmftH3SRG0xeMo7dX/T6bJ9ZZPxjKer\nif9TwLOAL1LcRHJjwyH1lKSPAy+guInksxRDPFsziqIcD74MONn2U8o3gqvK0VpDT9LV1WGO5dDH\n5bYPbDCsnpL0aGAOm05kNrQrqA1bGauTiR9A0m4UIydOofj4+XGKMdN3NxpYj0jajmLI4/EU0zd8\nw/ZpzUbVG5Xb4n9k++ll28DdFj9Zkt4KvI3iYuC9I80U5YOFtt/aVGy9VNa9zwJmUtxQeRjF4uRD\ne41mVBmrOgcRDGAZqxVD/LaE7bsoRvZcBPw2cBxwtaQ/bzSwHrH9AMUdyhdR9I5f2mxEPbVB0k6U\n/8EkPZ4WrMZl+93lvO5/b3u38mtX249tS9IvnQUcQnHn7hEUQ6vvaDakrWN7tu39K99HHs8etKQP\nHRvOOaK8YesUijlDPgEcanttWTK4ATivyfi2lqSRnv7zgW8BH6UYxtoW7wQuBfaR9GngObTkJi4A\n22+VNINiSt9tK+1DWwoZ5X7b90tC0g62b5TUmgkEh6GM1cnED/wR8I+j/xi275V0akMx9dJJFMM6\n/7SNF3htf72cZvowio/TZ9le33BYPSPpPRRTUtwAPFg2m+KaTRuslvQoiqUlvyHpdh4eYj3Uxitj\nUVxzGxidrfG3VTnfy+XlR+hWKheV/wywyJsu6tEKkm4CntbGN+3RyiHWu1MMQNjQdDxbq1wk6BDg\n+7YPlnQA8Le2x1z8qSmdrPFLOkzSEkn3SNog6cHKreRDzfaDwEPlZGZt9V6KKRtukPQFSX9czkza\nFiuB7ZoOYipJmiZpb+Bmip7xXg2H1Cv3j8yCO1LGAgaujNXVUs8HKT5Kf56H72x9YqMR9dY9wLWS\nvkFlLnC3Z+m+bwPfLj/dvAB4HXABxSLlQ6uy5OK9wDWSrqBy0botf79yAMU7gF8CI/fSmGJG0mE3\nFGWsTpZ6KsMBl7tcp7U6NHDYSRpz0RXbrVnftBzVcyzFRexnAF+1PdQjssb7u41oy99P0grgmYM2\nR32vDXIZq6s9/nslbU/Rq/o74Oe0qOxl+1/LxLiv7ZuajqfXJH0OOJRiZM8HgW+34S7stiT2GlZR\nTFfcSuUn0T0pylhQlLEGatGgrvb496P4mLk98EaKd+UP2V7RaGA9Uk5J8V5ge9uzJR0MvGvYJ6Eb\nIekPKC5gPzjhwUOovEA4+j/mnRSLsP/NsPeUJX2Mou59CZuWsoZ+6dPxylgjlYVB0cnED7+Zkx+P\nWqawDcqhji8AvlW5s/U620/Z/JmDTdJf2f678vHLbX++su9vbb+tueh6p/wU+iDFyCUorkftDPwC\nONz2sU3F1guS3jFWu4d86VMYnjJWpxK/JFG8G59BUdoRxZzu59l+V5Ox9ZKk79s+bNSUBssHrdcx\nWdU5bMaYz2aT7WE21msZaVNl+cJhp2JRcmzf03QsvSLpmxTTMA/03Fhdq/G/keIuz0Ns3wwgaX/g\nw5LeaPsfG42ud66X9EpgWjlz5ZnAVQ3H1Asa5/FY28NsmioL6qhYSGdkEZOBTih1SHoK8EngMeX2\neooJ99qwtvBKiuUyB7qM1ZoLmjWdBJw4kvQBbK8EXk0xpLMt/hx4MsU/vAsp1gJ9Q6MR9YbHeTzW\n9jA7DfiYpJtVrDr2MeB1knYB3t1oZL2xEHiT7f1s7wf8BfCRCc4ZFv8FfIPi+uGula+B0rVSz7h1\n7jbUwMdSzhtyh1vwh5b0IMV9CeKRM1juaLtVNz2N3IRnu1UjYMaaSbUNs6tWDXoZq2ulns2NpR2o\ncbZbQtLbgc+Vk17tQDE750HAg5JeafvyZiPcOm73mq1IerXtT0l606h2YPDKBVthpaT/S1HugeIT\n98oG4+mZYSljdS3xHzTO1AyiMpPeEDseOKd8/BqKUt7jKO5K/ldgqBN/B+xSfh+40kCPvRb4a+BL\n5fZ3y7Y2GCljfRNA0vMpyljPbjKo0TpV6mm7UaN4vgh83fY/l9utGfUSMaiGpYzVtR5/2/26/Kj5\nS+AI4M2VfTs3E1LUVZbqxmPb52xm/8CT9H7bbyhnV31Ej7MlNxgORRkrib9dzqJYVWw6xXoDI0NW\nXwT8qMnAopaxppjeBTgVeCwPl/GG1UgyfG+jUUytoShjpdTTQpL2L4epVttmV4exxmCTtCvFG/mp\nFIvqvM+9/tmcAAAFL0lEQVT22maj6g1JZ9n+p4naYuok8bfQOHd+LrP9u03FFPVIegzwJuBVFBfk\n/8n27c1G1Vvj/Psc6tlxh62MlVJPi5Sr/TwZ2F1SdcWf3WjHqKVWk/T3wMsoRoY8dVDHgG8pSScC\nrwRmS1pU2bUrcFszUfXMUJWx0uNvEUnzgJcCLwGq/7HuBi6y3YZpG1pL0kMUd1tvZNNeoygu7g77\nQjP7AbMp7j4+u7LrbmD5oM9vU8ewlLGS+FtI0rNsf6/pOCK6ZljKWF2bq6crjpO0m6TtJF0haZ2k\nVzcdVAS0c81rSSeW9f3ZkhZVvr7JAJaxUuNvp6Ns/5Wk44BbKOrG3wE+1WhUEYU2rnl9FcVKfnsA\n76u03w0sbySizUjib6eRycpeDHze9p0j871EDALbKyRNK1dR+7ikHwFvbTquLWX7ZxSLqj+r6Vjq\nSOJvp69IuhG4D/izcrWx+xuOKWJEa9e8lnQYcB7wOxRTM08DfjVoF+ZzcbelyvHgd9p+UNLOwG62\nf9F0XBFq8ZrXkpYyRhnL9kB9mknibyFJYy4qY/sT/Y4lokskLbU9t7rU6SCO6kmpp50OqTzeEXgh\ncDWQxB+Nk/Qc4J3AflRykO39m4qph4aijJUefwdIehTFDVxHNx1LRHn96Y3AMuDBkXbbtzYWVI8M\nSxkrib8DJG0HXG972IfMRQtI+oHtZzYdR5el1NNCoyaK2gY4kGKGx4hB8M1yXqIvUUxRAYDtq5sL\nqTeGpYyVHn+LSHoCsCebvqFvpJjr5ee2f9pIYBEV5d2so9n2C/oeTI8NSxkrib9FJH0VeKvta0e1\nPxX4W9vHNhNZREHSNsAf227lJ9BhKWMl8beIpCW2Dxln37W2n9rvmCJGGxny2HQcU0HSeyhu2hro\nMlZq/O3yqM3s26lvUURs3uWS3gx8lspyk7YHbjKzLTDS26++sRkYqDJWevwtIulC4ErbHxnVfhpw\npO3jm4ks4mGSxloC1IN2AXSyhqmMlcTfIpL2BC4GNlBcXIKi57E9cFymbIiYWsNSxkribyFJRwBP\nKTevt31lk/FEVJVzR70J2Nf2fElzgCfZ/mrDoW21ssa/ngEvYyXxR0RfSfosxSfSk20/pXwjuMr2\nwQ2HttWGpYyVi7sR0W+Pt318ufg6tu9VSxaMsD276RjqGLjJgyKi9TZI2ony7nJJj6cy9HGYSdpZ\n0v+RtLDcniPpD5uOa7Qk/ojot3cClwL7SPo0cAXwlkYj6p2PUwyueHa5vQb4m+bCGVtq/BHRd5Ie\nCxxGMZ3I922vbziknqjMx/+bOfgl/dj2QU3HVpUef0T0laQrbN9q+xLbX7W9XtIVTcfVI0NRxsrF\n3YjoC0k7AjsDe0h6NEVvH2A3YEZjgfXWO9m0jPUc4JRGIxpDSj0R0ReSzgLeAOxNUfseSfx3AR+x\n/cGmYuulYShjJfFHRF9J+nPb5zUdx1Qoy1gvnKitaSn1RERf2T5P0rOBWWy6WMnQrgk9bGWsJP6I\n6CtJnwQeD1zDw4uVGBjaxA/8KQ+XsZaxaRlr4EpYKfVERF9J+glwoFuYfIaljJUef0T023XAXsDP\nmw6k14aljJXEHxH9tgdwg6QfsukqVS9pLqTeGJYyVko9EdFXkp43Vrvtb/c7ll4bljJWevwR0Vdt\nSPCbMRRlrCT+iOgLSXdTTmUwehfFnPW79TmkqTAUZayUeiIiemRYylhJ/BERHZNST0TEVhq2MlZ6\n/BERHZP5+CMiOiaJPyKiY5L4IyI6Jok/IqJj/gdgKxDSldFwkQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1199a23c8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "feature_columns = list([\n",
    "    #\"State\", \n",
    "    #\"AccountLength\", \n",
    "    #\"AreaCode\", \n",
    "    #\"Phone\", \n",
    "    \"InternationalPlan\", \n",
    "    #\"VMailPlan\", \n",
    "    #\"VMailMessage\",\n",
    "    \"DayMins\", \n",
    "    \"EveMins\", \n",
    "    \"NightMins\", \n",
    "    #\"InternationalMins\",\n",
    "    #\"DayCalls\", \n",
    "    #\"EveCalls\", \n",
    "    #\"NightCalls\", \n",
    "    \"InternationalCalls\",\n",
    "    #\"DayCharge\", \"EveCharge\", \"NightCharge\", \n",
    "    #\"InternationalCharge\", \n",
    "    \"CustServCalls\"\n",
    "])\n",
    "label_columns = \"Churn\"\n",
    "features = customer_all_df[feature_columns].values\n",
    "labels = customer_all_df[label_columns].values\n",
    "\n",
    "print(feature_columns)\n",
    "#churn_tree = testFeaturesWithDecisionTree(features, labels)\n",
    "ran_forest = testFeaturesWithRandomForest(features, labels)\n",
    "\n",
    "feature_fav_series= pd.Series(ran_forest.feature_importances_,feature_columns)\n",
    "feature_fav_series.sort_values(ascending=False).plot(kind='bar',title='Feature Importance')    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[3930,   31],\n",
       "       [ 173,  483]])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Print Confusion matrix \n",
    "pred = ran_forest.predict(features)\n",
    "df_confusion = metrics.confusion_matrix(labels, pred)\n",
    "df_confusion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX8AAAEmCAYAAACKxZBYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xm8VVX9//HXm8EriuRAIuKsqAElCl9wDocUzRK1FDPn\npNJMKy3N+ooVX83SzDT76s8UzVJySCIcELUURaZQwCEpIRkcsK8DBgSXz++PvS4ernfYBw7ce89+\nP33sx91n7b3XXude/Jx1PnvttRURmJlZsbRr6QaYmdn65+BvZlZADv5mZgXk4G9mVkAO/mZmBeTg\nb2ZWQA7+1upJ6iTpj5LekfT7tajnJEkPV7JtLUHSA5JObel2WNvm4G8VI+kLkqZIWixpYQpS+1eg\n6s8B3YAtIuLza1pJRNwREYdVoD2rkTRIUki6r175Hqn88Zz1DJf0m+b2i4gjImLkGjbXDHDwtwqR\n9E3gGuB/yAL1dsD1wGcrUP32wN8iYkUF6lpX3gT2kbRFSdmpwN8qdQJl/P+sVYT/Idlak/QR4AfA\nORFxb0S8HxHLI2JMRHw77VMj6RpJC9JyjaSatG2QpHmSviXpjfSt4fS07TLgv4ET0jeKM+v3kCXt\nkHrYHdLr0yT9Q9J7kl6RdFJJ+ZMlx+0raXJKJ02WtG/Jtscl/VDShFTPw5K6NvFr+A/wB2BoOr49\ncAJwR73f1c8lvSrpXUlTJR2QygcD3y15n8+WtGOEpAnAv4GdUtmX0vYbJN1TUv+PJY2XpNx/QCsk\nB3+rhH2ADYH7mtjnEmBvoC+wBzAA+F7J9q2AjwA9gDOB6yVtFhGXkn2buCsiOkfEzU01RNLGwLXA\nERGxCbAvML2B/TYH/pT23QK4GvhTvZ77F4DTgS2BDYALmjo3cBtwSlo/HJgJLKi3z2Sy38HmwG+B\n30vaMCIerPc+9yg55mRgGLAJMLdefd8CPp4+2A4g+92dGp63xZrh4G+VsAWwqJm0zEnADyLijYh4\nE7iMLKjVWZ62L4+IscBiYLc1bM9KoI+kThGxMCJmNbDPp4GXI+L2iFgREb8DXgQ+U7LPLRHxt4hY\nAowiC9qNioingM0l7Ub2IXBbA/v8JiLeSue8Cqih+fd5a0TMSscsr1ffv8l+j1cDvwHOjYh5zdRn\n5uBvFfEW0LUu7dKIrVm91zo3la2qo96Hx7+BzuU2JCLeJ0u3fAVYKOlPknbP0Z66NvUoef3aGrTn\nduBrwEE08E1I0gWSXkipprfJvu00lU4CeLWpjRHxDPAPQGQfUmbNcvC3SngaWAYMaWKfBWQXbuts\nx4dTInm9D2xU8nqr0o0R8VBEfAroTtabvylHe+raNH8N21TnduBsYGzqla+S0jLfBo4HNouITYF3\nyII2QGOpmiZTOJLOIfsGsSDVb9YsB39baxHxDtlF2eslDZG0kaSOko6QdGXa7XfA9yR9NF04/W+y\nNMWamA4cKGm7dLH54roNkrpJOjrl/peRpY9WNlDHWGDXNDy1g6QTgF7AmDVsEwAR8QrwSbJrHPVt\nAqwgGxnUQdJ/A11Ktr8O7FDOiB5JuwI/Ar5Ilv75tqQm01Nm4OBvFZLy198ku4j7Jlmq4mtkI2Ag\nC1BTgOeAGcC0VLYm5xoH3JXqmsrqAbtdascC4F9kgfirDdTxFnAU2QXTt8h6zEdFxKI1aVO9up+M\niIa+1TwEPEg2/HMusJTVUzp1N7C9JWlac+dJabbfAD+OiGcj4mWyEUO3142kMmuMPCjAzKx43PM3\nMysgB38zswJy8DczKyAHfzOzAmrqphxrhjp0Cm2wSUs3wypoz49t19JNsAqaO3cOixYtqtg8R+27\nbB+xYknu/WPJmw9FxOBKnb+SHPzXgjbYhJrdjm/pZlgFTXjmupZuglXQfgP7V7S+WLGkrP/nl06/\nvrm7t1uMg7+ZWW6CKplV28HfzCwvAVUyW7aDv5lZOdzzNzMrGkG79i3diIpw8DczK4fTPmZmBSOc\n9jEzKx65529mVkju+ZuZFZB7/mZmReObvMzMisc3eZmZFZR7/mZmReO0j5lZ8Qho7zt8zcyKxzl/\nM7OicdrHzKyY3PM3Mysg9/zNzApGntvHzKyY3PM3MyugKun5V8dHmJnZepFG++RdmqtN2lDSJEnP\nSpol6bJUPlzSfEnT03JkyTEXS5ot6SVJh5eU95M0I227Vmr6U8o9fzOzclS2578MODgiFkvqCDwp\n6YG07WcR8dPVT61ewFCgN7A18IikXSOiFrgBOAt4BhgLDAYeoBHu+ZuZ5VX3JK8K9fwjszi97JiW\naOKQo4E7I2JZRLwCzAYGSOoOdImIiRERwG3AkKbO7eBvZpZbeoB73gW6SppSsgz7UI1Se0nTgTeA\ncRHxTNp0rqTnJP1a0maprAfwasnh81JZj7Rev7xRDv5mZuUor+e/KCL6lyw31q8uImojoi+wDVkv\nvg9ZCmcnoC+wELiq0m/Dwd/MrBx1Y/3zLGWIiLeBx4DBEfF6+lBYCdwEDEi7zQe2LTlsm1Q2P63X\nL2+Ug7+ZWV6q+Gifj0raNK13Aj4FvJhy+HWOAWam9dHAUEk1knYEegKTImIh8K6kvdMon1OA+5s6\nt0f7mJmVo7KjfboDIyW1J+uMj4qIMZJul9SX7OLvHODLABExS9Io4HlgBXBOGukDcDZwK9CJbJRP\noyN9wMHfzKwszQyfL0tEPAfs2UD5yU0cMwIY0UD5FKBP3nM7+JuZ5ZQ9wrc67vB18Dczy0tpqQIO\n/mZmuck9fzOzInLwNzMroHbtqmOEvIO/mVlezvmbmRWPnPM3MysmB38zswJy8DczKyAHfzOzovEF\nXzOzYnLP38ysYDzax8ysoBz8zcyKRqB2Dv5mZoXjnr+ZWQE5+JuZFYwv+JqZFVV1xH4HfzOz3OS0\nj7VhNRt04JGbz2eDDTrQoX177nvkr/zoV2P5+K49+MUlQ9m4Uw1zF7zF6ZeM5L33l9K/9/Zc9/0T\nAZBgxK/GMvqx5wDY82PbcuNlJ9OppiMPTZjFt668uyXfmtWzdOlSDj3oQP6zbBkraldwzLGf4/uX\nXsY9d/+eET8czosvvMATT02iX//+Ld3UNsPB39qsZf9ZweBh1/L+kv/QoUM7Hv31N3l4wvNc/Z3P\nc9HP7uPJqbM55ei9+caph/CDX/6JWX9fwH4nXUlt7Uq26tqFZ+66mD/9ZSa1tSu59rsncM4Pf8uk\nGXP4w3Vf5bD9evHwhOdb+i1aUlNTw4PjHqVz584sX76cgz+5P4cdfgS9e/fhzlH38rWzv9zSTWxz\nKhn8JW0I/AWoIYvHd0fEpZI2B+4CdgDmAMdHxP+lYy4GzgRqga9HxEOpvB9wK9AJGAucFxHR2Lmr\n45E0Vrb3l/wHgI4d2tOhQ3sigl2225Inp84G4NGJLzLkkL4ALFm6nNralQDUbNCRun9PW3XtwiYb\nb8ikGXMA+O2YSXxm0CfW8zuxpkiic+fOACxfvpwVy5cjid0/9jF23W23Fm5dG6UyluYtAw6OiD2A\nvsBgSXsDFwHjI6InMD69RlIvYCjQGxgM/FJS+1TXDcBZQM+0DG7qxA7+BdWunZh450X8c/wVPDrx\nRSbPnMsL/1i4Kngf+6m92KbbZqv2/68+2zP17kuY8vvv8vURd1Jbu5Ktt9yU+W+8vWqf+a+/zdZb\nbrre34s1rba2loH9+rLd1lty8KGfYsDAgS3dpDZNUu6lOZFZnF52TEsARwMjU/lIYEhaPxq4MyKW\nRcQrwGxggKTuQJeImJh6+7eVHNOgdRb8JYWkq0peXyBpeDPHDEmfbI1tP0XSTEkzJP1V0gWp/HFJ\nTlqWYeXKYO+hV7DL4d+jf5/t6bVzd748/A6GHX8AE+74Np03quE/y2tX7T955lz6fW4E+3/xSi48\n4zBqNnDGsK1o3749z0ydzuw585gyeRKzZs5s6Sa1WeUE/hT8u0qaUrIMa6DO9pKmA28A4yLiGaBb\nRCxMu7wGdEvrPYBXSw6fl8p6pPX65Y1al/8HLwOOlXR5RCzKecwQYAzwoaSxpCOA84HDImKBpBrg\nlEo0VFL7iKhtfs/q887iJfx5yt84bN9eXHP7eD5z9vUA7LLdlhxxQO8P7f/SK6+z+N/L6L3L1ix4\n4216lPT0e3TblAUl3wSsddl000355KCDePjhB+ndp09LN6fNKvMB7osiosmOaYo9fSVtCtwnqU+9\n7SGp0dz9mlqXaZ8VwI3AN+pvkLSDpEclPSdpvKTtJO0LfBb4iaTpknaud9jFwAURsQAgfe25qWT7\n5yVNkvQ3SQek85wm6bqS846RNCitL5Z0laRngX0kzZF0maRp6ZvF7hX8XbQqXTfrzEc6dwJgw5qO\nHDJwd16a8zof3SzLDUviorMO56a7nwRg+623oH377J/Kdt03Y7cdt2Lugrd4bdG7vPf+UgZ8fAcA\nvnDUAMb8+bn1/4asUW+++SZvv519IC9ZsoTxj4xjt92q9p/2+lHZnP8qEfE28BhZrv71lMoh/Xwj\n7TYf2LbksG1S2fy0Xr+8Uev6u/v1wHOSrqxX/gtgZESMlHQGcG1EDJE0GhgTEQ2NF+wDTG3iXB0i\nYoCkI4FLgUObadvGwDMR8S1YdQV/UUTsJels4ALgS/UPSl/bsq9uHTs3c4rWaauuXbjpByfTvl07\n2rUT94ybxgNPzOScEwfx5RMOBOD+R6dz2/0TAdh3z5244PTDWL6ilpUrg/P+5y7eevt9AM67fBQ3\nXvZFOtV05OEJz/PQkx7p05q8tnAhZ51xKrW1tayMlRz3ueM58tNHcf8f7uOb55/Lojff5NijP80n\n9ujLH8c+1NLNbRMqPNrno8DyiHhbUifgU8CPgdHAqcAV6ef96ZDRwG8lXQ1sTXZhd1JE1Ep6N10s\nfoYsK/KLJs/dxEigtSJpcUR0lvQDYDmwBOgcEcMlLQK6R8RySR2BhRHRVdKtNBL8Jf0L2DEi3mlg\n2+PAJRExQVI3YEJE7CLpNKB/RHwt7TcG+GlEPC5pBVBTl+6RNAfYLyLmSxoIjIiIJj9A2m20ZdTs\ndvya/YKsVfq/ydc1v5O1GfsN7M/UqVMqFq1rtuoZ25x0be79/3H1kVObSvtI+gTZBd32ZJmYURHx\nA0lbAKOA7YC5ZEM9/5WOuQQ4gyy7cn5EPJDK+/PBUM8HgHObGuq5Pq7aXQNMA25Zy3pmAf2ARxvZ\nviz9rOWD97WC1VNbG5asL20gz99QHWZmQMrmVPAer4h4DtizgfK3gEMaOWYEMKKB8ilkGZJc1vlQ\nz/RpNYrspoQ6T5GNVQU4CXgirb8HbNJIVZeTXQ/YCkDSBpI+lJapZw7ZhZR2krYFBpT/DszM6pQ9\n2qfVWl/j/K8Cupa8Phc4XdJzwMnAean8TuDCNIxztQu+ETEWuA54RNIssm8TXZo57wTgFbLRQ9em\nY8zM1piUf2nN1llqIyI6l6y/DmxU8noucHADx0wAGh3nHxG30ED6KCIGlawvIrslmpTvOqm59qXX\nO5SsTwEGYWZWT2vv0eflvLaZWV5toEefl4O/mVlOIpsapRo4+JuZlcHB38ysaJz2MTMrnmycf3VE\nfwd/M7PcWv/4/bwc/M3MylAlsd/B38ysHO75m5kVjS/4mpkVjy/4mpkVVJXEfgd/M7NyuOdvZlY0\n8h2+ZmaFU+mHubQkB38zs9x8k5eZWSFVSex38DczK4d7/mZmRVNFN3mtr2f4mpm1eXU3eVXqAe6S\ntpX0mKTnJc2SdF4qHy5pvqTpaTmy5JiLJc2W9JKkw0vK+0makbZdq2Ya4J6/mVkZKpz2WQF8KyKm\nSdoEmCppXNr2s4j4ab1z9wKGAr2BrYFHJO0aEbXADcBZwDPAWGAw8EBjJ3bP38ysDFL+pTkRsTAi\npqX194AXgB5NHHI0cGdELIuIV4DZwABJ3YEuETExIgK4DRjS1Lkd/M3MylBm2qerpCkly7Am6t0B\n2JOs5w5wrqTnJP1a0maprAfwaslh81JZj7Rev7xRDv5mZnmV0etPPf9FEdG/ZLmxwWqlzsA9wPkR\n8S5ZCmcnoC+wELiq0m/FOX8zs5yEKj69g6SOZIH/joi4FyAiXi/ZfhMwJr2cD2xbcvg2qWx+Wq9f\n3ij3/M3MytBOyr00J43IuRl4ISKuLinvXrLbMcDMtD4aGCqpRtKOQE9gUkQsBN6VtHeq8xTg/qbO\n7Z6/mVkZKjzOfz/gZGCGpOmp7LvAiZL6AgHMAb4MEBGzJI0CnicbKXROGukDcDZwK9CJbJRPoyN9\nwMHfzCy3LJdfuegfEU+S3T5Q39gmjhkBjGigfArQJ++5HfzNzMpQJTM6O/ibmZXDc/uYmRVQlcR+\nB38zs7xENtyzGjj4m5mVwTl/M7OiyTlbZ1vg4G9mlpOA9lXS9XfwNzMrQ5V0/B38zczK4bSPmVnB\n5J2nvy1w8DczK0OeCdvaAgd/M7MyVEfod/A3MytL1ef8Jf2RbDrRBkXEZ9dJi8zMWilRjJu8ftrE\nNjOz4inCTV4R8ef12RAzs7agSmJ/8zl/ST2By4FewIZ15RGx0zpsl5lZq1NNd/jmeYbvLWRPkl8B\nHATcBvxmXTbKzKy1Ukr95FlaszzBv1NEjAcUEXMjYjjw6XXbLDOz1kllLK1ZnqGeyyS1A16W9DVg\nPtB53TbLzKz1karnJq88Pf/zgI2ArwP9yJ40f+q6bJSZWWtVN8VDnqU1azb4R8TkiFgcEfMi4vSI\nODYiJq6PxpmZtTaVzPlL2lbSY5KelzRL0nmpfHNJ4yS9nH5uVnLMxZJmS3pJ0uEl5f0kzUjbrlUz\nDcgz2ucxGrjZKyIObvadmZlVmQr36FcA34qIaZI2AaZKGgecBoyPiCskXQRcBHxHUi9gKNAb2Bp4\nRNKuEVFLNjDnLOAZYCwwGHigsRPnyflfULK+IXBcarCZWaEIVTTnHxELgYVp/T1JLwA9gKOBQWm3\nkcDjwHdS+Z0RsQx4RdJsYICkOUCXuqyMpNuAIaxN8I+IqfWKJkialPfNmZlVjfJz+V0lTSl5fWNE\n3Nhg1dIOwJ5kPfdu6YMB4DWgW1rvAZSm3eelsuVpvX55o/KkfTYvedmO7KLvR5o7rgj22H07Hpvw\n85ZuhlXQ6+8sbekmWAUtr210erI1Vub4/UUR0T9HnZ2Be4DzI+Ld0nNEREiq+BvJk/aZSpbzF1m6\n5xXgzEo3xMysLcgzRLIckjqSBf47IuLeVPy6pO4RsVBSd+CNVD4f2Lbk8G1S2fy0Xr+8UXnex8ci\nYqeI2DEiekbEYcDkHMeZmVWVuukd8i7N1pd18W8GXoiIq0s2jeaDIfWnAveXlA+VVCNpR6AnMCml\niN6VtHeq85SSYxqUp+f/FLBXvbKnGygzM6t6FZ7aZz+ye6dmSJqeyr4LXAGMknQmMBc4HiAiZkka\nBTxPlok5J430ATgbuBXoRHaht9GLvdD0fP5bkV0w6CRpTz64W7kL2U1fZmaFkt28VdHRPk/S+EwQ\nhzRyzAhgRAPlU4A+ec/dVM//cLKxptsAV5U08F2yTyYzs8Kpkkk9m5zPfyQwUtJxEXHPemyTmVmr\n1dqnbcgrzwXffpI2rXshaTNJP1qHbTIza5Wyxzgq99Ka5Qn+R0TE23UvIuL/gCPXXZPMzFqvdmUs\nrVme0T7tJdWk24mR1AmoWbfNMjNrnVp5hz63PMH/DmC8pFvIvvWcRjbXhJlZoagNpHPyyjO3z48l\nPQscSnan70PA9uu6YWZmrVGVxP5cPX+A18kC/+fJpnfw6B8zKxwBHapkrGdTN3ntCpyYlkXAXWTP\n8T1oPbXNzKzVKULP/0XgCeCoiJgNIOkb66VVZmatkarnJq+mRiMdS/aQgcck3STpEFr/A+nNzNYp\nlfFfa9Zo8I+IP0TEUGB34DHgfGBLSTdIOmx9NdDMrLXIbvLKv7RmeR7g/n5E/DYiPkM2z89fyR4n\nZmZWONUS/POO9gFW3d17Y1rMzAqnkrN6tqSygr+ZWZHVpX2qgYO/mVle5T/AvdVy8DczK0Nhpncw\nM7NM9gzflm5FZTj4m5nlJtq18vH7eTn4m5nlJJzzNzMrnjYwfj+vKslemZmtH5V8jKOkX0t6Q9LM\nkrLhkuZLmp6WI0u2XSxptqSXJB1eUt5P0oy07VrluBnBwd/MLKe6tE/eJYdbgcENlP8sIvqmZSyA\npF7AUKB3OuaXktqn/W8AzgJ6pqWhOlfj4G9mVoZK9vwj4i/Av3Ke+mjgzohYFhGvALOBAZK6A10i\nYmJEBHAbMKTZ95HzpGZmRtk9/66SppQsw3Ke5lxJz6W00GaprAfwask+81JZj7Rev7xJDv5mZjmJ\nLGjmXYBFEdG/ZMkzL9oNwE5AX7Jp9a+q7LvIeLSPmVleWvcTu0XE66tOJ90EjEkv5wPbluy6TSqb\nn9brlzfJPX8zszKojGWN6s9y+HWOAepGAo0GhkqqkbQj2YXdSRGxEHhX0t5plM8pwP3Nncc9fzOz\nnAS0r2DPX9LvgEFk1wbmAZcCgyT1BQKYA3wZICJmSRoFPA+sAM6JiNpU1dlkI4c6AQ+kpUkO/mZm\nZahk1iciTmyg+OYm9h8BjGigfArQp5xzO/ibmeUmP8zFzKxo6kb7VAMHfzOzMrjnb2ZWQNUR+h38\nzczyWw/j/NcXB38zs5yc8zczKyj3/M3MCqg6Qr+Dv5lZbpW+w7clOfibmZWhSmK/g7+ZWX5CVZL4\ncfA3MyuDe/5mZgWTDfWsjujv4G9mllf+B7O3eg7+ZmZlcPA3MyugarngWy13Ktta+NqXv0TP7buz\nT/89VpWdcfKJHDCwHwcM7Mcndt+ZAwb2A2Dq5EmryvcfuBdj7v9DSzXbmlFbW8uRB+3NGSceC8Cs\nGc8y5PADOWLQQD5zyH5MnzYZgOnTJnPEoIEcMWgggz85gAf/1OwTAAtLQDvlX1oz9/yNE08+hbO+\ncjZfOev0VWW/vv13q9a/d9EFdOnyEQA+1rsPj014hg4dOvDawoUcsPdeDP70UXTo4H9Krc0t/3sd\nu/TcjcXvvQfAFZddwnkXXsJBhx7OY+Me5PLhl3DX6IfZbffe/PGRCXTo0IE3XlvIEYMGcujhn/bf\ntBHu+VvV2G//A9ls880b3BYR3HfP3Rx3/FAANtpoo1VBYdmypVUzz0m1WbhgHo+Oe5ChX/zgAx2J\nxe+9C8C7775Dt62y54R3Wu1vusx/02a0k3IvrZk/2q1JT014gi237MbOu/RcVTZl0jOc+9WzePWf\nc/nV/xvpHmIr9INLLuTiS0ewePHiVWWXjvgJp3z+M/zPpRezcuVK7nngsVXb/jp1Et/++leYP++f\nXP3Lm/03bURd2qcatKmev6StJN0p6e+SpkoaK2mYpDEt3bZqdc+ouzju+BNWK+s/YCBPT32O8U9M\n5Gc/vYKlS5e2UOusIeMfGssWXbfk4333Wq38N7fcyPd/dCVPPzeb7//oSr5z3ldXbduz3wDGTZjG\n6HFPcsM1P/HftFEq67/WrM0Ef2XfRe8DHo+InSOiH3Ax0G0t63UXpxErVqxgzOj7OOa44xvcvtvu\nH2Pjzp15YdbM9dwya8qUSU/zyINj2G/P3Th32Ck89eTjnP+V07nnzjsYfNQQAD599HE8O23Kh47d\nZdfd2WjjzvzthVnru9ltQxrnn3dptjrp15LekDSzpGxzSeMkvZx+blay7WJJsyW9JOnwkvJ+kmak\nbdcqR+6uzQR/4CBgeUT8qq4gIp4FngA6S7pb0ouS7qh745LmSOqa1vtLejytD5d0u6QJwO2STpN0\nr6QH0y/8yvX+7lqhxx99hJ677kaPbbZZVTZ3ziusWLECgH/+cy4vv/QS222/Qwu10Bryne//kIkz\n/s6Ev77EL268jX33H8Q1v7qFLbfqzsQJTwDw1BOPs8NOuwDw6tw5q/6m816dy99ffoltttu+xdrf\n2qmMJYdbgcH1yi4CxkdET2B8eo2kXsBQoHc65peS2qdjbgDOAnqmpX6dH9KWer19gKmNbNuT7Bey\nAJgA7Ac82Ux9vYD9I2KJpNOAvqmeZcBLkn4REa/WP0jSMGAYwDbbbrcGb6P1OfPUk5jwlz/z1luL\n6L3L9lz0vUs5+bQzuPfuURz3+aGr7fv0UxP4+VVX0qFDR9q1a8dPr7mOLbp2baGWWzmu+Nn1XPbd\nC1lRu4Kamhouv/o6ACY/8xQ3/PyndOjYkXZqxw9/8nM238J/04ZkOf/KpXMi4i+SdqhXfDQwKK2P\nBB4HvpPK74yIZcArkmYDAyTNAbpExEQASbcBQ4AHmjp3Wwr+TZkUEfMAJE0HdqD54D86IpaUvB4f\nEe+kOp4Htgc+FPwj4kbgRoA99+ofa9/0lnfzyDsaLP/ljb/+UNnQL3yRoV/44rpuklXIPvsfyD77\nHwjAf+29H2MefepD+xx7/Bc49vgvrO+mtVllhv6ukkrzazemGNKUbhGxMK2/xgep7R7AxJL95qWy\n5Wm9fnmT2lLwnwV8rpFty0rWa/ngfa3gg9TWhvWOeT9nHWZmHygv+i+KiP5reqqICEnrpJPZlnL+\njwI1Ke0CgKRPAAc0ccwcoF9aP27dNc3MimI9jPZ5XVJ3gPTzjVQ+H9i2ZL9tUtn8tF6/vEltJvhH\nRADHAIemoZ6zgMvJvhY15jLg5+lrV+16aKaZVblKjvZpxGjg1LR+KnB/SflQSTWSdiS7sDsppYje\nlbR3GuxySskxjWpTqY2IWAA0NO7wppJ9vlay/gSwawP1DK/3+layq+51r49a68aaWVWq5Oh9Sb8j\nu7jbVdI84FLgCmCUpDOBuaSYFxGzJI0CnidLaZ8TEXWd2rPJYlgnsgu9TV7shTYW/M3MWpKgotNf\nRMSJjWw6pJH9RwAjGiifQjYiMjcHfzOzvPwwFzOzYqqS2O/gb2ZWliqJ/g7+Zma5tf4J2/Jy8Dcz\nK4Nz/mZmBVPGhG2tnoO/mVk5qiT6O/ibmZXBOX8zswJyzt/MrGh8k5eZWTE57WNmVjDZ3D4t3YrK\ncPA3MytDlcR+B38zs7JUSfR38DczK4Nz/mZmBeScv5lZAVVJ7HfwNzMrS5VEfwd/M7OcsondqiP6\nO/ibmeXx3YInAAAHnElEQVQlaFcdsd/B38ysLFUS/Nu1dAPMzNoOlfVfrhqlOZJmSJouaUoq21zS\nOEkvp5+blex/saTZkl6SdPiavhMHfzOzMkj5lzIcFBF9I6J/en0RMD4iegLj02sk9QKGAr2BwcAv\nJbVfk/fh4G9mlpPKXNbC0cDItD4SGFJSfmdELIuIV4DZwIA1OYGDv5lZOcqL/l0lTSlZhjVQYwCP\nSJpasr1bRCxM668B3dJ6D+DVkmPnpbKy+YKvmVkZyhzquagkldOY/SNivqQtgXGSXizdGBEhKcpt\nZ3Pc8zczK0Olc/4RMT/9fAO4jyyN87qk7tn51B14I+0+H9i25PBtUlnZHPzNzMpQyZy/pI0lbVK3\nDhwGzARGA6em3U4F7k/ro4Ghkmok7Qj0BCatyftw2sfMLK/KP8axG3Cfsko7AL+NiAclTQZGSToT\nmAscDxARsySNAp4HVgDnRETtmpzYwd/MrCyVi/4R8Q9gjwbK3wIOaeSYEcCItT23g7+ZWU7C0zuY\nmRWS5/M3Mysgz+ppZlZE1RH7HfzNzMpRJbHfwd/MLK81mLCt1XLwNzMrg3P+ZmZFVB2x38HfzKwc\nVRL7HfzNzMrhnL+ZWcEI0a5Kor9n9TQzKyD3/M3MylAlHX8HfzOzcniop5lZ0fgmLzOz4sn7hK62\nwMHfzKwcVRL9HfzNzMrgnL+ZWQE5529mVkBVEvsd/M3MyqEq6fo7+JuZ5SSqJ+2jiGjpNrRZkt4E\n5rZ0O9aDrsCilm6EVVRR/qbbR8RHK1WZpAfJfnd5LYqIwZU6fyU5+FuzJE2JiP4t3Q6rHP9NzRO7\nmZkVkIO/mVkBOfhbHje2dAOs4vw3LTjn/M3MCsg9fzOzAnLwNzMrIAf/KiApJF1V8voCScObOWaI\npF5NbD9F0kxJMyT9VdIFqfxxSR4i2EpI2krSnZL+LmmqpLGShkka09Jts9bNwb86LAOOlVTOzSdD\ngAaDv6QjgPOBwyLi48DewDtr3cqs7vaVqMdA2TwD9wGPR8TOEdEPuBjotpb1+s7/AnDwrw4ryEZv\nfKP+Bkk7SHpU0nOSxkvaTtK+wGeBn0iaLmnneoddDFwQEQsAImJZRNxUsv3zkiZJ+pukA9J5TpN0\nXcl5x0galNYXS7pK0rPAPpLmSLpM0rT0zWL3Cv4uiuQgYHlE/KquICKeBZ4AOku6W9KLku5IHxSk\n333XtN5f0uNpfbik2yVNAG5Pf897JT0o6WVJV673d2frlIN/9bgeOEnSR+qV/wIYGRGfAO4Aro2I\np4DRwIUR0Tci/l7vmD7A1CbO1SEiBpB9O7g0R9s2Bp6JiD0i4slUtigi9gJuAC7IUYd9WFN/pz3J\n/j69gJ2A/XLU1ws4NCJOTK/7AicAHwdOkLTt2jXXWhMH/yoREe8CtwFfr7dpH+C3af12YP8KnO7e\n9HMqsEOO/WuBe9ayDivPpIiYFxErgenk+x2PjoglJa/HR8Q7EbEUeB7Yfh2001qIg391uQY4k6yn\nvTZmAf2a2L4s/azlg5lhV7D6v6cNS9aXRkRtjjqsPE39nZaVrDf2d9qQ1b2fsw6rAg7+VSQi/gWM\nIvsAqPMUMDStn0SWDwZ4D9ikkaouJ7sesBWApA0kfamZ088B+kpql9IDA8p/B1amR4EaScPqCiR9\nAjigiWPm8MEHxnHrrmnW2jn4V5+rWH3K2XOB0yU9B5wMnJfK7wQuTMM4V7vgGxFjgeuARyTNAqYB\nXZo57wTgFbL0wLXpGFuHIrs9/xjg0DTUcxbZB/drTRx2GfBzSVPIevNWUJ7ewcysgNzzNzMrIAd/\nM7MCcvA3MysgB38zswJy8DczKyAHf2tzJNWmOYlmSvq9pI3Woq5BdTNgSvqspIua2HdTSWevwTmG\n182KatZaOPhbW7QkzUnUB/gP8JXSjcqU/W87IkZHxBVN7LIpUHbwN2uNHPytrXsC2CXNXvqSpNuA\nmcC2kg6T9HSaPfT3kjoDSBqcZrucBhxbV1HpzKSSukm6T9KzadkXuALYOX3r+Ena70JJk9OsqZeV\n1HVJmvX0SWC39fbbMMvJc3VYm5XmnT8CeDAV9QROjYiJadri75HNUvm+pO8A30xTE98EHAzMBu5q\npPprgT9HxDHpGQSdgYuAPhHRN53/sHTOAYCA0ZIOJJsjZyjZrJgdyO52bmqWVLP1zsHf2qJOkqan\n9SeAm4GtgbkRMTGV7002RfGENJX9BsDTwO7AKxHxMoCk3wDD+LCDgVMA0qR070jarN4+h6Xlr+l1\nZ7IPg02A+yLi3+kco9fq3ZqtAw7+1hYtqet910kBvnRWSgHjSuamr9tvtePWkoDLI+J/653j/Aqe\nw2ydcM7fqtVEYD9JuwBI2ljSrsCLwA4lk9md2Mjx44GvpmPbp4fk1J8J9SHgjJJrCT0kbQn8BRgi\nqZOkTYDPVPi9ma01B3+rShHxJnAa8Ls0o+nTwO7pwSTDgD+lC75vNFLFecBBkmaQ5et7RcRbZGmk\nmZJ+EhEPkz0o5+m0393AJhExjexawrPAA8DkdfZGzdaQZ/U0Mysg9/zNzArIwd/MrIAc/M3MCsjB\n38ysgBz8zcwKyMHfzKyAHPzNzAro/wPDToFZGqVjcwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11a3c5d30>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_confusion_matrix(df_confusion, title='Confusion matrix', cmap=plt.cm.Blues):\n",
    "    #plt.matshow(df_confusion, cmap=cmap)\n",
    "    plt.imshow(df_confusion, cmap=cmap)\n",
    "    plt.title('Confusion Matrix')\n",
    "    plt.colorbar()\n",
    "    plt.xticks(np.arange(2), ('Not Churn', 'Churn'))\n",
    "    plt.yticks(np.arange(2), ('Not Churn', 'Churn'))\n",
    "    plt.tight_layout()\n",
    "    plt.ylabel('Actual')\n",
    "    plt.xlabel('Predicted')\n",
    "    \n",
    "    thresh = df_confusion.max() / 2\n",
    "    for i, j in itertools.product(range(df_confusion.shape[0]), range(df_confusion.shape[1])):\n",
    "        plt.text(j, i, df_confusion[i, j],\n",
    "                 horizontalalignment=\"center\",\n",
    "                 color=\"white\" if df_confusion[i, j] > thresh else \"black\")\n",
    "\n",
    "plot_confusion_matrix(df_confusion)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Random Forest prediction is more accurate than Decision Tree\n",
      "The top 3 features that impact customer churn are : (1) Day Mins (2) Cust Serv Calls (3) Even Mins\n",
      "When run multiple times with different split of training and testing data set,\n",
      "Decision Tree accuracy varies 1-3%, while Random Forest is more stable\n",
      "Area Code has 0% Importance - customer in different location has same preferences\n",
      "Calls (Day, Even, Night) do not matter much (near 0% Importance), only International Call matters\n",
      "Using the prediction model, Churn score estimated to be 13.593. That means around 13.5 out of every 100 customers could switch to other telecom service providers.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"Random Forest prediction is more accurate than Decision Tree\")\n",
    "print(\"The top 3 features that impact customer churn are : (1) Day Mins (2) Cust Serv Calls (3) Even Mins\")\n",
    "print(\"When run multiple times with different split of training and testing data set,\")\n",
    "print(\"Decision Tree accuracy varies 1-3%, while Random Forest is more stable\")\n",
    "print(\"Area Code has 0% Importance - customer in different location has same preferences\")\n",
    "print(\"Calls (Day, Even, Night) do not matter much (near 0% Importance), only International Call matters\")\n",
    "print(\"Using the prediction model, Churn score estimated to be \" + str(churn_score) + \". That means around 13.5 out of every 100 customers could switch to other telecom service providers.\")\n",
    "print(\"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}