model_selection.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from twosigmafunc import R_score, split_data, preprocess, add_diffs, add_nans\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn\n",
    "plt.rcParams['figure.figsize'] = (14.0, 8.0) # set default size of plots\n",
    "plt.rcParams['image.interpolation'] = 'nearest'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train, origin_features = preprocess()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sucessfully add 108 nan features\n"
     ]
    }
   ],
   "source": [
    "add_nans(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sucessfully add 108 diff features\n"
     ]
    }
   ],
   "source": [
    "add_diffs(train, origin_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>derived_0</th>\n",
       "      <th>derived_1</th>\n",
       "      <th>derived_2</th>\n",
       "      <th>derived_3</th>\n",
       "      <th>derived_4</th>\n",
       "      <th>fundamental_0</th>\n",
       "      <th>fundamental_1</th>\n",
       "      <th>fundamental_2</th>\n",
       "      <th>...</th>\n",
       "      <th>technical_35_diff</th>\n",
       "      <th>technical_36_diff</th>\n",
       "      <th>technical_37_diff</th>\n",
       "      <th>technical_38_diff</th>\n",
       "      <th>technical_39_diff</th>\n",
       "      <th>technical_40_diff</th>\n",
       "      <th>technical_41_diff</th>\n",
       "      <th>technical_42_diff</th>\n",
       "      <th>technical_43_diff</th>\n",
       "      <th>technical_44_diff</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>131062</th>\n",
       "      <td>0</td>\n",
       "      <td>167</td>\n",
       "      <td>-0.000837</td>\n",
       "      <td>0.005523</td>\n",
       "      <td>0.021095</td>\n",
       "      <td>0.002476</td>\n",
       "      <td>0.011752</td>\n",
       "      <td>-0.040645</td>\n",
       "      <td>-0.007395</td>\n",
       "      <td>-0.030291</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>131895</th>\n",
       "      <td>0</td>\n",
       "      <td>168</td>\n",
       "      <td>-0.000837</td>\n",
       "      <td>0.005523</td>\n",
       "      <td>0.021095</td>\n",
       "      <td>0.002476</td>\n",
       "      <td>0.011752</td>\n",
       "      <td>-0.040645</td>\n",
       "      <td>-0.007395</td>\n",
       "      <td>-0.030291</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.400551e-18</td>\n",
       "      <td>1.388941e-19</td>\n",
       "      <td>8.286639e-23</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>132728</th>\n",
       "      <td>0</td>\n",
       "      <td>169</td>\n",
       "      <td>-0.000837</td>\n",
       "      <td>0.005523</td>\n",
       "      <td>0.021095</td>\n",
       "      <td>0.002476</td>\n",
       "      <td>0.011752</td>\n",
       "      <td>-0.040645</td>\n",
       "      <td>-0.007395</td>\n",
       "      <td>-0.030291</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.400551e-18</td>\n",
       "      <td>1.388941e-19</td>\n",
       "      <td>8.286639e-23</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>133561</th>\n",
       "      <td>0</td>\n",
       "      <td>170</td>\n",
       "      <td>-0.230583</td>\n",
       "      <td>0.488096</td>\n",
       "      <td>0.935920</td>\n",
       "      <td>0.028222</td>\n",
       "      <td>-0.083071</td>\n",
       "      <td>-0.240929</td>\n",
       "      <td>-0.007395</td>\n",
       "      <td>0.212425</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.400551e-18</td>\n",
       "      <td>1.388941e-19</td>\n",
       "      <td>8.286639e-23</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>134393</th>\n",
       "      <td>0</td>\n",
       "      <td>171</td>\n",
       "      <td>-0.230583</td>\n",
       "      <td>0.488096</td>\n",
       "      <td>0.935920</td>\n",
       "      <td>0.028222</td>\n",
       "      <td>-0.083071</td>\n",
       "      <td>-0.240929</td>\n",
       "      <td>-0.007395</td>\n",
       "      <td>0.212425</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 328 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        id  timestamp  derived_0  derived_1  derived_2  derived_3  derived_4  \\\n",
       "131062   0        167  -0.000837   0.005523   0.021095   0.002476   0.011752   \n",
       "131895   0        168  -0.000837   0.005523   0.021095   0.002476   0.011752   \n",
       "132728   0        169  -0.000837   0.005523   0.021095   0.002476   0.011752   \n",
       "133561   0        170  -0.230583   0.488096   0.935920   0.028222  -0.083071   \n",
       "134393   0        171  -0.230583   0.488096   0.935920   0.028222  -0.083071   \n",
       "\n",
       "        fundamental_0  fundamental_1  fundamental_2        ...          \\\n",
       "131062      -0.040645      -0.007395      -0.030291        ...           \n",
       "131895      -0.040645      -0.007395      -0.030291        ...           \n",
       "132728      -0.040645      -0.007395      -0.030291        ...           \n",
       "133561      -0.240929      -0.007395       0.212425        ...           \n",
       "134393      -0.240929      -0.007395       0.212425        ...           \n",
       "\n",
       "        technical_35_diff  technical_36_diff  technical_37_diff  \\\n",
       "131062                0.0                0.0       0.000000e+00   \n",
       "131895                0.0                0.0       4.400551e-18   \n",
       "132728                0.0                0.0       4.400551e-18   \n",
       "133561                0.0                0.0       4.400551e-18   \n",
       "134393                0.0                0.0       0.000000e+00   \n",
       "\n",
       "        technical_38_diff  technical_39_diff  technical_40_diff  \\\n",
       "131062       0.000000e+00       0.000000e+00                0.0   \n",
       "131895       1.388941e-19       8.286639e-23                0.0   \n",
       "132728       1.388941e-19       8.286639e-23                0.0   \n",
       "133561       1.388941e-19       8.286639e-23                0.0   \n",
       "134393       0.000000e+00       0.000000e+00                0.0   \n",
       "\n",
       "        technical_41_diff  technical_42_diff  technical_43_diff  \\\n",
       "131062                0.0                0.0                0.0   \n",
       "131895                0.0                0.0                0.0   \n",
       "132728                0.0                0.0                0.0   \n",
       "133561                0.0                0.0                0.0   \n",
       "134393                0.0                0.0                0.0   \n",
       "\n",
       "        technical_44_diff  \n",
       "131062                0.0  \n",
       "131895                0.0  \n",
       "132728                0.0  \n",
       "133561                0.0  \n",
       "134393                0.0  \n",
       "\n",
       "[5 rows x 328 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.fillna(train.median(), inplace=True)\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Add some important features "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "train['id_diff'] = train.id.diff()\n",
    "train['tec20-30'] = train.technical_20 - train.technical_30\n",
    "train['tec123'] = train['tec20-30'] + train.technical_13\n",
    "train['y_past'] = train.y.shift()\n",
    "train['tec123_past'] = train.tec123.shift()\n",
    "train.loc[train.id_diff != 0, ['y_past', 'tec123_past']] = 0\n",
    "train.fillna(0, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lr_features = ['tec20-30', 'technical_20_diff']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import Ridge\n",
    "import twosigmafunc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "ridge_1 = Ridge(normalize=True)\n",
    "ridge_2 = Ridge(normalize=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "X_train, y_train, X_test, y_test = split_data(train, twosigmafunc.origin_features(train, ['id', 'timestamp', 'y', 'id_diff']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "low_y_cut = -0.075\n",
    "high_y_cut = 0.075\n",
    "y_is_above_cut = (y_train > high_y_cut)\n",
    "y_is_below_cut = (y_train < low_y_cut)\n",
    "y_is_within_cut = (~y_is_above_cut & ~y_is_below_cut)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train = X_train[y_is_within_cut]\n",
    "y_train = y_train[y_is_within_cut]\n",
    "num = y_train.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "791870"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training on 791870 samples\n",
      "r_score on train: 0.0223489496849 , improving: 99999.0223489\n",
      "r_score on test: 0.0246343621827 , improving: 99999.0246344\n",
      "training on 783951 samples\n",
      "r_score on train: 0.0231251924926 , improving: 0.000776242807643\n",
      "r_score on test: 0.025717726297 , improving: 0.00108336411427\n",
      "training on 776111 samples\n",
      "r_score on train: 0.0231663033699 , improving: 4.11108773756e-05\n",
      "r_score on test: 0.0258679325651 , improving: 0.000150206268154\n",
      "training on 768349 samples\n",
      "r_score on train: 0.0230078719005 , improving: -0.000158431469423\n",
      "r_score on test: 0.0258765475687 , improving: 8.61500361838e-06\n"
     ]
    }
   ],
   "source": [
    "better = True\n",
    "r_best_train = -99999\n",
    "r_best_test = -99999\n",
    "quant = 0.99\n",
    "limit_len = num * 0.9\n",
    "ix = X_train.index\n",
    "while better:\n",
    "    print('training on', len(ix), 'samples')\n",
    "    ridge_2.fit(X_train.loc[ix, lr_features], y_train.loc[ix])\n",
    "    y_pred_residual = ridge_1.predict(X_train.loc[ix, lr_features])\n",
    "    y_train_pred = ridge_2.predict(X_train[lr_features])\n",
    "    y_test_pred = ridge_2.predict(X_test[lr_features])\n",
    "    r = R_score(y_test_pred, y_test)\n",
    "    r_train = R_score(y_train_pred, y_train)\n",
    "    print('r_score on train:', r_train, ', improving:', r_train - r_best_train)\n",
    "    print('r_score on test:', r, ', improving:', r - r_best_test)\n",
    "    if r > r_best_test:\n",
    "        r_best_test = r \n",
    "    if r_train > r_best_train:\n",
    "        r_best_train = r_train\n",
    "        residual = np.abs(y_pred_residual - y_train[ix])\n",
    "        ix = residual[abs(residual) <= abs(residual).quantile(quant)].index\n",
    "        if len(ix) < limit_len:\n",
    "            break\n",
    "    else:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training on 791870 samples\n",
      "r_score on train: 0.0149244686904 , improving: 99999.0149245\n",
      "r_score on test: 0.0198638321117 , improving: 99999.0198638\n",
      "training on 783951 samples\n",
      "r_score on train: 0.0155413209696 , improving: 0.000616852279273\n",
      "r_score on test: 0.0208503996251 , improving: 0.000986567513408\n",
      "training on 776111 samples\n",
      "r_score on train: 0.0155900364915 , improving: 4.87155218904e-05\n",
      "r_score on test: 0.0209409604868 , improving: 9.05608617213e-05\n",
      "training on 768349 samples\n",
      "r_score on train: 0.0156443889891 , improving: 5.4352497612e-05\n",
      "r_score on test: 0.0211267717193 , improving: 0.000185811232517\n",
      "training on 760665 samples\n",
      "r_score on train: 0.0155475936357 , improving: -9.67953534738e-05\n",
      "r_score on test: 0.0210691261756 , improving: -5.76455437521e-05\n"
     ]
    }
   ],
   "source": [
    "better = True\n",
    "r_best_train = -99999\n",
    "r_best_test = -99999\n",
    "quant = 0.99\n",
    "limit_len = num * 0.9\n",
    "ix = X_train.index\n",
    "while better:\n",
    "    print('training on', len(ix), 'samples')\n",
    "    ridge_1.fit(np.array(X_train.loc[ix, lr_features[1]]).reshape(-1, 1), y_train.loc[ix])\n",
    "    y_pred_residual = ridge_1.predict(np.array(X_train.loc[ix, lr_features[1]]).reshape(-1, 1))\n",
    "    y_train_pred = ridge_1.predict(np.array(X_train[lr_features[1]]).reshape(-1,1))\n",
    "    y_test_pred = ridge_1.predict(np.array(X_test[lr_features[1]]).reshape(-1,1))\n",
    "    r = R_score(y_test_pred, y_test)\n",
    "    r_train = R_score(y_train_pred, y_train)\n",
    "    print('r_score on train:', r_train, ', improving:', r_train - r_best_train)\n",
    "    print('r_score on test:', r, ', improving:', r - r_best_test)\n",
    "    if r > r_best_test:\n",
    "        r_best_test = r \n",
    "    if r_train > r_best_train:\n",
    "        r_best_train = r_train\n",
    "        residual = np.abs(y_pred_residual - y_train[ix])\n",
    "        ix = residual[abs(residual) <= abs(residual).quantile(quant)].index\n",
    "        if len(ix) < limit_len:\n",
    "            break\n",
    "    else:\n",
    "        break"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# ETR features "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import json\n",
    "from sklearn.ensemble import ExtraTreesRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "try:\n",
    "    with open('features.json', 'r') as file:\n",
    "        features = json.load(file)\n",
    "except IOError as err:\n",
    "    raise err"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ExtraTreesRegressor(bootstrap=False, criterion='mse', max_depth=6,\n",
       "          max_features=0.6, max_leaf_nodes=None, min_impurity_split=1e-07,\n",
       "          min_samples_leaf=30, min_samples_split=2,\n",
       "          min_weight_fraction_leaf=0.0, n_estimators=128, n_jobs=-1,\n",
       "          oob_score=False, random_state=17, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "etr = ExtraTreesRegressor(n_jobs=-1,n_estimators=128, max_depth=6, \n",
    "                          max_features=0.6, min_samples_leaf=30,\n",
    "                          random_state=17, verbose=0)\n",
    "etr.fit(X_train[etr_features], y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Ridge(alpha=1.0, copy_X=True, fit_intercept=True, max_iter=None,\n",
       "   normalize=True, random_state=None, solver='auto', tol=0.001)"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ridge_1.fit(np.array(X_train[lr_features[1]]).reshape(-1, 1), y_train)    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Ridge(alpha=1.0, copy_X=True, fit_intercept=True, max_iter=None,\n",
       "   normalize=True, random_state=None, solver='auto', tol=0.001)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ridge_2.fit(X_train[lr_features], y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "del train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train['y_lr1'] = ridge_1.predict(np.array(X_train[lr_features[1]]).reshape(-1, 1))\n",
    "X_train['y_lr2'] = ridge_2.predict(X_train[lr_features])\n",
    "X_test['y_lr1'] = ridge_1.predict(np.array(X_test[lr_features[1]]).reshape(-1, 1))\n",
    "X_test['y_lr2'] = ridge_2.predict(X_test[lr_features])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train['y_etr'] = etr.predict(X_train[etr_features])\n",
    "X_test['y_etr'] = etr.predict(X_test[etr_features])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "residual = pd.DataFrame({\n",
    "        'residual_lr1':np.abs(X_train['y_lr1'] - y_train),\n",
    "        'residual_lr2':np.abs(X_train['y_lr2'] - y_train),\n",
    "        'residual_etr':np.abs(X_train['y_etr'] - y_train),\n",
    "    })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "residual['best'] = residual.idxmin(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>residual_etr</th>\n",
       "      <th>residual_lr1</th>\n",
       "      <th>residual_lr2</th>\n",
       "      <th>best</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>131062</th>\n",
       "      <td>0.006954</td>\n",
       "      <td>0.007190</td>\n",
       "      <td>0.007202</td>\n",
       "      <td>residual_etr</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>131895</th>\n",
       "      <td>0.002038</td>\n",
       "      <td>0.001868</td>\n",
       "      <td>0.001856</td>\n",
       "      <td>residual_lr2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>132728</th>\n",
       "      <td>0.017888</td>\n",
       "      <td>0.017643</td>\n",
       "      <td>0.017631</td>\n",
       "      <td>residual_lr2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>133561</th>\n",
       "      <td>0.012839</td>\n",
       "      <td>0.012853</td>\n",
       "      <td>0.012841</td>\n",
       "      <td>residual_etr</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>134393</th>\n",
       "      <td>0.025352</td>\n",
       "      <td>0.025311</td>\n",
       "      <td>0.025323</td>\n",
       "      <td>residual_lr1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        residual_etr  residual_lr1  residual_lr2          best\n",
       "131062      0.006954      0.007190      0.007202  residual_etr\n",
       "131895      0.002038      0.001868      0.001856  residual_lr2\n",
       "132728      0.017888      0.017643      0.017631  residual_lr2\n",
       "133561      0.012839      0.012853      0.012841  residual_etr\n",
       "134393      0.025352      0.025311      0.025323  residual_lr1"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "residual.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "model_features = etr_features + ['y_lr1', 'y_lr2', 'y_etr']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "model_etr = ExtraTreesClassifier(n_jobs=-1,n_estimators=128, max_depth=6, \n",
    "                          max_features=0.6,\n",
    "                          random_state=17, verbose=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ExtraTreesClassifier(bootstrap=False, class_weight=None, criterion='gini',\n",
       "           max_depth=6, max_features=0.6, max_leaf_nodes=None,\n",
       "           min_impurity_split=1e-07, min_samples_leaf=1,\n",
       "           min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "           n_estimators=128, n_jobs=-1, oob_score=False, random_state=17,\n",
       "           verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_etr.fit(X_train[model_features], residual.best)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "best_model_train = model_etr.predict(X_train[model_features])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.47803553613598193"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.mean(best_model_train == residual.best)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "residual_test = pd.DataFrame({\n",
    "        'residual_lr1':np.abs(X_test['y_lr1'] - y_test),\n",
    "        'residual_lr2':np.abs(X_test['y_lr2'] - y_test),\n",
    "        'residual_etr':np.abs(X_test['y_etr'] - y_test),\n",
    "    })"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "residual_test['best'] = residual_test.idxmin(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "best_model_test = model_etr.predict(X_test[model_features])\n",
    "probs = model_etr.predict_proba(X_test[model_features])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_basic = 0.75 * X_test.y_etr + 0.2 * X_test.y_lr2 + 0.05 * X_test.y_lr1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.47331326,  0.1480471 ,  0.37863964],\n",
       "       [ 0.48265545,  0.21398561,  0.30335894],\n",
       "       [ 0.4930417 ,  0.30018219,  0.20677612],\n",
       "       ..., \n",
       "       [ 0.44041989,  0.24420038,  0.31537973],\n",
       "       [ 0.45458804,  0.24376482,  0.30164714],\n",
       "       [ 0.47191509,  0.32809474,  0.19999016]])"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "probs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.025406067236916314"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "R_score(y_basic, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "y_model = probs[:,0] * X_test.y_etr + probs[:,2] * X_test.y_lr2 + probs[:,1] * X_test.y_lr1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.025596126615695682"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "R_score(y_model, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature</th>\n",
       "      <th>important</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>technical_20</td>\n",
       "      <td>0.245408</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>tec20-30</td>\n",
       "      <td>0.119768</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>tec123</td>\n",
       "      <td>0.068962</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>technical_20_diff</td>\n",
       "      <td>0.063568</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>y_lr1</td>\n",
       "      <td>0.062851</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>y_lr2</td>\n",
       "      <td>0.062715</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>technical_22</td>\n",
       "      <td>0.057385</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>tec123_past</td>\n",
       "      <td>0.053950</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>fundamental_25_nan</td>\n",
       "      <td>0.049425</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>technical_7</td>\n",
       "      <td>0.046965</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               feature  important\n",
       "10        technical_20   0.245408\n",
       "1             tec20-30   0.119768\n",
       "3               tec123   0.068962\n",
       "32   technical_20_diff   0.063568\n",
       "98               y_lr1   0.062851\n",
       "99               y_lr2   0.062715\n",
       "41        technical_22   0.057385\n",
       "6          tec123_past   0.053950\n",
       "12  fundamental_25_nan   0.049425\n",
       "23         technical_7   0.046965"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imp = pd.DataFrame({'feature':model_features, 'important':model_etr.feature_importances_}).sort_values('important', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "model_features = imp.loc[imp.important > imp.important.mean() / 10, 'feature']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "26"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(model_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ExtraTreesClassifier(bootstrap=False, class_weight=None, criterion='gini',\n",
       "           max_depth=6, max_features=0.6, max_leaf_nodes=None,\n",
       "           min_impurity_split=1e-07, min_samples_leaf=1,\n",
       "           min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "           n_estimators=128, n_jobs=-1, oob_score=False, random_state=17,\n",
       "           verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_etr.fit(X_train[model_features], residual.best)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.025590265591563088"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "probs = model_etr.predict_proba(X_test[model_features])\n",
    "y_model = probs[:,0] * X_test.y_etr + probs[:,2] * X_test.y_lr2 + probs[:,1] * X_test.y_lr1\n",
    "R_score(y_model, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.025728764241391578"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_model = (probs[:,0] + 0.75) / 2 * X_test.y_etr + (probs[:,2] + 0.2) / 2 * X_test.y_lr2 + (probs[:,1] + 0.05) / 2 * X_test.y_lr1\n",
    "R_score(y_model, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.025733699498178265"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_model = (probs[:,0] * 2 + 0.75) / 3 * X_test.y_etr + (probs[:,2] * 2 + 0.2) / 3 * X_test.y_lr2 + (probs[:,1] * 2 + 0.05) / 3 * X_test.y_lr1\n",
    "R_score(y_model, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "set `max_features = 0.8`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ExtraTreesClassifier(bootstrap=False, class_weight=None, criterion='gini',\n",
       "           max_depth=6, max_features=0.8, max_leaf_nodes=None,\n",
       "           min_impurity_split=1e-07, min_samples_leaf=1,\n",
       "           min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "           n_estimators=128, n_jobs=-1, oob_score=False, random_state=17,\n",
       "           verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_etr = ExtraTreesClassifier(n_jobs=-1,n_estimators=128, max_depth=6, max_features=0.8,\n",
    "                          random_state=17, verbose=0)\n",
    "model_etr.fit(X_train[model_features], residual.best)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.025740364077357233"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "probs = model_etr.predict_proba(X_test[model_features])\n",
    "y_model = (probs[:,0] + 0.75) / 2 * X_test.y_etr + (probs[:,2] + 0.2) / 2 * X_test.y_lr2 + (probs[:,1] + 0.05) / 2 * X_test.y_lr1\n",
    "R_score(y_model, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ExtraTreesClassifier(bootstrap=False, class_weight=None, criterion='gini',\n",
       "           max_depth=5, max_features=0.8, max_leaf_nodes=None,\n",
       "           min_impurity_split=1e-07, min_samples_leaf=1,\n",
       "           min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "           n_estimators=128, n_jobs=-1, oob_score=False, random_state=17,\n",
       "           verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_etr = ExtraTreesClassifier(n_jobs=-1,n_estimators=128, max_depth=5, max_features=0.8,\n",
    "                          random_state=17, verbose=0)\n",
    "model_etr.fit(X_train[model_features], residual.best)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.47610340081073915"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "best_model_train = model_etr.predict(X_train[model_features])\n",
    "np.mean(best_model_train == residual.best)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.025709521624548957"
      ]
     },
     "execution_count": 119,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "probs = model_etr.predict_proba(X_test[model_features])\n",
    "y_model = (probs[:,0] + 0.75) / 2 * X_test.y_etr + (probs[:,2] + 0.2) / 2 * X_test.y_lr2 + (probs[:,1] + 0.05) / 2 * X_test.y_lr1\n",
    "R_score(y_model, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ExtraTreesClassifier(bootstrap=False, class_weight=None, criterion='gini',\n",
       "           max_depth=5, max_features=0.8, max_leaf_nodes=None,\n",
       "           min_impurity_split=1e-07, min_samples_leaf=10,\n",
       "           min_samples_split=2, min_weight_fraction_leaf=0.0,\n",
       "           n_estimators=128, n_jobs=-1, oob_score=False, random_state=17,\n",
       "           verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 120,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model_etr = ExtraTreesClassifier(n_jobs=-1,n_estimators=128, max_depth=5, max_features=0.8,\n",
    "                          random_state=17, verbose=0, min_samples_leaf=10)\n",
    "model_etr.fit(X_train[model_features], residual.best)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.025707923593783009"
      ]
     },
     "execution_count": 121,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "probs = model_etr.predict_proba(X_test[model_features])\n",
    "y_model = (probs[:,0] + 0.75) / 2 * X_test.y_etr + (probs[:,2] + 0.2) / 2 * X_test.y_lr2 + (probs[:,1] + 0.05) / 2 * X_test.y_lr1\n",
    "R_score(y_model, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['technical_20',\n",
       " 'tec20-30',\n",
       " 'tec123',\n",
       " 'technical_20_diff',\n",
       " 'y_lr1',\n",
       " 'y_lr2',\n",
       " 'technical_22',\n",
       " 'tec123_past',\n",
       " 'fundamental_25_nan',\n",
       " 'technical_7',\n",
       " 'technical_40',\n",
       " 'y_etr',\n",
       " 'technical_30',\n",
       " 'fundamental_27_nan',\n",
       " 'y_past',\n",
       " 'technical_30_diff',\n",
       " 'fundamental_17_nan',\n",
       " 'technical_2',\n",
       " 'fundamental_63_nan',\n",
       " 'fundamental_59',\n",
       " 'fundamental_47_nan',\n",
       " 'fundamental_5_nan',\n",
       " 'technical_17',\n",
       " 'technical_9_nan',\n",
       " 'technical_11',\n",
       " 'technical_21']"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(model_features)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# Training a Network to combine 3 model results? "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "nn_features = features['nn_features']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "nn_features.extend(['y_lr1', 'y_lr2', 'y_etr'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "35"
      ]
     },
     "execution_count": 130,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(nn_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using Theano backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.wrappers.scikit_learn import KerasRegressor\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import KFold\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.pipeline import Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# define base mode\n",
    "def baseline_model():\n",
    "\t# create model\n",
    "\tmodel = Sequential()\n",
    "\tmodel.add(Dense(35, input_dim=35, init='normal', activation='relu'))\n",
    "\tmodel.add(Dense(1, init='normal', activation='relu'))\n",
    "\t# Compile model\n",
    "\tmodel.compile(loss='mean_squared_error', optimizer='adam', metrics=['accuracy'])\n",
    "\treturn model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "791870/791870 [==============================] - 36s - loss: 3.4753e-04 - acc: 0.0000e+00    \n",
      "Epoch 2/10\n",
      "791870/791870 [==============================] - 31s - loss: 3.4731e-04 - acc: 0.0000e+00    \n",
      "Epoch 3/10\n",
      "141440/791870 [====>.........................] - ETA: 24s - loss: 3.4600e-04 - acc: 0.0000e+00"
     ]
    }
   ],
   "source": [
    "baseline_model().fit(np.array(X_train[nn_features]), np.array(y_train), nb_epoch=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}