LCFS implementation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>duration</th>\n",
       "      <th>src_bytes</th>\n",
       "      <th>dst_bytes</th>\n",
       "      <th>land</th>\n",
       "      <th>wrong_fragment</th>\n",
       "      <th>urgent</th>\n",
       "      <th>hot</th>\n",
       "      <th>num_failed_logins</th>\n",
       "      <th>logged_in</th>\n",
       "      <th>num_compromised</th>\n",
       "      <th>root_shell</th>\n",
       "      <th>su_attempted</th>\n",
       "      <th>num_root</th>\n",
       "      <th>num_file_creations</th>\n",
       "      <th>num_shells</th>\n",
       "      <th>num_access_files</th>\n",
       "      <th>num_outbound_cmds</th>\n",
       "      <th>is_host_login</th>\n",
       "      <th>is_guest_login</th>\n",
       "      <th>count</th>\n",
       "      <th>srv_count</th>\n",
       "      <th>serror_rate</th>\n",
       "      <th>srv_serror_rate</th>\n",
       "      <th>rerror_rate</th>\n",
       "      <th>srv_rerror_rate</th>\n",
       "      <th>same_srv_rate</th>\n",
       "      <th>diff_srv_rate</th>\n",
       "      <th>srv_diff_host_rate</th>\n",
       "      <th>dst_host_count</th>\n",
       "      <th>dst_host_srv_count</th>\n",
       "      <th>dst_host_same_srv_rate</th>\n",
       "      <th>dst_host_diff_srv_rate</th>\n",
       "      <th>dst_host_same_src_port_rate</th>\n",
       "      <th>dst_host_srv_diff_host_rate</th>\n",
       "      <th>dst_host_serror_rate</th>\n",
       "      <th>dst_host_srv_serror_rate</th>\n",
       "      <th>dst_host_rerror_rate</th>\n",
       "      <th>dst_host_srv_rerror_rate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>494021.000000</td>\n",
       "      <td>4.940210e+05</td>\n",
       "      <td>4.940210e+05</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.0</td>\n",
       "      <td>494021.0</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "      <td>494021.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>47.979302</td>\n",
       "      <td>3.025610e+03</td>\n",
       "      <td>8.685324e+02</td>\n",
       "      <td>0.000045</td>\n",
       "      <td>0.006433</td>\n",
       "      <td>0.000014</td>\n",
       "      <td>0.034519</td>\n",
       "      <td>0.000152</td>\n",
       "      <td>0.148247</td>\n",
       "      <td>0.010212</td>\n",
       "      <td>0.000111</td>\n",
       "      <td>0.000036</td>\n",
       "      <td>0.011352</td>\n",
       "      <td>0.001083</td>\n",
       "      <td>0.000109</td>\n",
       "      <td>0.001008</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.001387</td>\n",
       "      <td>332.285690</td>\n",
       "      <td>292.906557</td>\n",
       "      <td>0.176687</td>\n",
       "      <td>0.176609</td>\n",
       "      <td>0.057433</td>\n",
       "      <td>0.057719</td>\n",
       "      <td>0.791547</td>\n",
       "      <td>0.020982</td>\n",
       "      <td>0.028997</td>\n",
       "      <td>232.470778</td>\n",
       "      <td>188.665670</td>\n",
       "      <td>0.753780</td>\n",
       "      <td>0.030906</td>\n",
       "      <td>0.601935</td>\n",
       "      <td>0.006684</td>\n",
       "      <td>0.176754</td>\n",
       "      <td>0.176443</td>\n",
       "      <td>0.058118</td>\n",
       "      <td>0.057412</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>707.746472</td>\n",
       "      <td>9.882181e+05</td>\n",
       "      <td>3.304000e+04</td>\n",
       "      <td>0.006673</td>\n",
       "      <td>0.134805</td>\n",
       "      <td>0.005510</td>\n",
       "      <td>0.782103</td>\n",
       "      <td>0.015520</td>\n",
       "      <td>0.355345</td>\n",
       "      <td>1.798326</td>\n",
       "      <td>0.010551</td>\n",
       "      <td>0.007793</td>\n",
       "      <td>2.012718</td>\n",
       "      <td>0.096416</td>\n",
       "      <td>0.011020</td>\n",
       "      <td>0.036482</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.037211</td>\n",
       "      <td>213.147412</td>\n",
       "      <td>246.322817</td>\n",
       "      <td>0.380717</td>\n",
       "      <td>0.381017</td>\n",
       "      <td>0.231623</td>\n",
       "      <td>0.232147</td>\n",
       "      <td>0.388189</td>\n",
       "      <td>0.082205</td>\n",
       "      <td>0.142397</td>\n",
       "      <td>64.745380</td>\n",
       "      <td>106.040437</td>\n",
       "      <td>0.410781</td>\n",
       "      <td>0.109259</td>\n",
       "      <td>0.481309</td>\n",
       "      <td>0.042133</td>\n",
       "      <td>0.380593</td>\n",
       "      <td>0.380919</td>\n",
       "      <td>0.230590</td>\n",
       "      <td>0.230140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.500000e+01</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>117.000000</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>255.000000</td>\n",
       "      <td>46.000000</td>\n",
       "      <td>0.410000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.200000e+02</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>510.000000</td>\n",
       "      <td>510.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>255.000000</td>\n",
       "      <td>255.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.032000e+03</td>\n",
       "      <td>0.000000e+00</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>511.000000</td>\n",
       "      <td>511.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>255.000000</td>\n",
       "      <td>255.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.040000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>58329.000000</td>\n",
       "      <td>6.933756e+08</td>\n",
       "      <td>5.155468e+06</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>30.000000</td>\n",
       "      <td>5.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>884.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>993.000000</td>\n",
       "      <td>28.000000</td>\n",
       "      <td>2.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>511.000000</td>\n",
       "      <td>511.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>255.000000</td>\n",
       "      <td>255.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            duration     src_bytes     dst_bytes           land  \\\n",
       "count  494021.000000  4.940210e+05  4.940210e+05  494021.000000   \n",
       "mean       47.979302  3.025610e+03  8.685324e+02       0.000045   \n",
       "std       707.746472  9.882181e+05  3.304000e+04       0.006673   \n",
       "min         0.000000  0.000000e+00  0.000000e+00       0.000000   \n",
       "25%         0.000000  4.500000e+01  0.000000e+00       0.000000   \n",
       "50%         0.000000  5.200000e+02  0.000000e+00       0.000000   \n",
       "75%         0.000000  1.032000e+03  0.000000e+00       0.000000   \n",
       "max     58329.000000  6.933756e+08  5.155468e+06       1.000000   \n",
       "\n",
       "       wrong_fragment         urgent            hot  num_failed_logins  \\\n",
       "count   494021.000000  494021.000000  494021.000000      494021.000000   \n",
       "mean         0.006433       0.000014       0.034519           0.000152   \n",
       "std          0.134805       0.005510       0.782103           0.015520   \n",
       "min          0.000000       0.000000       0.000000           0.000000   \n",
       "25%          0.000000       0.000000       0.000000           0.000000   \n",
       "50%          0.000000       0.000000       0.000000           0.000000   \n",
       "75%          0.000000       0.000000       0.000000           0.000000   \n",
       "max          3.000000       3.000000      30.000000           5.000000   \n",
       "\n",
       "           logged_in  num_compromised     root_shell   su_attempted  \\\n",
       "count  494021.000000    494021.000000  494021.000000  494021.000000   \n",
       "mean        0.148247         0.010212       0.000111       0.000036   \n",
       "std         0.355345         1.798326       0.010551       0.007793   \n",
       "min         0.000000         0.000000       0.000000       0.000000   \n",
       "25%         0.000000         0.000000       0.000000       0.000000   \n",
       "50%         0.000000         0.000000       0.000000       0.000000   \n",
       "75%         0.000000         0.000000       0.000000       0.000000   \n",
       "max         1.000000       884.000000       1.000000       2.000000   \n",
       "\n",
       "            num_root  num_file_creations     num_shells  num_access_files  \\\n",
       "count  494021.000000       494021.000000  494021.000000     494021.000000   \n",
       "mean        0.011352            0.001083       0.000109          0.001008   \n",
       "std         2.012718            0.096416       0.011020          0.036482   \n",
       "min         0.000000            0.000000       0.000000          0.000000   \n",
       "25%         0.000000            0.000000       0.000000          0.000000   \n",
       "50%         0.000000            0.000000       0.000000          0.000000   \n",
       "75%         0.000000            0.000000       0.000000          0.000000   \n",
       "max       993.000000           28.000000       2.000000          8.000000   \n",
       "\n",
       "       num_outbound_cmds  is_host_login  is_guest_login          count  \\\n",
       "count           494021.0       494021.0   494021.000000  494021.000000   \n",
       "mean                 0.0            0.0        0.001387     332.285690   \n",
       "std                  0.0            0.0        0.037211     213.147412   \n",
       "min                  0.0            0.0        0.000000       0.000000   \n",
       "25%                  0.0            0.0        0.000000     117.000000   \n",
       "50%                  0.0            0.0        0.000000     510.000000   \n",
       "75%                  0.0            0.0        0.000000     511.000000   \n",
       "max                  0.0            0.0        1.000000     511.000000   \n",
       "\n",
       "           srv_count    serror_rate  srv_serror_rate    rerror_rate  \\\n",
       "count  494021.000000  494021.000000    494021.000000  494021.000000   \n",
       "mean      292.906557       0.176687         0.176609       0.057433   \n",
       "std       246.322817       0.380717         0.381017       0.231623   \n",
       "min         0.000000       0.000000         0.000000       0.000000   \n",
       "25%        10.000000       0.000000         0.000000       0.000000   \n",
       "50%       510.000000       0.000000         0.000000       0.000000   \n",
       "75%       511.000000       0.000000         0.000000       0.000000   \n",
       "max       511.000000       1.000000         1.000000       1.000000   \n",
       "\n",
       "       srv_rerror_rate  same_srv_rate  diff_srv_rate  srv_diff_host_rate  \\\n",
       "count    494021.000000  494021.000000  494021.000000       494021.000000   \n",
       "mean          0.057719       0.791547       0.020982            0.028997   \n",
       "std           0.232147       0.388189       0.082205            0.142397   \n",
       "min           0.000000       0.000000       0.000000            0.000000   \n",
       "25%           0.000000       1.000000       0.000000            0.000000   \n",
       "50%           0.000000       1.000000       0.000000            0.000000   \n",
       "75%           0.000000       1.000000       0.000000            0.000000   \n",
       "max           1.000000       1.000000       1.000000            1.000000   \n",
       "\n",
       "       dst_host_count  dst_host_srv_count  dst_host_same_srv_rate  \\\n",
       "count   494021.000000       494021.000000           494021.000000   \n",
       "mean       232.470778          188.665670                0.753780   \n",
       "std         64.745380          106.040437                0.410781   \n",
       "min          0.000000            0.000000                0.000000   \n",
       "25%        255.000000           46.000000                0.410000   \n",
       "50%        255.000000          255.000000                1.000000   \n",
       "75%        255.000000          255.000000                1.000000   \n",
       "max        255.000000          255.000000                1.000000   \n",
       "\n",
       "       dst_host_diff_srv_rate  dst_host_same_src_port_rate  \\\n",
       "count           494021.000000                494021.000000   \n",
       "mean                 0.030906                     0.601935   \n",
       "std                  0.109259                     0.481309   \n",
       "min                  0.000000                     0.000000   \n",
       "25%                  0.000000                     0.000000   \n",
       "50%                  0.000000                     1.000000   \n",
       "75%                  0.040000                     1.000000   \n",
       "max                  1.000000                     1.000000   \n",
       "\n",
       "       dst_host_srv_diff_host_rate  dst_host_serror_rate  \\\n",
       "count                494021.000000         494021.000000   \n",
       "mean                      0.006684              0.176754   \n",
       "std                       0.042133              0.380593   \n",
       "min                       0.000000              0.000000   \n",
       "25%                       0.000000              0.000000   \n",
       "50%                       0.000000              0.000000   \n",
       "75%                       0.000000              0.000000   \n",
       "max                       1.000000              1.000000   \n",
       "\n",
       "       dst_host_srv_serror_rate  dst_host_rerror_rate  \\\n",
       "count             494021.000000         494021.000000   \n",
       "mean                   0.176443              0.058118   \n",
       "std                    0.380919              0.230590   \n",
       "min                    0.000000              0.000000   \n",
       "25%                    0.000000              0.000000   \n",
       "50%                    0.000000              0.000000   \n",
       "75%                    0.000000              0.000000   \n",
       "max                    1.000000              1.000000   \n",
       "\n",
       "       dst_host_srv_rerror_rate  \n",
       "count             494021.000000  \n",
       "mean                   0.057412  \n",
       "std                    0.230140  \n",
       "min                    0.000000  \n",
       "25%                    0.000000  \n",
       "50%                    0.000000  \n",
       "75%                    0.000000  \n",
       "max                    1.000000  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from time import time\n",
    "\n",
    "pd.set_option('display.max_rows', None)\n",
    "pd.set_option('display.max_columns', None)\n",
    "\n",
    "col_names = [\"duration\", \"protocol_type\", \"service\", \"flag\", \"src_bytes\", \"dst_bytes\", \"land\", \"wrong_fragment\", \"urgent\", \"hot\", \n",
    "             \"num_failed_logins\", \"logged_in\", \"num_compromised\", \"root_shell\", \"su_attempted\", \"num_root\", \"num_file_creations\", \n",
    "             \"num_shells\", \"num_access_files\", \"num_outbound_cmds\", \"is_host_login\", \"is_guest_login\", \"count\", \"srv_count\", \n",
    "             \"serror_rate\", \"srv_serror_rate\", \"rerror_rate\", \"srv_rerror_rate\", \"same_srv_rate\", \"diff_srv_rate\", \n",
    "             \"srv_diff_host_rate\", \"dst_host_count\",\"dst_host_srv_count\", \"dst_host_same_srv_rate\", \"dst_host_diff_srv_rate\", \n",
    "             \"dst_host_same_src_port_rate\", \"dst_host_srv_diff_host_rate\", \"dst_host_serror_rate\", \"dst_host_srv_serror_rate\", \n",
    "             \"dst_host_rerror_rate\", \"dst_host_srv_rerror_rate\", \"label\"]\n",
    "\n",
    "df26 = pd.read_csv(\"C:/Users/Rahul/BTP/Data/kddcup.data_10_percent_corrected\", names = col_names)\n",
    "df26.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>duration</th>\n",
       "      <th>protocol_type</th>\n",
       "      <th>service</th>\n",
       "      <th>flag</th>\n",
       "      <th>src_bytes</th>\n",
       "      <th>dst_bytes</th>\n",
       "      <th>land</th>\n",
       "      <th>wrong_fragment</th>\n",
       "      <th>urgent</th>\n",
       "      <th>hot</th>\n",
       "      <th>num_failed_logins</th>\n",
       "      <th>logged_in</th>\n",
       "      <th>num_compromised</th>\n",
       "      <th>root_shell</th>\n",
       "      <th>su_attempted</th>\n",
       "      <th>num_root</th>\n",
       "      <th>num_file_creations</th>\n",
       "      <th>num_shells</th>\n",
       "      <th>num_access_files</th>\n",
       "      <th>num_outbound_cmds</th>\n",
       "      <th>is_host_login</th>\n",
       "      <th>is_guest_login</th>\n",
       "      <th>count</th>\n",
       "      <th>srv_count</th>\n",
       "      <th>serror_rate</th>\n",
       "      <th>srv_serror_rate</th>\n",
       "      <th>rerror_rate</th>\n",
       "      <th>srv_rerror_rate</th>\n",
       "      <th>same_srv_rate</th>\n",
       "      <th>diff_srv_rate</th>\n",
       "      <th>srv_diff_host_rate</th>\n",
       "      <th>dst_host_count</th>\n",
       "      <th>dst_host_srv_count</th>\n",
       "      <th>dst_host_same_srv_rate</th>\n",
       "      <th>dst_host_diff_srv_rate</th>\n",
       "      <th>dst_host_same_src_port_rate</th>\n",
       "      <th>dst_host_srv_diff_host_rate</th>\n",
       "      <th>dst_host_serror_rate</th>\n",
       "      <th>dst_host_srv_serror_rate</th>\n",
       "      <th>dst_host_rerror_rate</th>\n",
       "      <th>dst_host_srv_rerror_rate</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>181</td>\n",
       "      <td>5450</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9</td>\n",
       "      <td>9</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.11</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>239</td>\n",
       "      <td>486</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>19</td>\n",
       "      <td>19</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.05</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>235</td>\n",
       "      <td>1337</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>29</td>\n",
       "      <td>29</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>219</td>\n",
       "      <td>1337</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>39</td>\n",
       "      <td>39</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.03</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>217</td>\n",
       "      <td>2032</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>6</td>\n",
       "      <td>6</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>49</td>\n",
       "      <td>49</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.02</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   duration  protocol_type  service  flag  src_bytes  dst_bytes  land  \\\n",
       "0         0              0        0     0        181       5450     0   \n",
       "1         0              0        0     0        239        486     0   \n",
       "2         0              0        0     0        235       1337     0   \n",
       "3         0              0        0     0        219       1337     0   \n",
       "4         0              0        0     0        217       2032     0   \n",
       "\n",
       "   wrong_fragment  urgent  hot  num_failed_logins  logged_in  num_compromised  \\\n",
       "0               0       0    0                  0          1                0   \n",
       "1               0       0    0                  0          1                0   \n",
       "2               0       0    0                  0          1                0   \n",
       "3               0       0    0                  0          1                0   \n",
       "4               0       0    0                  0          1                0   \n",
       "\n",
       "   root_shell  su_attempted  num_root  num_file_creations  num_shells  \\\n",
       "0           0             0         0                   0           0   \n",
       "1           0             0         0                   0           0   \n",
       "2           0             0         0                   0           0   \n",
       "3           0             0         0                   0           0   \n",
       "4           0             0         0                   0           0   \n",
       "\n",
       "   num_access_files  num_outbound_cmds  is_host_login  is_guest_login  count  \\\n",
       "0                 0                  0              0               0      8   \n",
       "1                 0                  0              0               0      8   \n",
       "2                 0                  0              0               0      8   \n",
       "3                 0                  0              0               0      6   \n",
       "4                 0                  0              0               0      6   \n",
       "\n",
       "   srv_count  serror_rate  srv_serror_rate  rerror_rate  srv_rerror_rate  \\\n",
       "0          8          0.0              0.0          0.0              0.0   \n",
       "1          8          0.0              0.0          0.0              0.0   \n",
       "2          8          0.0              0.0          0.0              0.0   \n",
       "3          6          0.0              0.0          0.0              0.0   \n",
       "4          6          0.0              0.0          0.0              0.0   \n",
       "\n",
       "   same_srv_rate  diff_srv_rate  srv_diff_host_rate  dst_host_count  \\\n",
       "0            1.0            0.0                 0.0               9   \n",
       "1            1.0            0.0                 0.0              19   \n",
       "2            1.0            0.0                 0.0              29   \n",
       "3            1.0            0.0                 0.0              39   \n",
       "4            1.0            0.0                 0.0              49   \n",
       "\n",
       "   dst_host_srv_count  dst_host_same_srv_rate  dst_host_diff_srv_rate  \\\n",
       "0                   9                     1.0                     0.0   \n",
       "1                  19                     1.0                     0.0   \n",
       "2                  29                     1.0                     0.0   \n",
       "3                  39                     1.0                     0.0   \n",
       "4                  49                     1.0                     0.0   \n",
       "\n",
       "   dst_host_same_src_port_rate  dst_host_srv_diff_host_rate  \\\n",
       "0                         0.11                          0.0   \n",
       "1                         0.05                          0.0   \n",
       "2                         0.03                          0.0   \n",
       "3                         0.03                          0.0   \n",
       "4                         0.02                          0.0   \n",
       "\n",
       "   dst_host_serror_rate  dst_host_srv_serror_rate  dst_host_rerror_rate  \\\n",
       "0                   0.0                       0.0                   0.0   \n",
       "1                   0.0                       0.0                   0.0   \n",
       "2                   0.0                       0.0                   0.0   \n",
       "3                   0.0                       0.0                   0.0   \n",
       "4                   0.0                       0.0                   0.0   \n",
       "\n",
       "   dst_host_srv_rerror_rate  label  \n",
       "0                       0.0      1  \n",
       "1                       0.0      1  \n",
       "2                       0.0      1  \n",
       "3                       0.0      1  \n",
       "4                       0.0      1  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cleanup_nums = {'protocol_type' : {'tcp' : 0, 'udp' : 1, 'icmp' : 2},\n",
    "                'service' : {'http' : 0, 'smtp' : 1, 'finger' : 2, 'domain_u' : 3, 'auth' : 4, 'telnet' : 5, 'ftp' : 6,\n",
    "                             'eco_i' : 7, 'ntp_u' : 8, 'ecr_i' : 9, 'other' : 10, 'private' : 11, 'pop_3' : 12, \n",
    "                             'ftp_data' : 13, 'rje' : 14, 'time' : 15, 'mtp' : 16, 'link' : 17, 'remote_job' : 18, \n",
    "                             'gopher' : 19, 'ssh' : 20, 'name' : 21, 'whois' : 22, 'domain' : 23, 'login' : 24, 'imap4' : 25,\n",
    "                             'daytime' : 26, 'ctf' : 27, 'nntp' : 28, 'shell' : 29, 'IRC' : 30, 'nnsp' : 31, 'http_443' : 32,\n",
    "                             'exec' : 33, 'printer' : 34, 'efs' : 35, 'courier' : 36, 'uucp' : 37, 'klogin' : 38, \n",
    "                             'kshell' : 39, 'echo' : 40, 'discard' : 41, 'systat' : 42, 'supdup' : 43, 'iso_tsap' : 44, \n",
    "                             'hostnames' : 45, 'csnet_ns' : 46, 'pop_2' : 47, 'sunrpc' : 48, 'uucp_path' : 49, \n",
    "                             'netbios_ns' : 50, 'netbios_ssn' : 51, 'netbios_dgm' : 52, 'sql_net' : 53, 'vmnet' : 54, \n",
    "                             'bgp' : 55, 'Z39_50' : 56, 'ldap' : 57, 'netstat' : 58, 'urh_i' : 59, 'X11' : 60, 'urp_i' : 61,\n",
    "                             'pm_dump' : 62, 'tftp_u' : 63, 'tim_i' : 64, 'red_i' : 64},\n",
    "                'flag' : {'SF' : 0, 'S1' : 1, 'REJ' : 2, 'S2' : 3, 'S0' : 4, 'S3' : 5, 'RSTO' : 6, 'RSTR' : 7, 'RSTOS0' : 8,\n",
    "                          'OTH' : 9, 'SH' : 10},\n",
    "                'label' : {'normal.' : 1, 'buffer_overflow.' : 0, 'loadmodule.' : 0, 'perl.' : 0, 'neptune.' : 0, \n",
    "                           'smurf.' : 0, 'guess_passwd.' : 0, 'pod.' : 0, 'teardrop.' : 0, 'portsweep.' : 0, 'ipsweep.' : 0,\n",
    "                           'land.' : 0, 'ftp_write.' : 0, 'back.' : 0, 'imap.' : 0, 'satan.' : 0, 'phf.' : 0, 'nmap.' : 0, \n",
    "                           'multihop.' : 0, 'warezmaster.' : 0, 'warezclient.' : 0, 'spy.' : 0, 'rootkit.' : 0}}\n",
    "\n",
    "df26.replace(cleanup_nums, inplace = True)\n",
    "df26.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df1 = df26\n",
    "\n",
    "from sklearn import preprocessing\n",
    "\n",
    "minmax_scale = preprocessing.MinMaxScaler().fit(df1[col_names])\n",
    "df1[col_names] = minmax_scale.transform(df1[col_names])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "corr = df1.corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "solution = df1['logged_in']\n",
    "df1.drop('logged_in', axis = 1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "srv_diff_host_rate\n",
      "same_srv_rate\n",
      "dst_host_srv_diff_host_rate\n",
      "duration\n",
      "dst_host_diff_srv_rate\n",
      "dst_host_same_srv_rate\n",
      "dst_host_srv_count\n",
      "num_access_files\n",
      "dst_bytes\n",
      "is_guest_login\n",
      "num_file_creations\n",
      "num_shells\n",
      "num_root\n",
      "su_attempted\n",
      "hot\n",
      "root_shell\n",
      "num_compromised\n",
      "urgent\n",
      "num_failed_logins\n",
      "dst_host_rerror_rate\n",
      "src_bytes\n",
      "land\n",
      "srv_rerror_rate\n",
      "rerror_rate\n",
      "dst_host_srv_rerror_rate\n",
      "diff_srv_rate\n",
      "wrong_fragment\n",
      "flag\n",
      "srv_serror_rate\n",
      "dst_host_serror_rate\n",
      "serror_rate\n",
      "dst_host_srv_serror_rate\n",
      "dst_host_same_src_port_rate\n",
      "protocol_type\n",
      "service\n",
      "srv_count\n",
      "dst_host_count\n",
      "count\n",
      "Garima\n",
      "Garima\n"
     ]
    }
   ],
   "source": [
    "solution = pd.DataFrame()\n",
    "\n",
    "while df1.shape[1] > 1:\n",
    "    maxval = -1000\n",
    "    maxind = -1\n",
    "    corr = df1.corr()\n",
    "    ind = -1\n",
    "    rpind = -1\n",
    "    name = 'Garima'\n",
    "    for rp in df1:\n",
    "        rpind += 1\n",
    "        if rp == 'label':\n",
    "            break\n",
    "    \n",
    "    for i in df1:\n",
    "        ind += 1\n",
    "        if i == 'label':\n",
    "            continue\n",
    "        val1 = corr.iloc[ind, rpind]\n",
    "        temp1 = df1[i]\n",
    "        temp1 = pd.DataFrame()\n",
    "        temp1 = temp1.append(solution, ignore_index = True)\n",
    "        corrtemp1 = temp1.corr()\n",
    "        n = solution.shape[1]\n",
    "        \n",
    "        tempind = 0\n",
    "        for j in temp1:\n",
    "            if tempind == 0:\n",
    "                tempind += 1\n",
    "                continue\n",
    "            val1 -= 0.5 / n * corrtemp1.iloc[0, tempind]\n",
    "        \n",
    "        \n",
    "        if val1 > maxval:\n",
    "            maxval = val1\n",
    "            maxind = ind\n",
    "            name = i\n",
    "        \n",
    "        temp1 = temp1.iloc[0 : 0]\n",
    "        corrtemp1 = corrtemp1.iloc[0 : 0]\n",
    "    \n",
    "    ind = 0\n",
    "    print (name)\n",
    "    \n",
    "    for i in df1:\n",
    "        if ind == maxind:\n",
    "            break\n",
    "        ind += 1\n",
    "    \n",
    "    temp2 = df1[i]\n",
    "    temp2 = pd.DataFrame()\n",
    "    solution = solution.append(temp2, ignore_index = True)\n",
    "    df1.drop(i, axis = 1, inplace = True)\n",
    "    \n",
    "    temp2 = temp2.iloc[0 : 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}