Anomaly_detection.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.decomposition import PCA\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.colors as pltc\n",
    "from sklearn.neighbors import LocalOutlierFactor\n",
    "from sklearn import svm\n",
    "from sklearn.preprocessing import StandardScaler, OneHotEncoder, LabelEncoder\n",
    "from sklearn.metrics import classification_report\n",
    "import pickle\n",
    "from random import sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## SECTION 1: DATA LOADING PART"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "headers = ['timestamp','duration','protocol', 'src_ip', 'src_port', 'direction', 'dst_ip' , 'dst_port'\n",
    "           ,'state', 'src_type', 'dst_type' , 'total_packets', 'bytes_bothdir', 'bytes_onedir' ]\n",
    "val_headers = headers.copy()\n",
    "val_headers.append('label')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "<string>:2: DtypeWarning: Columns (4,7) have mixed types.Specify dtype option on import or set low_memory=False.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 8min 47s\n",
      "Parser   : 121 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "df_train_original = pd.read_csv(\"training_data.csv\", names=headers, header=None).sample(frac=0.002, replace=False, random_state=42).fillna(-1)\n",
    "df_val_notsampled = pd.read_csv(\"valid_data_with_labels.csv\", names=val_headers, header=None).fillna(-1)\n",
    "df_test_original = pd.read_csv(\"test_data.csv\", names=headers, header=None).fillna(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "27764\n",
      "940062\n",
      "1053845\n"
     ]
    }
   ],
   "source": [
    "print(len(df_train_original))\n",
    "print(len(df_val_notsampled))\n",
    "print(len(df_test_original))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_val_notsampled.loc[df_val_notsampled['label'].str.contains(\"From-Botnet\"), 'label'] = -1\n",
    "df_val_notsampled.loc[df_val_notsampled['label'].str.contains(\"From-Botnet\")== False, 'label'] = 1\n",
    "df_val_notsampled.label = df_val_notsampled.label.astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "47003\n"
     ]
    }
   ],
   "source": [
    "# To sample anomaly and normal data evenly from validation set\n",
    "df_val_original = df_val_notsampled.copy()\n",
    "normal = df_val_original[df_val_original['label']==1].sample(frac=0.05, replace=False, random_state=42) #0.054\n",
    "anomaly= df_val_original[df_val_original['label']==-1].sample(frac=0.05, replace=False, random_state=42)\n",
    "df_val_original = pd.concat([normal, anomaly]) \n",
    "print(len(df_val_original))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## SECTION 2: FUNCTIONS FOR PROCESSING"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Preprocessing for calculation, and preparing for onehot \n",
    "def convert_to_str(df):\n",
    "    convert_to_str_list = ['src_port', 'dst_port']\n",
    "    for column in convert_to_str_list: \n",
    "        df[column]= df[column].apply(str)\n",
    "\n",
    "    df_obj = df.select_dtypes(['object'])\n",
    "    df[df_obj.columns] = df_obj.apply(lambda x: x.str.strip())\n",
    "    \n",
    "def apply_stat(df):\n",
    "    df['pps'] = df['total_packets']/ df['duration']\n",
    "    df['bps_onedir'] = df['bytes_onedir']/ df['total_packets']\n",
    "    df['bpp_onedir'] = df['bytes_onedir']/ df['duration']\n",
    "    df['bps_bothdir'] = df['bytes_bothdir']/ df['total_packets']\n",
    "    df['bpp_bothdir'] = df['bytes_bothdir']/ df['duration']  \n",
    "    \n",
    "def preprocess(flag, df_original):\n",
    "    groupby_columns = ['src_ip', 'dst_ip', 'src_port', 'dst_port', 'protocol']\n",
    "    df_original[\"stream_id\"] = df_original.groupby(groupby_columns).grouper.group_info[0]\n",
    "    df_original = df_original.sort_values(by=['stream_id','timestamp'])\n",
    "    df_original['timestamp']=df_original['timestamp'].apply(pd.to_datetime)\n",
    "    #df['endtime'] = df['timestamp'] + pd.to_timedelta(df['duration'], unit='s')\n",
    "    df_original.loc[df_original['duration']==0, 'duration'] = 0.000001\n",
    "    #apply_addinfo(df)    \n",
    "    convert_to_str(df_original)    \n",
    "    stream_id_grouped = df_original.groupby(['stream_id'])    \n",
    "    stream_id = stream_id_grouped.grouper.group_info[0]\n",
    "    stream_id = set(stream_id)\n",
    "    newdf = pd.DataFrame(stream_id, columns=['stream_id'])\n",
    "    newdf['src_ip'] = stream_id_grouped['src_ip'].first()\n",
    "    newdf['dst_ip'] = stream_id_grouped['dst_ip'].first()\n",
    "    newdf['src_port'] = stream_id_grouped['src_port'].first()\n",
    "    newdf['dst_port'] = stream_id_grouped['dst_port'].first()\n",
    "    newdf['protocol'] = stream_id_grouped['protocol'].first()\n",
    "    \n",
    "    if flag == 'val':\n",
    "        newdf['label'] = stream_id_grouped['label'].mean()\n",
    "\n",
    "    return newdf, df_original"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 0 ns\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "## FOR FEATURE3: average cumulative features\n",
    "\n",
    "src_time_diff_dict = {}\n",
    "src_cnt_dict = {}\n",
    "dst_time_diff_dict = {}\n",
    "dst_cnt_dict = {}\n",
    "srcdst_time_diff_dict = {}\n",
    "srcdst_cnt_dict = {}\n",
    "\n",
    "def get_time_diff(df_original):\n",
    "\n",
    "    df_original['src_time_diff']= df_original.groupby('src_ip')['timestamp'].apply(lambda s: s.diff().dt.seconds < 60)\n",
    "    df_original['dst_time_diff']= df_original.groupby('dst_ip')['timestamp'].apply(lambda s: s.diff().dt.seconds < 60)\n",
    "    df_original['srcdst_time_diff']= df_original.groupby(['src_ip', 'dst_port'])['timestamp'].apply(lambda s: s.diff().dt.seconds < 30) \n",
    "    \n",
    "def groupby_sum(df, df_original):\n",
    "    fields = ['bytes_bothdir','bytes_onedir', 'total_packets', 'duration']\n",
    "\n",
    "    for field in fields:\n",
    "        df[field] = df_original.groupby(['stream_id'])[field].sum()\n",
    "    \n",
    "    return df\n",
    "\n",
    "def apply_avg(df, df_original):\n",
    "    df['pps'] = df['total_packets']/df['duration']\n",
    "    df['bps_onedir'] = df['bytes_onedir']/ df['total_packets']\n",
    "    df['bpp_onedir'] = df['bytes_onedir']/ df['duration']\n",
    "    df['bps_bothdir'] = df['bytes_bothdir']/ df['total_packets']\n",
    "    df['bpp_bothdir'] = df['bytes_bothdir']/ df['duration']    \n",
    "    return df\n",
    "\n",
    "def cnt_timed_src(df, df_original):  \n",
    "    src_time_diff_dict = df_original.groupby('src_ip')['src_time_diff'].sum().to_dict()\n",
    "    #dst_cnt_dict = df_original.groupby('src_ip')['dst_ip'].count().to_dict()\n",
    "\n",
    "    df['cnt_timed_src']=df.apply(lambda x: src_time_diff_dict[x.src_ip], axis=1)\n",
    "    #df['cnt_dst']=df.apply(lambda x: dst_cnt_dict[x.src_ip], axis=1)\n",
    "    return df\n",
    "\n",
    "def cnt_timed_dst(df, df_original ):\n",
    "    dst_time_diff_dict = df_original.groupby('dst_ip')['dst_time_diff'].sum().to_dict()\n",
    "    #src_cnt_dict = df_original.groupby('dst_ip')['src_ip'].count().to_dict()\n",
    "    df['cnt_timed_dst']=df.apply(lambda x: dst_time_diff_dict[x.dst_ip], axis=1)\n",
    "    #df['cnt_src']=df.apply(lambda x: src_cnt_dict[x.dst_ip], axis=1)\n",
    "    return df\n",
    "\n",
    "def get_dict(dictionary):\n",
    "    newdict={}\n",
    "    for (k1, k2), count in dictionary.items():\n",
    "        if k1 in newdict:\n",
    "            if count>0:\n",
    "                newdict[k1] +=1\n",
    "        else:\n",
    "            newdict[k1] = 0\n",
    "    return newdict\n",
    "    \n",
    "def cnt_timed_srcdst(df, df_original):    \n",
    "    srcdst_time_diff_dict = df_original.groupby(['src_ip', 'dst_port' ])['srcdst_time_diff'].sum().to_dict() \n",
    "    srcdst_time_diff_dict = get_dict(srcdst_time_diff_dict)\n",
    "    #srcdst_cnt_dict = df_original.groupby(['src_ip', 'dst_port'])['src_ip'].count().to_dict()\n",
    "    #srcdst_cnt_dict = get_dict(srcdst_cnt_dict)\n",
    "    df['cnt_timed_srcdst']=df.apply(lambda x: srcdst_time_diff_dict[x.src_ip], axis=1)\n",
    "    #df['cnt_srcdst']=df.apply(lambda x: srcdst_cnt_dict[x.src_ip], axis=1)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 996 µs\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "## ONE HOT CATEGORY FIELD\n",
    "\n",
    "def apply_onehot(df):       \n",
    "    ohe    = OneHotEncoder(handle_unknown='ignore')\n",
    "    \n",
    "    columns_to_encode = [ 'protocol', 'src_ip','src_port', 'dst_ip',  'dst_port'] #, \n",
    "    df_ohe_lst = [] \n",
    "    df_ohe_lst.append(df)\n",
    "    \n",
    "    for column in columns_to_encode:\n",
    "        print('>> encoding: ', column)\n",
    "        colvalues_count = df[column].value_counts()     \n",
    "        len_ccolvalues_count = 0\n",
    "\n",
    "        if column == 'src_ip':\n",
    "            len_ccolvalues_count = 10        \n",
    "        elif column == 'src_port':\n",
    "            len_ccolvalues_count = 10\n",
    "        elif column == 'dst_ip':\n",
    "            len_ccolvalues_count = 10     \n",
    "        elif column == 'dst_port':\n",
    "            len_ccolvalues_count = 10     \n",
    "        elif column == 'protocol':\n",
    "            len_ccolvalues_count = 10          \n",
    "        else:        \n",
    "            len_ccolvalues_count = len(colvalues_count)    \n",
    "\n",
    "        fit_cols = colvalues_count[:len_ccolvalues_count].index.tolist()    \n",
    "        df_fit_cols = pd.DataFrame(fit_cols)    \n",
    "        ohe.fit(df_fit_cols)\n",
    "\n",
    "        encoded_data = ohe.transform(df[column].values.reshape(-1,1)).toarray()    \n",
    "        dfOneHot = pd.DataFrame(encoded_data, columns = [column+str(int(i)) for i in range(encoded_data.shape[1])]) \n",
    "        df_ohe_lst.append(dfOneHot)    \n",
    "\n",
    "    new_df = pd.concat(df_ohe_lst, axis=1)        \n",
    "    #new_df = pd.concat(ohe_df, axis=1) \n",
    "    #new_df = ohe_df.drop(columns_to_encode, axis=1)\n",
    "\n",
    "    return new_df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 0 ns\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "#basic features: existing numeric columns + pps, bps, bpp\n",
    "def get_feature1(df_original):      \n",
    "    df_original['starttime']=df_original['timestamp'].apply(pd.to_datetime)\n",
    "    df_original['endtime'] = df_original['starttime'] + pd.to_timedelta(df_original['duration'], unit='s')\n",
    "    df_original.loc[df_original['duration']==0, 'duration'] = 0.000001   \n",
    "    convert_to_str(df_original)  \n",
    "    apply_stat(df_original)     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 0 ns\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "## adding to feature1 (+ onehot field)\n",
    "def get_feature2(df_original): \n",
    "    df_original = df_original.reset_index(drop=True)\n",
    "    new_df = apply_onehot(df_original)   \n",
    "\n",
    "    return new_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 0 ns\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "## cumulative featrues mostly by grouped by [src_ip, dst_ip, src_port, dst_port, protocol]\n",
    "def get_feature3(flag, df_original):           \n",
    "\n",
    "    df, df_original = preprocess(flag, df_original)\n",
    "    df[['starttime', 'endtime']] = df_original.groupby(['stream_id']).agg(starttime=('timestamp', 'min'), endtime=('timestamp', 'max'))    \n",
    "    df = groupby_sum(df, df_original)    \n",
    "    df = apply_avg(df, df_original)  \n",
    "    get_time_diff(df_original)\n",
    "    df = cnt_timed_src(df, df_original)\n",
    "    df = cnt_timed_dst(df, df_original)\n",
    "    df = cnt_timed_srcdst(df, df_original)   \n",
    "    print(flag , 'DONE')\n",
    "    \n",
    "    return df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 0 ns\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "## cumulative featrues mostly by grouped by [src_ip, dst_ip, src_port, dst_port, protocol]\n",
    "def get_feature4(flag, df_original):\n",
    "    new_df = get_feature3(flag, df_original)\n",
    "    new_df = new_df.reset_index(drop=True)\n",
    "    new_df = apply_onehot(new_df)             \n",
    "    \n",
    "    return new_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## SECTION 3:FEATURE GENERATION PART\n",
    "## !! ONLY RUN FOR FEATURES YOU WANT TO GENERATE (Feature 1,2,3,4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### FEATURE GENERATION"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## GENERATE \"FEATURE1\"!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train len:  27764\n",
      "validation len:  47003\n",
      "test len:  1053845\n",
      "DONE\n"
     ]
    }
   ],
   "source": [
    "df_train_f1 = df_train_original.copy()\n",
    "df_val_f1 = df_val_original.copy()\n",
    "df_test_f1 = df_test_original.copy()\n",
    "\n",
    "get_feature1(df_train_f1) \n",
    "print('train len: ', len(df_train_f1))\n",
    "\n",
    "get_feature1(df_val_f1) \n",
    "print('validation len: ', len(df_val_f1))\n",
    "\n",
    "get_feature1(df_test_f1) \n",
    "print('test len: ', len(df_test_f1))\n",
    "\n",
    "df_train = df_train_f1.copy()\n",
    "df_val = df_val_f1.copy()\n",
    "df_test = df_test_f1.copy()\n",
    "print('DONE')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## GENERATE FEATURE2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ">> encoding:  protocol\n",
      ">> encoding:  src_ip\n",
      ">> encoding:  src_port\n",
      ">> encoding:  dst_ip\n",
      ">> encoding:  dst_port\n",
      "train len:  27764\n",
      ">> encoding:  protocol\n",
      ">> encoding:  src_ip\n",
      ">> encoding:  src_port\n",
      ">> encoding:  dst_ip\n",
      ">> encoding:  dst_port\n",
      "validation len:  47003\n",
      ">> encoding:  protocol\n",
      ">> encoding:  src_ip\n",
      ">> encoding:  src_port\n",
      ">> encoding:  dst_ip\n",
      ">> encoding:  dst_port\n",
      "test len:  1053845\n",
      "DONE\n"
     ]
    }
   ],
   "source": [
    "df_train_f2 = df_train_f1.copy()\n",
    "df_val_f2 = df_val_f1.copy()\n",
    "df_test_f2 = df_test_f1.copy()\n",
    "\n",
    "df_train_f2 = get_feature2(df_train_f2) \n",
    "print('train len: ', len(df_train_f2))\n",
    "\n",
    "df_val_f2 = get_feature2(df_val_f2) \n",
    "print('validation len: ', len(df_val_f2))\n",
    "\n",
    "df_test_f2 = get_feature2(df_test_f2) \n",
    "print('test len: ', len(df_test_f2))\n",
    "\n",
    "df_train = df_train_f2.copy()\n",
    "df_val = df_val_f2.copy()\n",
    "df_test = df_test_f2.copy()\n",
    "print('DONE')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## GENERATE FEATURE3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train DONE\n",
      "train len:  51503\n",
      "val DONE\n",
      "validation len:  41073\n",
      "test DONE\n",
      "test len:  870431\n",
      "DONE\n",
      "Wall time: 5min 27s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "df_train_f3 = df_train_original.copy()\n",
    "df_val_f3 = df_val_original.copy()\n",
    "df_test_f3 = df_test_original.copy()\n",
    "\n",
    "df_train_f3 = get_feature3('train', df_train_f3) \n",
    "print('train len: ', len(df_train_f3))\n",
    "\n",
    "df_val_f3 = get_feature3('val', df_val_f3) \n",
    "print('validation len: ', len(df_val_f3))\n",
    "\n",
    "df_test_f3 = get_feature3('test', df_test_f3) \n",
    "print('test len: ', len(df_test_f3))\n",
    "\n",
    "df_train = df_train_f3.copy()\n",
    "df_val = df_val_f3.copy()\n",
    "df_test = df_test_f3.copy()\n",
    "print('DONE')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# GENERATE FEATURE 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train DONE\n",
      ">> encoding:  protocol\n",
      ">> encoding:  src_ip\n",
      ">> encoding:  src_port\n",
      ">> encoding:  dst_ip\n",
      ">> encoding:  dst_port\n",
      "train len:  26551\n",
      "val DONE\n",
      ">> encoding:  protocol\n",
      ">> encoding:  src_ip\n",
      ">> encoding:  src_port\n",
      ">> encoding:  dst_ip\n",
      ">> encoding:  dst_port\n",
      "validation len:  41073\n",
      "test DONE\n",
      ">> encoding:  protocol\n",
      ">> encoding:  src_ip\n",
      ">> encoding:  src_port\n",
      ">> encoding:  dst_ip\n",
      ">> encoding:  dst_port\n",
      "test len:  870431\n",
      "DONE\n"
     ]
    }
   ],
   "source": [
    "df_train_f4 = df_train_original.copy()\n",
    "df_val_f4 = df_val_original.copy()\n",
    "df_test_f4 = df_test_original.copy()\n",
    "\n",
    "df_train_f4 = get_feature4('train', df_train_f4) \n",
    "print('train len: ', len(df_train_f4))\n",
    "\n",
    "df_val_f4 = get_feature4('val', df_val_f4) \n",
    "print('validation len: ', len(df_val_f4))\n",
    "\n",
    "df_test_f4 = get_feature4('test', df_test_f4) \n",
    "print('test len: ', len(df_test_f4))\n",
    "\n",
    "df_train = df_train_f4.copy()\n",
    "df_val = df_val_f4.copy()\n",
    "df_test = df_test_f4.copy()\n",
    "print('DONE')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "26551\n",
      "870431\n",
      "41073\n",
      "41073\n"
     ]
    }
   ],
   "source": [
    "print(len(df_train))\n",
    "print(len(df_test))\n",
    "print(len(df_val))\n",
    "print(len(df_val[df_val['label']==1]) + len(df_val[df_val['label']==-1]) )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## SECTION 4:SCALER NUMERICS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['stream_id', 'src_ip', 'dst_ip', 'src_port', 'dst_port', 'protocol',\n",
       "       'label', 'starttime', 'endtime', 'bytes_bothdir', 'bytes_onedir',\n",
       "       'total_packets', 'duration', 'pps', 'bps_onedir', 'bpp_onedir',\n",
       "       'bps_bothdir', 'bpp_bothdir', 'cnt_timed_src', 'cnt_timed_dst',\n",
       "       'cnt_timed_srcdst'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_val.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature1 = ['duration', 'total_packets', 'bytes_bothdir', 'bytes_onedir', 'pps','bps_onedir', 'bpp_onedir', 'bps_bothdir', 'bpp_bothdir']\n",
    "\n",
    "feature2 = ['protocol0', 'protocol1', 'protocol2', 'protocol3',\n",
    "       'protocol4', 'src_ip0', 'src_ip1', 'src_ip2', 'src_ip3', 'src_ip4',\n",
    "       'src_ip5', 'src_ip6', 'src_ip7', 'src_ip8', 'src_ip9', 'src_port0',\n",
    "       'src_port1', 'src_port2', 'src_port3', 'src_port4', 'src_port5',\n",
    "       'src_port6', 'src_port7', 'src_port8', 'src_port9', 'dst_ip0',\n",
    "       'dst_ip1', 'dst_ip2', 'dst_ip3', 'dst_ip4', 'dst_ip5', 'dst_ip6',\n",
    "       'dst_ip7', 'dst_ip8', 'dst_ip9', 'dst_port0', 'dst_port1', 'dst_port2',\n",
    "       'dst_port3', 'dst_port4', 'dst_port5', 'dst_port6', 'dst_port7',\n",
    "       'dst_port8', 'dst_port9']\n",
    "\n",
    "feature2.extend(feature1)\n",
    "\n",
    "feature3 = ['bytes_bothdir', 'bytes_onedir', 'total_packets', 'duration', \n",
    "             'pps', 'bps_onedir', 'bpp_onedir', 'bps_bothdir', 'bpp_bothdir', \n",
    "             'cnt_timed_src', 'cnt_timed_dst', 'cnt_timed_srcdst']\n",
    "\n",
    "\n",
    "selected_features = feature2\n",
    "columns_to_scale = feature3\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 992 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "scaler = StandardScaler()\n",
    "\n",
    "def apply_scaler(df):\n",
    "    df_columns_to_scale = df[columns_to_scale]\n",
    "    train_scaled_features  = scaler.fit_transform(df_columns_to_scale) \n",
    "    train_scaled_features_df = pd.DataFrame( train_scaled_features, index=df_columns_to_scale.index, columns=columns_to_scale)\n",
    "\n",
    "    return train_scaled_features_df\n",
    "\n",
    "df_train[columns_to_scale] = apply_scaler(df_train)\n",
    "df_val[columns_to_scale] = apply_scaler(df_val)\n",
    "df_test[columns_to_scale] = apply_scaler(df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_ori = df_train[selected_features]\n",
    "X_val_ori = df_val[selected_features]\n",
    "X_test_ori = df_test[selected_features]\n",
    "val_label_ori = df_val['label']\n",
    "val_label = np.array(val_label_ori)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>protocol0</th>\n",
       "      <th>protocol1</th>\n",
       "      <th>protocol2</th>\n",
       "      <th>protocol3</th>\n",
       "      <th>protocol4</th>\n",
       "      <th>src_ip0</th>\n",
       "      <th>src_ip1</th>\n",
       "      <th>src_ip2</th>\n",
       "      <th>src_ip3</th>\n",
       "      <th>src_ip4</th>\n",
       "      <th>...</th>\n",
       "      <th>dst_port9</th>\n",
       "      <th>duration</th>\n",
       "      <th>total_packets</th>\n",
       "      <th>bytes_bothdir</th>\n",
       "      <th>bytes_onedir</th>\n",
       "      <th>pps</th>\n",
       "      <th>bps_onedir</th>\n",
       "      <th>bpp_onedir</th>\n",
       "      <th>bps_bothdir</th>\n",
       "      <th>bpp_bothdir</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.241746</td>\n",
       "      <td>-0.012116</td>\n",
       "      <td>-0.013608</td>\n",
       "      <td>-0.007135</td>\n",
       "      <td>-0.255804</td>\n",
       "      <td>-0.194463</td>\n",
       "      <td>-0.198838</td>\n",
       "      <td>-0.449123</td>\n",
       "      <td>-0.201012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.243130</td>\n",
       "      <td>-0.014583</td>\n",
       "      <td>-0.013861</td>\n",
       "      <td>-0.007307</td>\n",
       "      <td>-0.240470</td>\n",
       "      <td>-0.231256</td>\n",
       "      <td>-0.197383</td>\n",
       "      <td>-0.543864</td>\n",
       "      <td>-0.198536</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.243131</td>\n",
       "      <td>-0.014830</td>\n",
       "      <td>-0.013883</td>\n",
       "      <td>-0.007314</td>\n",
       "      <td>1.964145</td>\n",
       "      <td>-0.066624</td>\n",
       "      <td>0.127602</td>\n",
       "      <td>-0.595328</td>\n",
       "      <td>0.099577</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.234292</td>\n",
       "      <td>-0.013597</td>\n",
       "      <td>-0.013786</td>\n",
       "      <td>-0.007258</td>\n",
       "      <td>-0.255831</td>\n",
       "      <td>-0.267425</td>\n",
       "      <td>-0.198841</td>\n",
       "      <td>-0.541525</td>\n",
       "      <td>-0.201017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.552437</td>\n",
       "      <td>-0.014090</td>\n",
       "      <td>-0.013824</td>\n",
       "      <td>-0.007282</td>\n",
       "      <td>-0.255833</td>\n",
       "      <td>-0.257447</td>\n",
       "      <td>-0.198841</td>\n",
       "      <td>-0.543864</td>\n",
       "      <td>-0.201018</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 54 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   protocol0  protocol1  protocol2  protocol3  protocol4  src_ip0  src_ip1  \\\n",
       "0        0.0        0.0        0.0        0.0        0.0      0.0      0.0   \n",
       "1        0.0        0.0        0.0        0.0        0.0      0.0      0.0   \n",
       "2        0.0        0.0        0.0        0.0        0.0      0.0      0.0   \n",
       "3        0.0        0.0        0.0        0.0        0.0      0.0      0.0   \n",
       "4        0.0        0.0        0.0        0.0        0.0      0.0      0.0   \n",
       "\n",
       "   src_ip2  src_ip3  src_ip4  ...  dst_port9  duration  total_packets  \\\n",
       "0      0.0      0.0      0.0  ...        1.0 -0.241746      -0.012116   \n",
       "1      0.0      0.0      0.0  ...        0.0 -0.243130      -0.014583   \n",
       "2      0.0      0.0      0.0  ...        0.0 -0.243131      -0.014830   \n",
       "3      0.0      0.0      0.0  ...        0.0 -0.234292      -0.013597   \n",
       "4      0.0      0.0      0.0  ...        0.0  4.552437      -0.014090   \n",
       "\n",
       "   bytes_bothdir  bytes_onedir       pps  bps_onedir  bpp_onedir  bps_bothdir  \\\n",
       "0      -0.013608     -0.007135 -0.255804   -0.194463   -0.198838    -0.449123   \n",
       "1      -0.013861     -0.007307 -0.240470   -0.231256   -0.197383    -0.543864   \n",
       "2      -0.013883     -0.007314  1.964145   -0.066624    0.127602    -0.595328   \n",
       "3      -0.013786     -0.007258 -0.255831   -0.267425   -0.198841    -0.541525   \n",
       "4      -0.013824     -0.007282 -0.255833   -0.257447   -0.198841    -0.543864   \n",
       "\n",
       "   bpp_bothdir  \n",
       "0    -0.201012  \n",
       "1    -0.198536  \n",
       "2     0.099577  \n",
       "3    -0.201017  \n",
       "4    -0.201018  \n",
       "\n",
       "[5 rows x 54 columns]"
      ]
     },
     "execution_count": 128,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_ori.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## SECTION 5: CLUSTERING UTIL FUNCTIONS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_pc(X, num_components=2, tsne=False):\n",
    "    if tsne:\n",
    "        reduced_data = TSNE(n_components=num_components).fit_transform(X)\n",
    "    else:\n",
    "        pca = PCA(n_components=num_components, random_state=42).fit(X)\n",
    "        print(\"PCA.explained variance_ratio_ :\",sum(pca.explained_variance_ratio_) )\n",
    "        reduced_data = pca.transform(X)\n",
    "    \n",
    "    return reduced_data\n",
    "\n",
    "def save(df, name, mode=\"outputs\"):\n",
    "    output_columns = [ 'starttime', 'endtime', 'src_ip', 'dst_ip', 'src_port', 'dst_port', 'protocol' ]\n",
    "    df = df[output_columns]\n",
    "    df.to_csv(mode+\"/\" + name + \".csv\")\n",
    "    \n",
    "def get_anomalies(original_df, labels):\n",
    "    original_df['label'] = labels\n",
    "    return original_df[original_df.label==-1]\n",
    "\n",
    "def clustering_print_results(original_df, labels, features, X=None,print_out=True, plot_out=False,label=\"label\"):\n",
    "\n",
    "    print(\"\\nExporting \"+label.upper()+\"!\")\n",
    "    original_df['label'] = labels\n",
    "\n",
    "    # group clusters\n",
    "    clusters = original_df.groupby('label')\n",
    "    clusters_log=pd.DataFrame(columns=original_df.columns)\n",
    "    print(\"Number of clusters:\", len(set(labels)))\n",
    "    \n",
    "    if (plot_out):\n",
    "        original_df['x'] = X[:, 0] \n",
    "        original_df['y'] = X[:, 1] \n",
    "        \n",
    "        # colors\n",
    "        all_colors = [k for k,v in pltc.cnames.items()]\n",
    "        colors = sample(all_colors, len(clusters))\n",
    "        fig, ax = plt.subplots()\n",
    "        indexColor = 0\n",
    "        for key,cluster in clusters:\n",
    "            cluster.plot(ax=ax, kind='scatter', x='x', y='y', alpha=0.5, s=10,label='Label: {:d}'.format(key), color=colors[indexColor])\n",
    "            indexColor+=1\n",
    "\n",
    "    if (print_out):\n",
    "        for key, cluster in clusters:\n",
    "            print('\\Label {:d}: {:d} data points'.format(key, len(cluster)))\n",
    "            clusters_log=pd.concat([clusters_log,cluster.head(100)],sort=False)\n",
    "\n",
    "    print(\"\\n\"+\"DONE.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "## SECTION 6: HYPERPARAMETER TUNING "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "#### HYPERPARAM TUNING UTIL FUNCTIONS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def perf_measure(y_actual, y_hat):\n",
    "    TP = 0\n",
    "    FP = 0\n",
    "    TN = 0\n",
    "    FN = 0\n",
    "\n",
    "    for i in range(len(y_hat)): \n",
    "        if y_actual[i]==y_hat[i]==-1:\n",
    "           TP += 1\n",
    "        if y_hat[i]==-1 and y_actual[i]!=y_hat[i]:\n",
    "           FP += 1\n",
    "        if y_actual[i]==y_hat[i]==1:\n",
    "           TN += 1\n",
    "        if y_hat[i]==1 and y_actual[i]!=y_hat[i]:\n",
    "           FN += 1\n",
    "\n",
    "    return(TP, FP, TN, FN)\n",
    "\n",
    "def print_stats(y_pred):\n",
    "    cnt_pos=0\n",
    "    cnt_neg=0\n",
    "    for y in y_pred:\n",
    "        if y == 1:\n",
    "            cnt_pos+=1\n",
    "        else:\n",
    "            cnt_neg+=1\n",
    "    print('Total: ', len(y_pred))\n",
    "    print('Normal: ', cnt_pos)\n",
    "    print('Anomaly: ', cnt_neg)\n",
    "    \n",
    "def print_result(flag, model, y_true, y_pred, best_acc, best_score, best_param):\n",
    "    (TP, FP, TN, FN) = perf_measure(y_true, y_pred)    \n",
    "    print(TP, FP, TN, FN)    \n",
    "    TPR = TP/(TP+FN)\n",
    "    FPR = FP/(FP+TN)\n",
    "    TPR_FPR_DIFF = TPR - FPR   \n",
    "    accuracy = (TP + TN)/(TP + TN + FP + FN)\n",
    "\n",
    "    #param_dict = model.get_params()\n",
    "    if flag == 'ocsvm':\n",
    "        param_dict = {'kernel': model.get_params()['kernel'], 'gamma': model.get_params()['gamma'], 'nu': model.get_params()['nu']}\n",
    "    else:\n",
    "        param_dict = {'max_samples': model.get_params()['max_samples'], 'contamination': model.get_params()['contamination']}\n",
    "        \n",
    "    if accuracy> 0.88 and TPR_FPR_DIFF > best_score:\n",
    "        best_acc = accuracy\n",
    "        best_score = TPR_FPR_DIFF\n",
    "        best_param = param_dict\n",
    "    \n",
    "    print('TPR_FPR_DIFF: ', TPR_FPR_DIFF, 'accuracy: ', accuracy)\n",
    "    print()    \n",
    "    \n",
    "    return best_acc, best_score, best_param"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [],
   "source": [
    "##  TUNING "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "26551\n",
      "41073\n"
     ]
    }
   ],
   "source": [
    "X_train_tuning = X_train_ori.copy()\n",
    "X_val_tuning = X_val_ori.copy()\n",
    "val_label_tuning = val_label.copy()\n",
    "\n",
    "X_train_tuning = X_train_tuning.reset_index(drop=True)\n",
    "X_val_tuning = X_val_tuning.reset_index(drop=True)\n",
    "print(len(X_train_tuning))\n",
    "print(len(X_val_tuning))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>protocol0</th>\n",
       "      <th>protocol1</th>\n",
       "      <th>protocol2</th>\n",
       "      <th>protocol3</th>\n",
       "      <th>protocol4</th>\n",
       "      <th>src_ip0</th>\n",
       "      <th>src_ip1</th>\n",
       "      <th>src_ip2</th>\n",
       "      <th>src_ip3</th>\n",
       "      <th>src_ip4</th>\n",
       "      <th>...</th>\n",
       "      <th>dst_port9</th>\n",
       "      <th>duration</th>\n",
       "      <th>total_packets</th>\n",
       "      <th>bytes_bothdir</th>\n",
       "      <th>bytes_onedir</th>\n",
       "      <th>pps</th>\n",
       "      <th>bps_onedir</th>\n",
       "      <th>bpp_onedir</th>\n",
       "      <th>bps_bothdir</th>\n",
       "      <th>bpp_bothdir</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>-0.241746</td>\n",
       "      <td>-0.012116</td>\n",
       "      <td>-0.013608</td>\n",
       "      <td>-0.007135</td>\n",
       "      <td>-0.255804</td>\n",
       "      <td>-0.194463</td>\n",
       "      <td>-0.198838</td>\n",
       "      <td>-0.449123</td>\n",
       "      <td>-0.201012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.243130</td>\n",
       "      <td>-0.014583</td>\n",
       "      <td>-0.013861</td>\n",
       "      <td>-0.007307</td>\n",
       "      <td>-0.240470</td>\n",
       "      <td>-0.231256</td>\n",
       "      <td>-0.197383</td>\n",
       "      <td>-0.543864</td>\n",
       "      <td>-0.198536</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.243131</td>\n",
       "      <td>-0.014830</td>\n",
       "      <td>-0.013883</td>\n",
       "      <td>-0.007314</td>\n",
       "      <td>1.964145</td>\n",
       "      <td>-0.066624</td>\n",
       "      <td>0.127602</td>\n",
       "      <td>-0.595328</td>\n",
       "      <td>0.099577</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-0.234292</td>\n",
       "      <td>-0.013597</td>\n",
       "      <td>-0.013786</td>\n",
       "      <td>-0.007258</td>\n",
       "      <td>-0.255831</td>\n",
       "      <td>-0.267425</td>\n",
       "      <td>-0.198841</td>\n",
       "      <td>-0.541525</td>\n",
       "      <td>-0.201017</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.552437</td>\n",
       "      <td>-0.014090</td>\n",
       "      <td>-0.013824</td>\n",
       "      <td>-0.007282</td>\n",
       "      <td>-0.255833</td>\n",
       "      <td>-0.257447</td>\n",
       "      <td>-0.198841</td>\n",
       "      <td>-0.543864</td>\n",
       "      <td>-0.201018</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 54 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   protocol0  protocol1  protocol2  protocol3  protocol4  src_ip0  src_ip1  \\\n",
       "0        0.0        0.0        0.0        0.0        0.0      0.0      0.0   \n",
       "1        0.0        0.0        0.0        0.0        0.0      0.0      0.0   \n",
       "2        0.0        0.0        0.0        0.0        0.0      0.0      0.0   \n",
       "3        0.0        0.0        0.0        0.0        0.0      0.0      0.0   \n",
       "4        0.0        0.0        0.0        0.0        0.0      0.0      0.0   \n",
       "\n",
       "   src_ip2  src_ip3  src_ip4  ...  dst_port9  duration  total_packets  \\\n",
       "0      0.0      0.0      0.0  ...        1.0 -0.241746      -0.012116   \n",
       "1      0.0      0.0      0.0  ...        0.0 -0.243130      -0.014583   \n",
       "2      0.0      0.0      0.0  ...        0.0 -0.243131      -0.014830   \n",
       "3      0.0      0.0      0.0  ...        0.0 -0.234292      -0.013597   \n",
       "4      0.0      0.0      0.0  ...        0.0  4.552437      -0.014090   \n",
       "\n",
       "   bytes_bothdir  bytes_onedir       pps  bps_onedir  bpp_onedir  bps_bothdir  \\\n",
       "0      -0.013608     -0.007135 -0.255804   -0.194463   -0.198838    -0.449123   \n",
       "1      -0.013861     -0.007307 -0.240470   -0.231256   -0.197383    -0.543864   \n",
       "2      -0.013883     -0.007314  1.964145   -0.066624    0.127602    -0.595328   \n",
       "3      -0.013786     -0.007258 -0.255831   -0.267425   -0.198841    -0.541525   \n",
       "4      -0.013824     -0.007282 -0.255833   -0.257447   -0.198841    -0.543864   \n",
       "\n",
       "   bpp_bothdir  \n",
       "0    -0.201012  \n",
       "1    -0.198536  \n",
       "2     0.099577  \n",
       "3    -0.201017  \n",
       "4    -0.201018  \n",
       "\n",
       "[5 rows x 54 columns]"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## without PCA\n",
    "X_train_tuning.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PCA.explained variance_ratio_ : 0.8505886924542188\n",
      "PCA.explained variance_ratio_ : 0.8495007721308345\n"
     ]
    }
   ],
   "source": [
    "X_train_tuning = get_pc(X_train_tuning, 5)\n",
    "X_val_tuning = get_pc(X_val_tuning, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "now... >>  1 / 40\n",
      "index:  1 , kernel:  rbf , gamma:  scale , nu:  0.05\n",
      "38007 2619 183 264\n",
      "TPR_FPR_DIFF:  0.0584123189535779 accuracy:  0.9298079030019721\n",
      "\n",
      "now... >>  2 / 40\n",
      "index:  2 , kernel:  rbf , gamma:  scale , nu:  0.07\n",
      "38053 2673 129 218\n",
      "TPR_FPR_DIFF:  0.040342324827947085 accuracy:  0.9296131278455433\n",
      "\n",
      "now... >>  3 / 40\n",
      "index:  3 , kernel:  rbf , gamma:  scale , nu:  0.1\n",
      "38079 2682 120 192\n",
      "TPR_FPR_DIFF:  0.03780969897032638 accuracy:  0.9300270250529545\n",
      "\n",
      "now... >>  4 / 40\n",
      "index:  4 , kernel:  rbf , gamma:  scale , nu:  0.13\n",
      "38085 2627 175 186\n",
      "TPR_FPR_DIFF:  0.05759531218728242 accuracy:  0.9315121856207241\n",
      "\n",
      "now... >>  5 / 40\n",
      "index:  5 , kernel:  rbf , gamma:  scale , nu:  0.15\n",
      "37923 2422 380 348\n",
      "TPR_FPR_DIFF:  0.1265243691767216 accuracy:  0.9325591020865288\n",
      "\n",
      "now... >>  6 / 40\n",
      "index:  6 , kernel:  rbf , gamma:  0.01 , nu:  0.05\n",
      "23825 1764 1038 14446\n",
      "TPR_FPR_DIFF:  -0.007016287596676829 accuracy:  0.605336839286149\n",
      "\n",
      "now... >>  7 / 40\n",
      "index:  7 , kernel:  rbf , gamma:  0.01 , nu:  0.07\n",
      "23870 1791 1011 14401\n",
      "TPR_FPR_DIFF:  -0.015476436863510967 accuracy:  0.6057750833881138\n",
      "\n",
      "now... >>  8 / 40\n",
      "index:  8 , kernel:  rbf , gamma:  0.01 , nu:  0.1\n",
      "23593 1653 1149 14678\n",
      "TPR_FPR_DIFF:  0.02653624212808492 accuracy:  0.6023908650451635\n",
      "\n",
      "now... >>  9 / 40\n",
      "index:  9 , kernel:  rbf , gamma:  0.01 , nu:  0.13\n",
      "23293 1470 1332 14978\n",
      "TPR_FPR_DIFF:  0.08400790105187528 accuracy:  0.5995422783823924\n",
      "\n",
      "now... >>  10 / 40\n",
      "index:  10 , kernel:  rbf , gamma:  0.01 , nu:  0.15\n",
      "25985 1396 1406 12286\n",
      "TPR_FPR_DIFF:  0.18075807507565927 accuracy:  0.666885788717649\n",
      "\n",
      "now... >>  11 / 40\n",
      "index:  11 , kernel:  rbf , gamma:  0.03 , nu:  0.05\n",
      "23171 711 2091 15100\n",
      "TPR_FPR_DIFF:  0.35169805305418805 accuracy:  0.6150512502130353\n",
      "\n",
      "now... >>  12 / 40\n",
      "index:  12 , kernel:  rbf , gamma:  0.03 , nu:  0.07\n",
      "22965 671 2131 15306\n",
      "TPR_FPR_DIFF:  0.3605909048156903 accuracy:  0.6110096657171378\n",
      "\n",
      "now... >>  13 / 40\n",
      "index:  13 , kernel:  rbf , gamma:  0.03 , nu:  0.1\n",
      "22765 626 2176 15506\n",
      "TPR_FPR_DIFF:  0.37142497293476245 accuracy:  0.6072358970613299\n",
      "\n",
      "now... >>  14 / 40\n",
      "index:  14 , kernel:  rbf , gamma:  0.03 , nu:  0.13\n",
      "31766 445 2357 6505\n",
      "TPR_FPR_DIFF:  0.6712128264579043 accuracy:  0.8307890828524822\n",
      "\n",
      "now... >>  15 / 40\n",
      "index:  15 , kernel:  rbf , gamma:  0.03 , nu:  0.15\n",
      "31783 438 2364 6488\n",
      "TPR_FPR_DIFF:  0.6741552425878402 accuracy:  0.8313734083217685\n",
      "\n",
      "now... >>  16 / 40\n",
      "index:  16 , kernel:  rbf , gamma:  0.05 , nu:  0.05\n",
      "33898 1031 1771 4373\n",
      "TPR_FPR_DIFF:  0.5177844725855401 accuracy:  0.8684293818323473\n",
      "\n",
      "now... >>  17 / 40\n",
      "index:  17 , kernel:  rbf , gamma:  0.05 , nu:  0.07\n",
      "34104 951 1851 4167\n",
      "TPR_FPR_DIFF:  0.5517181732865644 accuracy:  0.8753925936746768\n",
      "\n",
      "now... >>  18 / 40\n",
      "index:  18 , kernel:  rbf , gamma:  0.05 , nu:  0.1\n",
      "33964 846 1956 4307\n",
      "TPR_FPR_DIFF:  0.5855332843532126 accuracy:  0.8745404523653008\n",
      "\n",
      "now... >>  19 / 40\n",
      "index:  19 , kernel:  rbf , gamma:  0.05 , nu:  0.13\n",
      "33164 598 2204 5107\n",
      "TPR_FPR_DIFF:  0.6531379365582664 accuracy:  0.8611009665717138\n",
      "\n",
      "now... >>  20 / 40\n",
      "index:  20 , kernel:  rbf , gamma:  0.05 , nu:  0.15\n",
      "33060 583 2219 5211\n",
      "TPR_FPR_DIFF:  0.655773793307807 accuracy:  0.8589340929564434\n",
      "\n",
      "now... >>  21 / 40\n",
      "index:  21 , kernel:  rbf , gamma:  0.07 , nu:  0.05\n",
      "36607 1609 1193 1664\n",
      "TPR_FPR_DIFF:  0.38228791213255053 accuracy:  0.9203126141260682\n",
      "\n",
      "now... >>  22 / 40\n",
      "index:  22 , kernel:  rbf , gamma:  0.07 , nu:  0.07\n",
      "36365 1480 1322 1906\n",
      "TPR_FPR_DIFF:  0.4220031302739726 accuracy:  0.9175614150415115\n",
      "\n",
      "now... >>  23 / 40\n",
      "index:  23 , kernel:  rbf , gamma:  0.07 , nu:  0.1\n",
      "36060 1397 1405 2211\n",
      "TPR_FPR_DIFF:  0.4436553482526311 accuracy:  0.9121564044506123\n",
      "\n",
      "now... >>  24 / 40\n",
      "index:  24 , kernel:  rbf , gamma:  0.07 , nu:  0.13\n",
      "35717 1284 1518 2554\n",
      "TPR_FPR_DIFF:  0.47502128542659006 accuracy:  0.9065566187032844\n",
      "\n",
      "now... >>  25 / 40\n",
      "index:  25 , kernel:  rbf , gamma:  0.07 , nu:  0.15\n",
      "35331 1156 1646 2940\n",
      "TPR_FPR_DIFF:  0.5106169755116741 accuracy:  0.9002751199084557\n",
      "\n",
      "now... >>  26 / 40\n",
      "index:  26 , kernel:  rbf , gamma:  0.1 , nu:  0.05\n",
      "37775 2487 315 496\n",
      "TPR_FPR_DIFF:  0.09945949535928178 accuracy:  0.9273732135466122\n",
      "\n",
      "now... >>  27 / 40\n",
      "index:  27 , kernel:  rbf , gamma:  0.1 , nu:  0.07\n",
      "37886 2464 338 385\n",
      "TPR_FPR_DIFF:  0.11056828633977778 accuracy:  0.9306356974167945\n",
      "\n",
      "now... >>  28 / 40\n",
      "index:  28 , kernel:  rbf , gamma:  0.1 , nu:  0.1\n",
      "37860 2431 371 411\n",
      "TPR_FPR_DIFF:  0.1216662226899039 accuracy:  0.9308061256786697\n",
      "\n",
      "now... >>  29 / 40\n",
      "index:  29 , kernel:  rbf , gamma:  0.1 , nu:  0.13\n",
      "37739 2325 477 532\n",
      "TPR_FPR_DIFF:  0.15633468115390536 accuracy:  0.9304409222603657\n",
      "\n",
      "now... >>  30 / 40\n",
      "index:  30 , kernel:  rbf , gamma:  0.1 , nu:  0.15\n",
      "37423 2147 655 848\n",
      "TPR_FPR_DIFF:  0.21160382926740695 accuracy:  0.9270810508119689\n",
      "\n",
      "now... >>  31 / 40\n",
      "index:  31 , kernel:  rbf , gamma:  0.13 , nu:  0.05\n",
      "38122 2740 62 149\n",
      "TPR_FPR_DIFF:  0.018233764760129167 accuracy:  0.9296618216346505\n",
      "\n",
      "now... >>  32 / 40\n",
      "index:  32 , kernel:  rbf , gamma:  0.13 , nu:  0.07\n",
      "38163 2740 62 108\n",
      "TPR_FPR_DIFF:  0.01930507201627607 accuracy:  0.9306600443113481\n",
      "\n",
      "now... >>  33 / 40\n",
      "index:  33 , kernel:  rbf , gamma:  0.13 , nu:  0.1\n",
      "38209 2739 63 62\n",
      "TPR_FPR_DIFF:  0.020863914435970132 accuracy:  0.9318043483553673\n",
      "\n",
      "now... >>  34 / 40\n",
      "index:  34 , kernel:  rbf , gamma:  0.13 , nu:  0.13\n",
      "38189 2715 87 82\n",
      "TPR_FPR_DIFF:  0.028906636023038 accuracy:  0.9319017359335817\n",
      "\n",
      "now... >>  35 / 40\n",
      "index:  35 , kernel:  rbf , gamma:  0.13 , nu:  0.15\n",
      "38180 2699 103 91\n",
      "TPR_FPR_DIFF:  0.03438167801059466 accuracy:  0.9320721641954569\n",
      "\n",
      "now... >>  36 / 40\n",
      "index:  36 , kernel:  rbf , gamma:  0.15 , nu:  0.05\n",
      "38144 2771 31 127\n",
      "TPR_FPR_DIFF:  0.007745086503291088 accuracy:  0.9294426995836681\n",
      "\n",
      "now... >>  37 / 40\n",
      "index:  37 , kernel:  rbf , gamma:  0.15 , nu:  0.07\n",
      "38207 2773 29 64\n",
      "TPR_FPR_DIFF:  0.008677465681043883 accuracy:  0.9309278601514377\n",
      "\n",
      "now... >>  38 / 40\n",
      "index:  38 , kernel:  rbf , gamma:  0.15 , nu:  0.1\n",
      "38224 2773 29 47\n",
      "TPR_FPR_DIFF:  0.009121666250665794 accuracy:  0.9313417573588488\n",
      "\n",
      "now... >>  39 / 40\n",
      "index:  39 , kernel:  rbf , gamma:  0.15 , nu:  0.13\n",
      "38232 2774 28 39\n",
      "TPR_FPR_DIFF:  0.008973813875653103 accuracy:  0.9315121856207241\n",
      "\n",
      "now... >>  40 / 40\n",
      "index:  40 , kernel:  rbf , gamma:  0.15 , nu:  0.15\n",
      "38244 2747 55 27\n",
      "TPR_FPR_DIFF:  0.01892334152298414 accuracy:  0.9324617145083145\n",
      "\n",
      "==========================\n",
      "best score:  0.5106169755116741  || best ACC:  0.9002751199084557  || best param:  {'kernel': 'rbf', 'gamma': 0.07, 'nu': 0.15}\n",
      "Wall time: 5min 10s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "from sklearn.metrics import  accuracy_score, f1_score, classification_report\n",
    "import itertools \n",
    "import time\n",
    "\n",
    "best_acc = 0\n",
    "best_score = 0\n",
    "ocsvm_best_param = None\n",
    "\n",
    "kernel =  ['rbf'] #poly, 'linear'\n",
    "gamma= ['scale', 0.01, 0.03, 0.05, 0.07, 0.1, 0.13, 0.15 ] #0.001 auto, 'scale',\n",
    "nu = [0.05, 0.07, 0.1, 0.13, 0.15]\n",
    "parameters = [kernel, gamma, nu]\n",
    "\n",
    "ocsvm_allConfgs = list(itertools.product(*parameters))\n",
    "counter = 0\n",
    "\n",
    "for conf in ocsvm_allConfgs:\n",
    "    _kernel,_gamma,_nu = conf    \n",
    "    counter+=1\n",
    "    print('now... >> ', counter , '/', len(ocsvm_allConfgs))\n",
    "    print('index: ', counter,', kernel: ', _kernel, ', gamma: ', _gamma, ', nu: ', _nu)    \n",
    "    model_ocsvm=svm.OneClassSVM(nu=_nu, kernel=_kernel, gamma=_gamma)  \n",
    "    \n",
    "    #start = time.time()\n",
    "    model_ocsvm.fit(X_train_tuning)    \n",
    "    #end = time.time()\n",
    "    #print(\"fit: \", end - start)\n",
    "    \n",
    "    #start = time.time()\n",
    "    y_pred = model_ocsvm.predict(X_val_tuning)\n",
    "    #end = time.time()\n",
    "    #print(\"predict: \", end - start)\n",
    "    \n",
    "    y_true = val_label_tuning\n",
    "    best_acc, best_score, ocsvm_best_param = print_result('ocsvm', model_ocsvm, y_true, y_pred, best_acc, best_score, ocsvm_best_param)\n",
    "        \n",
    "print(\"==========================\") \n",
    "print(\"best score: \", best_score, ' || best ACC: ', best_acc, ' || best param: ', ocsvm_best_param ) \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "now... >>  1 / 20\n",
      "index:  1 , max_samples:  auto , contamination:  auto\n",
      "14389 720 2082 23882\n",
      "TPR_FPR_DIFF:  0.11901727324187578 accuracy:  0.40101770019234045\n",
      "\n",
      "now... >>  2 / 20\n",
      "index:  2 , max_samples:  auto , contamination:  0.1\n",
      "7331 296 2506 30940\n",
      "TPR_FPR_DIFF:  0.08591613388056338 accuracy:  0.23950040172376014\n",
      "\n",
      "now... >>  3 / 20\n",
      "index:  3 , max_samples:  auto , contamination:  0.3\n",
      "35424 2069 733 2847\n",
      "TPR_FPR_DIFF:  0.1872083272695676 accuracy:  0.880310666374504\n",
      "\n",
      "now... >>  4 / 20\n",
      "index:  4 , max_samples:  auto , contamination:  0.5\n",
      "38271 2802 0 0\n",
      "TPR_FPR_DIFF:  0.0 accuracy:  0.9317800014608136\n",
      "\n",
      "now... >>  5 / 20\n",
      "index:  5 , max_samples:  0.2 , contamination:  auto\n",
      "14480 806 1996 23791\n",
      "TPR_FPR_DIFF:  0.09070269016347243 accuracy:  0.4011394346651085\n",
      "\n",
      "now... >>  6 / 20\n",
      "index:  6 , max_samples:  0.2 , contamination:  0.1\n",
      "15377 1019 1783 22894\n",
      "TPR_FPR_DIFF:  0.03812367195136096 accuracy:  0.41779271053977063\n",
      "\n",
      "now... >>  7 / 20\n",
      "index:  7 , max_samples:  0.2 , contamination:  0.3\n",
      "38271 2802 0 0\n",
      "TPR_FPR_DIFF:  0.0 accuracy:  0.9317800014608136\n",
      "\n",
      "now... >>  8 / 20\n",
      "index:  8 , max_samples:  0.2 , contamination:  0.5\n",
      "38271 2802 0 0\n",
      "TPR_FPR_DIFF:  0.0 accuracy:  0.9317800014608136\n",
      "\n",
      "now... >>  9 / 20\n",
      "index:  9 , max_samples:  0.4 , contamination:  auto\n",
      "14886 708 2094 23385\n",
      "TPR_FPR_DIFF:  0.1362862627882513 accuracy:  0.4134102695201227\n",
      "\n",
      "now... >>  10 / 20\n",
      "index:  10 , max_samples:  0.4 , contamination:  0.1\n",
      "15457 1045 1757 22814\n",
      "TPR_FPR_DIFF:  0.030934941206230304 accuracy:  0.41910744284566503\n",
      "\n",
      "now... >>  11 / 20\n",
      "index:  11 , max_samples:  0.4 , contamination:  0.3\n",
      "38271 2802 0 0\n",
      "TPR_FPR_DIFF:  0.0 accuracy:  0.9317800014608136\n",
      "\n",
      "now... >>  12 / 20\n",
      "index:  12 , max_samples:  0.4 , contamination:  0.5\n",
      "38271 2802 0 0\n",
      "TPR_FPR_DIFF:  0.0 accuracy:  0.9317800014608136\n",
      "\n",
      "now... >>  13 / 20\n",
      "index:  13 , max_samples:  0.6 , contamination:  auto\n",
      "14734 652 2150 23537\n",
      "TPR_FPR_DIFF:  0.15230031158943846 accuracy:  0.4110729676429771\n",
      "\n",
      "now... >>  14 / 20\n",
      "index:  14 , max_samples:  0.6 , contamination:  0.1\n",
      "15242 972 1830 23029\n",
      "TPR_FPR_DIFF:  0.05136992988748057 accuracy:  0.4156501838190539\n",
      "\n",
      "now... >>  15 / 20\n",
      "index:  15 , max_samples:  0.6 , contamination:  0.3\n",
      "38271 2802 0 0\n",
      "TPR_FPR_DIFF:  0.0 accuracy:  0.9317800014608136\n",
      "\n",
      "now... >>  16 / 20\n",
      "index:  16 , max_samples:  0.6 , contamination:  0.5\n",
      "38271 2802 0 0\n",
      "TPR_FPR_DIFF:  0.0 accuracy:  0.9317800014608136\n",
      "\n",
      "now... >>  17 / 20\n",
      "index:  17 , max_samples:  0.8 , contamination:  auto\n",
      "14661 628 2174 23610\n",
      "TPR_FPR_DIFF:  0.15895817257709682 accuracy:  0.40987996980985075\n",
      "\n",
      "now... >>  18 / 20\n",
      "index:  18 , max_samples:  0.8 , contamination:  0.1\n",
      "15618 1078 1724 22653\n",
      "TPR_FPR_DIFF:  0.02336447996780766 accuracy:  0.4222238453485258\n",
      "\n",
      "now... >>  19 / 20\n",
      "index:  19 , max_samples:  0.8 , contamination:  0.3\n",
      "38271 2802 0 0\n",
      "TPR_FPR_DIFF:  0.0 accuracy:  0.9317800014608136\n",
      "\n",
      "now... >>  20 / 20\n",
      "index:  20 , max_samples:  0.8 , contamination:  0.5\n",
      "38271 2802 0 0\n",
      "TPR_FPR_DIFF:  0.0 accuracy:  0.9317800014608136\n",
      "\n",
      "==========================\n",
      "best score:  0.1872083272695676  || best ACC:  0.880310666374504  || best param:  {'max_samples': 'auto', 'contamination': 0.3}\n",
      "Wall time: 58.6 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "from sklearn.ensemble import IsolationForest\n",
    "\n",
    "max_samples =  ['auto', 0.2, 0.4, 0.6, 0.8] #auto poly, 'linear'\n",
    "contamination = ['auto', 0.1, 0.3, 0.5 ] #auto 0.001\n",
    "#bootstrap = [True, False]\n",
    "\n",
    "parameters = [max_samples, contamination]\n",
    "\n",
    "iforest_allConfgs = list(itertools.product(*parameters))\n",
    "counter = 0\n",
    "\n",
    "best_acc = 0\n",
    "best_score = 0\n",
    "iforest_best_param = None\n",
    "\n",
    "for conf in iforest_allConfgs:\n",
    "    _max_samples, _contam = conf    \n",
    "    counter+=1\n",
    "    print('now... >> ', counter , '/', len(iforest_allConfgs))\n",
    "    print('index: ', counter,', max_samples: ', _max_samples, ', contamination: ', _contam)\n",
    "    iforest = IsolationForest(max_samples=_max_samples, contamination=_contam)\n",
    "    iforest.fit(X_train_tuning)\n",
    "    y_pred = iforest.predict(X_val_tuning)\n",
    "    y_true = val_label_tuning\n",
    "    \n",
    "    best_acc, best_score, iforest_best_param = print_result('iforest', iforest, y_true, y_pred, best_acc, best_score, iforest_best_param)\n",
    "        \n",
    "print(\"==========================\") \n",
    "print(\"best score: \", best_score, ' || best ACC: ', best_acc, ' || best param: ', iforest_best_param) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## SECTION 7: Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "####################### FEATURE 1 Analysis #######################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 470,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'kernel': 'rbf', 'gamma': 0.01, 'nu': 0.15}"
      ]
     },
     "execution_count": 470,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ocsvm_best_param"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 471,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_samples': 0.4, 'contamination': 0.1}"
      ]
     },
     "execution_count": 471,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "iforest_best_param"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 472,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PCA.explained variance_ratio_ : 0.7122985042171565\n",
      "PCA.explained variance_ratio_ : 0.6991558955054943\n"
     ]
    }
   ],
   "source": [
    "X_train = get_pc(X_train_ori, 3)\n",
    "X_val = get_pc(X_val_ori,3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 473,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_ocsvm = svm.OneClassSVM(**ocsvm_best_param)   \n",
    "ocsvm_label = best_ocsvm.fit(X_train).predict(X_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 245,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_iforest = IsolationForest(**iforest_best_param)\n",
    "iforest_label = best_iforest.fit(X_train).predict(X_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "OCSVM: =============\n",
      "Total:  50763\n",
      "Normal:  3207\n",
      "Anomaly:  47556\n",
      "IFOREST: ============\n",
      "Total:  50763\n",
      "Normal:  12132\n",
      "Anomaly:  38631\n"
     ]
    }
   ],
   "source": [
    "y_pred_ocsvm = ocsvm_label \n",
    "y_pred_iforest = iforest_label \n",
    "print('Validation data: ')\n",
    "print('OCSVM: =============')\n",
    "print_stats(y_pred_ocsvm) \n",
    "print()\n",
    "print('IFOREST: ============')\n",
    "print_stats(y_pred_iforest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=========== OCSVM: FEATURE 1 ===========\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          -1       0.95      0.95      0.95     47581\n",
      "           1       0.26      0.26      0.26      3182\n",
      "\n",
      "    accuracy                           0.91     50763\n",
      "   macro avg       0.60      0.60      0.60     50763\n",
      "weighted avg       0.91      0.91      0.91     50763\n",
      "\n",
      "| TP:  45200 | FP:  2356 | TN:  826 | FN:  2381\n",
      "TPR:  0.9499590172547866\n",
      "FPR:  0.7404148334380892\n",
      "TPR - FPR :  0.20954418381669737\n"
     ]
    }
   ],
   "source": [
    "print('=========== OCSVM: FEATURE 1 ===========')\n",
    "print(classification_report(y_true, y_pred_ocsvm))\n",
    "(TP, FP, TN, FN) = perf_measure(y_true, y_pred_ocsvm)\n",
    "TPR = TP/(TP+FN)\n",
    "FPR = FP/(FP+TN)\n",
    "\n",
    "print('| TP: ', TP, '| FP: ', FP, '| TN: ', TN, '| FN: ', FN)\n",
    "print('TPR: ', TPR)\n",
    "print('FPR: ', FPR)\n",
    "print('TPR - FPR : ', TPR - FPR )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=========== IFOREST FEATURE 1 ===========\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          -1       0.95      0.77      0.85     47581\n",
      "           1       0.09      0.35      0.15      3182\n",
      "\n",
      "    accuracy                           0.74     50763\n",
      "   macro avg       0.52      0.56      0.50     50763\n",
      "weighted avg       0.89      0.74      0.80     50763\n",
      "\n",
      "| TP:  36566 | FP:  2065 | TN:  1117 | FN:  11015\n",
      "TPR:  0.7685000315251886\n",
      "FPR:  0.6489629164047769\n",
      "TPR - FPR :  0.11953711512041176\n"
     ]
    }
   ],
   "source": [
    "print('=========== IFOREST FEATURE 1 ===========')\n",
    "print(classification_report(y_true, y_pred_iforest))\n",
    "(TP, FP, TN, FN) = perf_measure(y_true, y_pred_iforest)\n",
    "TPR = TP/(TP+FN)\n",
    "FPR = FP/(FP+TN)\n",
    "\n",
    "print('| TP: ', TP, '| FP: ', FP, '| TN: ', TN, '| FN: ', FN)\n",
    "print('TPR: ', TPR)\n",
    "print('FPR: ', FPR)\n",
    "print('TPR - FPR : ', TPR - FPR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Findings ####\n",
    "1.When hyperparams were set in a small set, but trained on a largeset with the very hyperparams, no use at all\n",
    "The result is as below, hyperparams for trained train:validset (138820, 94006)\n",
    "    >> although the best result was :\n",
    "                best score:  0.509059860741818  \n",
    "                || best ACC:  0.882884799489416  \n",
    "                || best param:  {'kernel': 'rbf', 'gamma': 0.01, 'nu': 0.2}\n",
    "\n",
    "but actually trained on (13882, 9401) with the same best param...\n",
    "  \n",
    "2. PCA - more generalisaion with TPR-FPR\n",
    "===========OCSVM\n",
    "              precision    recall  f1-score   support\n",
    "\n",
    "          -1       0.94      0.99      0.97     88158\n",
    "           1       0.40      0.11      0.17      5848\n",
    "\n",
    "    accuracy                           0.93     94006\n",
    "   macro avg       0.67      0.55      0.57     94006\n",
    "weighted avg       0.91      0.93      0.92     94006\n",
    "\n",
    "87226 5216 632 932\n",
    "TPR:  0.9894280723246898\n",
    "FPR:  0.8919288645690835\n",
    "TPR - FPR :  0.09749920775560628"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PCA.explained variance_ratio_ : 0.9328201326385201\n"
     ]
    }
   ],
   "source": [
    "X_test = df_test[selected_features]\n",
    "X_test = get_pc(X_test, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 267,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=======>>>OCSVM CLUSTER RESULT\n",
      "\n",
      "Exporting LABEL...\n",
      "Number of clusters: 2\n",
      "\\Label -1: 1035114 data points\n",
      "\\Label 1: 18731 data points\n",
      "\n",
      "DONE.\n",
      "--------------------------------------\n",
      "Wall time: 1min\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEGCAYAAACKB4k+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAAAfZklEQVR4nO3df3Dc9Z3f8edbq9VvGXktYYxlW3ZsQoCAXSQwB8kkQAc3YeB6HSZOm4S2zDDTQC7X3OQS95qGZkovQzI3uQxHGwJpuZAew+Ryg3PJNRAIlyHlh+1gkoB/Kf6BFcu25LUsrX6upHf/2N1vVj+9kve7P+TXY8Yj7ff73d33GvN97ffz62vujoiICEBFsQsQEZHSoVAQEZGAQkFERAIKBRERCSgUREQkUFnsAi5Ec3Ozt7W1FbsMEZGysmfPnl53b5ltX1mHQltbG7t37y52GSIiZcXMjs21T81HIiISUCiIiEhAoSAiIoGy7lMQkYtDMpmkq6uLkZGRYpdSVmpqamhtbSUajeb8HIWCiJS8rq4uGhsbaWtrw8yKXU5ZcHfOnDlDV1cX69evz/l5aj4SkZI3MjLCihUrFAgLYGasWLFiwVdXulIQkVkNxuMM9fVR19REfSxW7HIUCIuwmL8zhYKIzDAYj3N01y7MDHenraOjJIJBwqfmIxGZYaivDzOjprERM2Oor6/YJRVdQ0NDzsc+9NBDfP3rXw/t9TO2bdtGU1MTd95554KfOxeFgojMUNfUhLszMjCAu1PX1FTskmQWn//85/nud7+b19dUKIjIDPWxGG0dHbRs3Kimo3n88Ic/5MYbb2TLli3cfvvtnDp1Ktj31ltvceutt7Jp0ya+/e1vB9u/9rWv0dHRwbXXXsuXv/zlC3r/2267jcbGxgt6jekUCiIyq/pYjJYNG8o2EAbjcXoOH2YwHg/tPW655RZee+013nzzTbZv384jjzwS7PvVr37Fj370I1599VW+8pWvcOLECZ5//nkOHTrEG2+8wd69e9mzZw8///nPZ7zu5s2bQ6v5fNTRLCJLTqE6yru6uvjYxz5Gd3c3Y2NjU+YD3H333dTW1lJbW8uHP/xh3njjDV555RWef/55tmzZAkAikeDQoUN88IMfnPK6e/fuzXutuQr1SsHM/qOZvW1mvzGzvzWzGjOLmdkLZnYo/XN51vE7zKzTzA6Y2R1h1iYiS1ehOso/85nP8OCDD/LrX/+ab33rW1PmBEwfDpoJqB07drB371727t1LZ2cn9913X07v9frrr7N582Y2b97Mzp078/o5soUWCma2GvhjoN3drwEiwHbgi8CL7r4JeDH9GDO7Kr3/amAb8JiZRcKqT0SWrkJ1lJ87d47Vq1cD8NRTT03Z99xzzzEyMsKZM2d4+eWX6ejo4I477uA73/kOiUQCgN/97necPn06p/e68cYbgzC566678vtBsoTdfFQJ1JpZEqgDTgA7gA+l9z8FvAx8AbgbeMbdR4EjZtYJ3AC8GnKNIrLEZDrK8zn5bmhoiNbW1uDx5z73OR566CHuueceVq9ezdatWzly5Eiw/4YbbuCjH/0o7777Ll/60pe4/PLLufzyy9m3bx833XQTkBqG+vTTT3PppZdOea/Nmzfn1IT0gQ98gP3795NIJGhtbeXJJ5/kjjsurJHF3P2CXmDeFzf7LPAwMAw87+7/xsz63L0p65iz7r7czB4FXnP3p9PbnwT+0d2/P+017wfuB1i7du31x47Nea8IEVki9u3bx/ve975il1GWZvu7M7M97t4+2/FhNh8tJ/Xtfz1wOVBvZp+Y7ymzbJuRWO7+uLu3u3t7S8usd5MTEZFFCrOj+XbgiLv3uHsS+AHwB8ApM1sFkP6ZaVDrAtZkPb+VVHOTiIgUSJih8C6w1czqLNUNfxuwD9gJ3Js+5l7gufTvO4HtZlZtZuuBTcAbIdYnIiLThNbR7O6vm9n3gV8C48CbwONAA/Csmd1HKjjuSR//tpk9C7yTPv4Bd58Iqz4REZkp1NFH7v5lYPo87lFSVw2zHf8wqY5pEREpAi1zISIiAYWCiEgOSmHp7EcffZSNGzdiZvT29i7o9XOlUBARKRM333wzP/3pT1m3bl1o76FQEBFZpEIvnb1lyxba2tryVf6sFAoisiQlDh/m1EsvkTh8OLT30NLZIiJlIHH4MIceewyLRPCJCTZ9+tM0bNiQ9/fR0tkiImVg8OhRLBKh9vLLsUiEwaNHQ3mfQi6dXSgKBRFZcurb2vCJCYZPnMAnJqgPqR2+kEtnF4qaj0RkyWnYsIFNn/40g0ePUt/Wlpemo1JYOvub3/wmjzzyCCdPnuTaa6/lIx/5CE888cQFf7ZsoS6dHbb29nbfvXt3scsQkZBp6ezFK5mls0VEpPwoFEREJKBQEJGyUM5N3cWymL8zhYKIlLyamhrOnDmjYFgAd+fMmTPU1NQs6HkafSQiJa+1tZWuri56enqKXUpZqampmTJiKhcKBREpedFodMpsYQmPmo9ERCSgUBARkYBCQUREAgoFEREJKBRERCSgUBARkYCGpIqIlJnBeJyhvj7qmpqoj8Xy+toKBRGRMjIYj3N0167gpj1tHR15DQY1H4mIlJGhvj7MjJrGRsyMob6+vL6+QkFEpIzUNTXh7owMDODu1DU15fX11XwkIlJG6mMx2jo61KcgIiIp9bFY3sMgQ81HIiISUCiIiEhAoSAiIgGFgoiIBBQKIiISUCiIiEhAQ1JFRAoozHWL8kGhICJSIGGvW5QPaj4SESmQsNctygeFgohIgYS9blE+hBoKZtZkZt83s/1mts/MbjKzmJm9YGaH0j+XZx2/w8w6zeyAmd0RZm0iIoWWWbeoZePGkmw6gvCvFP4K+L/ufiVwHbAP+CLwortvAl5MP8bMrgK2A1cD24DHzCwScn0iIgVVH4vRsmFDSQYChBgKZrYM+CDwJIC7j7l7H3A38FT6sKeAP0z/fjfwjLuPuvsRoBO4Iaz6RERkpjCvFDYAPcD/MrM3zewJM6sHVrp7N0D656Xp41cDx7Oe35XeJiIiBRJmKFQC/wz4H+6+BRgk3VQ0B5tlm884yOx+M9ttZrt7enryU6mIiADhhkIX0OXur6cff59USJwys1UA6Z+ns45fk/X8VuDE9Bd198fdvd3d21taWkIrXkTkYhRaKLj7SeC4mb03vek24B1gJ3Bvetu9wHPp33cC282s2szWA5uAN8KqT0REZgp7RvNngO+ZWRVwGPh3pILoWTO7D3gXuAfA3d82s2dJBcc48IC7T4Rcn4iIZAk1FNx9L9A+y67b5jj+YeDhMGsSEZG5aUaziIgEFAoiIhJQKIiISEChICIiAYWCiIgEFAoiIhJQKIiISEChICIiAYWCiIgEFAoiIhJQKIiISEChICIiAYWCiIgEFAoiIhJQKIiISEChICIiAYWCiIgEFAoiIhJQKIiISEChICIiAYWCiIgEFAoiIhJQKIiISEChICIiAYWCiIgEFAoiIhJQKIiISEChICIiAYWCiIgEFAoiIhJQKIiISEChICIiAYWCiIgEFAoiIhJQKIiISEChICIiAYWCiIgEzhsKZvagmS0vRDEiIlJcuVwpXAbsMrNnzWybmVnYRYmISHGcNxTc/T8Dm4AngX8LHDKz/25m78nlDcwsYmZvmtk/pB/HzOwFMzuU/rk869gdZtZpZgfM7I5FfSIREVm0nPoU3N2Bk+k/48By4Ptm9kgOT/8ssC/r8ReBF919E/Bi+jFmdhWwHbga2AY8ZmaRHD+HiIjkQS59Cn9sZnuAR4BfAO939/8AXA/8q/M8txX4KPBE1ua7gafSvz8F/GHW9mfcfdTdjwCdwA25fxQREblQlTkc0wz8kbsfy97o7pNmdud5nvsN4M+AxqxtK929O/0a3WZ2aXr7auC1rOO60tumMLP7gfsB1q5dm0P5IiKSq1z6FP7L9EDI2rdvtu0A6cA47e57cqxltg5sn+U9H3f3dndvb2lpyfGlRUQkF7lcKSzWzcBdZvYRoAZYZmZPA6fMbFX6KmEVcDp9fBewJuv5rcCJEOsTEZFpQpu85u473L3V3dtIdSC/5O6fAHYC96YPuxd4Lv37TmC7mVWb2XpSI57eCKs+ERGZKcwrhbl8FXjWzO4D3gXuAXD3t83sWeAdUiOcHnD3iSLUJyIlaDAeZ6ivj7qmJupjsWKXs2RZarRpeWpvb/fdu3cXuwwRCdlgPM7RXbswM9ydto4OBcMFMLM97t4+2z6tfSQiJW+orw8zo6axETNjqK+v2CUtWQoFESl5dU1NuDsjAwO4O3VNTcUuackqRp+CiMiC1MditHV0qE+hABQKIlIW6mMxhUEBqPlIREQCCgUREQkoFEREJKBQEBGRgEJBREQCCgUREQkoFEREJKBQEBGRgEJBREQCCgUREQkoFEREJKBQEBGRgEJBREQCCgUREQkoFEREJKBQEBGRgEJBREQCuvOayEVqMB7X7S1lBoWCyEVoMB7n6K5dmBnuTltHh4JBADUfiVyUhvr6MDNqGhsxM4b6+opdkpQIXSmIlIhCNufUNTXh7owMDODu1DU1hfp+Uj4UCiIloNDNOfWxGG0dHepTkBkUCiIlILs5Z2RggKG+vtBP1PWxmMJAZlCfgkgJUHOOlApdKYiUADXnSKlQKIiUCDXnSClQ85GIiAQUCiIiElAoiIhIQKEgIiIBhYKIiAQUCiIiEtCQVJGLnJbQlmwKBZGLmJbQlulCaz4yszVm9jMz22dmb5vZZ9PbY2b2gpkdSv9cnvWcHWbWaWYHzOyOsGoTkRQtoS3ThdmnMA78qbu/D9gKPGBmVwFfBF50903Ai+nHpPdtB64GtgGPmVkkxPpELnpac0mmC635yN27ge707wNmtg9YDdwNfCh92FPAy8AX0tufcfdR4IiZdQI3AK+GVaPIxU5rLsl0BelTMLM2YAvwOrAyHRi4e7eZXZo+bDXwWtbTutLbpr/W/cD9AGvXrg2xapHSEHZHsNZckmyhh4KZNQB/B/yJu/eb2ZyHzrLNZ2xwfxx4HKC9vX3GfpGlIBMEZsapgwfVESwFE2oomFmUVCB8z91/kN58ysxWpa8SVgGn09u7gDVZT28FToRZn0gpyh4RNNzfT2V1NY3NzQW7+Y5c3MIcfWTAk8A+d//LrF07gXvTv98LPJe1fbuZVZvZemAT8EZY9YmUquwRQdHqasZHR9URLAUT5pXCzcAngV+b2d70tv8EfBV41szuA94F7gFw97fN7FngHVIjlx5w94kQ6xMpSXVNTSRHRhgeGKAyGmXd9dcHgaCrBAlbmKOPXmH2fgKA2+Z4zsPAw2HVJFJOMv/z1F5yyXnDQLOSJV80o1mkxAz19RGtqaGxpSWnfgTNSpZ80oJ4IgU0GI/Tc/gwg/H4nMdkTyhLjowwkkjMe7xmJUs+KRRECiTzjb6ns5Oju3bNeaKvj8VYecUVWCTC6OAgPYcO0fnKK3Mer1nJkk9qPhIpkOxv9PM1Cw3G45w6eJDBs2cZjMeprq/H3Yl3dc16vGYlSz4pFEQKJNdv9JnwiFZXA+CTk2AGPvdcTc1KlnxRKIgUSK7f6DPhEamsJFpdTbSujuq6OmJr1sx6vEg+KRRECiiXb/TZ4XH5NddojoIUlEJBpASpOUiKRaEgEjJNLJNyolAQCZEmlkm50TwFkRCVwsSyXCbMiWToSkEKptybURZTf7EnlulKRRZKoSAFUe4np8XWX+yJZblOmBPJUPORFEQpNKNciAupvz4Wo2XDhqKcjIt9pSLlR1cKUhDFOjkNxuPEjx8HM2obG6eM+V9Ic1Cm/kRvL8nRUea6rWypNZEV+0pFyo9CQQoinyen7BMvMOU1s/cNnztH5y9+wUh/P5illo6oqcHdWbZyJYkzZ6iuq6O6oeG8zUGZReqO7dlDZXU1pw4enHGfg0I1kS00eDTnQRZCoSAFk4+T0/T7F0+Mj1MZjTI2NIS7M3zuHO6eCgH31LpBWcZHRgAYOXcOq6hgLJEAyKmt3d2pXbZszvb5QrTfl3vfjJQ+hYKUlcyJt6KyksF4HJ+cZHJ8nMnx8RnHzr18XHp/OjRGh4Zyas46XxNYIZrI1HEsYVMoSFnJnHgHz57FKiowM8ZHRy/oNRtWrJixLbsvItbaGlzlzNcEVoj2e3UcS9jM51mOt9S1t7f77t27i12GFFjmhB0/doyJZJL+kyeZGB+fc2npispKKioria1dy5mjR8n8m49Eo8HPhhUr2HjLLUG/ROcrr5A4cwZgyr656ilkR26pdWZL+TGzPe7ePts+XSlI2cl8a4+tWRM0p5w7eZKzx48zOjjI5MQEFZEIdU1NXHLZZUy607RqFc3r19N75Ah93d2p55w4QXJ4GIDxZDJoihnq62M8maQyHRoTWfumK0YbvzqOJUwKBSlb2SfH5vXr4aabzvuc5vXraV6/nsF4nERPD8P9/QDULFsWNMXUNTVRGY2mRi1N2zed2vhlqVEoyEWpPhZj4y23EO/qAndia9YEJ/PMvpMHDjA2NDTvxDO18ctSo1CQi9b5mmFG+vsxs1nnJGS/hiaHyVKiZS5kyZpvddDZ9mVvW8iyFpllLACtRiplT1cKMq/pM4T7urupaWiguqGhpL8Zz9cBPNs+gM5XXmEimSQSjbL6/e9fULOQJpXJUqFQkFkNxuPEu7qIHz1KtKaGgdOnGTp3jopIhMmJCS657LKclocolvk6gLP3JXp76d6/HzMjceYMldEow/39DA8MLKhZaKEdzhpWKqVKoSAzZL71jiYSDPf3U798eWrMvjuTySRUVDA5MRE0q4SxlMOFnjDn6wDOXtxuoLcXB0YTCSYnJiAaZXJigkRvL7HW1qBZKNf3G+jtZXyeBfMAeo8cCdZQqqyqKtlglYuT+hRkhsy33rrlywFInDkz9SQ3OYlPToYy2iYTSD2dnRzdtWvR7fOZDuCWjRtnnHQzi9tRUUHtsmU0NjdT3dBAdX09VlmJAcNnz7L/pZfoPXIk5/dbecUVjI+OEk0vmDdXX8axPXsYHRhg+Nw5xsfGym4ZcVnaFAoyQ+Zb7+T4OA0rVtC8fj2RqiosEkmdSJuaWL52bSjfcLObYSbGxujev/+CgmG24aSD8TinDh6EyUmG+/sZ6O2lsqqKDVu3Elu7lrrly0kODzM6MMCxPXtyfv/MgnkNzc1zdk4P9fURra4mEo0ymUwyPjqqYaxSUtR8tETks406e5hlpuO0qraW07/9LVW1tVQ3NLDqyitDXdsnu2nn6K5dOQfQfH8PmX0jiQRmRkNzc+o9V6wIPk/tJZdw7sQJJpJJKqJRKqurc24iy2XOQl1TE5GqKmovuYTk6Cjrrr9eTUdSUhQKS0AYI18yz89+3Y033zzlJjVhyARS9/79ONDY3JzzTOFcRxwl08tnA0SqqqYEXH0sxrrrr5/S5p/rN/lc5ixoXoOUOoXCEhDWUgvTX9fdc+54nU2uVzP1sRirrrySo7t2LWimcK4jjpKjo1TV19OwYsWUmczZ9V15662LOnHPNSFu+me/0KsekbAoFJaAsJZayOfrLuRqJnMyXHnFFQu6Mpmv3sxNeRLxOCP9/TSsWMFkMklszZo568sEYPZktlzqySy6l1mEbzFXcpkRStHqaiIaoSQFpFAoM7N9ewyrSSKfr5vr1cyFNIVNrxdSM4wzS1WYGUNnz1JRWUlyeJjKqqopK6POVl9mGe3RoSHGEgmWrVw570m698gRDrz8MgacOnAASHVAL3QOQ2aE0vjICLWXXKKF9qRgFAplZL4TZr5udTlb4OTjZJTrVUe+msKGz53j1MGDwZDP6ro6orW1qVFUpJbDTmaN/JmrvpMHDnCuuxurqGB8bCx1+895Op/7ursxoLqhgdFEgr7ublZdeeWCrriG+vqorK4mOTIyo06RsCkUykhYfQfTZy+HsUxDrlcduYZHpokms+RGpnkoc+OdzIkVIDk8zMjAAA2xGBUVFdQsWwbuU0b+zHaVcfytt+jp7Pz9rT7dGR8dpSISYWRggMF4fMbnaFq1ilMHDjCaSODpxwu94qpraqIyPUJpXCOUpMB057USNhiPc3zvXhK9vTQ0N9O8fn3QDLLQE/f0q4DBeJyTBw7Qf/Iko4kEkWiUseFhVqxbx+jgIPVZwzQLLbtWYMbJtPfIEfa9+CKTySQT4+NEa2pIjoxgFRWpSXUTE3O/uFnqz+QkFdEo0bq64HWqamupW76ciWSSgd7eVBhMThKpqqIiEqGqro6W97yHRE8PE8kko0NDrLnuOlqvvXbKW0zvU7jQvwMFguTbfHdeUyiUqMF4nHeef57+kyeDbctWrmT91q05d75mTk4VZvQePYqZUVVXx4q2Nt795S8ZzppcFYlGsYoKqhsbmUgmaWxuntJ2PtdJKh8nwNleJ/vqBTMSp0+THBsDUm30k+nfC6myro5ll17KaCLBYDyOmRGJRmm97joue+97dfKWslFWt+M0s23AXwER4Al3/2oh33+2b9SL+cZ2vm+75zPU18fo4GDqW206uEcHB6cMC53tPTK3puw/dYq+rq7fN31k6T18eMa2iWQSi0SYHB+ndtkyqurr6TtxgoP/9E+0vOc9qXb1aauKnjxwgO6336YiEgk6VTPBkDmpjw0OAlBVV0ftsmW4O6OJBGe7uhgfG6N22TLOnTzJYG8v7s6xEv6SMj40lAqpNAfGJyfp+e1vGenvn/XKTd/4pdyU1JWCmUWAg8A/B7qAXcDH3f2d2Y6/kCuFF7/xjUVWWVqGHn10zhvWX3RiMWzFChgbw4eGoK6O6IYNWH09FU1NVKTXcsq3hksvpbaxkVVXXz0lsOPHjxM/dmxR/TS5hEn2Mb/8wQ8YSyTy9pkWqy4W46ZPfWrO/YPxOEdeey21zEiZqIhGueHjH19SoV5OVwo3AJ3ufhjAzJ4B7gZmDYXFUiAsUfE4Pm2douTx49jq1VTU1xNtbw8lGBKnT5McGuLya64Bfj9KbCSRYKS/nxXr1jE5Pp7zwIBchuVmH3Py4EF8livCYhiKx3n1b/5m1mAYjMf5zY9/TKK3twiVLd5kMslr3/0uWz/5ySUVDHMptQXxVgPHsx53pbcFzOx+M9ttZrt7enoKWlzJUSCc3+QkTEyAGZMhrkZa1dBA5qo704xXnw6gobNnFzT5L5e7vmUfUyqBkDE0xwKCQ319jKSbE8uO+0Wzmm2phcJsi9BPOfO5++Pu3u7u7S0tLQUqq0TNs2a/pFVUQCQC7lSENNa/Ihqluq5uxpyHzCqzLZs2LajpKNeF9TLHWGVpXfDXzTPcuKa+vsDV5InZRTNXpNT6FG4CHnL3O9KPdwC4+1/Mdrz6FNSENEWh+xQiES657DJWXnHFlDWU4MI7mNWnUDoutj6FUguFSlIdzbcBvyPV0fyv3f3t2Y5fykNSRUTCUjYdze4+bmYPAj8hNST1O3MFgoiI5F9JhQKAu/8Y+HGx6xARuRiVWkeziIgUkUJBREQCCgUREQkoFEREJFBSQ1IXysx6gGNAM1Bec+enKvf6ofw/g+ovLtVfWOvcfdbZv2UdChlmtnuuMbfloNzrh/L/DKq/uFR/6VDzkYiIBBQKIiISWCqh8HixC7hA5V4/lP9nUP3FpfpLxJLoUxARkfxYKlcKIiKSBwoFEREJLJlQMLOvmdl+M/uVmf29mTUVu6aFMLN7zOxtM5s0s7IZ2mZm28zsgJl1mtkXi13PQpnZd8zstJn9pti1LIaZrTGzn5nZvvS/n88Wu6aFMLMaM3vDzN5K1/9fi13TYphZxMzeNLN/KHYtF2rJhALwAnCNu19L6p4MO4pcz0L9Bvgj4OfFLiRXZhYB/hr4F8BVwMfN7KriVrVg/xvYVuwiLsA48Kfu/j5gK/BAmf03GAVudffrgM3ANjPbWtySFuWzwL5iF5EPSyYU3P15d8/crPY1oLWY9SyUu+9z9wPFrmOBbgA63f2wu48BzwB3F7mmBXH3nwOz31S4DLh7t7v/Mv37AKkT0+r5n1U6PCVzy7ho+k9ZjX4xs1bgo8ATxa4lH5ZMKEzz74F/LHYRF4HVwPGsx12U0QlpqTGzNmAL8HqRS1mQdNPLXuA08IK7l1X9wDeAPwMmi1xHXpTcTXbmY2Y/BS6bZdefu/tz6WP+nNQl9fcKWVsucqm/zNgs28rqW95SYWYNwN8Bf+Lu/cWuZyHcfQLYnO4H/Hszu8bdy6KPx8zuBE67+x4z+1CRy8mLsgoFd799vv1mdi9wJ3Cbl+AEjPPVX4a6gDVZj1uBE0Wq5aJlZlFSgfA9d/9BsetZLHfvM7OXSfXxlEUoADcDd5nZR4AaYJmZPe3unyhyXYu2ZJqPzGwb8AXgLncfKnY9F4ldwCYzW29mVcB2YGeRa7qomJkBTwL73P0vi13PQplZS2akoJnVArcD+4ta1AK4+w53b3X3NlL//l8q50CAJRQKwKNAI/CCme01s/9Z7IIWwsz+pZl1ATcBPzKznxS7pvNJd+w/CPyEVAfns+7+dnGrWhgz+1vgVeC9ZtZlZvcVu6YFuhn4JHBr+t/93vS31nKxCviZmf2K1JeMF9y97Id1ljMtcyEiIoGldKUgIiIXSKEgIiIBhYKIiAQUCiIiElAoiIhIQKEgIiIBhYKIiAQUCiJ5ZGYd6Xt61JhZffoeAdcUuy6RXGnymkiemdl/I7UOTi3Q5e5/UeSSRHKmUBDJs/Q6ULuAEeAP0quAipQFNR+J5F8MaCC1FldNkWsRWRBdKYjkmZntJHUXuvXAKnd/sMglieSsrO6nIFLqzOxTwLi7/5/0Paz/n5nd6u4vFbs2kVzoSkFERALqUxARkYBCQUREAgoFEREJKBRERCSgUBARkYBCQUREAgoFEREJ/H8G3X/pKUG9nQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%time\n",
    "print('=========== OCSVM FEATURE 1 ===========')\n",
    "print('=======>>>OCSVM CLUSTER RESULT')\n",
    "ocsvm_test_label = best_ocsvm.fit(X_train).predict(X_test)\n",
    "# print results\n",
    "clustering_print_results(df_test, ocsvm_test_label, selected_features, X_test, True, True )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 266,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=======>>>IFOREST CLUSTER RESULT\n",
      "\n",
      "Exporting LABEL...\n",
      "Number of clusters: 2\n",
      "\\Label -1: 1033881 data points\n",
      "\\Label 1: 19964 data points\n",
      "\n",
      "DONE.\n",
      "--------------------------------------\n",
      "Wall time: 40.2 s\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEGCAYAAACKB4k+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAAAhsElEQVR4nO3de3hc9X3n8fd3RldLsi1k+SqML9gEm1A7ETcTsiFQcBoKbRdvaLZdtiVLmydQsiltcbtNWJ5m203YbJLNphsSsssm2SaEXnCa0pjQUJKCbeziQGSDfMNGvkiykGVpdJ2Z7/4xM6cj6+KRNFf583oeHmnOOTPzHeHnfOb3+53f75i7IyIiAhAqdAEiIlI8FAoiIhJQKIiISEChICIiAYWCiIgEygpdwEwsWLDAV6xYUegyRERKyp49e067e+N4+0o6FFasWMHu3bsLXYaISEkxs6MT7VP3kYiIBBQKIiISUCiIiEigpMcUROTCMDIyQltbG4ODg4UupaRUVVXR1NREeXl5xs9RKIhI0Wtra6Ouro4VK1ZgZoUupyS4O11dXbS1tbFy5cqMn6fuIxEpeoODgzQ0NCgQpsDMaGhomHLrSi0FERlXpKuVSPchaupXU9OwttDlKBCmYTp/M4WCiIwR6Wrl8IuPQigM8RirNj1YFMEguafuIxEZI9J9CEJhquc2QSiceHyBq62tzfjYhx9+mEcffTRnr5+yefNm5s+fz2233Tbl505EoSAiY9TUr4Z4jIGzbRCPJR5L0fm93/s9vvGNb2T1NRUKIjJGTcNaVm16kEWX3a6uo0l873vf45prrmHjxo3cfPPNtLe3B/t++tOf8v73v581a9bw1a9+Ndj+2c9+lquuuoorr7yST33qUzN6/5tuuom6uroZvca5FAoiMq6ahrUsvPQDJRsIka5WOg4+Q6SrNWfv8Z73vIcdO3bwyiuvcNddd/GZz3wm2Pfqq6/y/e9/n5deeolHHnmEEydOsH37dg4cOMCuXbvYu3cve/bs4YUXXhjzuhs2bMhZzeejgWYRmXXyNVDe1tbGhz70IU6ePMnw8PCo+QB33HEH1dXVVFdXc+ONN7Jr1y5+8pOfsH37djZu3AhAX18fBw4c4L3vfe+o1927d2/Wa81UTlsKZvYfzazFzH5mZn9hZlVmdpGZPWtmB5I/69OO32pmB83sDTO7NZe1icjsla+B8vvvv5/77ruP1157ja985Suj5gScezmomeHubN26lb1797J3714OHjzIPffck9F77dy5kw0bNrBhwwa2bduW1c+RLmehYGbLgN8Bmt39CiAM3AU8BDzn7muA55KPMbN1yf3rgc3Al80snKv6RGT2ytdAeU9PD8uWLQPgiSeeGLXv6aefZnBwkK6uLp5//nmuuuoqbr31Vr7+9a/T19cHwPHjx+no6Mjova655pogTG6//fbsfpA0ue4+KgOqzWwEmAOcALYC70vufwJ4HvgD4A7g2+4+BBwxs4PA1cBLOa5RRGaZ1EB5Niff9ff309TUFDz+xCc+wcMPP8yWLVtYtmwZ1157LUeOHAn2X3311Xzwgx/k2LFj/PEf/zFLly5l6dKl7N+/n+uuuw5IXIb6zW9+k4ULF456rw0bNmTUhXTDDTfw+uuv09fXR1NTE48//ji33jqzThZz9xm9wKQvbvYA8GlgANju7v/WzM64+/y0Y7rdvd7MvgTscPdvJrc/Djzj7k+d85r3AvcCLF++/N1Hj054rwgRmSX279/P5ZdfXugyStJ4fzsz2+PuzeMdn8vuo3oS3/5XAkuBGjP7tcmeMs62MYnl7o+5e7O7Nzc2jns3ORERmaZcDjTfDBxx9053HwH+CtgEtJvZEoDkz1SHWhtwcdrzm0h0N4mISJ7kMhSOAdea2RxLDMPfBOwHtgF3J4+5G3g6+fs24C4zqzSzlcAaYFcO6xMRkXPkbKDZ3Xea2VPAPwNR4BXgMaAWeNLM7iERHFuSx7eY2ZPAvuTxH3P3WK7qExGRsXJ69ZG7fwo4dx73EIlWw3jHf5rEwLSIiBSAlrkQEZGAQkFEJAPFsHT2l770JS699FLMjNOnT0/p9TOlUBARKRHXX389P/zhD7nkkkty9h4KBRGRacr30tkbN25kxYoV2Sp/XAoFEZmVPB4lHh3E49GcvYeWzhYRKQEej+IjvYDhMYfyOiyU/dPdbFw6W6EgIrNOonVgYGHwGB6P5iQU7r//fj7xiU9w++238/zzz/Pwww8H+yZbOvu3fuu3sl5Ltqj7SERmnUQAOHgM8JwEAuR36ex8USiIyKxjoTKsvA7CVViWuo5SS2en/vvc5z4XLJ19ww03sGDBglHHp5bOvvbaa4Ols2+55RY+/OEPc9111/HOd76TO++8k97e3jHvNdGYwhe/+EWamppoa2vjyiuv5CMf+ciMP9e5crp0dq41Nzf77t27C12GiOSYls6evqJZOltEREqPQkFERAIKBREpCaXc1V0o0/mbKRREpOhVVVXR1dWlYJgCd6erq4uqqqopPU/zFESk6KWuuOns7Cx0KSWlqqqKpqamKT1HoSAiRa+8vHzUbGHJHXUfiYhIQKEgIiIBhYKIiAQUCiIiElAoiIhIQKEgIiIBXZIqIlJiIl2tRLoPUVO/mpqGtVl9bYWCiEgJiXS1cvjFRyEUhniMVZsezGowqPtIRKSERLoPQShM9dwmCIUTj7NIoSAiUkJq6ldDPMbA2TaIxxKPs0jdRyIiJaSmYS2rNj2oMQUREUmoaVib9TBIUfeRiIgEFAoiIhJQKIiISEChICIiAYWCiIgEFAoiIhLQJakiInmUy3WLskGhICKSJ7letygb1H0kIpInuV63KBsUCiIieZLrdYuyIafdR2Y2H/gacAXgwG8CbwDfAVYAbwL/xt27k8dvBe4BYsDvuPsPclmfiEg+5XrdomzIdUvhC8Dfu/s7gJ8D9gMPAc+5+xrgueRjzGwdcBewHtgMfNnMwjmuT0Qkr2oa1rLw0g8UZSBADkPBzOYC7wUeB3D3YXc/A9wBPJE87Angl5K/3wF8292H3P0IcBC4Olf1iYjIWLlsKawCOoH/bWavmNnXzKwGWOTuJwGSPxcmj18GvJX2/LbkNhERyZNchkIZ8C7gz919IxAh2VU0ARtnm485yOxeM9ttZrs7OzuzU6mIiAC5DYU2oM3ddyYfP0UiJNrNbAlA8mdH2vEXpz2/CThx7ou6+2Pu3uzuzY2NjTkrXkTkQpSzUHD3U8BbZnZZctNNwD5gG3B3ctvdwNPJ37cBd5lZpZmtBNYAu3JVn4iIjJXrGc33A98yswrgMPAbJILoSTO7BzgGbAFw9xYze5JEcESBj7l7LMf1iYhImpyGgrvvBZrH2XXTBMd/Gvh0LmsSEZGJaUaziIgEFAoiIhJQKIiISEChICIiAYWCiIgEFAoiIhJQKIiISEChICIiAYWCiIgEFAoiIhJQKIiISEChICIiAYWCiIgEFAoiIhJQKIiISEChICIiAYWCiIgEFAoiIhJQKIiISEChICIiAYWCiIgEFAoiIhJQKIiISEChICIiAYWCiIgEFAoiIhJQKIiISEChICIiAYWCiIgEFAoiIhJQKIiISEChICIiAYWCiIgEFAoiIhJQKIiISEChICIiAYWCiIgEzhsKZnafmdXnoxgRESmsTFoKi4GXzexJM9tsZpbrokREpDDOGwru/p+ANcDjwL8HDpjZfzGz1Zm8gZmFzewVM/vb5OOLzOxZMzuQ/FmfduxWMztoZm+Y2a3T+kQiIjJtGY0puLsDp5L/RYF64Ckz+0wGT38A2J/2+CHgOXdfAzyXfIyZrQPuAtYDm4Evm1k4w88hIiJZkMmYwu+Y2R7gM8A/Ae90948C7wb+9Xme2wR8EPha2uY7gCeSvz8B/FLa9m+7+5C7HwEOAldn/lFERGSmyjI4ZgHwK+5+NH2ju8fN7LbzPPfzwO8DdWnbFrn7yeRrnDSzhcnty4Adace1JbeNYmb3AvcCLF++PIPyRUQkU5mMKXzy3EBI27d/vO0AycDocPc9GdYy3gC2j/Oej7l7s7s3NzY2ZvjSIiKSiUxaCtN1PXC7mf0CUAXMNbNvAu1mtiTZSlgCdCSPbwMuTnt+E3Aih/WJiMg5cjZ5zd23unuTu68gMYD8D+7+a8A24O7kYXcDTyd/3wbcZWaVZraSxBVPu3JVn4iIjJXLlsJE/gx40szuAY4BWwDcvcXMngT2kbjC6WPuHitAfSJShCJdrUS6D1FTv5qahrWFLmfWssTVpqWpubnZd+/eXegyRCTHIl2tHH7xUQiFIR5j1aYHFQwzYGZ73L15vH1a+0hEil6k+xCEwlTPbYJQOPFYckKhICJFr6Z+NcRjDJxtg3gs8VhyohBjCiIiU1LTsJZVmx7UmEIeKBREpCTUNKxVGOSBuo9ERCSgUBARkYBCQUREAgoFEREJKBRERCSgUBARkYBCQUREAgoFEREJKBRERCSgUBARkYBCQUREAgoFEREJKBRERCSgUBARkYBCQUREAgoFEREJKBRERCSgO6+JXKAiXa26vaWMoVAQuQBFulo5/OKjEApDPMaqTQ8qGARQ95HIBSnSfQhCYarnNkEonHgsgloKIkUjn905NfWrIR5j4GwbxGOJxyIoFESKQr67c2oa1rJq04MaU5AxFAoiRSC9O2fgbFviZJ3r1kLDWoWBjKExBZEioO4cKRZqKYgUAXXnSLFQKIgUCXXnSDFQ95GIiAQUCiIiElAoiIhIQKEgIiIBhYKIiAQUCiIiEtAlqSIXOC2hLekUCiIXMC2hLefKWfeRmV1sZj8ys/1m1mJmDyS3X2Rmz5rZgeTP+rTnbDWzg2b2hpndmqvaRCRBS2jLuXI5phAFftfdLweuBT5mZuuAh4Dn3H0N8FzyMcl9dwHrgc3Al80snMP6RC54WnNJzpWz7iN3PwmcTP7ea2b7gWXAHcD7koc9ATwP/EFy+7fdfQg4YmYHgauBl3JVo8iFTmsuybnyMqZgZiuAjcBOYFEyMHD3k2a2MHnYMmBH2tPaktvOfa17gXsBli9fnsOqRYpDrgeCteaSpMt5KJhZLfCXwMfd/ayZTXjoONt8zAb3x4DHAJqbm8fsF5kNUkFgFubUvqc0ECx5k9NQMLNyEoHwLXf/q+TmdjNbkmwlLAE6ktvbgIvTnt4EnMhlfSLFKP2KoMEzRymrmk/dwvV5u/mOXNhyefWRAY8D+939c2m7tgF3J3+/G3g6bftdZlZpZiuBNcCuXNUnUqzSrwgqq5rPyFCPBoIlb3LZUrge+HXgNTPbm9z2h8CfAU+a2T3AMWALgLu3mNmTwD4SVy59zN1jOaxPpCjV1K8mOtBN99k2ysprWHnNA7jHNBAseZHLq49+wvjjBAA3TfCcTwOfzlVNIiXFE0Nmc+avOG8YaFayZItmNIsUmUj3Icqq66lb9M6MxhE0K1mySQviieRRpKuVjoPPEOlqnfCY9All0YFuBnuPT3q8ZiVLNikURPIk9Y2+/Y1tHH7x0QlP9DUNa1m87k5C4SqGIh207/8bWn/0yYmP16xkySJ1H4nkSfo3+sm6hSJdrZza9xSR7kP0de6jsm4pHo/SdeyFcY/XrGTJJoWCSJ5k+o0+FR7llfMBiMeGMAuBTzjxU7OSJWsUCiJ5kuk3+lR4hMoqKauaR2XNIiprFtJwyQ15rlguROZeuitFNDc3++7duwtdhkjWpS9zoTkKkm1mtsfdm8fbp5aCSBFSd5AUikJBJMc0sUxKiUJBJIc0sUxKjeYpiORQMUwsy2TCnEiKWgqSN6XejTKd+gs9sUwtFZkqhYLkRamfnKZbf6EnlmU6YU4kRd1HkhfF0I0yEzOpv6ZhLQsv/UBBTsaFbqlI6VFLQfKiUCenSFcrXUd/DOZUz10+6pr/qXQHperv7WghOngGs/CE71dMXWSFbqlI6VEoSF5k8+SUfuIFRr1m+r7+M2/S+vwnGehpwwALhSmvbgCPM3dJM30dr1FRu4iq2sXn7Q5KLVJ3ZOcXKK+az6l9T425z0G+usimGjya8yBToVCQvMnGyWnU/Yt73iI2EiFcUcdwfwfxWIyBs8fwmIM5FgrhIwNAPHj+yMDbgNPffRALVzHYexyWvCujvnb3GNXzlk/YP5+P/vtSH5uR4qdQkJKSOvGGy6rp7dwPHiUWHSQ+0g+MXrJl/Ju5/ssx7nE8HmO4rz2j7qzzdYHlo4tMA8eSawoFKSmpE2/k7QOEwuXgYUYGz3JuIIzPxh5nULvwijFHpo9FNCx/b9DKmawLLB/99xo4llzTgnhSclIn7K7DzxId6afnxB7iIxHco+McbYTK5xAuq6Jh5c9z+uAzxD2GYYQqaoE4ZeVzqGtcz9obHwnGJVp/9El6O1sAo65xXbBvonryOZBbbIPZUnq0IJ7MKqlv7Q2X3BCsJHrmxG66jr7AUO8J4rERQuEKauovZf6yq4j7CPXLrqFx9S10HtpO9/Gdiee89SIjA2/jQHQkEnTFRLoPEU2OVRgQHe6b9IY4+e7j18Cx5JJCQUpW+smxcfUtrLnhD8/7nMbVt9C4+hYiXa30tb/KQM8xwCmbtzzoiqmpX01ZeQ0Dw28CRnXavnOpj19mG4WCXJBqGtay9sZH6Dr2ArjRcMkNwck8te9Ey3cZ6u9k0ZpfOO8NcdTHL7OFQkEuWOfrhhnsOQqh8LhzEtJfQ5PDZDbRMhcya022Ouh4+9K3TWVZi9QyFoBWI5WSp5aCTOrcGcLdx3dSXddEZd3iov5mPNkA8Hj7AFp/9Emiw32UVdTStPE3p9QtpEllMlsoFGRcka5Wuo69QNehH1JWXc/ZU3sZ6H6TUFkl8egg85ZdndHyEIUy2QBw+r7ejhaOt3wHsxC9nfsoq6hloOcYA2ePTalbaKoDzrqsVIqVQkHGSH3rHew9wUDPUWoa3kFf5+uJ2cMjEQiVEY8OBd0quVjKYaYnzMkGgNMXt+vteA2Aob6TxGNDOLXEY8P0drTQsPy9QbdQpu/X29HCyFDPhAvmAXQe2p5YQ6lyHmUVtUUbrHJh0piCjJH61ps4URl9p/dhodQ/lTjERxITxXJwtU0qkNrf2MbhFx+ddv98agB40WW3jznppha3s3A51fNXUrdwPZW1S6iqXUq4rBpw+t8+xL6//zidh7Zn/H6L191JdPAM5ZXzOLXvqQnHMt7c8XkGe9ro7z4czIEQKRYKBRkj9a03Fh2grnEdjas3Ey6vxcKVEKpkzkVruWjF+3LyDTe9GyY63Mfxlu/MKBjGu49BpKuVU/uewmMjDJw5Qm9HC2UVtay+YSsXrXwfNQ2XMdzfwcDZ4xzZ+YWM3989RtX8S6hbuH7CwelI9yHKquYTrqgmNjLAyFCPLmOVoqLuo1kim33U6ZdZmoVxj1Exp5H2N/6GipqFVNUuZtn6D+V0bZ/0rp3DLz6acQBN9ndI7RvsPQ6hcOLkDdQ0Xh58njnzV9Dz1kvEhgcIl1dTXjkv4y6yTOYs1NSvpqyiljnzVxEdPMOKax5Q15EUFYXCLJCLK19Sz09/3bU3PjLqJjW5kAqk4y3fAaBu4fqMZwpnesVRdKAbgAGgrKJ2VMDVNKxlxbUfH9Xnn+k3+UzmLGhegxQ7hcIskKulFs59XfdYxgOv475ehq2Zmoa1LFv/IQ6/+OiUZgpnfMXRYA8VtYuoa7xi1Ezm9PrW3/rfp3XinmhC3LmffaatHpFcUSjMArlaaiGbrzuV1kzqZLh43Z1TaplMVq9ZmMEzR4l0tdLf/SZ1C9cTH47QcMkNE9aXCsBUPamutPPVk1p0L7UI33Racp2HtvPmjs9TVjVfVyhJXikUSsx43x5z1SWR1VtoZtiamUlX2Ln1QmKGsVliqQpCZUTaX8XKqhju76CsonbUyqjj1ZdaRnso0slgbxvzlrx70pN056Ht7N/+u0CIUy1Pwi3/DffYlOcwHNn5BQbPHic8+DZz5q/SQnuSNwqFEjLZCTNbt7ocL3CycTLKtNWRra6w/jNvcmrfU8lLPg9TWbOQijkNhCvqwIzY8ADRwTOjVkYdr74TLd/lzImXMSsjOtSTmJ9RNfHgc/fxnUCIqrolDPaepPv4Tpat/9DUZkd3H6K8ch4j5V1j6hTJNYVCCcnZ2ME5s5dzsUxDpq2OTMMj1UWTWnLDLMxAz1vBjXf6uw9SVlkPHmO4/zSDZ9+idsE7CIXLqJ6/EvfYqCt/xmtlHPvnx+lo/R6x4QiY4R5nZLCbULicwbMniHS1jvkc9cuu4VTLkwz2ngTi1C+7ZsotruAKpfpVjAz16AolySvdea2IRbpaObr7K5xt/xl1C9ezcM3mZDfI1LtWzm0FRLpaObHvu5w98QqDvW2Ey+cw3N/JglU/z1DfqVGXaeZbeq3AmJNp56HttDxzP7GRIWIjfZRXNxAdfBtCFRAfSXybJ/0GzWm34bTyxN8vFiNUXkVFzUJiw33EYoNUVC9gzkWXEh/pp7ejhXhsEI+PUFZRh4UqqKxdwMK1v0hf+2tER/oZ7mtnefNvc/HG3xhV/7ljCjP9GygQJNsmu/OaQqFIRbpaee1vf5ueE3uAOBBi3tJ3s/o9D2U8+Jo6OYWsnNOHfgBmVNYsZMHqW3lz5xfpP3MIPA5AuGIeFiqjau5SYsN91C1856i+84lOUtk4AY73OumtFyzE2fZXiQ73ghvx+Ajx4Z5pv9f0lFM25yLmLXkXQ73H6Tv9OmZhyirmcPG772Xpui06eUvJKKnbcZrZZuALQBj4mrv/WT7ff7xv1NP5xna+b7vnfX73IYYi7ckvuYmJ54OR9lGXhY73HolbU75Mz4lX6D72Y+LRARKh8i86DjzD6G/SEBvuwcqqiY8MUj1/JZW1i+lu28H+Zx9i0WW/yJm2l8asKnpi33c5/tP/SyhcFQyqpoIhdVIf6k18hsqaxVTPuxj3GEO9pzh99B+JDfVRPW85PSf20Nf5Ku5wxEcy/hvn1wjR/na6Dj0TbHGMkaEoHW98j8EzR8dtuekbv5SaomopWGIVsVbg54E24GXgV91933jHz6Sl8IM/reVffbSFcHj8hcuiQ730de5jINKOOVTWLSJy+gBtr32L/jNHWLpuC4vX3kaovIr+M8dof2MbXcd+PK1aZmJF80e5pPne5Jo9EI+PMNx/mlCoHPcove37GBnpo7xyLqcPP8eJfd9l6botLF1/JxXVDcRiIwxHOhjqa4eQER3ooa/7AGfadtF/5khWa126bguX3fgIZpbV1wWIxWL845+vz/rrZqJu8buontvEsp/79VGB3XX0x3QdfnZa4zSZhEn6MS//xS8z1Hs0a59pumoWXMF7/sOOCfdHulo5+OP/yqn938ljVTMTKp/Lpt94flaFesl0H5nZdcDD7n5r8vFWAHf/0/GOn24opAKhrKxswhOUuyd7oh0wDMc9Ts+pVzn95j+y8qqPEi6rxAH3KCMD3ex7dmteg2FF80dZfd0DmE28hFXw/9djxGPDnDrw9yxe80HCZRWQ/OyJYxySP4f736an/acc/KdHsxYMS9dt4fKb/iQngQCJzxCNRgsTDFZGZe0i1n/gf4yalzDYd5KBM0dZsOpmYtEBFl12e0aT/zK5LDf9mJP7/hKP9ufq003ZRMEQ6Wpl79/cTV9y+ZKSEqrgPR/ZMWuCYbJQKLYF8ZYBb6U9bktuC5jZvWa228x2d3Z2TvuNwuHwpCeoxD7HsNS5E3enYs4CLrp4U3LVUE8eZ4RC5cxdvGHa9UzHRRdvmjQQIHHeN0ss1gZG/bKrsXAo0S0VHGPpDzEzwmXVVM+/JGu11jddm7NAgGTNE7T6cs6dqrqm5N84bZXZi9YAnlhQbwqT/zK561v6McUUCACR06+Pv737EEO9p/JcTZbEhy+Y1WyLLRTGO2uMasq4+2Pu3uzuzY2NjdN+o1gsxmStpMQ+w3FSh5kZw/2nefutF/F4PLE/+e06Hh/h7Km9065nOt5+60Xc45Me48kGQKJnzuk+vguPxUf9Vd191B/Z3YlFBxg4k73uiO62HZP+vWfK3YnFYuc/MAfC5XOorGkcM+chscrsehZd/ktTm4SX4cJ6qWOsbE5WP89M1Sx4x/jb61dTWbc4z9VkSajigpkrckF2H4HGFDSmkAXhOcxf+i6WrNsyag0lmPkAs8YUiofGFArIzMpIDDTfBBwnMdD8YXdvGe/42XxJqohIrpTMJanuHjWz+4AfkLgk9esTBYKIiGRfUYUCgLv/HfB3ha5DRORCVGwDzSIiUkAKBRERCSgUREQkoFAQEZFAUV2SOlVm1gkcBRYApwtczkyUev1Q+p9B9ReW6s+vS9x93Nm/JR0KKWa2e6JrbktBqdcPpf8ZVH9hqf7ioe4jEREJKBRERCQwW0LhsUIXMEOlXj+U/mdQ/YWl+ovErBhTEBGR7JgtLQUREckChYKIiARmTSiY2WfN7HUze9XM/trM5he6pqkwsy1m1mJmcTMrmUvbzGyzmb1hZgfN7KFC1zNVZvZ1M+sws58VupbpMLOLzexHZrY/+e/ngULXNBVmVmVmu8zsp8n6/3Oha5oOMwub2Stm9reFrmWmZk0oAM8CV7j7lSTuybC1wPVM1c+AXwFeKHQhmbLE7dz+J/ABYB3wq2a2rrBVTdn/ATYXuogZiAK/6+6XA9cCHyux/wdDwPvd/eeADcBmM7u2sCVNywPA/kIXkQ2zJhTcfbu7R5MPdwBNhaxnqtx9v7u/Ueg6puhq4KC7H3b3YeDbwB0FrmlK3P0F4O1C1zFd7n7S3f85+XsviRPTssmfVTw8oS/5sDz5X0ld/WJmTcAHga8VupZsmDWhcI7fBJ4pdBEXgGXAW2mP2yihE9JsY2YrgI3AzgKXMiXJrpe9QAfwrLuXVP3A54HfBya/YXqJKLqb7EzGzH4IjHfn7z9y96eTx/wRiSb1t/JZWyYyqb/EjHez5ZL6ljdbmFkt8JfAx939bKHrmQp3jwEbkuOAf21mV7h7SYzxmNltQIe77zGz9xW4nKwoqVBw95sn229mdwO3ATd5EU7AOF/9JagNuDjtcRNwokC1XLDMrJxEIHzL3f+q0PVMl7ufMbPnSYzxlEQoANcDt5vZLwBVwFwz+6a7/1qB65q2WdN9ZGabgT8Abnf3/kLXc4F4GVhjZivNrAK4C9hW4JouKGZmwOPAfnf/XKHrmSoza0xdKWhm1cDNwOsFLWoK3H2ruze5+woS//7/oZQDAWZRKABfAuqAZ81sr5n9r0IXNBVm9stm1gZcB3zfzH5Q6JrOJzmwfx/wAxIDnE+6e0thq5oaM/sL4CXgMjNrM7N7Cl3TFF0P/Drw/uS/+73Jb62lYgnwIzN7lcSXjGfdveQv6yxlWuZCREQCs6mlICIiM6RQEBGRgEJBREQCCgUREQkoFEREJKBQEBGRgEJBREQCCgWRLDKzq5L39Kgys5rkPQKuKHRdIpnS5DWRLDOzPyGxDk410Obuf1rgkkQyplAQybLkOlAvA4PApuQqoCIlQd1HItl3EVBLYi2uqgLXIjIlaimIZJmZbSNxF7qVwBJ3v6/AJYlkrKTupyBS7Mzs3wFRd/9/yXtYv2hm73f3fyh0bSKZUEtBREQCGlMQEZGAQkFERAIKBRERCSgUREQkoFAQEZGAQkFERAIKBRERCfx/V6w5wwZQVjkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%time\n",
    "print('=========== IFOREST FEATURE 1 ===========')\n",
    "print('=======>>>IFOREST CLUSTER RESULT')\n",
    "iforest_test_label = best_iforest.fit(X_train).predict(X_test)\n",
    "# print results\n",
    "clustering_print_results(df_test, iforest_test_label, selected_features, X_test, True, True )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 272,
   "metadata": {},
   "outputs": [],
   "source": [
    "sort_anomalies    =   ['total_packets', 'bytes_bothdir','bytes_onedir', 'duration']\n",
    "# get anomalies\n",
    "anomalies_ocsvm_result = get_anomalies(df_test, ocsvm_test_label)\n",
    "anomalies_ocsvm_result.sort_values(by=sort_anomalies,ascending=False)\n",
    "save(anomalies_ocsvm_result, \"ocsvm_anomalies\")\n",
    "\n",
    "anomalies_iforest_result = get_anomalies(df_test, iforest_test_label)\n",
    "anomalies_iforest_result.sort_values(by=sort_anomalies,ascending=False)\n",
    "save(anomalies_iforest_result, \"ifoest_anomalies\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 284,
   "metadata": {},
   "outputs": [],
   "source": [
    "####################### FEATURE 2 Analysis #######################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 390,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'kernel': 'rbf', 'gamma': 0.01, 'nu': 0.15}"
      ]
     },
     "execution_count": 390,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ocsvm_best_param"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 391,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_samples': 'auto', 'contamination': 0.3}"
      ]
     },
     "execution_count": 391,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "iforest_best_param"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PCA.explained variance_ratio_ : 0.9469287695925639\n",
      "PCA.explained variance_ratio_ : 0.9115256759324034\n"
     ]
    }
   ],
   "source": [
    "X_train = get_pc(X_train_ori, 10)\n",
    "X_val = get_pc(X_val_ori,10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_ocsvm = svm.OneClassSVM(**ocsvm_best_param)   \n",
    "ocsvm_label = best_ocsvm.fit(X_train).predict(X_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_iforest = IsolationForest(**iforest_best_param)\n",
    "iforest_label = best_iforest.fit(X_train).predict(X_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation data: \n",
      "OCSVM: =============\n",
      "Total:  47003\n",
      "Normal:  5528\n",
      "Anomaly:  41475\n",
      "\n",
      "IFOREST: ============\n",
      "Total:  47003\n",
      "Normal:  1828\n",
      "Anomaly:  45175\n"
     ]
    }
   ],
   "source": [
    "y_pred_ocsvm = ocsvm_label \n",
    "y_pred_iforest = iforest_label \n",
    "print('Validation data: ')\n",
    "print('OCSVM: =============')\n",
    "print_stats(y_pred_ocsvm) \n",
    "print()\n",
    "print('IFOREST: ============')\n",
    "print_stats(y_pred_iforest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=========== OCSVM: FEATURE 2 ===========\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          -1       0.97      0.91      0.94     44057\n",
      "           1       0.29      0.55      0.38      2946\n",
      "\n",
      "    accuracy                           0.89     47003\n",
      "   macro avg       0.63      0.73      0.66     47003\n",
      "weighted avg       0.93      0.89      0.90     47003\n",
      "\n",
      "| TP:  40153 | FP:  1322 | TN:  1624 | FN:  3904\n",
      "TPR:  0.9113875207118052\n",
      "FPR:  0.4487440597420231\n",
      "TPR - FPR :  0.4626434609697821\n"
     ]
    }
   ],
   "source": [
    "print('=========== OCSVM: FEATURE 2 ===========')\n",
    "print(classification_report(y_true, y_pred_ocsvm))\n",
    "(TP, FP, TN, FN) = perf_measure(y_true, y_pred_ocsvm)\n",
    "TPR = TP/(TP+FN)\n",
    "FPR = FP/(FP+TN)\n",
    "\n",
    "print('| TP: ', TP, '| FP: ', FP, '| TN: ', TN, '| FN: ', FN)\n",
    "print('TPR: ', TPR)\n",
    "print('FPR: ', FPR)\n",
    "print('TPR - FPR : ', TPR - FPR )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=========== IFOREST FEATURE 2 ===========\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          -1       0.94      0.96      0.95     44057\n",
      "           1       0.09      0.06      0.07      2946\n",
      "\n",
      "    accuracy                           0.91     47003\n",
      "   macro avg       0.51      0.51      0.51     47003\n",
      "weighted avg       0.89      0.91      0.90     47003\n",
      "\n",
      "| TP:  42395 | FP:  2780 | TN:  166 | FN:  1662\n",
      "TPR:  0.9622761422702408\n",
      "FPR:  0.9436524100475221\n",
      "TPR - FPR :  0.018623732222718736\n"
     ]
    }
   ],
   "source": [
    "print('=========== IFOREST FEATURE 2 ===========')\n",
    "print(classification_report(y_true, y_pred_iforest))\n",
    "(TP, FP, TN, FN) = perf_measure(y_true, y_pred_iforest)\n",
    "TPR = TP/(TP+FN)\n",
    "FPR = FP/(FP+TN)\n",
    "\n",
    "print('| TP: ', TP, '| FP: ', FP, '| TN: ', TN, '| FN: ', FN)\n",
    "print('TPR: ', TPR)\n",
    "print('FPR: ', FPR)\n",
    "print('TPR - FPR : ', TPR - FPR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PCA.explained variance_ratio_ : 0.9200865845868786\n"
     ]
    }
   ],
   "source": [
    "X_test = df_test[selected_features]\n",
    "X_test = get_pc(X_test, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 488,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "870431"
      ]
     },
     "execution_count": 488,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(iforest_test_label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=========== OCSVM FEATURE 2 ===========\n",
      "=======>>>OCSVM CLUSTER RESULT\n",
      "\n",
      "Exporting LABEL...\n",
      "Number of clusters: 2\n",
      "\\Label -1: 825584 data points\n",
      "\\Label 1: 228261 data points\n",
      "\n",
      "DONE.\n",
      "--------------------------------------\n",
      "Wall time: 46.1 s\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEGCAYAAACKB4k+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAAAgoUlEQVR4nO3dfXAcd53n8fd39CxbtkeW5CfZkROU4DxhJ3LiPJgLSUy8JCQsRwovB5e7C5Wqq4RlF4oF3x5HdvfY5YCiForjjkC4SxF2U6nAXswCR5ynSyAPfkicOI6c2Ov4QbZsPY1lydLoab73x4yakTSyRw+jmZE+r6rUaHq6e76tWP3p/v26f23ujoiICEAo2wWIiEjuUCiIiEhAoSAiIgGFgoiIBBQKIiISKMx2AVNRVVXldXV12S5DRCSv7N69u83dq1N9ltehUFdXx65du7JdhohIXjGzI+N9puYjEREJKBRERCSgUBARkUBe9ymIyNwwMDBAU1MT0Wg026XkldLSUmpraykqKkp7GYWCiOS8pqYmKioqqKurw8yyXU5ecHfa29tpampi9erVaS+n5iMRyXnRaJTFixcrECbAzFi8ePGEz650piCT0tIWobUjQnVlmJqqcLbLkTlAgTBxk/mdKRRkwlraImx/cSehkBGLOZs2rlcwiMwSaj6SCWvtiBAKGeGFFYRCRmtHJNsliWTc/Pnz0573wQcf5Nvf/nbG1j9s8+bNLFq0iDvuuGPCy45HoSATVl0ZJhZzIp1dxGJOdaXOEkSy4Utf+hI//elPp3WdCgWZsJqqMJs2rufKNRep6UjmtF/+8pdce+21rFu3jltvvZVTp04Fn73xxhvcfPPN1NfX86Mf/SiY/q1vfYv169dz5ZVX8rWvfW1K33/LLbdQUVExpXWMplCQSampCnPZxRcqECRntbRF2PfuIVraMte8eeONN/LKK6/w+uuvs2XLFr75zW8Gn7355pv86le/4uWXX+av//qvOXHiBE899RQHDhxgx44d7Nmzh927d/PCCy+MWe/atWszVvP5qKNZRGadmboYoqmpiU9+8pM0NzfT398/4n6Au+66i7KyMsrKyvjQhz7Ejh07+N3vfsdTTz3FunXrAOju7ubAgQN88IMfHLHePXv2THut6cromYKZ/bmZ7TOzt8zsH82s1MwqzWy7mR1IvIaT5t9qZgfN7B0zuy2TtYnI7DVTF0N87nOf44EHHmDv3r388Ic/HHFPwOjLQc0Md2fr1q3s2bOHPXv2cPDgQe699960vuvVV19l7dq1rF27lm3btk3rdiTLWCiY2QrgT4EGd78cKAC2AF8BnnH3euCZxHvM7NLE55cBm4EfmFlBpuoTkdlrpi6G6OzsZMWKFQA88sgjIz578skniUajtLe38/zzz7N+/Xpuu+02fvKTn9Dd3Q3A8ePHaWlpSeu7rr322iBM7rzzzundkCSZbj4qBMrMbAAoB04AW4GbEp8/AjwPfBm4C3jM3fuA98zsIHAN8HKGaxSRWWb4YojpvMGyp6eH2tra4P0XvvAFHnzwQe6++25WrFjBhg0beO+994LPr7nmGm6//XaOHj3KV7/6VZYvX87y5ctpbGzkuuuuA+KXoT766KPU1NSM+K61a9em1YS0ceNG9u/fT3d3N7W1tTz88MPcdtvUGlnM3ae0gnOu3OzzwNeBXuApd/83Znba3RclzRNx97CZfR94xd0fTUx/GPiNuz8xap33AfcBrFq16uojR8Z9VoSIzBKNjY2sWbMm22XkpVS/OzPb7e4NqebPZPNRmPjR/2pgOTDPzD59rkVSTBuTWO7+kLs3uHtDdXXKp8mJiMgkZbKj+VbgPXdvdfcB4BfA9cApM1sGkHgdblBrAlYmLV9LvLlJRERmSCZD4SiwwczKLd4NfwvQCGwD7knMcw/wZOLnbcAWMysxs9VAPbAjg/WJiMgoGetodvdXzewJ4DVgEHgdeAiYDzxuZvcSD467E/PvM7PHgbcT89/v7kOZqk9ERMbK6NVH7v41YPR93H3EzxpSzf914h3TIiKSBRrmQkREAgoFEZE05MLQ2d///vd53/veh5nR1tY2ofWnS6EgIpInbrjhBp5++mkuuOCCjH2HQkFEZJJmeujsdevWUVdXN13lp6RQEJHZqScK7afjrxmiobNFRPJBTxSOniA+UILDquVQXjrtX6Ohs0VE8kFvFDAoLY6/9mbmbGEmh86eKQoFEZl9ykoBh2h//LVs+s8SYGaHzp4pCgURmX3KS+NNRtXhaWs6Gh46e/i/73znO8HQ2Rs3bqSqqmrE/MNDZ2/YsCEYOvvDH/4wn/rUp7juuuu44oor+MQnPkFXV9eY7xqvT+F73/setbW1NDU1ceWVV/LZz352yts1WkaHzs60hoYG37VrV7bLEJEM09DZk5czQ2eLiEj+USiIiEhAoSAieSGfm7qzZTK/M4WCiOS80tJS2tvbFQwT4O60t7dTWjqxTnbdvCYiOW/4ipvW1tZsl5JXSktLqa2tndAyCgURyXlFRUUj7haWzFHzkYiIBBQKIiISUCiIiEhAoSAiIgGFgoiIBBQKIiIS0CWpIjJGS1uE1o4I1ZVhaqrC2S5HZpBCQURGaGmLsP3FnYRCRizmbNq4XsEwh6j5SERGaO2IEAoZ4YUVhEJGa0ck2yXJDFIoiMgI1ZVhYjEn0tlFLOZUV+osYS5R85GIjFBTFWbTxvXqU5ijFAoiMkZNlcJgrlLzkYiIBBQKIiISUCiIiEhAoSAiIgGFgoiIBBQKIiIS0CWpInOQxjaS8SgUROYYjW0k56LmI5E5RmMbybkoFETmGI1tJOeS0VAws0Vm9oSZ7TezRjO7zswqzWy7mR1IvIaT5t9qZgfN7B0zuy2TtYnMVcNjG1255iI1HckYmT5T+C7wf939/cAHgEbgK8Az7l4PPJN4j5ldCmwBLgM2Az8ws4IM1ycyJ9VUhbns4gsVCDJGxkLBzBYAHwQeBnD3fnc/DdwFPJKY7RHgY4mf7wIec/c+d38POAhck6n6RERkrEyeKVwItAL/y8xeN7Mfm9k8YIm7NwMkXmsS868AjiUt35SYNoKZ3Wdmu8xsV2trawbLFxGZezIZCoXAVcD/cPd1wFkSTUXjsBTTfMwE94fcvcHdG6qrq6enUhERATIbCk1Ak7u/mnj/BPGQOGVmywASry1J869MWr4WOJHB+kREZJSMhYK7nwSOmdkliUm3AG8D24B7EtPuAZ5M/LwN2GJmJWa2GqgHdmSqPhERGSvTdzR/DviZmRUDh4B/TzyIHjeze4GjwN0A7r7PzB4nHhyDwP3uPpTh+kREJElGQ8Hd9wANKT66ZZz5vw58PZM1iYjI+HRHs4iIBBQKIiISUCiIiEhAoSAiIgGFgoiIBBQKIiISUCiIiEhAoSAiIgGFgoiIBBQKIiISUCiIiEhAoSAiIgGFgoiIBBQKIiISUCiIiEhAoSAiIgGFgoiIBBQKIiISUCiIiEhAoSAiIgGFgoiIBBQKIiISUCiIiEhAoSAiIgGFgoiIBBQKIiISUCiIiEhAoSAiIgGFgoiIBBQKIiISUCiIiEhAoSAiIgGFgoiIBBQKIiISUCiIiEhAoSAiIgGFgoiIBM4bCmb2gJmFZ6IYERHJrnTOFJYCO83scTPbbGaW6aJERCQ7zhsK7v6fgXrgYeDfAQfM7G/N7KJ0vsDMCszsdTP758T7SjPbbmYHEq/hpHm3mtlBM3vHzG6b1BaJiMikpdWn4O4OnEz8NwiEgSfM7JtpLP55oDHp/VeAZ9y9Hngm8R4zuxTYAlwGbAZ+YGYFaW6HiIhMg3T6FP7UzHYD3wR+D1zh7v8RuBr41+dZtha4Hfhx0uS7gEcSPz8CfCxp+mPu3ufu7wEHgWvS3xQREZmqwjTmqQI+7u5Hkie6e8zM7jjPsn8P/AVQkTRtibs3J9bRbGY1iekrgFeS5mtKTBvBzO4D7gNYtWpVGuWLiEi60ulT+C+jAyHps8ZU0wESgdHi7rvTrCVVB7an+M6H3L3B3Ruqq6vTXLWIiKQjnTOFyboBuNPMPgKUAgvM7FHglJktS5wlLANaEvM3ASuTlq8FTmSwPhERGSVjN6+5+1Z3r3X3OuIdyM+6+6eBbcA9idnuAZ5M/LwN2GJmJWa2mvgVTzsyVZ+IiIyVyTOF8XwDeNzM7gWOAncDuPs+M3sceJv4FU73u/tQFuoTyUstbRFaOyJUV4apqdL9pjI5Fr/aND81NDT4rl27sl2GSNa1tEXY/uJOQiEjFnM2bVyvYJBxmdlud29I9ZnGPhKZBVo7IoRCRnhhBaGQ0doRyXZJkqcUCiKzQHVlmFjMiXR2EYs51ZU6S5DJyUafgohMs5qqMJs2rlefgkyZQkFklqipUhjI1Kn5SEREAgoFEREJKBRERCSgUBARkYBCQUREAgoFEREJKBRERCSgUBARkYBCQUREAgoFEREJKBRERCSgUBARkYBCQUREAgoFEREJKBRERCSgUBARkYBCQUREAnrymsg0aGmL6FGYMisoFESmqKUtwvYXdxIKGbGYs2njegWD5C01H4lMUWtHhFDICC+sIBQyWjsi2S5JZNJ0piCz2kw061RXhonFnEhnF7GYU12pswTJXwoFmbVmqlmnpirMpo3r1acgs4JCQWat5GadSGcXrR2RjO2wa6oUBjI7qE9BZi0164hMnM4UZNZSs47IxCkUZFZTs47IxKj5SEREAgoFEREJKBRERCSgUBARkYBCQUREAgoFEREJ6JJUkRmmYbYllykURGaQhtmWXJex5iMzW2lmz5lZo5ntM7PPJ6ZXmtl2MzuQeA0nLbPVzA6a2TtmdlumahPJFg2zLbkuk30Kg8AX3X0NsAG438wuBb4CPOPu9cAzifckPtsCXAZsBn5gZgUZrE9kxmk8Jsl1GWs+cvdmoDnxc5eZNQIrgLuAmxKzPQI8D3w5Mf0xd+8D3jOzg8A1wMuZqlFkpmk8Jsl1M9KnYGZ1wDrgVWBJIjBw92Yzq0nMtgJ4JWmxpsS00eu6D7gPYNWqVRmsWmSs6egk1nhMkssyHgpmNh/4OfBn7n7GzMadNcU0HzPB/SHgIYCGhoYxn4tMp+QQANRJLLNeRkPBzIqIB8LP3P0XicmnzGxZ4ixhGdCSmN4ErExavBY4kcn6RM5l9JVCq1cunbGH9ohkSyavPjLgYaDR3b+T9NE24J7Ez/cATyZN32JmJWa2GqgHdmSqPpHzGX2lkIM6iWXWy+SZwg3AZ4C9ZrYnMe0/Ad8AHjeze4GjwN0A7r7PzB4H3iZ+5dL97j6UwfpEzmn0lUL1dSupr1vJwcPHxrZriswS5p6//7wbGhp8165d2S5DZrHRHcsTvflMdy9LLjKz3e7ekOoz3dEscg6jrxRKblI6X7+C7l6WfKQB8WTOammLsO/dQ7S0pXdXcUtbhNNnuunpjabVr6C7lyUf6UxB5qTJNAMNz49D5cIKKuaX0x7pHLd5SHcvSz5SKMicNJFmoNHz90b7OHjkOGWlJRw/2Ubt0iqKi4vHBIvuXpZ8pFCQOWmiR/HJ8/f29lFeVkJBYQFmUFBYEDQPjd7x6+5lyTcKBZmTJnoUnzx/yEK89ta79PX34w5Dg0MUhArUPCSzgkJB5qyJHsUnz784vDAIiJjH1Dwks4ZCQWQS1Cwks5VCQWQcuvFM5iKFgkgKuvFM5irdvCazykRvSBtPrtx4Nl3bI5IunSnIpOVa88p0Ht3nwo1nOluRbFAoyKTk4g5rojeknUsu3Hg2ndsjki6FgkxKLu6wpvvoPttXGOXC2YrMPQoFmZRc3GHVVIW56vKL2ffuISrmlQfTRz9Sc/h5CPV1K0fs9HOtOSwXzlZk7lEoyKTM5A4r1TMNUn3vy7v38txLr9HTGyVUEKLx4GFWLlvC0ROnCJnR2d1NbCjGwGCM4uJC9h84wkc33Tip5yRMx3akI9tnKzL3KBRk0mZihzV6Z33V5Rfz2lvv0t/fT2dXN/PKy2iPnKH7bA9ne6PBcmbQ1d3D8ZNtxDxGLOYjPhuKFdF1tido9sp0c1gu9sGIpKJQkJw2emd9uKmZ/v5+mlva6TjdyVAs9ZMDhx8oODg09omu7vHp7n9o9sp0c1gu9sGIpKJQkJw2emddV7uMQ0dP0NMbHTcQzqeoqJDq8EI237Qh2DFnujksF/tgRFJRKEhOS7WzPn2mi23NLedczoDi4iIGBgaDCbGYs3D+PBYumM/mmzawpr5uTDt/po7e1Wks+UKhIDlv9M56QcU8llRXcrqzi76+foYGhygqKaa8tITSshKqw4tYUh3mVGsHZ3uizJ9XxuFjJygtLeWK918UP1r32Iy386vTWPKBQkHyTnVlmIp55XR391I0r5Cl1YuDq4iSJe/0ly+tAWdE843a+UXGUihI3qmpCvPRW28c936D5Pk2bVwfzFe5cMGYZx+onV9kJIWC5KWJNMW8d+wkoZBx+NjJEU1EaucXGUujpErOS2ek0PHmSW4i6uvvZ/fe/SPmqakKc9nFF54zEDRSqcwlOlOQEZKvxmmPdHK4qZlFC+azoGJeVo6m0+kMPtc8w5eCvnf0BCdOtdPX10/H6a60O5V105nMNQoFCTQeOMyzL+2mvLSEtkgnbR2dFBUWMDA4RF3tUhZUzJ/xnWI6ncHnm2fxogUcaWqmsLCA7p5eSkqK0+5UnmxndK6NoySSLoWCAPGd2LO/303kTBcdpzuJdHbj7gwODWIYnV1nJ7QzHb3uye4g07npa7x5ho/yO7u6GRgcpKiwkL7+AXp7+6iuHH8MpWQhC9Fx+gzRaB/FxcVpdUa3tEX45dO/o69/gJLiIj5669gro0RylUJBgPgRcVlZCT3RKJHOM5gZITOGYjHA6YlGOX6yjZBNrBtqqs0v6XQGjzfP8FF+xbxyTra0UV5WSnlZKTdffzUA21/cSV9/P729fdx8w9Wsqa8bU/trb71LWWkJPb19bLjq8rRqP3j4GM2n2ikpKaIjcoaDh48pFCRvKBQEiB9tlxQXszi8EMfp6u7FgN5oH5WLKlixrIahwSFiHpvQetNtfjnXUXs6Vxqlmqe6MkxPb5TmU+0UFBRQWFjAzdfHd/773j1EX38/7ZFO+voHePal3SwOLxyxjuHaly+pCm54S4cTH3SPxOvkBuMQyQ6FQp7JRFv18Dqvuvzi4Dr+5E7mQ0ebCYWMglDBhK/lT6f5Z6JnE6l+B6mm1VSFef/7LqCvf4AlVWH6BwaDHXt1ZZje3j76+gcoLiqivLRkTGBNdryi+rqV7D9whL6BASoXLqC+bmXavy+RbFMo5JFMXAkz3jprqsJBc8pFF9SeN4jGC6t0mn8m0pmbql5g3N9Lfd1KDh87GQ+EpB17TVWYm2+4OuhYT9VfMNn7GGqqwnx0043qaJa8pFDII5kYliGddZ6v+eZ8YXW+5SdyRJ6qXiCY1nyqjd1793P1Fe8Pvne8Hfua+joWhxcGdzynkovjFQ0HcMhCY+7QFpkqhUIeycTwy9Oxzoke6adq5kn3iHy8emMxp/lUG00n24j29XPoyImg8zi543n4+5KNd8fzRJrqWtoiI4bTeO2td897b8VkziSGA7i/v5+mk22sWFpFSXGx7p+QaaNQyAPJO5DpHpZhOoZ6SDdYznVGke4R+Xj1XnX5xex8420WzCuju6d3ROcxjN+8NF6gTeSy0uF5m0+1YwblZaUsWjCfZYkO6tEhOZVmwOF6CwoLMIPCwgJCIdNgfjJtFAo5LtUO5LKLL5zW70i1Q57IkWy6wTJdzV+j6x2+dPRsT5Tm1g6KiwspLSlmYHCQg4ePsXDB/HG/N1WgtbRFeOHV12lqbqG8rPS8l5W2dkTi4VFSBICFjJ5o37ghOZXfw3C9Q4ND8SfIDQ5N6gIAkfEoFHLcTA/v3NIW4cDhY+w/eITystK0j2TTOdKfTFPV+a40+pcjTex8o5H+vgHcoLAwRDTaR2wohsecxgNHuL7hinG/d3SgQfys4sTJVrq6ewiFjJCFxvQ5JNcQX84509VDYWGI2oU18e8cp71/Kk12yfVuUJ+CZIBCIcdNZAfS0hbhhR17aD7VxvKaKjZeu3bMEfV4R/Pxo+13eG3vfvr6B3GP8YFL6+kfiB9tT0eT1fAO7cDhY1ga8zceOMyzv99NWVkJHnNWLK3GDJqaW7GQcbKlnY7OM7g7sZhTUlSEhUJgRm80yvyK8vgZlse46vKL2ffuITDjwOFjtEc6OXr8JF1ne6iYX86ptgjHTpyis+tssL5YLEbH6S4Ann5xJ799/lWi/VHcQ5jHcCAWi7FieQ1nz/YS7e9jqHeIUEEBp890cd3VV4zZnsNNzdTVLptSk10udn7L7GHu+XtrTUNDg+/atSvbZaQleYcw+s7Z80mnKaelLcI//J+nOHr8ZHBUW125kD/efFPw2Mlfbv8dfQMDlBQVcX3DFUQ6zwQdo8+9tJtDR08k7mCOW1gRf3RlSVER4UULxpw1jFfXubY1nbb64bOVnXveprsninuM3mhf4q4wwMEs3kQzWklREf2Dgwz/uy4IhaiYV060vy+4LHWm1C6vYWBgkJ6eKF1ne847f1FRAYvDC2ltOz3i/0NpSRFDQzEGBocoLChg/vwySouLCS9awAUrlgTPrk71eFGRVMxst7s3pPws10LBzDYD3wUKgB+7+zfGm3eqofDb//cKT7+4i8GhIQAKC0J8bPO/Ch7acr4d+Xh/gKOnNx44zBO/fj5+d6vDJz5yU9rBkO4f+Uu73uQ3z73C2d5e8Pj+s7iogGU1Vax5Xx2n2iO8c+AI/YODwfYOS+xnUyotLqK0tITLL7mQM11nKSgsoLy0hBVLq3n7wOExO/eXd+/lV8++RCwWAzeuvvISLq1fTaTzDGe6e2htj3Do6AmKi4roOnuWqvAiqhcvoqy0hN6+Pg4fix+9GzAwOATuuiP4PMyM8rISbr/5+uBGQ43oKueSN6FgZgXAu8AmoAnYCfyJu7+dav6phMKf/9V3J1smAJeEF3HD8iWsXlBBaWFhWs0hIlMxfKI0FItxqjfK3rYO9rS209rbC8DiRQtZvrQqGJbjyjUXTftFCcnGHLC0d9Jy+BiNx5qJRfsIFxZRFgqxZF4Z8wsLMZvBv5KCAgpGT1teA/Wr/vC+Jwq9USgrBeDswSMURboSyw3B0OgVxJsLv/Xa3uB3nm1XXXExn/n4H014uXOFQq71KVwDHHT3QwBm9hhwF5AyFCbri3/zvSktf0l4EZ96fz3zi2b4H7oIEAqFWFExnyXzyrlo0QJ+fuA9Wnt76U0MWghQkuaIrpM1+qq4j6y9lPLjLRSd7WVDxQKswgmFQoQgO38j7rjZyIO1Ey3x1/pV8UA4egIwGBykv7uH8mh/0syhlI8gC4VCfLnhA/y3XW/kRDC8tvddgEkFw3hy7clrK4BjSe+bEtMCZnafme0ys12tra2T+pKptitfsGA+RSH7w6hnIjPMiP/xlhcVUpU40q1buYzapVUsq1mc8aaj5KviQiGjvz3CwOAg0aEhQgYkdsg5d9DU0Rl/7Y0CBqXFMDgEfQNpr8LMgt95LviXw8endX25Fgqp/gWN2IO7+0Pu3uDuDdXV1ZP6klBoav9Qj5zpZiDm8Q4CkSxwIAb0DAxCaQkXrFjC/HnlFBcXB0N8ZNLoq+KKF4cpKiyktKCAmBP0BeVS8zQAlfGbGeNNRg7RfigsgMQ9Julwd9p6o5mpbxIuqltx/pkmINf6FK4DHnT32xLvtwK4+9+lmn8qfQpf/JvvTemMQX0KMtOS+xRaeqO0hoxVa9eweMWSrFx1pD6F7MtEn0KuhUIh8Y7mW4DjxDuaP+Xu+1LNn0+XpIqI5Iq86Wh290EzewD4LfFLUn8yXiCIiMj0y6lQAHD3XwO/znYdIiJzUa51NIuISBYpFEREJKBQEBGRgEJBREQCOXVJ6kSZWStwZBKLVgFt01xONs2m7ZlN2wLanlw3V7fnAndPefdvXofCZJnZrvGu0c1Hs2l7ZtO2gLYn12l7xlLzkYiIBBQKIiISmKuh8FC2C5hms2l7ZtO2gLYn12l7RpmTfQoiIpLaXD1TEBGRFBQKIiISmJOhYGbfMrP9Zvammf2TmS3Kdk1TYWZ3m9k+M4uZWd5eXmdmm83sHTM7aGZfyXY9U2FmPzGzFjN7K9u1TAczW2lmz5lZY+Lf2uezXdNkmVmpme0wszcS2/JX2a5pOphZgZm9bmb/PJX1zMlQALYDl7v7lcSf37A1y/VM1VvAx4EXsl3IZJlZAfDfgT8CLgX+xMwuzW5VU/K/gc3ZLmIaDQJfdPc1wAbg/jz+/9MH3OzuHwDWApvNbEN2S5oWnwcap7qSORkK7v6Uuw8m3r4C1Gaznqly90Z3fyfbdUzRNcBBdz/k7v3AY8BdWa5p0tz9BaAj23VMF3dvdvfXEj93Ed/5TO9zIGeIx3Un3hYl/svrK27MrBa4HfjxVNc1J0NhlP8A/CbbRQgrgGNJ75vI053ObGdmdcA64NUslzJpiaaWPUALsN3d83ZbEv4e+Avij+6ekpx7yM50MbOngaUpPvpLd38yMc9fEj8t/tlM1jYZ6WxPnkv1AN+8PnqbjcxsPvBz4M/c/Uy265ksdx8C1ib6E//JzC5397zs/zGzO4AWd99tZjdNdX2zNhTc/dZzfW5m9wB3ALd4Htyscb7tmQWagJVJ72uBE1mqRVIwsyLigfAzd/9FtuuZDu5+2syeJ97/k5ehANwA3GlmHwFKgQVm9qi7f3oyK5uTzUdmthn4MnCnu/dkux4BYCdQb2arzawY2AJsy3JNkmBmBjwMNLr7d7Jdz1SYWfXwFYdmVgbcCuzPalFT4O5b3b3W3euI/908O9lAgDkaCsD3gQpgu5ntMbP/me2CpsLM/tjMmoDrgF+Z2W+zXdNEJTr+HwB+S7wT83F335fdqibPzP4ReBm4xMyazOzebNc0RTcAnwFuTvzN7EkcmeajZcBzZvYm8YOR7e4+pcs4ZxMNcyEiIoG5eqYgIiIpKBRERCSgUBARkYBCQUREAgoFEREJKBRERCSgUBARkYBCQWQamdn6xHM6Ss1sXmK8/suzXZdIunTzmsg0M7P/SnwMmjKgyd3/LssliaRNoSAyzRJjN+0EosD1iRE5RfKCmo9Epl8lMJ/4+FqlWa5FZEJ0piAyzcxsG/Enx60Glrn7A1kuSSRts/Z5CiLZYGb/Fhh0939IPHf6JTO72d2fzXZtIunQmYKIiATUpyAiIgGFgoiIBBQKIiISUCiIiEhAoSAiIgGFgoiIBBQKIiIS+P+lXwJuTDvIaAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%time\n",
    "print('=========== OCSVM FEATURE 2 ===========')\n",
    "print('=======>>>OCSVM CLUSTER RESULT')\n",
    "ocsvm_test_label = best_ocsvm.fit(X_train).predict(X_test)\n",
    "# print results\n",
    "clustering_print_results(df_test, ocsvm_test_label, selected_features, X_test, True, True )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=========== IFOREST FEATURE 2 ===========\n",
      "=======>>>IFOREST CLUSTER RESULT\n",
      "\n",
      "Exporting LABEL...\n",
      "Number of clusters: 2\n",
      "\\Label -1: 978155 data points\n",
      "\\Label 1: 75690 data points\n",
      "\n",
      "DONE.\n",
      "--------------------------------------\n",
      "Wall time: 37.7 s\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEGCAYAAACKB4k+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAAAhMklEQVR4nO3dfXRc9X3n8fd3ZmQ9+0G2ZGwJWQ6oBEyo1QgwkKQhdhdvwsHd5LAlu2HZlnM47YY8NH0K7WbD9mzSHtLTp5OmDYFsOZAl5STpwSVpAjhw2CwYYscO4BgjYWMhP0m2ZOtp9DAz3/1jxrcjaWSPpBnNjPR5nZMjzZ07d75Xwfdz7+/3u79r7o6IiAhAqNAFiIhI8VAoiIhIQKEgIiIBhYKIiAQUCiIiEogUuoD5WLNmjbe0tBS6DBGRkrJ3797T7l6f6b2SDoWWlhb27NlT6DJEREqKmR2d6T01H4mISEChICIiAYWCiIgESrpPQUSWhomJCbq7uxkdHS10KSWloqKCpqYmysrKsv6MQkFEil53dze1tbW0tLRgZoUupyS4O2fOnKG7u5uNGzdm/Tk1H4lI0RsdHWX16tUKhFkwM1avXj3rqytdKcicdA310D3cS1N1Pc01DYUuR5YABcLszeVvplCQWesa6uHRjmcJmZFw587WbQoGkUVCzUcya93DvYTMWFu5ipAZ3cO9hS5JJO9qamqyXvf+++/nL/7iL/K2/fO2b9/OypUrufXWW2f92ZkoFGTWmqrrSbhzKtpPwp2m6ox3y4tInv3BH/wBjz76aE63qVCQWWuuaeDO1m386rpr1HQkS9q//Mu/cP3119PW1sa2bds4depU8N7Pf/5zPvShD9Ha2so3vvGNYPlXvvIVrr32Wq655hq++MUvzuv7t27dSm1t7by2MZVCQeakuaaBG9duUiBI0eoa6uHFUwfoGurJ23e8733vY/fu3ezbt4877riDBx54IHjv1Vdf5fvf/z4vvfQSf/qnf8rx48d5+umn6ejo4JVXXmH//v3s3buXF154Ydp2N2/enLeaL0YdzSKy6CzUYIju7m5+4zd+gxMnTjA+Pj7pfoAdO3ZQWVlJZWUlN998M6+88go/+clPePrpp2lrawNgaGiIjo4OPvCBD0za7v79+3Nea7byeqVgZr9rZgfM7HUze9zMKsyszsyeMbOO1M9VaevfZ2adZnbIzG7JZ20isngt1GCIT33qU9x777289tprfP3rX590T8DU4aBmhrtz3333sX//fvbv309nZyd33313Vt/18ssvs3nzZjZv3szOnTtzuh/p8hYKZtYIfBpod/ergTBwB/B5YJe7twK7Uq8xs6tS728CtgNfM7NwvuoTkcVroQZDnDt3jsbGRgAeeeSRSe89+eSTjI6OcubMGZ5//nmuvfZabrnlFr75zW8yNDQEwLFjx+jpya556/rrrw/C5LbbbsvtjqTJd/NRBKg0swmgCjgO3Ad8MPX+I8DzwB8BO4Bvu/sYcMTMOoHrgJfyXKOILDLnB0Pk8gbLkZERmpqagtef+9znuP/++7n99ttpbGxky5YtHDlyJHj/uuuu4yMf+QhdXV184QtfYP369axfv56DBw9yww03AMlhqI899hgNDZPr27x5c1ZNSO9///t54403GBoaoqmpiYcffphbbplfI4u5+7w2cMGNm30G+BIQBZ529/9sZmfdfWXaOv3uvsrMvgrsdvfHUssfBv7V3b8zZZv3APcANDc3v/fo0RmfFSEii8TBgwe58sorC11GScr0tzOzve7enmn9fDYfrSJ59r8RWA9Um9knLvSRDMumJZa7P+ju7e7eXl+v8fEiIrmUz47mbcARd+919wnge8CNwCkzWweQ+nm+Qa0buDTt800km5tERGSB5DMUuoAtZlZlyW74rcBBYCdwV2qdu4AnU7/vBO4ws3Iz2wi0Aq/ksT4REZkibx3N7v6ymX0H+BkQA/YBDwI1wBNmdjfJ4Lg9tf4BM3sC+EVq/U+6ezxf9YmIyHR5HX3k7l8Ept7HPUbyqiHT+l8i2TEtIiIFoGkuREQkoFAQEclCMUyd/dWvfpXLL78cM+P06dOz2n62FAoiIiXipptu4tlnn2XDhg15+w6FgojIHC301NltbW20tLTkqvyMFAoisij1+iHeSDxFrx/K23do6mwRkRLQ64d4wb+MeRgnzgf4Y+rtipx/j6bOFhEpAWe8A/MwK60Z8zBnvCMv37OQU2cvFIWCiCw6q60VtzhnvQu3OKutNS/fs5BTZy8UNR+JyKJTb1fwAf6YM3Sw2lpz0nRUDFNn/+3f/i0PPPAAJ0+e5JprruHDH/4wDz300Lz3LV1ep87Ot/b2dt+zZ0+hyxCRPNPU2XNXNFNni4hI6VEoiIhIQKEgIiWhlJu6C2UufzOFgogUvYqKCs6cOaNgmAV358yZM1RUVMzqcxp9JCJFr6mpie7ubnp7ewtdSkmpqKiYNGIqGwoFESl6ZWVlk+4WlvxR85GIiAQUCiIiElAoiIhIQKEgIiIBhYKIiAQUCiIiEtCQVBGZpmuoh+7hXpqq62muabj4B2TRUCiIyCRdQz082vEsITMS7tzZuk3BsISo+UhEJuke7iVkxtrKVYTM6B7WXcRLiUJBRCZpqq4n4c6paD8Jd5qq6wtdkiwgNR+JyCTNNQ3c2bpNfQpLlEJBRKZprmlQGCxRaj4SEZGAQkFERAIKBRERCSgUREQkoFAQEZGAQkFERAIakiqyBGluI5mJQkFkidHcRnIhaj4SWWI0t5FciEJBZInR3EZyIXltPjKzlcBDwNWAA78FHAL+CWgB3gb+o7v3p9a/D7gbiAOfdvcf5bM+kaVIcxvJheT7SuFvgB+6+7uBXwYOAp8Hdrl7K7Ar9Rozuwq4A9gEbAe+ZmbhPNcnsiQ11zRw49pNCgSZJm+hYGbLgQ8ADwO4+7i7nwV2AI+kVnsE+PXU7zuAb7v7mLsfATqB6/JVn4iITJfPK4V3Ab3A/zazfWb2kJlVA2vd/QRA6uf5U5VG4J20z3enlk1iZveY2R4z29Pbqw4yEZFcymcoRIBfAf7e3duAYVJNRTOwDMt82gL3B9293d3b6+vVQSYikkv5DIVuoNvdX069/g7JkDhlZusAUj970ta/NO3zTcDxPNYnIiJT5C0U3P0k8I6ZXZFatBX4BbATuCu17C7gydTvO4E7zKzczDYCrcAr+apPRESmy/cdzZ8CvmVmy4DDwG+SDKInzOxuoAu4HcDdD5jZEySDIwZ80t3jea5PRETS5DUU3H0/0J7hra0zrP8l4Ev5rElERGamO5pFRCSgUBARkYBCQUREAgoFEREJKBRERCSgUBARkYBCQUREAgoFEREJKBRERCSgUBARkYBCQUREAgoFEREJKBRERCSgUBARkYBCQUREAgoFEREJKBRERCSgUBARkYBCQUREAgoFEREJKBRERCSgUBARkYBCQUREAgoFEREJKBRERCSgUBARkYBCQUREAgoFEREJKBRERCSgUBARkYBCQUREAgoFEREJKBRERCSgUBARkYBCQUREAgoFEREJKBRERCRw0VAws3vNbNVCFCMiIoWVzZXCJcBPzewJM9tuZpbvokREpDAuGgru/t+BVuBh4L8CHWb2ZTO7LJsvMLOwme0zs6dSr+vM7Bkz60j9XJW27n1m1mlmh8zsljntkYiIzFlWfQru7sDJ1P9iwCrgO2b2QBYf/wxwMO3154Fd7t4K7Eq9xsyuAu4ANgHbga+ZWTjL/RARkRzIpk/h02a2F3gA+H/Ae9z9d4D3Ah+7yGebgI8AD6Ut3gE8kvr9EeDX05Z/293H3P0I0Alcl/2uiIjIfEWyWGcN8FF3P5q+0N0TZnbrRT7718AfArVpy9a6+4nUNk6YWUNqeSOwO2297tSySczsHuAegObm5izKFxGRbGXTp/A/pgZC2nsHMy0HSAVGj7vvzbKWTB3YnuE7H3T3dndvr6+vz3LTIiKSjWyuFObqJuA2M/swUAEsN7PHgFNmti51lbAO6Emt3w1cmvb5JuB4HusTEZEp8nbzmrvf5+5N7t5CsgP5x+7+CWAncFdqtbuAJ1O/7wTuMLNyM9tIcsTTK/mqT0REpsvnlcJM/hx4wszuBrqA2wHc/YCZPQH8guQIp0+6e7wA9YmUpK6hHrqHe2mqrqe5puHiHxDJwJKjTUtTe3u779mzp9BliBRc11APj3Y8S8iMhDt3tm5TMMiMzGyvu7dnek9zH4ksAt3DvYTMWFu5ipAZ3cO9hS5JSpRCQWQRaKquJ+HOqWg/CXeaqjUyT+amEH0KIpJjzTUN3Nm6TX0KMm8KBZFFormmQWEg86bmIxERCSgUREQkoFAQEZGAQkFERAIKBRERCSgUREQkoFAQEZGAQkFERAIKBRERCSgUREQkoFAQEZGAQkFERAIKBRERCSgUREQkoFAQEZGAQkFERAIKBRERCejJayI50DXUo0dhyqKgUBCZp66hHh7teJaQGQl37mzdpmCQkqXmI5F56h7uJWTG2spVhMzoHu4tdEkic6YrBVnUFqJZp6m6noQ7p6L9JNxpqq7Py/eILASFgixaC9Ws01zTwJ2t29SnIIuCQkEWrfRmnVPRfrqHe/N2wG6uaVAYyKKgPgVZtNSsIzJ7ulKQRUvNOiKzp1CQRU3NOiKzo+YjEREJKBRERCSgUBARkYBCQUREAgoFEREJKBRERCSgIakiC0zTbEsxUyiILCBNsy3FLm/NR2Z2qZk9Z2YHzeyAmX0mtbzOzJ4xs47Uz1Vpn7nPzDrN7JCZ3ZKv2kQKRdNsS7HLZ59CDPg9d78S2AJ80syuAj4P7HL3VmBX6jWp9+4ANgHbga+ZWTiP9YksOM3HJMUub81H7n4COJH6fdDMDgKNwA7gg6nVHgGeB/4otfzb7j4GHDGzTuA64KV81Siy0DQfkxS7BelTMLMWoA14GVibCgzc/YSZnf9X0QjsTvtYd2rZ1G3dA9wD0NzcnMeqRabLRSex5mOSYpb3UDCzGuC7wGfdfcDMZlw1wzKftsD9QeBBgPb29mnvi+RSeggA6iSWRS+voWBmZSQD4Vvu/r3U4lNmti51lbAO6Ekt7wYuTft4E3A8n/WJXMjUkUJXr2pZsIf2iBRKPkcfGfAwcNDd/zLtrZ3AXanf7wKeTFt+h5mVm9lGoBV4JV/1iVzM1JFC4OoklkUvn1cKNwF3Aq+Z2f7Usj8G/hx4wszuBrqA2wHc/YCZPQH8guTIpU+6ezyP9Ylc0NSRQm1rWmlb08q+051kaNkUWRTMvXT/425vb/c9e/YUugxZxKZ2LM/25jPdvSzFyMz2unt7pvd0R7PIBUwdKZTepHSxfgXdvSylSBPiyZLVNdTDi6cO0DXUc/GVU+v3Rs8yOBHNql9Bdy9LKdKVgixJc2kGOr++O1xSWUddeS3HR87M2Dyku5elFCkUZEmaTTPQ1PWHJqLs73uLmkgFnYPHaa1tpDJSPi1YdPeylCKFgixJsz2LT19/aGKU2rJKIqEI5kZZOBI0D0098OvuZSk1CgVZkmZ7Fp++fshC7Dq2j5HYKG7ORDxGxMJqHpJFQaEgS9Zsz+LT119ftToIiIQn1Dwki4ZCQWQO1Cwki5VCQWQGuvFMliKFgkgGuvFMlirdvCaLymxvSJtJsdx4lqv9EcmWrhRkzoqteSWXZ/fFcOOZrlakEBQKMifFeMCa7Q1pF1IMN57lcn9EsqVQkDkpxgNWrs/uCz3CqBiuVmTpUSjInBTjAau5poGtjW28ePIAq8prg+VTH6l5/nkIbWtaJx30i605rBiuVmTp0fMUZM4W6iCa6ZkGmb5359GX+KfDzzMwPkLIQqyrquOKFU28cfYdQmb0RgdIeILxxATlkWX80vImfueqW+f0nIRc7IdIoeh5CpIXC9G8MvVgvbWxjV3H9hGNjdE7do6Vy6o5MdxH//ggZydGgs+FMPrGB+k4e4wECWIkJr0X8zj94wNBs1e+m8OKsQ9GJBOFghS1qQfrA31vE42NcXjwJCdH+pgg8xNbE6nHZU4Qm/bgzATOeCJGwgmalPLdHFaMfTAimSgUpKhNPVhvqmvh1f4jDE4MzxgI6TI1jlaGylhfvYbfuuKW4MCc7/b7YuyDEclEoSBFLdPBuid6ljfPdl/wc4ZRFV7GaGICHAxIkGD1shXUV67gN6+4hS0NV05r58/X2bs6jaVUKBSk6E09WK+pWM6G2rX0RPsZiY8Ti8WoiCyjdlklNZFKGqvraaldy9uDJxkYH2FleTWv9x2lJlLB+9ZdnTpbTyx4O3+hh7iKZEOhICWnqbqeuvJa+seGWBkuY2PdumAUUbr0g/7lK9bjzqTmG7Xzi0ynUJCS01zTwG9feeuM9xukr3dn67ZgvbVVddOefaB2fpHJFApSkmbTFPN6/9uEzHi9/+ikJiK184tMp1lSpehlM1PoTOukNxGNxEZ5pnvvpHWaaxq4ce2mCwaCZiqVpURXCjJJ+mic4yNnOND3NvWVK1lTsbwgZ9PZdAZfaJ3zQ0Ff6zvC4cETRGPjnIz2Z92prJvOZKlRKEhgd89BHn/rOWojlRwbOcPxkdMsC5Uxlphg08oNrKlYseAHxWw6gy+2zrqqOg6cPUpZKELf+CCVkWVZdyrPtTNaU1pIqVIoCJA8iD3e+Rynov2coI+eaD8JnLF4jBBG7+gAVZHyOY3Qmc8BMpubvmZa5/xZ/unRc4zHJ1gWLmM0Ns7QxChN1fVZ1RWyECej/QxPjFIZKc+qM7prqId/OPgU0dgYlZFyfvvK6SOjRIqVQkGA5BlxTVkF5yaW0RM9i2FELETM4ySAoYkROgePE7LZdUPNt/klm87gmdY5f5a/qryWw4MnqY1Usbyqio9fdjMAj3Y8y0hslKGJUT5++c1sabhyWu27ju2jJlLB4ESUWzdsyar2fac7eWvgBFWRco6P9LHvdKdCQUqGQkGA5Nl2VaSCxqrVAPSPDYIbw7Eol1TV0bqikYl4jIQnLrKlybJtfrnQWXs2I40yrdNUXc/gRJS3Bk5QFgqzLBLh45clD/4vnjrASGyUYyNnGI2N8/hbz7G+avWkbZyv/bLl64Mb3rLjmCV/S/4s3ZmIZelRKJSYfLRVn9/m1sa2YBx/eifza31HCJkRsfCsx/Jn0/wz26uJTH+DTMuaaxq4vv4KRmKjbKhZy2h8PDiwN1XXMzQxymhsPHk3dKRyWmDNdb6itjWt7O45RDQ+yiXhOtrWtGb99xIpNIVCCcnHSJiZttlc0xA0p2xefdlFg2imsMqm+Wc2nbmZ6gVm/Lu0rWnl9f6jqUD4twN7c00DH7/85qBjPVN/wVzvY2iuaeB3rrpVHc1SkhQKJSQf0zJks82LNd9cLKwu9vnZnJFnqhcIlh0eOMEz3Xv5tab3Bt8704F9S8OVrK9aHdzxnEkxzld0PoBDFpp2h7bIfCkUSkg+pl/OxTZne6afqZkn2zPymepNuHN44AQdg8cYjo3yat+RoPM4veP5/Pelm+mO59k01XUN9UyaTmPXsX0XvbdiLlcS5wM4GhujY/AYl9eupypSofsnJGcUCiUg/QCS62kZcjHVQ7bBcqErimzPyGeqd2tjGz9856fULaulf3xoUucxzNy8NFOgzWZY6fl13xo4gRmsKKumvmIF71q+LmNIzqcZ8Hy9ZeEI5kZZqIyQmSbzk5xRKBS5TAeQG9duyul3ZDogz+ZMNttgyVXz19R6zw8dHRgf4e3Bk5SHl1FdVsFYfIJ9pzupr1wx4/dmCrSuoR6+c/gF3jzXTW1Z1UWHlXYP9xKNjVEVKQeSz24YjEVnDMn5/B3O1zsRj+HmTCQmKAvNfgCAyEwUCkVuoad3TjaDdPBy7yFqyyqzPpPN5kx/Lk1VFxtptP/MW/yoew/R2DgJEkRCEYZio8Q8QdwT7O55gx0tN8z4vVMDDZJXFZ0Dx+gbGyRkIUJmTO1zSK+hqboeB/pGB4iEI/zS8qbUd2Zu759Pk116vepTkHxQKBS52RxAuoZ6+O6R/8vhc8d514r1fGzj+6edUc90Np882/4ZzxzbR3RijIQ5H1x3DaPxcfad7sxJk9W/TWXdQfJ8+sJ29xzk8c7nqCmrIOHQumI9hvHmuWOEDA4PnuJUtI+EOwkSVIXLMTNCwNBElLryGsJmJDzB1sY2Xjx5ADNj3+kOjo+c4WB/F/1jg9RVLOfo4CneONdFX3SAOE7cE8RIcCLaRwh4tHMX3zz0NMOxUcwdN0h48mluv7SikbPjQwzGRklMxDliIXqiZ7ltww3T9udA39tsqmuZV5NdMXZ+y+Jh7qV7Y017e7vv2bOn0GVkJf2AMPXO2YvJpimna6iHL+9/nINnu/DUWW1TVT2fvvrXg8dO/v0vniIaH6UyXMGOlhs4NdLP+Y7Rxzuf57X+w0z4vz33uKF8BWsqVlARLueSqpXTrhpmqutC+5pNW/35q5UfdO/h3NgQCRIMTYwmT9Yt+SOEMRAbmfZ3qA6XE42Pk0j9DcoIs6q8luH4KGOxcWLM7ua7uQoBV6xoYjQ2wbmJYc6ND016onQIplVSYWWsr67j2NBpxlJrG1AdqSAWjzPuE0SIsKqihqpwOZdU1XHVqg3EE/Hgb605lyQbZrbX3dszvldsoWBm24G/AcLAQ+7+5zOtO99Q+Mc3n+axN59ljBgA5RbhU5t2BA9tudiBfKZ/gFOX7+45yF+9/l3MDTfnd6/+WNbBkO0/8ifffpGH3vwhA+PDeCoWKkPL2Lh8HVvq303XUA8v9x5iND7OhMdIbxAJY8RnGJJZE66gqqyC963dxJnRAcpCEWrLqrh8xXp29xycdnDfefQlHnzjByQ8gePc0vhetqy9ilMj/fSNDdA11Mtr/UeoCC+jb2yQ9VWr2VBTT3WkiuF4lAN9b9M3NggO4x7DIQi54vovtXiEMWrKqrjn3R8ObjTUjK5yISUTCmYWBt4Efg3oBn4KfNzdf5Fp/fmEwvuf+lzG5Tdu/SvC4TltsmhFozUMnl1PJDxOVe1pQpZgcGAt1TW9VFQOJc++Hc71NXKsq53oyEoAVq56B4Cz/ZcSHa5j1ZojbHjXi8E24vEy4rEyEoky+k+30N/XQnRkJdHhugLu7YWtWnOE5stepLLqLIlEiESsnOjIKo6/s5n+0xsLXd68NFau5rLl64JpOX513TU5H5SQLtPJzw/f2cORgeP0jJ5jLD7OxAJdmWXjYy3v47NXfzR4nV4/JE8Snzn+s0KVNyf/rvFX+ELbJ2b9uQuFQrH1KVwHdLr7YQAz+zawA8gYCnN181O/n3H5jVv/ikiEYN6axaKqaoiqqjcn7Vd55ZFp+7lqzTGqas4xNLCWcHiMispBMKgbPMqZ3hbedcULlJVNBOuHIxNQnnxdVfNzlq86TnR4DUcPX1eUwbBqzRGu/OXvT9oHGKW69hzLV57kjde2l3QwDIwnJy0EqIpU5HVE0tRRce+p28jDb/4weZVXpL779k8A+OzVH51U/8B4lK7hHrqGS+8hSk8fS4bYXIJhJsX25LVG4J20192pZQEzu8fM9pjZnt7e3jl9yUztyuHw4gsESO7T1P3KtJ9mULYsSjgyTqRsNHUlsIxwZJyVde8QDscu+B1ly6K4h6isOpvbHciR5StPEA5PZHwvFI6xfOWJBa4odwx4z+oWWmsbeVfturw3HaWPiguZ8bPTHUzEZ/7vo1js6e0AJtcfjY9yZnSgwJXN3b7Tb+V0e8UWCpkOyZPat9z9QXdvd/f2+vq5nQlFZtjteDzZjLLYuE/fr0z76Q4T45XEY8uITVQQDk8QjowTjy3jbN+lxOMzX1ie/6xZImh+KjYDZ9cRj5dlfC8RjzBwdt0CV5QbTVWruWrlBlYuq6EyUh5M8ZHX75wyKu5X1rRSFi62hofp2uuTkxOm118ZrmB1xfICVzZ3bWsuy+n2iq1P4Qbgfne/JfX6PgB3/7NM68+nT+Hmp34/4xWD+hTUp1AqNlav5b9tuq1go47Up1B4+ehTKLZQiJDsaN4KHCPZ0fyf3P1ApvVLaUiqiEixKJmOZnePmdm9wI9IDkn95kyBICIiuVdUoQDg7j8AflDoOkRElqJi62gWEZECUiiIiEhAoSAiIgGFgoiIBIpqSOpsmVkvcHQOH10DnM5xOYW0mPZnMe0LaH+K3VLdnw3unvHu35IOhbkysz0zjdEtRYtpfxbTvoD2p9hpf6ZT85GIiAQUCiIiEliqofBgoQvIscW0P4tpX0D7U+y0P1MsyT4FERHJbKleKYiISAYKBRERCSzJUDCzr5jZG2b2qpn9s5mtLHRN82Fmt5vZATNLmFnJDq8zs+1mdsjMOs3s84WuZz7M7Jtm1mNmrxe6llwws0vN7DkzO5j6b+0zha5prsyswsxeMbOfp/blfxa6plwws7CZ7TOzp+aznSUZCsAzwNXufg3J5zfcV+B65ut14KPAC4UuZK7MLAz8HfDvgauAj5vZVYWtal7+Edhe6CJyKAb8nrtfCWwBPlnC//+MAR9y918GNgPbzWxLYUvKic8AB+e7kSUZCu7+tLuff6DsbqCpkPXMl7sfdPdDha5jnq4DOt39sLuPA98GdhS4pjlz9xeAvkLXkSvufsLdf5b6fZDkwafxwp8qTp40lHpZlvpfSY+4MbMm4CPAQ/Pd1pIMhSl+C/jXQhchNALvpL3upkQPOoudmbUAbcDLBS5lzlJNLfuBHuAZdy/ZfUn5a+APYf7PPy26h+zkipk9C1yS4a0/cfcnU+v8CcnL4m8tZG1zkc3+lDjLsKykz94WIzOrAb4LfNbdBwpdz1y5exzYnOpP/Gczu9rdS7L/x8xuBXrcfa+ZfXC+21u0oeDu2y70vpndBdwKbPUSuFnjYvuzCHQDl6a9bgKOF6gWycDMykgGwrfc/XuFricX3P2smT1Psv+nJEMBuAm4zcw+DFQAy83sMXf/xFw2tiSbj8xsO/BHwG3uPlLoegSAnwKtZrbRzJYBdwA7C1yTpJiZAQ8DB939Lwtdz3yYWf35EYdmVglsA94oaFHz4O73uXuTu7eQ/Hfz47kGAizRUAC+CtQCz5jZfjP7h0IXNB9m9h/MrBu4Afi+mf2o0DXNVqrj/17gRyQ7MZ9w9wOFrWruzOxx4CXgCjPrNrO7C13TPN0E3Al8KPVvZn/qzLQUrQOeM7NXSZ6MPOPu8xrGuZhomgsREQks1SsFERHJQKEgIiIBhYKIiAQUCiIiElAoiIhIQKEgIiIBhYKIiAQUCiI5ZGbXpp7TUWFm1an5+q8udF0i2dLNayI5Zmb/i+QcNJVAt7v/WYFLEsmaQkEkx1JzN/0UGAVuTM3IKVIS1Hwkknt1QA3J+bUqClyLyKzoSkEkx8xsJ8knx20E1rn7vQUuSSRri/Z5CiKFYGb/BYi5+/9JPXf6RTP7kLv/uNC1iWRDVwoiIhJQn4KIiAQUCiIiElAoiIhIQKEgIiIBhYKIiAQUCiIiElAoiIhI4P8D6uS58g/PozwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%time\n",
    "print('=========== IFOREST FEATURE 2 ===========')\n",
    "print('=======>>>IFOREST CLUSTER RESULT')\n",
    "iforest_test_label = best_iforest.fit(X_train).predict(X_test)\n",
    "# print results\n",
    "clustering_print_results(df_test, iforest_test_label, selected_features, X_test, True, True )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>timestamp</th>\n",
       "      <th>duration</th>\n",
       "      <th>protocol</th>\n",
       "      <th>src_ip</th>\n",
       "      <th>src_port</th>\n",
       "      <th>direction</th>\n",
       "      <th>dst_ip</th>\n",
       "      <th>dst_port</th>\n",
       "      <th>state</th>\n",
       "      <th>src_type</th>\n",
       "      <th>...</th>\n",
       "      <th>dst_port3</th>\n",
       "      <th>dst_port4</th>\n",
       "      <th>dst_port5</th>\n",
       "      <th>dst_port6</th>\n",
       "      <th>dst_port7</th>\n",
       "      <th>dst_port8</th>\n",
       "      <th>dst_port9</th>\n",
       "      <th>label</th>\n",
       "      <th>x</th>\n",
       "      <th>y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2012/12/01 03:36:40.878146</td>\n",
       "      <td>13.342922</td>\n",
       "      <td>tcp</td>\n",
       "      <td>138.11.74.16</td>\n",
       "      <td>6667</td>\n",
       "      <td>&lt;?&gt;</td>\n",
       "      <td>201.30.94.155</td>\n",
       "      <td>1027</td>\n",
       "      <td>PA_PA</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1.877140</td>\n",
       "      <td>0.540756</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2012/12/01 03:51:49.918983</td>\n",
       "      <td>11.884443</td>\n",
       "      <td>tcp</td>\n",
       "      <td>138.11.74.16</td>\n",
       "      <td>6667</td>\n",
       "      <td>&lt;?&gt;</td>\n",
       "      <td>201.30.94.155</td>\n",
       "      <td>1027</td>\n",
       "      <td>PA_PA</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1.849994</td>\n",
       "      <td>0.481309</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2012/12/01 04:07:21.899558</td>\n",
       "      <td>12.332861</td>\n",
       "      <td>tcp</td>\n",
       "      <td>138.11.74.16</td>\n",
       "      <td>6667</td>\n",
       "      <td>&lt;?&gt;</td>\n",
       "      <td>201.30.94.155</td>\n",
       "      <td>1027</td>\n",
       "      <td>PA_PA</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1.869314</td>\n",
       "      <td>0.499861</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2012/12/01 04:38:46.872983</td>\n",
       "      <td>12.834812</td>\n",
       "      <td>tcp</td>\n",
       "      <td>138.11.74.16</td>\n",
       "      <td>6667</td>\n",
       "      <td>&lt;?&gt;</td>\n",
       "      <td>201.30.94.155</td>\n",
       "      <td>1027</td>\n",
       "      <td>PA_PA</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1.867158</td>\n",
       "      <td>0.522169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2012/12/01 04:53:55.866280</td>\n",
       "      <td>12.204415</td>\n",
       "      <td>tcp</td>\n",
       "      <td>138.11.74.16</td>\n",
       "      <td>6667</td>\n",
       "      <td>&lt;?&gt;</td>\n",
       "      <td>201.30.94.155</td>\n",
       "      <td>1027</td>\n",
       "      <td>PA_PA</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1.861056</td>\n",
       "      <td>0.494133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1053840</th>\n",
       "      <td>2012/12/10 23:24:28.388451</td>\n",
       "      <td>-0.188116</td>\n",
       "      <td>tcp</td>\n",
       "      <td>228.91.109.140</td>\n",
       "      <td>6059</td>\n",
       "      <td>-&gt;</td>\n",
       "      <td>135.31.242.15</td>\n",
       "      <td>443</td>\n",
       "      <td>S_RA</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1.813706</td>\n",
       "      <td>-0.021952</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1053841</th>\n",
       "      <td>2012/12/10 23:24:29.509241</td>\n",
       "      <td>-0.158789</td>\n",
       "      <td>tcp</td>\n",
       "      <td>157.51.106.143</td>\n",
       "      <td>1864</td>\n",
       "      <td>-&gt;</td>\n",
       "      <td>162.58.154.76</td>\n",
       "      <td>22</td>\n",
       "      <td>S_</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1.799352</td>\n",
       "      <td>-0.020764</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1053842</th>\n",
       "      <td>2012/12/10 23:24:30.040003</td>\n",
       "      <td>-0.162865</td>\n",
       "      <td>tcp</td>\n",
       "      <td>157.51.106.143</td>\n",
       "      <td>1865</td>\n",
       "      <td>-&gt;</td>\n",
       "      <td>140.18.193.101</td>\n",
       "      <td>22</td>\n",
       "      <td>S_</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1.758258</td>\n",
       "      <td>-0.019321</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1053843</th>\n",
       "      <td>2012/12/10 23:24:30.892397</td>\n",
       "      <td>-0.188522</td>\n",
       "      <td>tcp</td>\n",
       "      <td>228.91.109.140</td>\n",
       "      <td>6060</td>\n",
       "      <td>-&gt;</td>\n",
       "      <td>135.31.242.15</td>\n",
       "      <td>443</td>\n",
       "      <td>S_RA</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1.822083</td>\n",
       "      <td>-0.022361</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1053844</th>\n",
       "      <td>2012/12/10 23:24:32.068060</td>\n",
       "      <td>-0.159834</td>\n",
       "      <td>tcp</td>\n",
       "      <td>157.51.106.143</td>\n",
       "      <td>1867</td>\n",
       "      <td>-&gt;</td>\n",
       "      <td>226.90.25.126</td>\n",
       "      <td>22</td>\n",
       "      <td>S_</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1.764958</td>\n",
       "      <td>-0.019476</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1053845 rows × 70 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                          timestamp   duration protocol          src_ip  \\\n",
       "0        2012/12/01 03:36:40.878146  13.342922      tcp    138.11.74.16   \n",
       "1        2012/12/01 03:51:49.918983  11.884443      tcp    138.11.74.16   \n",
       "2        2012/12/01 04:07:21.899558  12.332861      tcp    138.11.74.16   \n",
       "3        2012/12/01 04:38:46.872983  12.834812      tcp    138.11.74.16   \n",
       "4        2012/12/01 04:53:55.866280  12.204415      tcp    138.11.74.16   \n",
       "...                             ...        ...      ...             ...   \n",
       "1053840  2012/12/10 23:24:28.388451  -0.188116      tcp  228.91.109.140   \n",
       "1053841  2012/12/10 23:24:29.509241  -0.158789      tcp  157.51.106.143   \n",
       "1053842  2012/12/10 23:24:30.040003  -0.162865      tcp  157.51.106.143   \n",
       "1053843  2012/12/10 23:24:30.892397  -0.188522      tcp  228.91.109.140   \n",
       "1053844  2012/12/10 23:24:32.068060  -0.159834      tcp  157.51.106.143   \n",
       "\n",
       "        src_port direction          dst_ip dst_port  state  src_type  ...  \\\n",
       "0           6667       <?>   201.30.94.155     1027  PA_PA       0.0  ...   \n",
       "1           6667       <?>   201.30.94.155     1027  PA_PA       0.0  ...   \n",
       "2           6667       <?>   201.30.94.155     1027  PA_PA       0.0  ...   \n",
       "3           6667       <?>   201.30.94.155     1027  PA_PA       0.0  ...   \n",
       "4           6667       <?>   201.30.94.155     1027  PA_PA       0.0  ...   \n",
       "...          ...       ...             ...      ...    ...       ...  ...   \n",
       "1053840     6059        ->   135.31.242.15      443   S_RA       0.0  ...   \n",
       "1053841     1864        ->   162.58.154.76       22     S_       0.0  ...   \n",
       "1053842     1865        ->  140.18.193.101       22     S_       0.0  ...   \n",
       "1053843     6060        ->   135.31.242.15      443   S_RA       0.0  ...   \n",
       "1053844     1867        ->   226.90.25.126       22     S_       0.0  ...   \n",
       "\n",
       "         dst_port3  dst_port4  dst_port5  dst_port6 dst_port7 dst_port8  \\\n",
       "0              0.0        0.0        0.0        0.0       0.0       0.0   \n",
       "1              0.0        0.0        0.0        0.0       0.0       0.0   \n",
       "2              0.0        0.0        0.0        0.0       0.0       0.0   \n",
       "3              0.0        0.0        0.0        0.0       0.0       0.0   \n",
       "4              0.0        0.0        0.0        0.0       0.0       0.0   \n",
       "...            ...        ...        ...        ...       ...       ...   \n",
       "1053840        0.0        0.0        0.0        1.0       0.0       0.0   \n",
       "1053841        1.0        0.0        0.0        0.0       0.0       0.0   \n",
       "1053842        1.0        0.0        0.0        0.0       0.0       0.0   \n",
       "1053843        0.0        0.0        0.0        1.0       0.0       0.0   \n",
       "1053844        1.0        0.0        0.0        0.0       0.0       0.0   \n",
       "\n",
       "         dst_port9  label         x         y  \n",
       "0              0.0     -1 -1.877140  0.540756  \n",
       "1              0.0     -1 -1.849994  0.481309  \n",
       "2              0.0     -1 -1.869314  0.499861  \n",
       "3              0.0     -1 -1.867158  0.522169  \n",
       "4              0.0     -1 -1.861056  0.494133  \n",
       "...            ...    ...       ...       ...  \n",
       "1053840        0.0     -1 -1.813706 -0.021952  \n",
       "1053841        0.0     -1 -1.799352 -0.020764  \n",
       "1053842        0.0     -1 -1.758258 -0.019321  \n",
       "1053843        0.0     -1 -1.822083 -0.022361  \n",
       "1053844        0.0     -1 -1.764958 -0.019476  \n",
       "\n",
       "[1053845 rows x 70 columns]"
      ]
     },
     "execution_count": 118,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "sort_anomalies    =   ['total_packets', 'bytes_bothdir','bytes_onedir', 'duration']\n",
    "# get anomalies\n",
    "anomalies_ocsvm_result = get_anomalies(df_test, ocsvm_test_label)\n",
    "anomalies_ocsvm_result.sort_values(by=sort_anomalies,ascending=False)\n",
    "save(anomalies_ocsvm_result, \"ocsvm_f2_anomalies\")\n",
    "\n",
    "anomalies_iforest_result = get_anomalies(df_test, iforest_test_label)\n",
    "anomalies_iforest_result.sort_values(by=sort_anomalies,ascending=False)\n",
    "save(anomalies_iforest_result, \"ifoest_f2_anomalies\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 285,
   "metadata": {},
   "outputs": [],
   "source": [
    "####################### FEATURE 3 Analysis #######################"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 438,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'kernel': 'rbf', 'gamma': 0.15, 'nu': 0.2}"
      ]
     },
     "execution_count": 438,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ocsvm_best_param"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 439,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'max_samples': 0.2, 'contamination': 0.1}"
      ]
     },
     "execution_count": 439,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "iforest_best_param"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PCA.explained variance_ratio_ : 0.8505886924542188\n",
      "PCA.explained variance_ratio_ : 0.8495007721308345\n"
     ]
    }
   ],
   "source": [
    "X_train = get_pc(X_train_ori, 5)\n",
    "X_val = get_pc(X_val_ori, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_ocsvm = svm.OneClassSVM(**ocsvm_best_param)   \n",
    "ocsvm_label = best_ocsvm.fit(X_train).predict(X_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_iforest = IsolationForest(**iforest_best_param)\n",
    "iforest_label = best_iforest.fit(X_train).predict(X_val)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation data: \n",
      "OCSVM: =============\n",
      "Total:  41073\n",
      "Normal:  4586\n",
      "Anomaly:  36487\n",
      "\n",
      "IFOREST: ============\n",
      "Total:  41073\n",
      "Normal:  1230\n",
      "Anomaly:  39843\n"
     ]
    }
   ],
   "source": [
    "y_pred_ocsvm = ocsvm_label \n",
    "y_pred_iforest = iforest_label \n",
    "print('Validation data: ')\n",
    "print('OCSVM: =============')\n",
    "print_stats(y_pred_ocsvm) \n",
    "print()\n",
    "print('IFOREST: ============')\n",
    "print_stats(y_pred_iforest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=========== OCSVM: FEATURE 3 ===========\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          -1       0.97      0.92      0.95     38271\n",
      "           1       0.36      0.59      0.45      2802\n",
      "\n",
      "    accuracy                           0.90     41073\n",
      "   macro avg       0.66      0.76      0.70     41073\n",
      "weighted avg       0.93      0.90      0.91     41073\n",
      "\n",
      "| TP:  35331 | FP:  1156 | TN:  1646 | FN:  2940\n",
      "TPR:  0.923179430900682\n",
      "FPR:  0.4125624553890079\n",
      "TPR - FPR :  0.5106169755116741\n"
     ]
    }
   ],
   "source": [
    "print('=========== OCSVM: FEATURE 3 ===========')\n",
    "print(classification_report(y_true, y_pred_ocsvm))\n",
    "(TP, FP, TN, FN) = perf_measure(y_true, y_pred_ocsvm)\n",
    "TPR = TP/(TP+FN)\n",
    "FPR = FP/(FP+TN)\n",
    "\n",
    "print('| TP: ', TP, '| FP: ', FP, '| TN: ', TN, '| FN: ', FN)\n",
    "print('TPR: ', TPR)\n",
    "print('FPR: ', FPR)\n",
    "print('TPR - FPR : ', TPR - FPR )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=========== IFOREST FEATURE 3 ===========\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "          -1       0.94      0.98      0.96     38271\n",
      "           1       0.23      0.10      0.14      2802\n",
      "\n",
      "    accuracy                           0.92     41073\n",
      "   macro avg       0.58      0.54      0.55     41073\n",
      "weighted avg       0.89      0.92      0.90     41073\n",
      "\n",
      "| TP:  37327 | FP:  2516 | TN:  286 | FN:  944\n",
      "TPR:  0.9753338036633482\n",
      "FPR:  0.8979300499643112\n",
      "TPR - FPR :  0.07740375369903696\n"
     ]
    }
   ],
   "source": [
    "print('=========== IFOREST FEATURE 3 ===========')\n",
    "print(classification_report(y_true, y_pred_iforest))\n",
    "(TP, FP, TN, FN) = perf_measure(y_true, y_pred_iforest)\n",
    "TPR = TP/(TP+FN)\n",
    "FPR = FP/(FP+TN)\n",
    "\n",
    "print('| TP: ', TP, '| FP: ', FP, '| TN: ', TN, '| FN: ', FN)\n",
    "print('TPR: ', TPR)\n",
    "print('FPR: ', FPR)\n",
    "print('TPR - FPR : ', TPR - FPR)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PCA.explained variance_ratio_ : 0.8692339371946369\n"
     ]
    }
   ],
   "source": [
    "X_test = df_test[selected_features]\n",
    "X_test = get_pc(X_test, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=========== OCSVM FEATURE 3 ===========\n",
      "=======>>>OCSVM CLUSTER RESULT\n",
      "\n",
      "Exporting LABEL...\n",
      "Number of clusters: 2\n",
      "\\Label -1: 761112 data points\n",
      "\\Label 1: 109319 data points\n",
      "\n",
      "DONE.\n",
      "--------------------------------------\n",
      "Wall time: 1min 55s\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEGCAYAAACKB4k+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAAAiiklEQVR4nO3dfZAc9X3n8fd3Znb2WUJarXjQgiTbCjGksHS3CHzYrjM4QTEu5EsVFeUuOd0dVfwDPruScoKcS+z4jhyFk1SScrgLBl9UthNKl8SFYucSsGwX5zqDWMWyDQhZClhi0dNqJVY7O9rpefjeH9PTzO6OtA/anofdz6uKmumenp7vLtr+zO/36/61uTsiIiIAiUYXICIizUOhICIiEYWCiIhEFAoiIhJRKIiISCTV6AKuxJo1a3zDhg2NLkNEpKUcOHDgrLv313qtpUNhw4YNDA0NNboMEZGWYmbHLvWauo9ERCSiUBARkYhCQUREIi09piAiy0M+n2d4eJjJyclGl9JSOjo6GBgYoK2tbc7vUSiISNMbHh6mt7eXDRs2YGaNLqcluDujo6MMDw+zcePGOb9P3Uci0vQmJyfp6+tTIMyDmdHX1zfv1pVCQUSaRpANyJzNEGSDGa8pEOZvIb8zdR+JSFMIsgGjx0aj5b71faS70g2saHlSS0FEmkKlddDW0TZluVn09PTMedvPfe5z/MEf/EFs+6/Ytm0bV111FR/72Mfm/d5LUSiISFOotAryk/kpy3Jpn/70p/nKV76yqPtUKIhIU0h3pelb30dvf2/LdB393d/9HbfddhtbtmzhIx/5CKdPn45e++EPf8idd97Jpk2b+NKXvhSt/8IXvsCtt97KLbfcwmc/+9kr+vy77rqL3t7eK9rHdAoFEWka6a40PWt6FiUQLjdovVg+8IEP8MILL/CDH/yAHTt28Nhjj0Wv/ehHP+Kb3/wm3//+9/n85z/PiRMnePbZZzly5Aj79+/n4MGDHDhwgOeff37Gfjdv3hxbzbPRQLOILDn1GrQeHh7ml3/5lzl58iRBEEy5HmD79u10dnbS2dnJhz/8Yfbv38/3vvc9nn32WbZs2QJAJpPhyJEjfOhDH5qy34MHDy56rXOlloKILDn1GrT+xCc+wUMPPcSPf/xj/vzP/3zKNQHTTwc1M9ydXbt2cfDgQQ4ePMjRo0e5//775/RZL774Ips3b2bz5s3s3bt3UX+OagoFEVly6jVoPTY2xrp16wDYvXv3lNeeeeYZJicnGR0d5bvf/S633nord999N1/+8pfJZDIAvPXWW5w5c2ZOn3XbbbdFYXLvvfcu7g9SRd1HIrLkVAatg2xAuiu9KKGQzWYZGBiIln/913+dz33uc9x3332sW7eO22+/nTfeeCN6fevWrdxzzz0cP36c3/md3+G6667juuuu49ChQ7z//e8HyqehfvWrX2Xt2rVTPmvz5s1z6kL64Ac/yGuvvUYmk2FgYICnnnqKu++++4p+TnP3K9pBIw0ODrpusiOy9B06dIj3vve9jS6jJdX63ZnZAXcfrLW9uo9ERCQSWyiY2Y1mdrDqvwtm9ikzW21mz5nZkfBxVdV7dpnZUTM7bGZX1gYSEZF5iy0U3P2wu292983AvwSywNeBh4F97r4J2BcuY2Y3ATuAm4FtwONmloyrPhERmale3Ud3Af/s7seA7UBlmH438PHw+XbgaXfPufsbwFFga53qExER6hcKO4C/Cp9f7e4nAcLHyrD7OuDNqvcMh+umMLMHzGzIzIZGRkZiLFlEZPmJPRTMLA3cC/zv2TatsW7GqVHu/oS7D7r7YH9//2KUKCIioXq0FH4R+Cd3r8wUddrMrgUIHytXbgwD11e9bwA4UYf6RERm1QxTZ3/xi1/kPe95D2bG2bNn57X/uapHKPwK73QdAewFdobPdwLPVK3fYWbtZrYR2ATsr0N9IiIt4Y477uBb3/oW69evj+0zYg0FM+sCfh7426rVjwI/b2ZHwtceBXD3V4A9wKvAPwAPunsxzvpERK5EvafO3rJlCxs2bFis8muKdZoLd88CfdPWjVI+G6nW9o8Aj8RZk4gsD2cPH+bckSOs3rSJNTfeGMtnVKbONjOefPJJHnvsMf7wD/8QKE+d/cILLzAxMcGWLVu45557ePnll6Ops92de++9l+eff37GLKlzneYiDpr7SESWnLOHD/N/f//3SSSTlIpFPviZz8QSDJo6W0SkBZw7coREMsnKG24gkUxy7siRWD6nnlNn14tCQUSWnNWbNlEqFhk7fpxSscjqTZti+Zx6Tp1dL+o+EpElZ82NN/LBz3xmUccUmmHq7D/90z/lscce49SpU9xyyy189KMf5cknn7zin62aps4WkaanqbMXTlNni4jIgikUREQkolAQkZbQyl3djbKQ35lCQUSaXkdHB6OjowqGeXB3RkdH6ejomNf7dPaRiDS9gYEBhoeH0XT589PR0THljKm5UCiISNNra2ubcrWwxEfdRyIiElEoiIhIRKEgIiIRhYKIiEQUCiIiElEoiIhIJO7bcV5lZn9tZq+Z2SEze7+ZrTaz58zsSPi4qmr7XWZ21MwOm9ndcdYmUi3IBmTOZgiyQaNLEWmouFsKfwL8g7v/LPA+4BDwMLDP3TcB+8JlzOwmYAdwM7ANeNzMkjHXJ0KQDRg9Nsr4yDijx0YVDLKsxRYKZrYC+BDwFIC7B+7+NrAdqNyNYjfw8fD5duBpd8+5+xvAUWBrXPWJVFRCoK2jbcqyyHIUZ0vhXcAI8L/M7Adm9qSZdQNXu/tJgPCxcneJdcCbVe8fDteJxCrdlQYgP5mfsiyyHMUZCingXwD/w923ABOEXUWXYDXWzZj9ysweMLMhMxvSPCiyGNJdafrW99Hb30vf+j6FgixrcYbCMDDs7i+Gy39NOSROm9m1AOHjmartr696/wBwYvpO3f0Jdx9098H+/v7YipflJd2VpmdNjwJBlr3YQsHdTwFvmlnl5qh3Aa8Ce4Gd4bqdwDPh873ADjNrN7ONwCZgf1z1iYjITHHPkvoJ4GtmlgZeB/4j5SDaY2b3A8eB+wDc/RUz20M5OArAg+5ejLk+ERGpEmsouPtBoNbNoe+6xPaPAI/EWZOIiFyarmgWEZGIQkFERCIKBRERiSgUREQkolAQCWlSPJH4T0kVaQmVSfEqdGWzLFdqKYigSfFEKhQKImhSPJEKdR+J8M6keEE2IN2VVijIsqVQEAkpDETUfSQiIlUUCiIiElEoiIhIRKEgIiIRhYKIiEQUCiIiElEoiIhIRKEgIiKRWEPBzH5qZj82s4NmNhSuW21mz5nZkfBxVdX2u8zsqJkdNrO746xNRERmqkdL4cPuvtndK/dqfhjY5+6bgH3hMmZ2E7ADuBnYBjxuZsk61CciIqFGdB9tB3aHz3cDH69a/7S759z9DeAosLX+5YmILF9xh4IDz5rZATN7IFx3tbufBAgf14br1wFvVr13OFw3hZk9YGZDZjY0MjISY+kiIstP3BPi3eHuJ8xsLfCcmb12mW2txjqfscL9CeAJgMHBwRmvi4jIwsXaUnD3E+HjGeDrlLuDTpvZtQDh45lw82Hg+qq3DwAn4qxPRESmii0UzKzbzHorz4FfAF4G9gI7w812As+Ez/cCO8ys3cw2ApuA/XHVJyIiM8XZfXQ18HUzq3zOX7r7P5jZS8AeM7sfOA7cB+Dur5jZHuBVoAA86O7FGOsTEZFpYgsFd38deF+N9aPAXZd4zyPAI3HVJCIil6crmkVEJKJQEBGRiEJBREQiCgUREYkoFEREJKJQEBGRiEJBREQiCgUREYkoFEREJKJQEBGRiEJBREQiCgUREYkoFEREJKJQEBGRiEJBREQiCgUREYkoFEREJBJ7KJhZ0sx+YGbfCJdXm9lzZnYkfFxVte0uMztqZofN7O64axMRkanq0VL4JHCoavlhYJ+7bwL2hcuY2U3ADuBmYBvwuJkl61CfiIiEYg0FMxsA7gGerFq9HdgdPt8NfLxq/dPunnP3N4CjwNY46xMRkanibin8MfCbQKlq3dXufhIgfFwbrl8HvFm13XC4bgoze8DMhsxsaGRkJJaiRUSWq9hCwcw+Bpxx9wNzfUuNdT5jhfsT7j7o7oP9/f1XVKOIiEyVinHfdwD3mtlHgQ5ghZl9FThtZte6+0kzuxY4E24/DFxf9f4B4ESM9YmIyDSxtRTcfZe7D7j7BsoDyN92918F9gI7w812As+Ez/cCO8ys3cw2ApuA/XHVJyIiM8XZUriUR4E9ZnY/cBy4D8DdXzGzPcCrQAF40N2LDahPRGTZMvcZ3fYtY3Bw0IeGhhpdhohISzGzA+4+WOu1WbuPzOyh6gvMRERk6ZrLmMI1wEtmtsfMtplZrbOERERkCZg1FNz9v1Ae9H0K+A/AETP7fTN7d8y1iYhInc3p7CMvDzycCv8rAKuAvzazx2KsTURE6mzWs4/M7D9TPnX0LOXpKj7t7nkzSwBHKF+xLCIiS8BcTkldA/ySux+rXunupfCqZRERWSJmDQV3/93LvHboUq+JiEjr0U12REQkolAQEZGIQkGWtCAbkDmbIcgGjS5FpCU0Yu4jkboIsgGjx0aj5b71faS70g2sSKT5qaUgS1alddDW0TZlWUQuTaEgS1alVZCfzE9ZFpFLU/eRLFnprjR96/sIsgHprrRCQWQOFAqypCkMROZH3UciIhJRKIiISEShICIikdhCwcw6zGy/mf3QzF4xs98L1682s+fM7Ej4uKrqPbvM7KiZHTazu+OqTUREaouzpZAD7nT39wGbgW1mdjvwMLDP3TcB+8JlzOwmYAdwM7ANeNzMkjHWJyIi08QWCl6WCRfbwv8c2A7sDtfvBj4ePt8OPO3uOXd/AzgKbI2rPhERmSnWMQUzS5rZQeAM8Jy7vwhc7e4nAcLHteHm64A3q94+HK6bvs8HzGzIzIZGRkbiLF9EZNmJNRTcvejum4EBYKuZ/dxlNrdau6ixzyfcfdDdB/v7+xepUhERgTqdfeTubwPfpTxWcNrMrgUIH8+Emw0D11e9bQA4UY/6RESkLM6zj/rN7KrweSfwEeA1YC/lez4TPj4TPt8L7DCzdjPbCGwC9sdVn4iIzBTnNBfXArvDM4gSwB53/4aZfR/YY2b3A8eB+wDc/RUz2wO8ChSAB929GGN9IiIyjbnP6LZvGYODgz40NNToMkREWoqZHXD3wVqv6YpmERGJKBRERCSiUBARkYhCQUREIgoFERGJKBRERCSiUBCpEmQDMmczBNmg0aWINITu0SwSCrIBo8dGo+W+9X26v7MsO2opiIQqrYO2jrYpy1eyP7U6pNWopSASqrQK8pP5KcsLoVaHtCqFgkgo3ZWmb30fQTYg3ZW+4lCAcqsjP5mP9inS7BQKIlWuNAyq9wOL0+oQqSeFgkgMFrPVIVJPCgWRmCgMpBXp7CMREYkoFEREJKJQEBGRSJz3aL7ezL5jZofM7BUz+2S4frWZPWdmR8LHVVXv2WVmR83ssJndHVdtInOlC9BkuYmzpVAAfsPd3wvcDjxoZjcBDwP73H0TsC9cJnxtB3AzsA14PLy/s0hDVC5AGx8ZZ/TYqIJBloXYQsHdT7r7P4XPx4FDwDpgO7A73Gw38PHw+XbgaXfPufsbwFFga1z1icxmsae9EGkFdRlTMLMNwBbgReBqdz8J5eAA1oabrQPerHrbcLhu+r4eMLMhMxsaGRmJtW5pbVfa9aML0GQ5iv06BTPrAf4G+JS7XzCzS25aY53PWOH+BPAEwODg4IzXRWBx5h7SBWiyHMXaUjCzNsqB8DV3/9tw9WkzuzZ8/VrgTLh+GLi+6u0DwIk465Ola7G6ftJdaXrW9CgQZNmI8+wjA54CDrn7H1W9tBfYGT7fCTxTtX6HmbWb2UZgE7A/rvpkaVPXj8jCxNl9dAfwa8CPzexguO4zwKPAHjO7HzgO3Afg7q+Y2R7gVcpnLj3o7sUY65MlTF0/IgsTWyi4+/eoPU4AcNcl3vMI8EhcNcnyojAQmT9NiCcSCrJBeezBAF94qFT2o1CSVqRQEOGds5WKhSLBREB7dzuJVGLeZy3pjmvS6jT3kQjvnJ1kCcNxLGFT1s93P7rgTVqVWgqybE3v5ikWihRyBSiBl8rBsJBrG0BnPUnrUijIklerj396N0/v2l7MjEQqQTKVpGt1F92runXBmyw7CgVZ0i7Vx1/dzZOfzHNx7CKJZIL27nbyk3na2tsWfEBXGEgr05iCLGmX6uOf3s3TubJzyrIO6rJcqaUgS9rl+vhTHSmKQZGeNT309PWQ7kyr20eWPYWCLGm1+viDbMDI6yNMZiYxjEJQIN2ZVhiIoFCQZWD6wT7IBhQLRRLJcu9pqVCKQkNkudOYgiw76a40yVSSUrGEF51EKqFAEAmppSDLTrorTf+7+smez+L4gk49FVmqFAqyLGn8QKQ2dR+JVLnSW3iKtDq1FERCmsxORC0FaSFxf4vXZHYiailIi6jHt/hWmMxO92qQuMV5j+Yvm9kZM3u5at1qM3vOzI6Ej6uqXttlZkfN7LCZ3R1XXdKa6vEtvnKhW29/76KEzmK3bCrBOD4yzuixUbVkJBZxdh/9BbBt2rqHgX3uvgnYFy5jZjcBO4Cbw/c8bmbJGGuTFlOvb/HprjQ9a3qiK58zZzNkRjM1Hy93UI7jAK7uLamHOO/R/LyZbZi2ejvwr8Pnu4HvAr8Vrn/a3XPAG2Z2FNgKfD+u+qS1xD0l9fRumSAbcOonp8hlc5QKJdq72ykGRSxllIISqfYUqfYUa9+9tmYt02dhXYwrpluhe0taX73HFK5295MA7n7SzNaG69cBL1RtNxyuE4nUCoPF6GOvNV4xdnqMybFJ3B2A3Hiu/GL4UMwXyWVyHM8cp727nbb2tnILI5xUj/KN2xb1AK57NUg9NMtAs9VY5zU3NHsAeADghhtuiLMmaXKLNfhc61t9kA2iQLicYq5INpcFIHM2Q7o7TVt7uXund20v+OJeKKcwkLjV+5TU02Z2LUD4eCZcPwxcX7XdAHCi1g7c/Ql3H3T3wf7+/liLlea2WH3stbplFnLgLZVKFIJCVA9OND4h0irqHQp7gZ3h853AM1Xrd5hZu5ltBDYB++tcm7SYxepjr3XW0cqrV5JMz+9ch0QiQSqdUp+/tLTYuo/M7K8oDyqvMbNh4LPAo8AeM7sfOA7cB+Dur5jZHuBVoAA86O7FuGqTpWGx+thrjUuku9Jcc+M1nDl6pnyQv0xPkiWM7r5uVl6zUjfqkZZnc+k3bVaDg4M+NDTU6DKkhc02LvH2W29z7q1zUSh4ySFRvgeDF8sr011prrvpOoWAtAwzO+Dug7Ve0zQXsqzNNi7RtaqLdGcaHNydjt4Orn7P1XR0d5BMJ2nrbCPVntI1A7JkNMvZRyINMdu4RLorzdp3r2Xi/ASGlUMi3Obc8XMkUgmSqaRaCbJkKBRkWZvLuET1BW2VFkFPX4/GD2RJUijIsnKpQeXZDuqXGnto5TDQ5HpSi0JBlo35XOxWa9oLWNxpKxYiyAZMnJ+gGBRJpVNTurPmux/dO0JqUSjIkjCXb71zPbBXDpilQolSscSq61c1xbxDQTbgzD+fITeeo5gvgkGqPUXXVV30rOmhp69nXvuCxoecNB+FgrS8ysGyVCiRSCVmTFIXjQXMMh9RZbtCrkCpUCKYDCgVS5w7fo5rbrymofMOBdmAsVNj5dpKpfIUHA75i3kuBBfIni9PtTHXYGiGkJPmpFCQljdxfoJgIsCSRiFXYOL8RHSQm95NUms+okqXTPZ8lkQyQbFQpBAUKOaLmBlYeZtGTVlR3XIpBAVKxdKU1y1hOM7FsYvzCgVNrie1KBSk5Rnlg2L1Y8X0bpLKfETVr48eGyU/mSefy9OzuvxaW3sbFyYvAFCYLNSesnEWizWQW/kZ2nvaASh5iezb2fIFdV7++Q2jc2XnvParMJBaFArS8rpWdZUHXwtF0p1pulZ1Ra/N1k0SHXC72ynkCuSyOdra22jvbqcr14W7U8gXysEwD4s5kFv9MyRSCVauXUlHbwfFoEiyPQkl6FzZOa8xBZFLUShIy0t3pel/V3/Nb+WzdZNUlkvFEunuND2re6JQmTg/QW4ih2FkzmWiM30Wc1B7rj9f5WfAYPzMePTayrUr9W1fFpVCQeomjvPiq/dZ3S1UbbbP61zRieN0r+qesl3Xqi6K+SLt3e2UiqXoQD+XFsBiD+RWfobM2Qygs4YkPgoFqYvq7pRioRh9I5/LRWOXCpLqs47MjJ7+nhkH9svta3oXT/eq7uh5ZjRDLpPDzKKB3flcrxDXQK7OGpK4KRSkLioH00QywcWxi1woXuDihYuzXkB2uW/llbOOMCjkChQLRSYvTNbcZ2Y0M2Wuoqg7hqkHeIALpy/w9sm3gXK30squlawaWDVln3M5KMcxkKuzhiRumiVV6qJy8MpN5HCc9q7ymTSXm110thlMK2cbeak8hXUynaRUKDF2amzKtkE2YPSno1FLoVgoTvmGXznAY+WuobdPvU0xKJanx3aPTletvKdvfR8dKzroXDG/s32uVJANou4j3dFN4qKWgtRF5WBaOcBWd8lc7j0w81t59VQP6Y40hXzYShifhBLkgzwXxy5iSWNyfLIcHIXyxV7FfJH8xTyWNPo39NOxoiOa/TTIBuRzeYpB+f5OlbApBAXOv3We8ZFxOlZ2AJDL5EimkrO2dhZLrausdbaRxEGhIHVT6e7oXtU9o28/yAbkgzy58RypdIoVV68AmHLQrmxbmeqh5CXw8g1vgOimN7nx3Ky1ZM5kyIxkaOtsI92ZJtWR4uL4RXITM9/rRSe4WK5xcnwSS1h5DGNND4VcgbFTY+W7rs3xzKS5qt5X5UrrycwkpVKJ3E9yXPMz19DT1zNlO4CRn44QZAJ61vbQv0H3MZf5UShIXQTZgLHTY5w/cR7Pz363v+qxBAAS5S6T9s52Lo5fpJQvXfYWmXPikM/myWfzTIxOzLotlMcYcKAE42fHSSQSOM7osVF61/YyfmZ8Tt/mK62d6YFXfXCvHph39/IFa6Firsipw6dYed1KMiMZvOQkUuHV2BfL11Sc++k5gLoHg2Zfnb/5/s7i/B03XSiY2TbgT4Ak8KS7P9rgkuQKZUYznHztZPlAvlCl8Nu9Za48DK5U+GMUgyJFilH31oWRC+XXDHDIjmfpWd3DymtWks/lyYxkSLaV79Z2cewihckCjkfdUpUuqUKuQLFUHtPo6Olg4vxE9JnVikExOvBfyoVTFxZ0LcOU+aL8nZ/pcgeh8yfOkzmToVgs0t5VPo2XBOVxkBKQgJXXrGT1wOopXYHVQXjh9AUKQYH23vboau1URyqqofI7qz7LLMgGZM9nyQd52tJtpDpSFCYLFIJC+e54HW2z1l6rlloH3fkcjOe67XwvdIx7htumCgUzSwJ/Bvw8MAy8ZGZ73f3Vxf6sn3zvJ+V+5gbqWNnB5NhkQ2sAaOtqo7e/tzxwOoc/nvkIsgGnf3L6ygKh2rT/ZReOv86xb3+TMwdfZPzkWwRjY5Ry2drvlbrZ8Z3XZq4swdiJMcZOjJHuTpNqT1HKlygWixTyBcwt6grkVPlMNQAzI939zunAiUSCiXMTrH33WgBGXh/h4oWL5VOTkxZNc1Iqlsr7MOjs7SSRSlzyAFp9oC0VyycYJFNJ4J2D7nynXp/PtjD3a0/inuG2qUIB2AocdffXAczsaWA7sKih0AyBADRFIEC5C+XcsXMk2hJ0rei67B/PfGXPZynk5jdFxFxdOP46Q3/8eUZePoDn87F8hizM0x/+2drBEAomgvLpxFW8OvHDsSJLGCUvUSwUyy2HhGHJcnhUDo7FwjsTF7o77h4tl7xEggSWKAfF5aZLh/KBNjtWnleqo6djykF3Pgfj+Ww732tP4r5WpdlCYR3wZtXyMHBb9QZm9gDwAMANN9ywoA9phkBoRl70Wf945r3PGPt6xt86Ru7Ceby4SK0QaSqVA7th5anCKwf9opPoSET/PpOpJIEH74SGlf8Ne8lJJMotBS+V/23P5cCcTCVx9xkH3fkcjOe77XyuPYn7WpVmC4Vac1FOOaq4+xPAEwCDg4MLOuJYyhQMNVjSZv3jma/qq4QXW++69bSvWIUlE3ipGNvnSB0lINWWontNd9SSTqQSrOhfEY0VTB9T6H9X/xWPKUw/0MLMMYX5HIwXcqCfz99cnIP4zRYKw8D1VcsDwInF/pCf+cDPNEUX0nIYU0h3pdm4dSNvHHgDFvm4veKGdzH4qd/VmEITulzXUaorRSqVKt81blUXhVyBIBuQakuVZ6edx8SDsHj/Xqfv51LTl8z1s1r17Ctzb55vzGaWAn4C3AW8BbwE/Ft3f6XW9oODgz40NFTHCkVEWp+ZHXD3wVqvNVVLwd0LZvYQ8I+UT0n98qUCQUREFl9ThQKAu/898PeNrkNEZDnShHgiIhJRKIiISEShICIiEYWCiIhEmuqU1PkysxHgWKPrmGYNcLbRRVyCalsY1bYwqm1h6lHbenevOX1uS4dCMzKzoUud/9toqm1hVNvCqLaFaXRt6j4SEZGIQkFERCIKhcX3RKMLuAzVtjCqbWFU28I0tDaNKYiISEQtBRERiSgUREQkolCIgZn9VzP7kZkdNLNnzey6RtdUYWZfMLPXwvq+bmZXNbqmCjO7z8xeMbOSmTX8dEEz22Zmh83sqJk93Oh6qpnZl83sjJm93OhaqpnZ9Wb2HTM7FP6//GSja6owsw4z229mPwxr+71G1zSdmSXN7Adm9o1G1aBQiMcX3P0Wd98MfAP43QbXU+054Ofc/RbK967Y1eB6qr0M/BLwfKMLMbMk8GfALwI3Ab9iZjc1tqop/gLY1ugiaigAv+Hu7wVuBx5sot9bDrjT3d8HbAa2mdntjS1phk8ChxpZgEIhBu5+oWqxG2K8UfE8ufuz7l4IF1+gfHe7puDuh9z9cKPrCG0Fjrr76+4eAE8D2xtcU8TdnwfONbqO6dz9pLv/U/h8nPIBbl1jqyrzsky42Bb+1zR/m2Y2ANwDPNnIOhQKMTGzR8zsTeDf0VwthWr/Cfg/jS6iSa0D3qxaHqZJDm6twsw2AFuAFxtcSiTsnjkInAGec/emqQ34Y+A3gVIji1AoLJCZfcvMXq7x33YAd/9td78e+BrwUDPVFm7z25Sb+l9rttqahNVY1zTfKpudmfUAfwN8alrLuaHcvRh26w4AW83s5xpcEgBm9jHgjLsfaHQtTXfntVbh7h+Z46Z/CXwT+GyM5UwxW21mthP4GHCX1/lClXn83hptGLi+ankAONGgWlqKmbVRDoSvufvfNrqeWtz9bTP7LuVxmWYYrL8DuNfMPgp0ACvM7Kvu/qv1LkQthRiY2aaqxXuB1xpVy3Rmtg34LeBed882up4m9hKwycw2mlka2AHsbXBNTc/MDHgKOOTuf9ToeqqZWX/lbDsz6wQ+QpP8bbr7LncfcPcNlP+tfbsRgQAKhbg8GnaJ/Aj4BcpnFDSLLwK9wHPhKbP/s9EFVZjZvzGzYeD9wDfN7B8bVUs4GP8Q8I+UB0v3uPsrjapnOjP7K+D7wI1mNmxm9ze6ptAdwK8Bd4b/vg6G336bwbXAd8K/y5cojyk07NTPZqVpLkREJKKWgoiIRBQKIiISUSiIiEhEoSAiIhGFgoiIRBQKIiISUSiIiEhEoSCyiMzs1vBeFR1m1h3O298U8+uIzIUuXhNZZGb23yjPX9MJDLv7f29wSSJzplAQWWThXEkvAZPAv3L3YoNLEpkzdR+JLL7VQA/lOaY6GlyLyLyopSCyyMxsL+U7tW0ErnX3ut5PQ+RK6H4KIovIzP49UHD3vwzv8/z/zOxOd/92o2sTmQu1FEREJKIxBRERiSgUREQkolAQEZGIQkFERCIKBRERiSgUREQkolAQEZHI/wcgwoKTDLteVwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%time\n",
    "print('=========== OCSVM FEATURE 3 ===========')\n",
    "print('=======>>>OCSVM CLUSTER RESULT')\n",
    "ocsvm_test_label = best_ocsvm.fit(X_train).predict(X_test)\n",
    "# print results\n",
    "clustering_print_results(df_test, ocsvm_test_label, selected_features, X_test, True, True )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "=========== IFOREST FEATURE 3 ===========\n",
      "=======>>>IFOREST CLUSTER RESULT\n",
      "\n",
      "Exporting LABEL...\n",
      "Number of clusters: 2\n",
      "\\Label -1: 870424 data points\n",
      "\\Label 1: 7 data points\n",
      "\n",
      "DONE.\n",
      "--------------------------------------\n",
      "Wall time: 29.7 s\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEGCAYAAACKB4k+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAAAjQElEQVR4nO3df3Rc5X3n8fd3RjMjWRLYMpIxFsROcVNwSuxUGFianAZC7JYUpz3h1E3bZbf08A9k09OetLD9kbRbdjmkzWl7stkNhWy9TVqOmzaLk3QTHNIcTlrAkcEBjGHtQgLCxiNkY+vn/Lrf/WPuXEbSyPphXc2M9HmdozNz79y5+kq27mfu8zz3uebuiIiIACTqXYCIiDQOhYKIiEQUCiIiElEoiIhIRKEgIiKRlnoXcD4uuugi37hxY73LEBFpKgcPHnzT3btrvdbUobBx40b6+/vrXYaISFMxsx/N9Jqaj0REJKJQEBGRiEJBREQiTd2nICIrQ6FQYGBggImJiXqX0lRaW1vp7e0llUrN+T0KBRFpeAMDA3R2drJx40bMrN7lNAV3Z2hoiIGBATZt2jTn96n5SEQa3sTEBGvXrlUgzIOZsXbt2nmfXelMQUQaRj6bIz+YJ92dJt2TmfSaAmH+FvI7UyiISEPIZ3Oc2j9Ubr8IoOumtdOCQeKn5iMRaQj5wTwkILUmBYlwuYF0dHTMedtPf/rT/Omf/mls+6/YuXMnq1ev5sMf/vC83zsThYKINIR0dxoCKJwuQBAuyzl98pOf5G/+5m8WdZ8KBRFpCOmeDF03raXjqs6maTr62te+xjXXXMO2bdv44Ac/yMmTJ6PXfvCDH3DDDTewefNm/uqv/ipa/5nPfIarr76aq666ik996lPn9f1vvPFGOjs7z2sfUykURKRhpHsydGzpXJRAyGdzjBweJp/NLUJltf30T/80Tz75JM888wy7d+/m/vvvj1579tln+cY3vsETTzzBH//xH3P8+HEeffRRjh49yoEDBzh06BAHDx7k8ccfn7bfrVu3xlbzbNTRLCLLzlJ1Wg8MDPBLv/RLnDhxgnw+P+l6gF27dtHW1kZbWxsf+MAHOHDgAN/73vd49NFH2bZtGwAjIyMcPXqU97///ZP2e+jQoUWvda50piAiy85SdVp//OMf56677uK5557jC1/4wqRrAqYOBzUz3J177rmHQ4cOcejQIY4dO8btt98+p+/11FNPsXXrVrZu3cq+ffsW9eeoplAQkWVnqTqtz5w5w4YNGwDYs2fPpNceeeQRJiYmGBoa4rvf/S5XX301O3bs4Itf/CIjIyMAvP7662Sz2Tl9r2uuuSYKk1tuuWVxf5Aqaj4SkWWn0mk904VwCzE2NkZvb2+0/Fu/9Vt8+tOf5tZbb2XDhg1ce+21vPLKK9Hr27dv5+abb+bVV1/lD/7gD7jkkku45JJLOHLkCNdddx1QHob6pS99iZ6enknfa+vWrXNqQnrf+97Hiy++yMjICL29vTz00EPs2LHjvH5Oc/fz2kE99fX1uW6yI7L8HTlyhCuuuKLeZTSlWr87Mzvo7n21tlfzkYiIRGILBTN7l5kdqvo6a2a/aWZdZrbfzI6Gj2uq3nOPmR0zs5fM7PzOgUREZN5iCwV3f8ndt7r7VuCngDHgq8DdwGPuvhl4LFzGzK4EdgNbgJ3A580sGVd9IiIy3VI1H90I/Ju7/wjYBVS66fcAHwmf7wIedvecu78CHAO2L1F9IiLC0oXCbuDvwufr3P0EQPhY6XbfALxW9Z6BcN0kZnaHmfWbWf/g4GCMJYuIrDyxh4KZpYFbgL+fbdMa66YNjXL3B9y9z937uru7F6NEEREJLcWZws8CT7t7Zaaok2a2HiB8rFy5MQBcWvW+XuD4EtQnIjKrRpg6+3Of+xyXX345Zsabb745r/3P1VKEwi/zdtMRwD7gtvD5bcAjVet3m1nGzDYBm4EDS1CfiEhTuP766/n2t7/NO97xjti+R6yhYGargJuAf6xafR9wk5kdDV+7D8DdDwN7gReAbwJ3unspzvpERM7HUk+dvW3bNjZu3LhY5dcU6zQX7j4GrJ2ybojyaKRa298L3BtnTSKyMowezzF2Iseq9RnaL4nn3gyVqbPNjAcffJD777+fP/uzPwPKU2c/+eSTjI6Osm3bNm6++Waef/75aOpsd+eWW27h8ccfnzZL6lynuYiD5j4SkWVn9HiOV746iBm4w6Zf6I4lGDR1tohIExg7kcMMWi9KYVZejsNSTp29VBQKIrLsrFqfwR0m3izgXl6Ow1JOnb1UFAoisuy0X5Jh0y9003PNBYvWdFSZOrvy9dnPfjaaOvt973sfF1100aTtK1NnX3vttdHU2R/60If42Mc+xnXXXcdP/uRP8tGPfpTh4eFp32um23H+5V/+Jb29vQwMDHDVVVfxG7/xG+f9c02lqbNFpOFp6uyF09TZIiKyYAoFERGJKBREpCk0c1N3vSzkd6ZQEJGG19raytDQkIJhHtydoaEhWltb5/U+XbwmIg2vMuJG0+XPT2trK729vfN6j0JBRBpeKpWadLWwxEfNRyIiElEoiIhIRKEgIiIRhYKIiEQUCiIiElEoiIhIJO7bca42s6+Y2YtmdsTMrjOzLjPbb2ZHw8c1VdvfY2bHzOwlM9sRZ20i1fLZHCOHh8ln45l3X6RZxH2m8BfAN939J4D3AEeAu4HH3H0z8Fi4jJldCewGtgA7gc+bWTLm+kTIZ3Oc2j/EyLPDnNo/pGCQFS22UDCzC4D3Aw8BuHve3d8CdgGVu1HsAT4SPt8FPOzuOXd/BTgGbI+rPpGK/GAeEpBak4JEuCyyQsV5pvBOYBD4X2b2jJk9aGbtwDp3PwEQPvaE228AXqt6/0C4TiRW6e40BFA4XYAgXBZZoeIMhRbgvcD/cPdtwChhU9EMrMa6abNfmdkdZtZvZv2aB0UWQ7onQ9dNa+m4qpOum9aS7onn1o0izSDOUBgABtz9qXD5K5RD4qSZrQcIH7NV219a9f5e4PjUnbr7A+7e5+593d3dsRUvK0u6J0PHlk4Fgqx4sYWCu78BvGZm7wpX3Qi8AOwDbgvX3QY8Ej7fB+w2s4yZbQI2Awfiqk9ERKaLe5bUjwNfNrM08DLwHykH0V4zux14FbgVwN0Pm9leysFRBO5091LM9YmISJVYQ8HdDwG1bg594wzb3wvcG2dNIiIyM13RLCIiEYWCiIhEFAoiIhJRKIiISET3aBYJ5bM58oN50t1pXa8gK5ZCQYS3J8UjAQToymZZsdR8JIImxROpUCiIoEnxRCrUfCTC25PiqU9BVjqFgkgo3ZNRGMiKp+YjERGJKBRERCSiUBARkYhCQUREIgoFERGJKBRERCSiUBARkYhCQUREIrGGgpn90MyeM7NDZtYfrusys/1mdjR8XFO1/T1mdszMXjKzHXHWJiIi0y3FmcIH3H2ru1fu1Xw38Ji7bwYeC5cxsyuB3cAWYCfweTNLLkF9IiISqkfz0S5gT/h8D/CRqvUPu3vO3V8BjgHbl748EZGVK+5QcOBRMztoZneE69a5+wmA8LEnXL8BeK3qvQPhuknM7A4z6zez/sHBwRhLFxFZeeKeEO96dz9uZj3AfjN78RzbWo11Pm2F+wPAAwB9fX3TXhcRkYWL9UzB3Y+Hj1ngq5Sbg06a2XqA8DEbbj4AXFr19l7geJz1iYjIZLGFgpm1m1ln5TnwIeB5YB9wW7jZbcAj4fN9wG4zy5jZJmAzcCCu+kREZLo4m4/WAV81s8r3+Vt3/6aZfR/Ya2a3A68CtwK4+2Ez2wu8ABSBO929FGN9IiIyRWyh4O4vA++psX4IuHGG99wL3BtXTSIicm66ollERCIKBRERiSgUREQkolAQEZGIQkFERCIKBRERiSgUREQkolAQEZGIQkFERCIKBRERiSgUREQkolAQEZGIQkFERCIKBRERiSgUREQkolAQEZGIQkFERCKxh4KZJc3sGTP7erjcZWb7zexo+Limatt7zOyYmb1kZjvirk1ERCZbijOFTwBHqpbvBh5z983AY+EyZnYlsBvYAuwEPm9mySWoT0REQrGGgpn1AjcDD1at3gXsCZ/vAT5Stf5hd8+5+yvAMWB7nPWJiMhkcZ8p/DnwO0BQtW6du58ACB97wvUbgNeqthsI101iZneYWb+Z9Q8ODsZStIjIShVbKJjZh4Gsux+c61tqrPNpK9wfcPc+d+/r7u4+rxpFRGSylhj3fT1wi5n9HNAKXGBmXwJOmtl6dz9hZuuBbLj9AHBp1ft7geMx1iciIlPEdqbg7ve4e6+7b6Tcgfwdd/9VYB9wW7jZbcAj4fN9wG4zy5jZJmAzcCCu+kREZLo4zxRmch+w18xuB14FbgVw98Nmthd4ASgCd7p7qQ71iYisWOY+rdm+afT19Xl/f3+9yxARaSpmdtDd+2q9NmvzkZndVX2BmYiILF9z6VO4GPi+me01s51mVmuUkIiILAOzhoK7/z7lTt+HgP8AHDWz/2pmPxZzbSIissTmNPrIyx0Pb4RfRWAN8BUzuz/G2kREZInNOvrIzP4T5aGjb1KeruKT7l4wswRwlPIVyyIisgzMZUjqRcAvuvuPqle6exBetSwiIsvErKHg7n94jteOzPSaiIg0H91kR0REIgoFERGJ1GOaC5Elk8/myA/mSXenSfdk6l2OSMNTKMiylc/mOLV/qHw+HEDXTWsVDCKzUPORLFv5wTwkILUmBYlwWUTOSaEgy1a6Ow0BFE4XIAiXReSc1Hwky1a6J0PXTWvVpyAyDwoFWdbSPRmFgcg8qPlIREQiCgUREYkoFEREJBJbKJhZq5kdMLMfmNlhM/ujcH2Xme03s6Ph45qq99xjZsfM7CUz2xFXbSIiUlucZwo54AZ3fw+wFdhpZtcCdwOPuftm4LFwGTO7EtgNbAF2Ap83s2SM9YmIyBSxhYKXjYSLqfDLgV3AnnD9HuAj4fNdwMPunnP3V4BjwPa46hMRkeli7VMws6SZHQKywH53fwpY5+4nAMLHnnDzDcBrVW8fCNdN3ecdZtZvZv2Dg4Nxli8isuLEGgruXnL3rUAvsN3M3n2Oza3WLmrs8wF373P3vu7u7kWqVEREYIlGH7n7W8B3KfcVnDSz9QDhYzbcbAC4tOptvcDxpahPRETK4hx91G1mq8PnbcAHgReBfZTv+Uz4+Ej4fB+w28wyZrYJ2AwciKs+ERGZLs5pLtYDe8IRRAlgr7t/3cyeAPaa2e3Aq8CtAO5+2Mz2Ai8AReBOdy/FWJ+IiExh7tOa7ZtGX1+f9/f317sMEZGmYmYH3b2v1mu6ollERCIKBRERiSgUREQkolAQEZGIQkFERCIKBRERieh2nCJV8tmc7uksK5pCQSSUz+Y4tX+ofP4cQNdNaxUMsuKo+UgklB/MQwJSa1KQCJfPZ3/ZHCOHh8lnc4tUoUj8dKYgEkp3pyGAwukCBOHyAumsQ5qVQkEklO7J0HXT2kXpU6g+6yicLpT3qVCQJqBQEKmS7sksysF7Mc86RJaSQkEkBot51iGylBQKIjFZrLMOkaWk0UciIhJRKIiISEShICIikTjv0Xypmf2zmR0xs8Nm9olwfZeZ7Tezo+Hjmqr33GNmx8zsJTPbEVdtInOlC9BkpYnzTKEI/La7XwFcC9xpZlcCdwOPuftm4LFwmfC13cAWYCfw+fD+ziJ1UbkAbeTZYU7tH1IwyIoQWyi4+wl3fzp8PgwcATYAu4A94WZ7gI+Ez3cBD7t7zt1fAY4B2+OqT2Q2iz3thUgzWJI+BTPbCGwDngLWufsJKAcH0BNutgF4reptA+G6qfu6w8z6zax/cHAw1rqluZ1v048uQJOVKPbrFMysA/gH4Dfd/ayZzbhpjXU+bYX7A8ADAH19fdNeF4HFmXtIF6DJShTrmYKZpSgHwpfd/R/D1SfNbH34+nogG64fAC6tensvcDzO+mT5Wqymn3RPho4tnQoEWTHiHH1kwEPAEXf/bNVL+4Dbwue3AY9Urd9tZhkz2wRsBg7EVZ8sb2r6EVmYOJuPrgd+DXjOzA6F6/4zcB+w18xuB14FbgVw98Nmthd4gfLIpTvdvRRjfbKMqelHZGHMvXmb5fv6+ry/v7/eZYiINBUzO+jufbVe04R4IqHK/ZktAR42OS3kDEP3eZZmplAQ4e3RSkGuRO71HJneDIl0ct6jlnTHNWl2mvtIhLdHK1mLgYElbUGjlnTBmzQ7nSnIijWpmac7TTBWojhcwIuOlxxLzn/UkkY9SbNTKMiyV6uNf2ozT+d7OwFIpBOke5Ks+vF2Vm1u1wVvsuIoFGRZm6mNv7qZp3C6wPgPx0msSpLZ0ErhdIGW1akFH9B1xzVpZupTkGVtpjb+qc08bRvb1Owjgs4UZJk7Vxt/am0LxZESHVs6aL+ik9TatJp9ZMVTKMiyVquNP5/N8ebXsuRO5MCgdKZIam1azT4iKBRkBZh6sM8P5glyJRKZBOCUckE5NBQIIupTkJUn3Z0mkUkS5AKCvJPMJNSHIBLSmYKsOOmeDBf9fA9jx0bBWdDQU5HlSqEgK5L6D0RqUyiIVNFkdrLSKRREQprMTkQdzdJE8tkcI4eHyWdz8exfk9mJ6ExBmsNSfIpvhsns1LwlcYvzHs1fNLOsmT1fta7LzPab2dHwcU3Va/eY2TEze8nMdsRVlzSnpfgUX7nQreOqzkUJncU+s6kE48izw5zaPxTbGZOsbHGeKfw18Dngf1etuxt4zN3vM7O7w+XfNbMrgd3AFuAS4Ntm9uO6R7NULNWn+OpRSVPvxDb18Vyf1uM4s5k6iZ8uuJM4xBYK7v64mW2csnoX8DPh8z3Ad4HfDdc/7O454BUzOwZsB56Iqz5pLnFPST21WSafzZH9yhvkhnIEYwGZi9MU3yqRWGWUhgNSXS20dKa46Od7atYSxwG8GZq3pPktdZ/COnc/AeDuJ8ysJ1y/AXiyaruBcJ1IpNa1BYvRxl7rU/3Zp88w/so4FB2A3A/LTTWlEaAIuZESuUSO8RPjtPZkaOlK0bHl7Un1LNzXYh7Ada8GWQqN0tFsNdZ5zQ3N7gDuALjsssvirEka3GI10dT6VF/I5qJAmKQYPjpQgmCoxNjQGCRh9NkR0hdnaFmTim7cM1sz03zpojuJ21IPST1pZusBwsdsuH4AuLRqu17geK0duPsD7t7n7n3d3d2xFiuNbbE6n2s1y6R6MrU/qszEISgGFEeKUT0eQMeWTh3EpaksdSjsA24Ln98GPFK1freZZcxsE7AZOLDEtUmTWaw29lqjji5474UkV8/jRNoh0ZKgpaNFbf7S1GJrPjKzv6PcqXyRmQ0AnwLuA/aa2e3Aq8CtAO5+2Mz2Ai9QPkG/UyOPZDaL1cZeq18i3ZNh3a0XM/h/TlI4VYASMzRogqWNVVvaufDq1bpRjzQ9c5/hf3oT6Ovr8/7+/nqXIU1stn6Jt/71FG9971Q5FICg6FgKggnHcwEAqZ4M63/lEoWANA0zO+jufbVe0zQXsqLN1i+x6vJ20mszeABegtZLWunZdTHp7gzJ9hZSXWlSF7ZoSgxZNhpl9JFIXczWLxHde+HoKFgYEuEZwanvnCLZliCRSar/QJYNhYKsaHPpl6gMA630PQC0X9Gp/gNZlhQKsqLM1Kk820F9pr6HZg4DTa4ntSgUZMWYz8Vu06a9aJB5h/LZHGNHRymOFGnpbJnUnDXf/ejeEVKLQkGWhbl86p3rgb1ywAzyJUpjAV03dDXEvEP5bI43v5ZlYmCCYKQESWi5sIW2zavo2NJJ+xWdc99Xg4ScNB6FgjS9ysGylAtIZhLTJqmrnu30XAf2ynbFMwWCfInCYIEgH3DqO6dYd+vFdZ13KJ/NcfbgGQrDRYJigBcdilDIFigOn2XsyBg9H2XOwdAIISeNSaEgTW/s6Ci5N3Ik0kbxtDN2dHTS9NfVzSS15iOqNMmMvThKYlWSYKxEcbhIabyImWHhUNV6TVkx6czlTDG6PqLCWgwMxn84PvdQ0OR6MgOFgjQ/I7za2MB90pxFU5tJKvMRRa+HB9zCmQLFUwXar2gHkmRWt1AYykOLUXizUD7LmKfF6sit/AyZ9a0ABIWgPINr4FAAw8ChbWPbvPbb7B3lEg+FgjS9VZe3M3ZklCBXItWVYtXl7dFrszWTRAfcdWmKpwvkT+ZouTBNZn2G0tlVeNEpjZbK75+HxezIrf4ZEukkF167mrbLximOlGjpTOJBORDm06cgMhOFgjS9ygVmtT6Vz9ZMUjngBnknc3GGVVe0R6EydmSU/Mk8WPl5ZaTPYnZqz/Xnq/wMloDhp4ffbg57/4X6tC+LSqEgSyaOcfHV+6xuFqo2WzNJ66ZWcFi1efLwzlU/0U4pF5BZlybIe3Th2lzOABa7I7fyM4wcHtaoIYmVQkGWRHVzSjBWij6Rz+WisZmCpHrUkQEd7+mcdmA/176mNvGs2vx2s9PokWFyJyYwymcRlQP7XM8A4urI1aghiZtCQZZE5WCaSBvjL+cIciUmXpmY9QKyc30qr4w6sgQU3ypSygdM/LD2PkePDE+aq6hywJ56gAc4+8wZzj55BoAgH9C5Ls2a93W9vc85HpTj6MjVqCGJm2ZJlSVR+YSbO5kHh/S6zKx3S5v1zmrhqCMvOg60rEoQ5EucPXiGfDb39n6yOYa+9SaFkznyJ8uBVDmoVh/gLVFuGjp74C1KZ4sE4yW86Iy/NMrYsVHy2Vx0UG69rLXc7LSE8tlcufkI3dFN4qMzBVkSlYNp5XqA6iaZGd8zQ1NJ9VQPqa4UpdEiNlpi4rUJvAjFs0XGXx4nkTYmXh/HA8cnyhd7lYZLFLIFLGWs3dFN68bWaPbT/GCe4tk8peESBOA5hwQUR0uc+ZfTjBwapvWdbRiQez1HYlVy1rOdxVLrKmuNNpI4KBRkyVSaU1Ztbp/Wtp8fzFM6W2Di9QmSnS1csO1CgEkH7cq2lakevFi+iCsYDcpnDLnyDaNyP5yYtZaRp4cZeW6YVHeadFea1JoUudfGyb0RnmEkie625mMB+WIeTuaZeH0CSxqWNDre3UHxbJGzB89wwU9dOOeRSXNVva9yYBXIHZ8gyAWc/EqOdeEVzNXbAQx9M0tuME/nuztZu6PnvGqQlUehIEsin81x9ukzvPUvp/GRYNbtT39zaPKKNLS/u5N0T5rx42MEIwEElL8WqgCF43kKx/OMPj9Se5vKjQnDlivPB3gCKMLICyPRhWOn9g/R+d5Ohp8entOn+crZztTAqz64V3fMe+CMHx2L6imdKvHG359g9TXjjDw7TFB0km1JSuMlitnyNRWnXi//Dpc6GDT76vzN93cW5++44W7HaWY7gb+g/FntQXe/b6ZtdTvO5jB6ZJg3Hj5OcPZ8juChJOUgaJT/tqnwMQDSQIG3r7BOQ/u7Orjw6tUU3yow/NwwibYkme404y+Pl5vFHFJdKdre2RY1SRVPFwiKAUGuRObi1nI/Qm6mAqpUruT2t2toWdfChl+/bN4Hjur5ojzsb5k6PchUZ544xfBzwwRjAel1GYKxEqSN0SPDMA5k4MKrV7O6qtN+ahCefeYMpeEirRta8aD8c6TWpKIaKr+z6lFm+WyOsWOjFM8WabmgJRo4UBwu0tLRQqorNWvttWqpddCdz8F4rtvO90LHxbgw8ly342yoUDCzJPD/gJuAAeD7wC+7+wu1tj+fUDj2+y/io/X92Vt/rI2Jfxufdbux8YBc3kkmoVSCQhESCUinoC2TIJWyWfcxowSkelJ0vOcC2t7RNqc/nvnIZ3O8/oVXKZ4qLsr+ZBlIQmp9mpbOFoKxgKAQUBwpQMnw8SAa/pLIlJ9Ywkivz5DP5gHHWhJkLi5fsAiEzYnjBOMBlk5E2RjkAiyTwJJG62WtJNLJGQ+gU4dMAyRWJScddOc79fpctx05PMzIs8NRmHVc1TnjNTcL2b6Wc4VCozUfbQeOufvLAGb2MLALqBkKC9UIgQDMORCGzjg4lKZ80E4YTKQD1lx4HsEQQOGNAqdPDnGmI0nbprZz/vHM19ixUQWCTFaCwkCeQqVNLpq7qupvMoDAA0galAJKY0W85CRSRiJtlHJBNBotyJXKfwwJcHe85FjSIGm4B1AqB0NlBFvNUKga6TZ+Oo9jZDa0TroWZT5Xqc9n2/leexL3tSqNFgobgNeqlgeAa6o3MLM7gDsALrvssgV9k0YIhLnK5cu12gzH/JJDoejnd7YA4OWJ1mb741nIfkXOaer/kfDDj1l59lcShgflA70HTpB3Ul2J6GCYyCQhyJeHFbdYedZYwEsB1lIOhHJQzHwAnTS/VCYJTD/ozudgPK9t53ntSdzXqjRaKNQ6sk36L+PuDwAPQLn5aEHfpN2aJhgyaWNk3JmplS9pkGo5z0AAMEikErP+8czXqs3tb4/kEZmqcqWUE/XDJNtb6NjSwfirE5g5iXSSjq2d0afuqX0KF/18z3n3KUw90ML0PoX5HIwXcqCfz8E9zhluGy0UBoBLq5Z7geOL/U0u/5OfaIgmpLn0KaxqKzdKNmufQronwzt+55386M9fLnc2yorXsi5Fsj1Jy4UttG9up3imSC6bI9neQmZ9Zl4TD8LiHSCn7mem6Uvm+r2adWryRutobqHc0Xwj8DrljuaPufvhWttr9JGIyPw1TUezuxfN7C7gW5QbHb44UyCIiMjia6hQAHD3fwL+qd51iIisRJoQT0REIgoFERGJKBRERCSiUBARkUhDDUmdLzMbBH5U7zqmuAh4s95FzEC1LYxqWxjVtjBLUds73L271gtNHQqNyMz6Zxr/W2+qbWFU28KotoWpd21qPhIRkYhCQUREIgqFxfdAvQs4B9W2MKptYVTbwtS1NvUpiIhIRGcKIiISUSiIiEhEoRADM/svZvasmR0ys0fN7JJ611RhZp8xsxfD+r5qZqvrXVOFmd1qZofNLDCzug8XNLOdZvaSmR0zs7vrXU81M/uimWXN7Pl611LNzC41s382syPhv+Un6l1ThZm1mtkBM/tBWNsf1bumqcwsaWbPmNnX61WDQiEen3H3q9x9K/B14A/rXE+1/cC73f0qyveuuKfO9VR7HvhF4PF6F2JmSeC/Az8LXAn8spldWd+qJvlrYGe9i6ihCPy2u18BXAvc2UC/txxwg7u/B9gK7DSza+tb0jSfAI7UswCFQgzc/WzVYjsNdKdid3/U3Yvh4pOU727XENz9iLu/VO86QtuBY+7+srvngYeBXXWuKeLujwOn6l3HVO5+wt2fDp8PUz7AbahvVWVeNhIupsKvhvnbNLNe4GbgwXrWoVCIiZnda2avAb9CY50pVPt14P/Wu4gGtQF4rWp5gAY5uDULM9sIbAOeqnMpkbB55hCQBfa7e8PUBvw58DtAUM8iFAoLZGbfNrPna3ztAnD333P3S4EvA3c1Um3hNr9H+VT/y41WW4OodePrhvlU2ejMrAP4B+A3p5w515W7l8Jm3V5gu5m9u84lAWBmHway7n6w3rU03J3XmoW7f3COm/4t8A3gUzGWM8lstZnZbcCHgRt9iS9Umcfvrd4GgEurlnuB43WqpamYWYpyIHzZ3f+x3vXU4u5vmdl3KffLNEJn/fXALWb2c0ArcIGZfcndf3WpC9GZQgzMbHPV4i3Ai/WqZSoz2wn8LnCLu4/Vu54G9n1gs5ltMrM0sBvYV+eaGp6ZGfAQcMTdP1vveqqZWXdltJ2ZtQEfpEH+Nt39HnfvdfeNlP+vfacegQAKhbjcFzaJPAt8iPKIgkbxOaAT2B8Omf2f9S6owsx+wcwGgOuAb5jZt+pVS9gZfxfwLcqdpXvd/XC96pnKzP4OeAJ4l5kNmNnt9a4pdD3wa8AN4f+vQ+Gn30awHvjn8O/y+5T7FOo29LNRaZoLERGJ6ExBREQiCgUREYkoFEREJKJQEBGRiEJBREQiCgUREYkoFEREJKJQEFlEZnZ1eK+KVjNrD+ftb4j5dUTmQheviSwyM/sTyvPXtAED7v7f6lySyJwpFEQWWThX0veBCeDfuXupziWJzJmaj0QWXxfQQXmOqdY61yIyLzpTEFlkZraP8p3aNgHr3X1J76chcj50PwWRRWRm/x4ouvvfhvd5/lczu8Hdv1Pv2kTmQmcKIiISUZ+CiIhEFAoiIhJRKIiISEShICIiEYWCiIhEFAoiIhJRKIiISOT/AxTFxKMVb2eZAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%time\n",
    "print('=========== IFOREST FEATURE 3 ===========')\n",
    "print('=======>>>IFOREST CLUSTER RESULT')\n",
    "iforest_test_label = best_iforest.fit(X_train).predict(X_test)\n",
    "# print results\n",
    "clustering_print_results(df_test, iforest_test_label, selected_features, X_test, True, True )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## SECTION 8: Save anomalies to csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "870431"
      ]
     },
     "execution_count": 190,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Already exported\n",
    "#sort_anomalies    =   ['total_packets', 'bytes_bothdir','bytes_onedir', 'duration']\n",
    "# get anomalies\n",
    "#anomalies_ocsvm_result = get_anomalies(df_test, ocsvm_test_label)\n",
    "#anomalies_ocsvm_result.sort_values(by=sort_anomalies,ascending=False)\n",
    "#save(anomalies_ocsvm_result, \"ocsvm_f3_anomalies\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [],
   "source": [
    "anomalies_iforest_result = get_anomalies(df_test, iforest_test_label)\n",
    "anomalies_iforest_result.sort_values(by=sort_anomalies,ascending=False)\n",
    "save(anomalies_iforest_result, \"ifoest_f3_anomalies\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## TASK2 GENERATING ADVERSARIAL SAMPLES =>>> Moved to another separate notebook file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 674,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DONE\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "headers = ['timestamp','duration','protocol', 'src_ip', 'src_port', 'direction', 'dst_ip' , 'dst_port'\n",
    "           ,'state', 'src_type', 'dst_type' , 'total_packets', 'bytes_bothdir', 'bytes_onedir', 'label']\n",
    "\n",
    "df_train_original = pd.read_csv(\"training_data_with_labels.csv\", names=headers, header=None).sample(frac=0.05, replace=False, random_state=42).fillna(-1)\n",
    "#df_val_notsampled = pd.read_csv(\"A2_2/valid_data_with_labels.csv\", names=val_headers, header=None).fillna(-1)\n",
    "df_test_original = pd.read_csv(\"test_data_with_labels.csv\", names=headers, header=None).fillna(-1)\n",
    "\n",
    "print('DONE')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 675,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "98908\n"
     ]
    }
   ],
   "source": [
    "print(len(df_train_original))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 676,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_headers = ['timestamp','duration','protocol', 'src_ip', 'src_port', 'direction', 'dst_ip' , 'dst_port'\n",
    "           ,'state', 'src_type', 'dst_type' , 'total_packets', 'bytes_bothdir', 'bytes_onedir']\n",
    "\n",
    "df_train_original['label']= df_train_original['label'].apply(str)\n",
    "df_train_original.loc[df_train_original['label'].str.contains(\"From-Botnet\"), 'label'] = -1\n",
    "df_train_original.loc[df_train_original['label'].str.contains(\"From-Botnet\")== False, 'label'] = 1\n",
    "df_train_original.label = df_train_original.label.astype(int)\n",
    "\n",
    "df_test_original['label']= df_test_original['label'].apply(str)\n",
    "df_test_original.loc[df_test_original['label'].str.contains(\"From-Botnet\"), 'label'] = -1\n",
    "df_test_original.loc[df_test_original['label'].str.contains(\"From-Botnet\")== False, 'label'] = 1\n",
    "df_test_original.label = df_test_original.label.astype(int)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 678,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "98908\n",
      "574383\n"
     ]
    }
   ],
   "source": [
    "X_train = df_train_original\n",
    "X_test = df_test_original\n",
    "print(len(X_train))\n",
    "print(len(X_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 679,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "53954\n",
      "44954\n"
     ]
    }
   ],
   "source": [
    "print(len(X_train[X_train['label']==1]))\n",
    "print(len(X_train[X_train['label']==-1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 681,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generatingi features... \n",
      "val DONE\n",
      "train len:  70584\n",
      "PCA.explained variance_ratio_ : 0.6232298575880069\n",
      "DONE\n",
      "Wall time: 32.6 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "print('Generatingi features... ')\n",
    "X_train = get_feature3('val', X_train) \n",
    "print('train len: ', len(X_train))\n",
    "Y = X_train['label']\n",
    "\n",
    "feature3 = ['bytes_bothdir', 'bytes_onedir', 'total_packets', 'duration', \n",
    "             'pps', 'bps_onedir', 'bpp_onedir', 'bps_bothdir', 'bpp_bothdir', \n",
    "             'cnt_timed_src', 'cnt_timed_dst', 'cnt_timed_srcdst']\n",
    "\n",
    "selected_features = feature3\n",
    "columns_to_scale = feature3\n",
    "\n",
    "X_train[columns_to_scale] = apply_scaler(X_train)\n",
    "X_train = X_train[selected_features]\n",
    "X = get_pc(X_train, 3)\n",
    "Y = np.array(Y)\n",
    "print('DONE')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 684,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\kebin\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:40: RuntimeWarning: overflow encountered in exp\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0 cost: 265939.7362045402 error rate: 0.34964014507537117\n",
      "epoch: 200 cost: 3463993.5325673562 error rate: 0.36794457667460045\n",
      "epoch: 400 cost: 3486488.2836368647 error rate: 0.36795874419131813\n",
      "epoch: 600 cost: 3496335.754784658 error rate: 0.36795874419131813\n",
      "epoch: 800 cost: 3502254.413517331 error rate: 0.36795874419131813\n"
     ]
    }
   ],
   "source": [
    "class logistic_regression2(object):\n",
    "    def fit(self, X, Y, learning_rate=0.0000003, epoch=1000):\n",
    "        X = np.array(X, dtype=\"float32\")\n",
    "        Y = np.array(Y, dtype=\"float32\")\n",
    "\n",
    "        N, D = X.shape\n",
    "        Y = Y.reshape(N, 1)\n",
    "\n",
    "        dlt = np.zeros([N, 1], dtype=\"float32\")\n",
    "        dW = np.zeros([1, D], dtype=\"float32\")\n",
    "        db = 0\n",
    "        self.weight = np.zeros([D, 1], dtype=\"float32\")\n",
    "        self.beta = 0\n",
    "\n",
    "        c = []\n",
    "\n",
    "        for n in range(epoch):\n",
    "            dlt = self.predict(X).T - Y\n",
    "            dW = np.matmul(dlt.T, X).T\n",
    "            db = dlt.sum() / N\n",
    "\n",
    "            self.weight -= learning_rate * dW\n",
    "            self.beta -= learning_rate * db\n",
    "            if n % 200 == 0:\n",
    "                c_new = cost(self.predict(X).T, Y)\n",
    "                c.append(c_new)\n",
    "                err = error_rate(self.predict_class(X).T, Y)\n",
    "                print(\"epoch:\", n, \"cost:\", c_new, \"error rate:\", err)\n",
    "\n",
    "    def predict(self, X):\n",
    "        z = np.matmul(self.weight.T, X.T) + self.beta\n",
    "        return 1 / (1 + np.exp(-z))\n",
    "\n",
    "    def predict_class(self, X):\n",
    "        predictclass = self.predict(X)\n",
    "        return (predictclass >= 0.5) * 1\n",
    "                \n",
    "    def predict(self, X):\n",
    "        z = np.matmul(self.weight.T, X.T) + self.beta\n",
    "        return 1 / (1 + np.exp(-z))\n",
    "\n",
    "    def predict_class(self, X):\n",
    "        predictclass = self.predict(X)\n",
    "        return (predictclass >= 0.5) * 1\n",
    "                \n",
    "def error_rate(P, Y):\n",
    "    return np.mean(Y != P)\n",
    "\n",
    "def adversial_samples(X, Y, model, epsilon=0.00001):\n",
    "    dlt = model.predict(X).T - Y.reshape(X.shape[0], 1)\n",
    "    direction = np.sign(np.matmul(dlt, model.weight.T))\n",
    "    return X + epsilon * direction, Y\n",
    "\n",
    "def cost(Y_hat, Y):\n",
    "    Y_hat = Y_hat.flatten()\n",
    "    Y = Y.flatten()\n",
    "    cost1 = 0\n",
    "    elp = 0.0000000000000000000000000000000000000000000001\n",
    "    for i in range(len(Y)):\n",
    "        cost1 -= Y[i] * np.log(Y_hat[i] + elp) + (1 - Y[i]) * np.log(1 - Y_hat[i] + elp)\n",
    "    return cost1\n",
    "\n",
    "\n",
    "\n",
    "model = logistic_regression2()\n",
    "model.fit(X, Y, learning_rate=0.0005,  epoch=1000) #20000\n",
    "\n",
    "#model = LogisticRegression()\n",
    "#model.fit(X, Y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 686,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\kebin\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:40: RuntimeWarning: overflow encountered in exp\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error Rate without adversarial examples: 0.3679870792247535\n",
      "Error Rate with adversarial examples, epsilon = 0.2: 0.3697296837810269\n",
      "Error Rate with adversarial examples, epsilon = 0.5: 0.3697296837810269\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAAA5vElEQVR4nO3de3xU1bn4/8/KhSRAIgkEhEQEOePhlhAgXApI5SL69QIIUkC+cgdbkN5srC0FqajnVCz2Z1VqLQqnIqBwROzxZzEGBHpoIUiEgEBQbgGEEDAXkpDJ5Pn+sSfDhEwu5EKSnef9es1rz6xZe++1B+aZlbXXfrYREZRSStmLX303QCmlVO3T4K6UUjakwV0ppWxIg7tSStmQBnellLIhDe5KKWVDlQZ3Y8xtxpitxpivjDEHjTE/cZcvMcacMcakuB/3e63zK2PMMWPMEWPMvXV5AEoppcoylc1zN8a0B9qLyBfGmFBgLzAW+AGQKyIvXVe/O7AW6A90ABKBO0XEVfvNV0op5UulPXcROSciX7if5wBfAVEVrDIGWCciV0XkOHAMK9ArpZS6SQJupLIxphPQG/gXMBh4whgzFUgGnhSRy1iB/59eq6VT8Y8Bbdq0kU6dOt1IU5RSqsnbu3fvRRGJ9PVelYO7MaYlsBH4qYhkG2NWAEsBcS9/D8wEjI/Vy4z9GGPmAnMBOnbsSHJyclWbopRSCjDGnCzvvSrNljHGBGIF9jUi8t8AInJeRFwiUgy8ybWhl3TgNq/Vo4Gz129TRP4sIvEiEh8Z6fOHRymlVDVVZbaMAVYCX4nIcq/y9l7VHgZS3c83A5OMMUHGmM6AA9hde01WSilVmaoMywwGHgMOGGNS3GW/BiYbY+KwhlxOAI8DiMhBY8x7wCGgCJivM2WUUurmqjS4i8hOfI+jf1zBOs8Dz9egXTidTtLT0ykoKKjJZpRSPgQHBxMdHU1gYGB9N0XVkRuaLXMzpaenExoaSqdOnbBGhpRStUFEyMzMJD09nc6dO9d3c1QdabDBvaCgQAO7UnXAGEPr1q3JyMio76Y0QWlAIhQcgKzLcKkV+MfCrSMhzFGre2qwwR3QwK5UHdHvVn1IA1ZCvgsyvgZnPlz9Er74AtgE8b+A2HtqbW+aOKwC6enpjBkzBofDQZcuXfjJT35CYWGhz7pnz57lkUceqXSb999/P99991212rNkyRJeeukln+VRUVHExcXhcDgYN24chw4dqnR7q1at4uzZMrNUlVJ1IhGIgOyzUCBwLBOu+MO/F0OBP3zyEqSl1dreNLiXQ0QYN24cY8eOJS0tjaNHj5Kbm8vChQvL1C0qKqJDhw5s2LCh0u1+/PHHtGrVqtbb+7Of/YyUlBTS0tKYOHEiw4cPr/TPbg3uSt1MZ4AwKLwM57+DZkFACDQvgOBwCCuCxMRa25sG93IkJSURHBzMjBkzAPD39+fll1/mrbfeIi8vj1WrVjFhwgQeeughRo0axYkTJ+jZsycAeXl5/OAHPyA2NpaJEycyYMAAzxW4nTp14uLFi5w4cYJu3boxZ84cevTowahRo8jPzwfgzTffpF+/fvTq1Yvx48eTl5d3Q22fOHEio0aN4t133wXg2WefpV+/fvTs2ZO5c+ciImzYsIHk5GSmTJlCXFwc+fn5PusppWpLFJANzcKhIBsCA6GZE/JCgHwIbAu12NmyTXBPS4MVK+A3v7GWNf3r5uDBg/Tt27dUWVhYGB07duTYsWMA7Nq1i9WrV5OUlFSq3uuvv054eDj79+9n0aJF7N27t5w2pzF//nwOHjxIq1at2LhxIwDjxo1jz549fPnll3Tr1o2VK1fecPv79OnD4cOHAXjiiSfYs2cPqamp5Ofn87e//Y1HHnmE+Ph41qxZQ0pKCiEhIT7rKaVqy0jgEoR1gOZ+4PcdNLsKZ28Bvzz4Lho6dKi1vdkiuKelwcqVkJMD0dHWcuXKmgV4EfF50sm7/J577iEiIqJMnZ07dzJp0iQAevbsSWxsrM99dO7cmbi4OAD69u3LiRMnAEhNTeWuu+4iJiaGNWvWcPDgwWq1v8TWrVsZMGAAMTExJCUllbu9qtZTSlWHA5gFIXfAv/WGHD84GAbZbSAzBjL8YeTIWtubLYJ7YiJERECrVuDnZy0jImo2fNWjR48yycyys7M5ffo0Xbp0AaBFixY+163qcEZQUJDnub+/P0VFRQBMnz6dV199lQMHDvDMM89U60Kuffv20a1bNwoKCpg3bx4bNmzgwIEDzJkzx+f2qlpPKVUTDuBH0G4N3LEZsqbBiX+D5nfArFngqL3pkLYI7mfOQFhY6bKwsJoNX40YMYK8vDz+67/+CwCXy8WTTz7J9OnTad68eYXrDhkyhPfeew+AQ4cOceDAgRvad05ODu3bt8fpdLJmzZobbvvGjRvZsmULkydP9gToNm3akJubW+qkb2hoKDk5OQAV1lNK1QGHA370I1i61FrWYmAHmwT3qCjIzi5dlp1ds+ErYwwffPAB77//Pg6HgzvvvJPg4GBeeOGFStedN28eGRkZxMbG8rvf/Y7Y2FhuueWWKu976dKlDBgwgHvuuYeuXbtWaZ2XX37ZMxXynXfeISkpicjISFq1asWcOXOIiYlh7Nix9OvXz7PO9OnT+eEPf0hcXBxBQUHl1lNKNT6V3mbvZoiPj5frh0C++uorunXrVqX1S8bcIyKsHnt2Nly6VOt/5VSZy+XC6XQSHBzM119/zYgRIzh69CjNmjW7+Y1Rqhw38h1TDZMxZq+IxPt6r0FfoVpVDocVyBMTrSGaDh3g4YfrJ7CDNRVy2LBhOJ1ORIQVK1ZoYFdK3VS2CO5gBfL6CubXCw0N1TtLKaXqlS3G3JVSSpWmwV0ppWxIg7tSStmQBnellLIhDe6V+OCDDzDGePK0+HL33Xc3iBOoycnJ/PjHP66wzrZt23jwwQd9lt9yyy3ExcV5Hom1mKGuqrwTsNWW6dOne1I9xMXFMWjQoGptxztdc8uWLWuxhVW3evVqHA4HDoeD1atX+6yzfPlyunfvTmxsLCNGjODkyZM3uZWqIbDNbJm6snbtWoYMGcK6detYsmRJne7L5XLh7+9frXWLioqIj48nPt7nlNcqueuuu2ybLGzZsmVVyrdfkY8/Lve2wTfFpUuX+O1vf0tycjLGGPr27cvo0aMJDw8vVa93794kJyfTvHlzVqxYwVNPPcX69evrqdWqvmjPvQK5ubn84x//YOXKlaxbt85Tnp+fz6RJkzwpfUtS9ZZ8kUqsWrWKBQsWAPDOO+/Qv39/4uLiePzxx3G5XIDVA1y8eDEDBgxg165dPP30055e1y9+8QsAPvroIwYMGEDv3r0ZOXIk58+fB6ybdMydO5dRo0YxderUUr3y3bt3M2jQIHr37s2gQYM4cuRItT6DPXv2EBsbS0FBAVeuXKFHjx6kpqaSm5vLiBEj6NOnDzExMXz44YeA1fPu2rUrs2fPpmfPnkyZMoXExEQGDx6Mw+Fg9+7dnrY/9thjDB8+HIfDwZtvvllm3y6Xi4SEBPr160dsbCxvvPEGAOfOnWPo0KHExcXRs2dPduzYUa1jK68N5W2/JF2zNxEhISGBnj17EhMT4wmi27Zt4+677+aRRx6ha9euTJkypcYplP/+9797ktWFh4dzzz338Mknn5SpN2zYME+KjIEDB5Kenl6j/apGSkTq/dG3b1+53qFDh8qUVeToxaPy+u7XZeFnC+X13a/L0YtHb2h9X/7617/KzJkzRUTke9/7nuzdu1dERH7/+9/LjBkzRETkyy+/FH9/f9mzZ49cuHBBunTp4ln/vvvukx07dsihQ4fkwQcflMLCQhER+dGPfiSrV68WERFA1q9fLyIimZmZcuedd0pxcbGIiFy+fFlERC5duuQpe/PNN+XnP/+5iIg888wz0qdPH8nLyxMRka1bt8oDDzwgIiJZWVnidDpFROTTTz+VcePGlanjbevWrRIWFia9evXyPI4dOyYiIgsXLpQnn3xS5s2bJy+88IKIiDidTsnKyhIRkYyMDOnSpYsUFxfL8ePHxd/fX/bv3y8ul0v69OkjM2bMkOLiYtm0aZOMGTPG0/bY2FjJy8uTjIwMiY6OljNnzsjx48elR48eIiLyxhtvyNKlS0VEpKCgQPr27SvffPONvPTSS/Lcc8+JiEhRUZFkZ2dX+O84bdo06dSpk+e4Hn300QrbUN72b7/9dsnIyBARkRYtWoiIyIYNG2TkyJFSVFQk3377rdx2221y9uxZz+d5+vRpcblcMnDgQNmxY0eZtr344oulPvOSx4IFC8rUXbZsmefzEBF59tlnZdmyZRUe+/z580ut4+1Gv2Oq4QGSpZy4aothmbTMNFbuW0lESATRYdFkX81m5b6VzOo9C0fr6l/ZtHbtWn76058CMGnSJNauXUufPn3Yvn27Z2w7NjbWk9I3MjKSO+64g3/+8584HA6OHDnC4MGDee2119i7d68nX0t+fj5t27YFrGyQ48ePB6x88cHBwcyePZsHHnjA0wtPT09n4sSJnDt3jsLCwlJ3rB89ejQhISFl2p6VlcW0adNIS0vDGIPT6az0eMsbllm8eDH9+vUjODiYV155BbA6Bb/+9a/Zvn07fn5+nDlzxvMXRefOnYmJiQGs7JojRozAGENMTIwnrTHAmDFjCAkJISQkhGHDhrF7925PCmSALVu2sH//fk8Ss6ysLNLS0ujXrx8zZ87E6XQyduzYUuuUp7xhGV9tuJHt79y5k8mTJ+Pv70+7du34/ve/z549ewgLC6N///5ER0cDEBcXx4kTJxgyZEip9RMSEkhISKi0/eA722hF90J95513SE5O5vPPP6/S9pW92GJYJvGbRCJCImgV3Ao/40er4FZEhESQ+E31TwhmZmaSlJTE7Nmz6dSpE8uWLWP9+vWeL1h5X6qJEyfy3nvvsXHjRh5++GGMMYgI06ZNIyUlhZSUFI4cOeIZvw8ODvaMswcEBLB7927Gjx/Ppk2buO+++wBYsGABTzzxBAcOHOCNN94olYq3vLTDixYtYtiwYaSmpvLRRx/VKH3vpUuXyM3NJScnx7OdNWvWkJGRwd69e0lJSaFdu3ae97xTGfv5+Xle+/n5edIaQ9nP8PrXIsIf//hHz+d2/PhxRo0axdChQ9m+fTtRUVE89thjnsyd1eGrDTeyfV8Bt0R5KZ29LVu2rNRJ7JKHrxPj0dHRnD592vM6PT2dDuVkx0tMTOT5559n8+bNpdqhmg5bBPczOWcICyqd8zcsKIyzOdXP+bthwwamTp3KyZMnOXHiBKdPn6Zz587s3LmToUOHelLxpqamsn//fs9648aNY9OmTaxdu5aJEycCVvrgDRs2cOHCBcAKlr5mMOTm5pKVlcX999/PH/7wB1JSUgCrxxoVFQVQ7gyJ63mvs2rVqmp9BiXmzp3L0qVLmTJlCr/85S8922/bti2BgYFs3bq1WjMyPvzwQwoKCsjMzGTbtm1lMlHee++9rFixwvNXx9GjR7ly5QonT56kbdu2zJkzh1mzZvHFF18AMHXqVM+Yfk3aUN72fRk6dCjr16/H5XKRkZHB9u3b6d+/f5X3n5CQ4Pnx8n6U/IV0/eexZcsWLl++zOXLl9myZQv33ntvmXr79u3j8ccfZ/PmzZ6/EFXTY4thmajQKLKvZtMquJWnLPtqNh1Cq5/zd+3atTz99NOlysaPH8+7777L8uXLmTFjBrGxscTFxZX6MoeHh9O9e3cOHTrkKe/evTvPPfcco0aNori4mMDAQF577TVuv/32UtvPyclhzJgxFBQUICK8/PLLgHXib8KECURFRTFw4ECOHz9eafufeuoppk2bxvLlyxk+fHiVjnnHjh2lhiB+85vfkJeXR0BAAI8++igul4tBgwaRlJTElClTeOihh4iPjycuLq7KqYm99e/fnwceeIBTp06xaNEiOnToUGrYZvbs2Zw4cYI+ffogIkRGRrJp0ya2bdvGsmXLCAwMpGXLlp6e9f79+2nfvr3PfSUkJPDcc895Xpf8CPhqw+rVq31u35eHH36YXbt20atXL4wxvPjii9x6660VTp2troiICBYtWuT5EVy8eLHnTmCLFy8mPj6e0aNHk5CQQG5uLhMmTACgY8eObN68udbboxo2e6T89RpzDwsKI/tqNpfyL9V4zF3VnSVLltCyZUvPjKCays7OZtasWbz//vv11obGRlP+Nn4Vpfy1xbCMo7WDWb1nEdoslDPZZwhtFqqBvYkJCwu7ocCulN3ZoueulLpx+h1r/Gzfc1dKKVWaBnellLIhDe5KKWVDlQZ3Y8xtxpitxpivjDEHjTE/cZdHGGM+NcakuZfhXuv8yhhzzBhzxBhTdiKuUkqpOlWVnnsR8KSIdAMGAvONMd2Bp4HPRMQBfOZ+jfu9SUAP4D7gdWNM9VIdNgCa8vfm0pS/FatKyt9Vq1YRGRnpOd6//OUvN7mVqiGo9CImETkHnHM/zzHGfAVEAWOAu93VVgPbgF+6y9eJyFXguDHmGNAf2FXbjb8ZNOWvPTSllL9gpcF49dVX66GVqqG4oTF3Y0wnoDfwL6CdO/CX/ACUXOccBZz2Wi3dXdboaMpfTfnbGFP+KgVUPeUv0BLYC4xzv/7uuvcvu5evAf/Xq3wlMN7H9uYCyUByx44dy6SyvNF0pBePXpTdr++WzxZ+Jrtf3y0Xj168ofV90ZS/mvK3Mab8ffvtt+XWW2+VmJgYGT9+vJw6dcrn56Ipfxs/apry1xgTCGwE1ojIf7uLzxtj2ovIOWNMe+CCuzwduM1r9WigTAYvEfkz8GewLmKqSjvKk5mWyb6V+wiJCCEsOoyr2VfZt3IfvWf1prWjdbW3qyl/LZryt/ztN8SUvw899BCTJ08mKCiIP/3pT0ybNo2kpKQq7UPZR1Vmyxis3vdXIrLc663NwDT382nAh17lk4wxQcaYzoADuLFUfTfom8RvCIkIIbhVMMbPENwqmJCIEL5J/Kba29SUv9doyt/GlfK3devWnn3PmTOHvXv3lttGZV9VGXMfDDwGDDfGpLgf9wP/CdxjjEkD7nG/RkQOAu8Bh4BPgPki4qqT1rvlnMkhKKx0zuqgsCByzuZUe5ua8vcaTfnbuFL+njt3zvN88+bNmmKgiarKbJmdQHm3exlRzjrPA8/XoF03JDQqlKvZVwluFewpu5p9ldAOodXepqb81ZS/jTXl7yuvvMLmzZsJCAggIiKixj/uqnGyReIw7zH3oLAgrmZfJf9Sfo3H3FXd0ZS/9U8ThzV+tk8c1trRmt6zetMstBnZZ7JpFtpMA3sToyl/lSrNFj13pdSN0+9Y42f7nrtSSqnSNLgrpZQNaXBXSikb0uCulFI2pMG9Aunp6YwZMwaHw0GXLl34yU9+QmFhoc+6Z8+erVLWQe+0sTdqyZIlvPTSSz7Lo6KiiIuLw+FwMG7cOA4dOlTp9latWsXZs2UyQyilbECDezlEhHHjxjF27FjS0tI4evQoubm5LFy4sEzdoqIiOnTo4MmBUpGPP/6YVq1a1Xp7f/azn5GSkkJaWhoTJ05k+PDhZGRkVLiOBnel7EuDezmSkpIIDg5mxowZgJUb5OWXX+att94iLy+PVatWMWHCBB566CFGjRpV6iYTeXl5/OAHP/CkBB4wYIDnZh4laWNPnDhBt27dmDNnDj169GDUqFGe1MFvvvkm/fr1o1evXowfP568vLwbavvEiRMZNWoU7777LgDPPvss/fr1o2fPnsydOxcRYcOGDSQnJzNlyhTi4uLIz8/3WU8p1TjZJ7inpcGKFfCb31jLtLQabe7gwYP07du3VFlYWBgdO3bk2LFjAOzatYvVq1eXybj3+uuvEx4ezv79+1m0aFG5iZvS0tKYP38+Bw8epFWrVmzcuBGw8tPs2bOHL7/8km7durFy5cobbn+fPn08l8A/8cQT7Nmzh9TUVPLz8/nb3/7GI488Qnx8PGvWrCElJYWQkBCf9ZRSjZM9gntaGqxcCTk5EB1tLVeurFGAFxGfmR+9y0tunHC9nTt3MmnSJAB69uzpSQl8vZJbvwH07dvXk1clNTWVu+66i5iYGNasWcPBgwer1f4SW7duZcCAAcTExJCUlFTu9qpaTynV8NkjuCcmQkQEtGoFfn7WMiLCKq+mHj16lLkvanZ2NqdPn6ZLly5A+el2qzqcUV5K2OnTp/Pqq69y4MABnnnmmWql6923bx/dunWjoKCAefPmsWHDBg4cOMCcOXN8bq+q9ZRSjYM9gvuZMxAWVrosLAxqcLJwxIgR5OXleTICulwunnzySaZPn07z5s0rXHfIkCG89957ABw6dIgDBw7c0L5zcnJo3749TqfTk1r4RmzcuJEtW7YwefJkT4Bu06YNubm5pU76hoaGkpNjpUWuqJ5SqvGxR3CPioLs7NJl2dng40YGVWWM4YMPPuD999/H4XBw5513EhwczAsvvFDpuvPmzSMjI4PY2Fh+97vfERsbyy233FLlfS9dupQBAwZwzz33VDmV7ssvv+yZCvnOO++QlJREZGQkrVq1Ys6cOcTExDB27NhSOdOnT5/OD3/4Q+Li4ggKCiq3nlKq8bFH4rCSMfeICKvHnp0Nly7BrFngcNRBiyvmcrlwOp0EBwfz9ddfM2LECI4ePUqzZs1ueluUKo8mDmv8KkocVqV7qDZ4DocVyBMTrSGaDh3g4YfrJbCDNRVy2LBhOJ1ORIQVK1ZoYFdK3VT2CO5gBfJ6CubXCw0NLXMyVimlbiZ7jLkrpZQqRYO7UkrZkAZ3pZSyIQ3uSillQxrcK/HBBx9gjPHkafHl7rvvbhAnUJOTk/nxj39cYZ1t27bx4IMP+iy/5ZZbiIuL8zwSa3CFb3V5J2CrLdOnT/ekeoiLi2PQoEHV2o53uuaWLVvWYgurbvXq1TgcDhwOB6tXr/ZZZ/ny5XTv3p3Y2FhGjBjByZMnb3IrVUNgn9kydWTt2rUMGTKEdevWsWTJkjrdl8vlwt/fv1rrFhUVER8fT3y8zymvVXLXXXfZNlnYsmXLqpRvvyIff/xxLbWmei5dusRvf/tbkpOTMcbQt29fRo8eTXh4eKl6vXv3Jjk5mebNm7NixQqeeuop1q9fX0+tVvVFe+4VyM3N5R//+AcrV65k3bp1nvL8/HwmTZrkSelbkqq35ItUYtWqVSxYsACAd955h/79+xMXF8fjjz+Oy+UCrB7g4sWLGTBgALt27eLpp5/29Lp+8YtfAPDRRx8xYMAAevfuzciRIzl//jxg3aRj7ty5jBo1iqlTp5bqle/evZtBgwbRu3dvBg0axJEjR6r1GezZs4fY2FgKCgq4cuUKPXr0IDU1ldzcXEaMGEGfPn2IiYnhww8/BKyed9euXZk9ezY9e/ZkypQpJCYmMnjwYBwOB7t37/a0/bHHHmP48OE4HA7efPPNMvt2uVwkJCTQr18/YmNjeeONNwA4d+4cQ4cOJS4ujp49e7Jjx45qHVt5bShv+yXpmr2JCAkJCfTs2ZOYmBhPEN22bRt33303jzzyCF27dmXKlCk1TqH897//3ZOsLjw8nHvuuYdPPvmkTL1hw4Z5UmQMHDiQ9PT0Gu1XNVIiUu+Pvn37yvUOHTpUpqxCWUdFjrwukrLQWmYdvbH1ffjrX/8qM2fOFBGR733ve7J3714REfn9738vM2bMEBGRL7/8Uvz9/WXPnj1y4cIF6dKli2f9++67T3bs2CGHDh2SBx98UAoLC0VE5Ec/+pGsXr1aREQAWb9+vYiIZGZmyp133inFxcUiInL58mUREbl06ZKn7M0335Sf//znIiLyzDPPSJ8+fSQvL09ERLZu3SoPPPCA9XFkZYnT6RQRkU8//VTGjRtXpo63rVu3SlhYmPTq1cvzOHbsmIiILFy4UJ588kmZN2+evPDCCyIi4nQ6JSsrS0REMjIypEuXLlJcXCzHjx8Xf39/2b9/v7hcLunTp4/MmDFDiouLZdOmTTJmzBhP22NjYyUvL08yMjIkOjpazpw5I8ePH5cePXqIiMgbb7whS5cuFRGRgoIC6du3r3zzzTfy0ksvyXPPPSciIkVFRZKdnV3hv+O0adOkU6dOnuN69NFHK2xDedu//fbbJSMjQ0REWrRoISIiGzZskJEjR0pRUZF8++23ctttt8nZs2c9n+fp06fF5XLJwIEDZceOHWXa9uKLL5b6zEseCxYsKFN32bJlns9DROTZZ5+VZcuWVXjs8+fPL7WOtxv+jqkGB0iWcuKqPYZlstPg65UQFAHNo8GZbb3uMgvCqn9h09q1a/npT38KwKRJk1i7di19+vRh+/btnrHt2NhYT0rfyMhI7rjjDv75z3/icDg4cuQIgwcP5rXXXmPv3r2efC35+fm0bdsWsLJBjh8/HrDyxQcHBzN79mweeOABTy88PT2diRMncu7cOQoLC+ncubOnjaNHjyYkJKRM27Oyspg2bRppaWkYY3A6nZUeb3nDMosXL6Zfv34EBwfzyiuvAFan4Ne//jXbt2/Hz8+PM2fOeP6i6Ny5MzExMYCVXXPEiBEYY4iJifGkNQYYM2YMISEhhISEMGzYMHbv3u1JgQywZcsW9u/f70lilpWVRVpaGv369WPmzJk4nU7Gjh1bap3ylDcs46sNN7L9nTt3MnnyZPz9/WnXrh3f//732bNnD2FhYfTv35/o6GgA4uLiOHHiBEOGDCm1fkJCAgkJCZW2H3xnG/WVlrrEO++8Q3JyMp9//nmVtq/sxR7B/dtEK7A3a2W9Lll+m1jt4J6ZmUlSUhKpqakYY3C5XBhjePHFF4Hyv1QTJ07kvffeo2vXrjz88MMYYxARpk2bxn/8x3+UqR8cHOwZZw8ICGD37t189tlnrFu3jldffZWkpCQWLFjAz3/+c0aPHs22bdtKjf2Xl3Z40aJFDBs2jA8++IATJ05w9913V+tzAGusNzc3F6fTSUFBAS1atGDNmjVkZGSwd+9eAgMD6dSpkyezpHcqYz8/P89rPz8/T1pjKPsZXv9aRPjjH//IvffeW6ZN27dv53/+53947LHHSEhIYOrUqdU6Nl9tGDp0aJW37yvgligvpbO3ZcuW+cz8OXToUM8PaYno6Gi2bdvmeZ2enl7uv2tiYiLPP/88n3/+eal2qKbDHmPu+Wcg8LqUv4FhkF/9lL8bNmxg6tSpnDx5khMnTnD69Gk6d+7Mzp07GTp0qOcLmZqayv79+z3rjRs3jk2bNrF27VomTpwIWOmDN2zYwIULFwArWPqawZCbm0tWVhb3338/f/jDH0hJSQGsHmtUVBRAuTMkrue9zqpVq6r1GZSYO3cuS5cuZcqUKfzyl7/0bL9t27YEBgaydevWas3I+PDDDykoKCAzM5Nt27aVyUR57733smLFCs9fHUePHuXKlSucPHmStm3bMmfOHGbNmsUXX3wBwNSpUz1j+jVpQ3nb92Xo0KGsX78el8tFRkYG27dvp3///lXef0JCAikpKWUe1wf2ks9jy5YtXL58mcuXL7NlyxafP3z79u3j8ccfZ/PmzZ6/EFXTY4+ee0iUNRRT0mMH63VI9VP+rl27lqeffrpU2fjx43n33XdZvnw5M2bMIDY2lri4uFJf5vDwcLp3786hQ4c85d27d+e5555j1KhRFBcXExgYyGuvvcbtt99eavs5OTmMGTOGgoICRISXX34ZsE78TZgwgaioKAYOHMjx48crbf9TTz3FtGnTWL58OcOHD6/SMe/YsaPUEMRvfvMb8vLyCAgI4NFHH8XlcjFo0CCSkpKYMmUKDz30EPHx8cTFxVU5NbG3/v3788ADD3Dq1CkWLVpEhw4dSg3bzJ49mxMnTtCnTx9EhMjISDZt2sS2bdtYtmwZgYGBtGzZ0pNzf//+/bRv397nvhISEnjuuec8r0t+BHy1YfXq1T6378vDDz/Mrl276NWrl+cvu1tvvbXCqbPVFRERwaJFizw/gosXL/bcCWzx4sXEx8czevRoEhISyM3NZcKECQB07NiRzZs313p7VMNmj5S/3mPugWFWYL96qcZj7qruLFmyhJYtW3pmBNVUdnY2s2bN4v3336+3NjQ2mvK38aso5a89hmXCHFYgDwiFvDPWUgN7kxIWFnZDgV0pu6t0WMYY8xbwIHBBRHq6y5YAc4AMd7Vfi8jH7vd+BcwCXMCPReTvddDussIcGswbkbq+IKyxtEGpulKVnvsq4D4f5S+LSJz7URLYuwOTgB7udV43xlTvkkullFLVVmlwF5HtwKUqbm8MsE5ErorIceAYUPWpA0oppWpFTcbcnzDG7DfGvGWMKUluEQWc9qqT7i5TSil1E1U3uK8AugBxwDng9+5yX1f2+JyOY4yZa4xJNsYkZ2Rk+KqilFKqmqoV3EXkvIi4RKQYeJNrQy/pwG1eVaMBn1cSicifRSReROIjIyOr04ybQlP+3lya8rdiVUn5u2rVKiIjIz3H+5e//OUmt1I1BNW6iMkY015EzrlfPgykup9vBt41xiwHOgAO4MYuGWxgNOWvPTSllL9gpcF49dVX66GVqqGotOdujFkL7AL+3RiTboyZBbxojDlgjNkPDAN+BiAiB4H3gEPAJ8B8EXHVWevrmKb81ZS/jTHlr1KAjVL+ylEReV1EFrqXmvJXU/5amlrK37fffltuvfVWiYmJkfHjx8upU6d8fi6a8rfxw/Ypf0kDVgIRWMP82e7Xs7BGhqpHU/5aNOVv+dtviCl/H3roISZPnkxQUBB/+tOfmDZtGklJSVXah7IPe6QfIBErsLfCOqRW7tfVPyFYkvJ39uzZdOrUiWXLlrF+/XrPF6yylL8bN24sk/K3JOPfkSNHPOP3vlL+jh8/nk2bNnHffda1YwsWLOCJJ57gwIEDvPHGG57UulB5yt/U1FQ++uijUuvcqJKUvzk5OZ7teKf8TUlJoV27dnWW8rfkczt+/DijRo3ypOSNioriscceqzCxV2UqSvlble37Crglqpry1/skdsnD14nx6OhoTp++NtM4PT2dDh3KJsdr3bq1Z99z5sxh79695bZR2ZdNgvsZ4LqUv4RRzkSdKtGUv9doyt/GlfL33LlznuebN2/W5GBNlE2GZaKwhmJaeZVlY03YqR5N+aspfxtryt9XXnmFzZs3ExAQQERERI1/3FXjZI+Uv6XG3MOwAvslajrmruqOpvytf5ryt/Gzf8pfHFiBPBRriCYUDexNi6b8Vao0mwzLgBXINZg3Fg0h3W5DaINSdcUmPXellFLeGnRwbwjnA5SyI/1u2V+DDe7BwcFkZmbqf0KlapmIkJmZSXBwcH03RdWhBjvmHh0dTXp6OpoOWKnaFxwc7Ll6VtlTgw3ugYGBpS6zV0opVXUNdlhGKaVU9WlwV0opG9LgrpRSNqTBXSmlbEiDu1JK2ZAGd6WUsiEN7kopZUMa3JVSyoY0uCullA1pcFdKKRvS4K6UUjakwV0ppWxIg7tSStmQBnellLIhDe5KKWVDGtyVUsqGNLgrpZQNaXBXSikb0uCulFI2pMFdKaVsSIO7UkrZUKXB3RjzljHmgjEm1asswhjzqTEmzb0M93rvV8aYY8aYI8aYe+uq4UoppcpXlZ77KuC+68qeBj4TEQfwmfs1xpjuwCSgh3ud140x/rXWWqWUUlVSaXAXke3ApeuKxwCr3c9XA2O9yteJyFUROQ4cA/rXTlOVUkpVVXXH3NuJyDkA97KtuzwKOO1VL91dppRS6iaq7ROqxkeZ+KxozFxjTLIxJjkjI6OWm6GUUk1bdYP7eWNMewD38oK7PB24zateNHDW1wZE5M8iEi8i8ZGRkdVshlJKKV+qG9w3A9Pcz6cBH3qVTzLGBBljOgMOYHfNmqiUUupGBVRWwRizFrgbaGOMSQeeAf4TeM8YMws4BUwAEJGDxpj3gENAETBfRFx11HallFLlqDS4i8jkct4aUU7954Hna9IopZRSNaNXqCqllA1pcFdKKRvS4K6UUjakwV0ppWxIg7tSStmQBnel7Ozw27AzHv51m7U8/HZ9t0jdJJVOhVRKNVKH34aCn0LkVQhwQVEG5P8UDgNdZ9Rz41Rd0567UnZ1+bcQnms9L2hmLcNzrXJle9pzV8quIs9AQQAUub/mRQFQIFa5sj3tuStlVwaQ6xK1Xv9a2ZYGd6Xs6kIHCCmyxtsptpYhRZDRob5bpm4CDe5K2VX4ErjUHCiGIKe1vNTcKle2p2PuStlV1xnWzJiM1yDwAjjbQpv5OlOmidDgrpSddZ0BaDBvinRYRimlbEiDu1KNmV6BqsqhwV2pxurw25C9EPxywHmrtcxeqAFeARrclWq8Lr4GRaEgrcD4WcuiUKtcNXka3JVqrALPQ3FY6bLiMGtmjGryNLgr1Vg524Ffdukyv2xryqNq8jS4K9VYtZkPATlgvgMptpYBOVa5avI0uCvVWHWdAWHPQ3Goe4gm1HqtFykp9CImpRo3vUhJlUN77kopZUPac1eq3qQBiVBwANJPwr5sONkSQgfA8CngcNR3A1UjpsFdqXqRBqyEfBecPADnz4DjCrTPg4Ak2P8qnH0Svv+r+m6oaqR0WEapepEIRED2WbiYC35OaJUJkU7IDoHwApD/1KtNVbVpcFeqXpwBwqDwMuQ5IeISXPUHfz8QfygOgKwAvdpUVZsGd6XqRRSQDc3CoXkgBBUCfnDVgCkGp4GiFnq1qao2De5K1YuRwCUI6wBtWkKRHwQVwSU/wAVXAqG1v15tqqpNg7tS9cIBzIKQO6BTLMi/QxZQWAz5LaBNC2h+Va82VdWms2WUqjcO6xEMdMM6eRr0GrS8AM7WEKa3xFPVV6Pgbow5AeQALqBIROKNMRHAeqATcAL4gYhcrlkzlbKpw29bJ00Dz1uJwPQep6qW1MawzDARiRORePfrp4HPRMQBfOZ+rZS6nt5sQ9WhuhhzHwOsdj9fDYytg30o1fjpzTZUHappcBdgizFmrzFmrrusnYicA3Av9XS/Ut4Ovw27OkGPveA4Cm33gV+G9Z7ebEPVkpqeUB0sImeNMW2BT40xh6u6ovvHYC5Ax44da9gMpRqJw29DyEzoDPgDxUCIE4JPQDoggTr9UdWKGvXcReSse3kB+ADoD5w3xrQHcC99dkNE5M8iEi8i8ZGRkTVphlIN0+G3YWc8/Os2a1kS2G8DAgGD9Q0MBsIEIs7qzTZUral2cDfGtDDGhJY8B0YBqcBmYJq72jTgw5o2UqlGx9fJ0lB3YPfmhzW42Rzwd+nNNlStqcmwTDvgA2NMyXbeFZFPjDF7gPeMMbOAU8CEmjdTqUbm4mvgV3KyFGiWaX1jvBmswO6HNTzzbS8YooFd1Y5qB3cR+Qbo5aM8ExhRk0Yp1egFnrd67Mb9OuqS73ol7xeiwzGqVukVqkrVFu8LklpegitXwa/YCuxtxKpTTNnBUAG+6Ax3aa9d1R4N7krVBs8Ye6jVY88rBMcFaywdrB56ycOFFeBLeu1fBsNd39RDo5WdaXBXqjZc/q2Vkz3gIhQEQn6INQumZDzdxbWpj7iXAuyLgAGZ9dNmZWuaFVKpmjr8Ntx2GhArsAe4oFOm1XUqBvLdj6tWFQCONofkGRrYVZ3RnrtSNXH4bXA9ASHFEFhope11BoK5ar0vXnWdWEMxV4AeV256U1XTosFdqeo6/Dbk/wQ65llBOxQIK4T8QuubVRLY/bGGZQKw6p0MB71uT9UxHZZRqrouL4GIPCji2li6HxCEVVYI5GIF+SCswJ7uBy1/Xw+NVU2N9tyVqq62ZyE/AAr9oL3TfVcDrAD/XQBkBUPLgmuzYjKiIPwZvQJV3RQa3JWqroBiaFl0bVqjYA3BCHA+GlxtoPA8DDhl1f+3emupaoJ0WEapG1WSsje8GFpybe66P9ZJ03OBUNwW/LI1w6OqNxrclboRJRcrRXwLF/2sYB7CtTF3A2S0B/OdZnhU9UqHZZS6ERdfgyAXRLinOhZijbMLkBEAzYrAX8AZCmFP6/i6qjca3JWqjHfOmDvSrStPg7CGYgrddfL8IOMOKA6FIcn111al3DS4K1WR/50Jt/4VosUagomg9Bz2ZkAeEFxsDcOE6f3gVcOgwV2p65X01Ft8DXd+B1cM5IZAhzzrDkri9QjASg52Hr3RhmpQNLgr5c07u2PLHOtkaXOBkDwriBdz7fZ4JQnBioHTffVGG6pB0eCulLeSOygFXoQo17V5682xnsO1qY8l0x/90VkxqsHR4K6Ut8Dz0DX92vx17+EXvMq8b7qRiw7HqAZH57krVeLzwRCXbiUAK0kZUDIEI16vi7BOrhZipfE9F3KzW6pUpbTnrhRYgb3//1onTH3xwwrq+Vi99mKgwMD5FpD77zerlUpVmQZ3pQBi/9eau14eAxQAWQaK/SC9t5VeQK9CVQ2UDsuopq0kT0xYJfUEuAx82wauBFlj88WhOv1RNVjac1dNV8m0x387d22MvTxFYE2LCQD/V2GABnTVsGlwV02LdyqBlpcgKg9aV7KOC8jA6rFrT101Ejoso5oOzwVKOeB3BbrkQftK1nECmcDZ9laPXQO7aiS0566ajpILlFqchm75lf/vd2FNdcxsrj121ehocFdNR+B5cPmDowqBvUQW2mNXjZIOy6imo9gPup20UvZWpuQq1JyK5kcq1XBpcFf2dvht2BkP+8IpvvMUxbdcuwbJ18PDBRxpDt/dbg3nKNXINPJhmTTIfxcyPoPTmfBVBLhGwPAp4HDUd+NUfft8MPT4X4iG4iCuJf6qQDHgVwwcbg45PUGKreEcpRqZRhzc06DgJTiTAt9cghZB0O88HPwY/noWHkuwQYBPAxKh4ACkn4R92XCyJYQO0B+wyuy4A753HPys0ZgqE+CQO7CD3uRaNVqNeFgmEXIuwLeFEBgKtIDCFnB7IbTLgMTE+m5gDaUBKyH/GzhxAM58BXech45OkG3w12WQlnZd/RVQMA+OPQApHeG7ELjqD7n+kBIAm26HU+OBeOA29/Jtr/WXQEEM5LSEXAOFBoque1w1cMHAdwZcBooNVh9h/E37ZCpzcdNgigcdpzjgBgO7mzOgudVj15tcq0asznruxpj7gP8P64/hv4jIf9buHs5AQQHkOaF5C6uoMBCa50HIVTh+tnZ3d9MlAhGQ/SVczIXiVlAsEJUNudHXfsAcDq79ELjg5AEISIWeWeAUK3OhP9ANaH4KIk5BZjA0bwdXU8B/JuyeCSFQHIt1c4pmpVtScl8Kz4s21zW12AX8N6Uu8ywGUoDDIAJFRRA04y3gNazbFrUD5gNDKPevk3sHQ6dj1947+i0EFsDtftZt75qLu8EOYChkzcPVEiJGV/9TF4B2OZBz3uqx602uVSNVJ8HdGOOP9S2+B0gH9hhjNovIodrbSxQEH4TmgeB0QrNAaOaEvADID4IOHWpvV/XiDBANhZev/YAVCjTPB0Ig5LLXD9h1PwR9rli9asSKVkVYwbYzkAM4CyD7FLjECuTx7h5uc98tKfZdXDE/oI/1MFjJFouLZyIC/q5wcO0Fv5lW3QCsdnQGboPifCDgEzAg2WCKrXLuuLbp4lIN2wusg9DKswhUxhgo8gsjcMCpGm5JqfpVV8My/YFjIvKNiBQC64AxtbuLkRDaFm5tBs4c4Ao0uwInm8H5SBg5snZ3d9NFAdnQLNzrB8wJeSFA/nU/YGeAsGs/BIEuK1p7/+sWYb2+6l66xAqOTqxsiOUE9tpmDBQHXLb2648V9b1ypxcHAi2wgn0QmCDr0LxT8Vbrx6aKRKCosLJ8BEo1fHUV3KOA016v091ltcgBwb+AqPuhawdAILkd5Nxvk5OpI4FLENYB2rQEv++sS+bPhEHhpet+wK7/IfC3pnx4R8FArNcl07Zd7mUQVsCvaZf3Rnnfyeh6/ly749FNPuXvKgKK593cnSpVB+rqq+MrVEipCsbMBeYCdOzYsZq7cUDIM9DxGegIDK7mZhokBzALQhKhUx4EhMG+HDgdCC2HwGPes2VGAivdPwSn4GwL6JR1LbgHYAXME1i5VIRrveZAIBXrb62bHeCr4ia2yemEwkuvENp+wc3bqVJ1pK6CezrWKGmJaKDUGU4R+TPwZ4D4+PhSgV+VcFiPYODf3I9y63n9EKSHQeoB6JQBQVfhqsARYLefNQvkHqx/kVxgG3AOuBMIr+vj8VKVvC4lN6eu4wAvAs48CA4VgipLJKZUI1FXwX0P4DDGdMYaEJ4EPFpH+1JAmR8Cb3/4D/jTn+DyZQgPhys/hAm3wq7fQs4ZyCnC789QPJvS6W9Lhm5K7iMKpe8lWhMurDsb+UoF4OJaUC+i/Fvf1ZDTCbmnLtLa0Rr/0LrZh1L1xYjUTafZGHM/8Aes/tdbIvJ8eXXj4+MlOTm5TtqhbkQasAb4F1dznWQc7Mzp/72LgOBu3DHyDlo7WsOxt+HQTIrvpWzQLZmZ8x7wA6yug9ePgGdKZNb/hW83QXYudAdu4doYu9OaLSN+YJwgThAXmFvAhFj1/EzNT6o6nRAUpH8wqsbNGLNXROJ9vldXwf1GaHBX1eO+gjczFVIvwcB1uAKsGTnl8fzAaGBXNlBRcG/E6QeUcg9FtQa+D7C2Kulj8K9KJaUauUacfkAppVR5NLgrpZQNaXBXSikb0uCulFI2pMFdKaVsqEFMhTTGZAAn62n3bYCL9bTvm6UpHCPocdqNHmflbheRSF9vNIjgXp+MMcnlzRO1i6ZwjKDHaTd6nDWjwzJKKWVDGtyVUsqGNLi7M1PaXFM4RtDjtBs9zhpo8mPuSillR9pzV0opG2qSwd0Ys8wYc9gYs98Y84ExppXXe78yxhwzxhwxxtxbj82sMWPMBGPMQWNMsTEm/rr3bHOcAMaY+9zHcswY83R9t6e2GGPeMsZcMMakepVFGGM+NcakuZc38zYrdcIYc5sxZqsx5iv3/9mfuMttc6zGmGBjzG5jzJfuY/ytu7xOjrFJBnfgU6CniMQCR4FfARhjumPdWKQHcB/wujGmMecQTAXGAdu9C+12nO62vwb8H6wM8ZPdx2gHq7D+jbw9DXwmIg7gM/frxq4IeFJEugEDgfnuf0M7HetVYLiI9ALigPuMMQOpo2NsksFdRLaISJH75T+xbjoHMAZYJyJXReQ4cAzr7qKNkoh8JSJHfLxlq+PEavsxEflGRAqBdVjH2OiJyHbg0nXFY4DV7uergbE3s011QUTOicgX7uc5wFdYd363zbGKJdf9suQOxkIdHWOTDO7XmQn8/+7nUcBpr/fS3WV2Y7fjtNvxVKadiJwDKygCbeu5PbXKGNMJ6A38C5sdqzHG3xiTAlwAPhWROjtG296swxiTCNzq462FIvKhu85CrD8H15Ss5qN+g55OVJXj9LWaj7IGfZyVsNvxNFnGmJbARuCnIpJtKrqtViMkIi4gzn2e7wNjTM+62pdtg7uIjKzofWPMNOBBYIRcmw+aDtzmVS0aOFs3LawdlR1nORrdcVbCbsdTmfPGmPYics4Y0x6rF9joGWMCsQL7GhH5b3exLY9VRL4zxmzDOp9SJ8fYJIdljDH3Ab8ERotIntdbm4FJxpggY0xnrPu47a6PNtYxux3nHsBhjOlsjGmGdbJ4cz23qS5tBqa5n08DyvsLrdEwVhd9JfCViCz3ess2x2qMiSyZmWeMCQFGAoepq2MUkSb3wDqBeBpIcT/+5PXeQuBr4Ajwf+q7rTU8zoexerVXgfPA3+14nO7juR9r5tPXWENS9d6mWjqutcA5wOn+t5yFddfYz7DuEP4ZEFHf7ayF4xyCNZS23+t7eb+djhWIBfa5jzEVWOwur5Nj1CtUlVLKhprksIxSStmdBnellLIhDe5KKWVDGtyVUsqGNLgrpZQNaXBXSikb0uCulFI2pMFdKaVs6P8BhdjTnoS5Hx0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "print('Error Rate without adversarial examples:', error_rate(model.predict_class(X).T, Y.reshape(X.shape[0], 1)))\n",
    "\n",
    "Xadv, Y = adversial_samples(X, Y, model, epsilon=0.2)\n",
    "print('Error Rate with adversarial examples, epsilon = 0.2:', error_rate(model.predict(Xadv).T, Y.reshape(Xadv.shape[0], 1)))\n",
    "Xadv2, Y = adversial_samples(X, Y, model, epsilon=0.5)\n",
    "print('Error Rate with adversarial examples, epsilon = 0.5:', error_rate(model.predict(Xadv2).T, Y.reshape(Xadv2.shape[0], 1)))\n",
    "\n",
    "plt.scatter(X[:9999, 0], X[:9999, 1], color='blue', alpha=0.4, label='Original Data')\n",
    "plt.scatter(Xadv[:9999, 0], Xadv[:9999, 1], color='green', alpha=0.4, label='Adversarial Examples, Epsilon = 0.2')\n",
    "plt.scatter(Xadv2[:9999, 0], Xadv2[:9999, 1], color='purple', alpha=0.4, label='Adversarial Examples, Epsilon = 0.5')\n",
    "plt.scatter(X[10000:, 0], X[10000:, 1], color='red', alpha=0.4, label='Original Data')\n",
    "plt.scatter(Xadv[10000:, 0], Xadv[10000:, 1], color='orange', alpha=0.4, label='Adversarial Examples, Epsilon = 0.2')\n",
    "plt.scatter(Xadv2[10000:, 0], Xadv2[10000:, 1], color='yellow', alpha=0.4, label='Adversarial Examples, Epsilon = 0.5')\n",
    "plt.legend(loc='best')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 664,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\kebin\\Anaconda3\\lib\\site-packages\\ipykernel_launcher.py:40: RuntimeWarning: overflow encountered in exp\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Error Rate without adversarial examples: 0.3697296837810269\n",
      "Error Rate with adversarial examples, epsilon = 0.2: 0.3697296837810269\n",
      "Error Rate with adversarial examples, epsilon = 0.5: 0.3697296837810269\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/d3fzzAAAACXBIWXMAAAsTAAALEwEAmpwYAAA5vElEQVR4nO3de3xU1bn4/8/KhSRAIgkEhEQEOePhlhAgXApI5SL69QIIUkC+cgdbkN5srC0FqajnVCz2Z1VqLQqnIqBwROzxZzEGBHpoIUiEgEBQbgGEEDAXkpDJ5Pn+sSfDhEwu5EKSnef9es1rz6xZe++1B+aZlbXXfrYREZRSStmLX303QCmlVO3T4K6UUjakwV0ppWxIg7tSStmQBnellLIhDe5KKWVDlQZ3Y8xtxpitxpivjDEHjTE/cZcvMcacMcakuB/3e63zK2PMMWPMEWPMvXV5AEoppcoylc1zN8a0B9qLyBfGmFBgLzAW+AGQKyIvXVe/O7AW6A90ABKBO0XEVfvNV0op5UulPXcROSciX7if5wBfAVEVrDIGWCciV0XkOHAMK9ArpZS6SQJupLIxphPQG/gXMBh4whgzFUgGnhSRy1iB/59eq6VT8Y8Bbdq0kU6dOt1IU5RSqsnbu3fvRRGJ9PVelYO7MaYlsBH4qYhkG2NWAEsBcS9/D8wEjI/Vy4z9GGPmAnMBOnbsSHJyclWbopRSCjDGnCzvvSrNljHGBGIF9jUi8t8AInJeRFwiUgy8ybWhl3TgNq/Vo4Gz129TRP4sIvEiEh8Z6fOHRymlVDVVZbaMAVYCX4nIcq/y9l7VHgZS3c83A5OMMUHGmM6AA9hde01WSilVmaoMywwGHgMOGGNS3GW/BiYbY+KwhlxOAI8DiMhBY8x7wCGgCJivM2WUUurmqjS4i8hOfI+jf1zBOs8Dz9egXTidTtLT0ykoKKjJZpRSPgQHBxMdHU1gYGB9N0XVkRuaLXMzpaenExoaSqdOnbBGhpRStUFEyMzMJD09nc6dO9d3c1QdabDBvaCgQAO7UnXAGEPr1q3JyMio76Y0QWlAIhQcgKzLcKkV+MfCrSMhzFGre2qwwR3QwK5UHdHvVn1IA1ZCvgsyvgZnPlz9Er74AtgE8b+A2HtqbW+aOKwC6enpjBkzBofDQZcuXfjJT35CYWGhz7pnz57lkUceqXSb999/P99991212rNkyRJeeukln+VRUVHExcXhcDgYN24chw4dqnR7q1at4uzZMrNUlVJ1IhGIgOyzUCBwLBOu+MO/F0OBP3zyEqSl1dreNLiXQ0QYN24cY8eOJS0tjaNHj5Kbm8vChQvL1C0qKqJDhw5s2LCh0u1+/PHHtGrVqtbb+7Of/YyUlBTS0tKYOHEiw4cPr/TPbg3uSt1MZ4AwKLwM57+DZkFACDQvgOBwCCuCxMRa25sG93IkJSURHBzMjBkzAPD39+fll1/mrbfeIi8vj1WrVjFhwgQeeughRo0axYkTJ+jZsycAeXl5/OAHPyA2NpaJEycyYMAAzxW4nTp14uLFi5w4cYJu3boxZ84cevTowahRo8jPzwfgzTffpF+/fvTq1Yvx48eTl5d3Q22fOHEio0aN4t133wXg2WefpV+/fvTs2ZO5c+ciImzYsIHk5GSmTJlCXFwc+fn5PusppWpLFJANzcKhIBsCA6GZE/JCgHwIbAu12NmyTXBPS4MVK+A3v7GWNf3r5uDBg/Tt27dUWVhYGB07duTYsWMA7Nq1i9WrV5OUlFSq3uuvv054eDj79+9n0aJF7N27t5w2pzF//nwOHjxIq1at2LhxIwDjxo1jz549fPnll3Tr1o2VK1fecPv79OnD4cOHAXjiiSfYs2cPqamp5Ofn87e//Y1HHnmE+Ph41qxZQ0pKCiEhIT7rKaVqy0jgEoR1gOZ+4PcdNLsKZ28Bvzz4Lho6dKi1vdkiuKelwcqVkJMD0dHWcuXKmgV4EfF50sm7/J577iEiIqJMnZ07dzJp0iQAevbsSWxsrM99dO7cmbi4OAD69u3LiRMnAEhNTeWuu+4iJiaGNWvWcPDgwWq1v8TWrVsZMGAAMTExJCUllbu9qtZTSlWHA5gFIXfAv/WGHD84GAbZbSAzBjL8YeTIWtubLYJ7YiJERECrVuDnZy0jImo2fNWjR48yycyys7M5ffo0Xbp0AaBFixY+163qcEZQUJDnub+/P0VFRQBMnz6dV199lQMHDvDMM89U60Kuffv20a1bNwoKCpg3bx4bNmzgwIEDzJkzx+f2qlpPKVUTDuBH0G4N3LEZsqbBiX+D5nfArFngqL3pkLYI7mfOQFhY6bKwsJoNX40YMYK8vDz+67/+CwCXy8WTTz7J9OnTad68eYXrDhkyhPfeew+AQ4cOceDAgRvad05ODu3bt8fpdLJmzZobbvvGjRvZsmULkydP9gToNm3akJubW+qkb2hoKDk5OQAV1lNK1QGHA370I1i61FrWYmAHmwT3qCjIzi5dlp1ds+ErYwwffPAB77//Pg6HgzvvvJPg4GBeeOGFStedN28eGRkZxMbG8rvf/Y7Y2FhuueWWKu976dKlDBgwgHvuuYeuXbtWaZ2XX37ZMxXynXfeISkpicjISFq1asWcOXOIiYlh7Nix9OvXz7PO9OnT+eEPf0hcXBxBQUHl1lNKNT6V3mbvZoiPj5frh0C++uorunXrVqX1S8bcIyKsHnt2Nly6VOt/5VSZy+XC6XQSHBzM119/zYgRIzh69CjNmjW7+Y1Rqhw38h1TDZMxZq+IxPt6r0FfoVpVDocVyBMTrSGaDh3g4YfrJ7CDNRVy2LBhOJ1ORIQVK1ZoYFdK3VS2CO5gBfL6CubXCw0N1TtLKaXqlS3G3JVSSpWmwV0ppWxIg7tSStmQBnellLIhDe6V+OCDDzDGePK0+HL33Xc3iBOoycnJ/PjHP66wzrZt23jwwQd9lt9yyy3ExcV5Hom1mKGuqrwTsNWW6dOne1I9xMXFMWjQoGptxztdc8uWLWuxhVW3evVqHA4HDoeD1atX+6yzfPlyunfvTmxsLCNGjODkyZM3uZWqIbDNbJm6snbtWoYMGcK6detYsmRJne7L5XLh7+9frXWLioqIj48nPt7nlNcqueuuu2ybLGzZsmVVyrdfkY8/Lve2wTfFpUuX+O1vf0tycjLGGPr27cvo0aMJDw8vVa93794kJyfTvHlzVqxYwVNPPcX69evrqdWqvmjPvQK5ubn84x//YOXKlaxbt85Tnp+fz6RJkzwpfUtS9ZZ8kUqsWrWKBQsWAPDOO+/Qv39/4uLiePzxx3G5XIDVA1y8eDEDBgxg165dPP30055e1y9+8QsAPvroIwYMGEDv3r0ZOXIk58+fB6ybdMydO5dRo0YxderUUr3y3bt3M2jQIHr37s2gQYM4cuRItT6DPXv2EBsbS0FBAVeuXKFHjx6kpqaSm5vLiBEj6NOnDzExMXz44YeA1fPu2rUrs2fPpmfPnkyZMoXExEQGDx6Mw+Fg9+7dnrY/9thjDB8+HIfDwZtvvllm3y6Xi4SEBPr160dsbCxvvPEGAOfOnWPo0KHExcXRs2dPduzYUa1jK68N5W2/JF2zNxEhISGBnj17EhMT4wmi27Zt4+677+aRRx6ha9euTJkypcYplP/+9797ktWFh4dzzz338Mknn5SpN2zYME+KjIEDB5Kenl6j/apGSkTq/dG3b1+53qFDh8qUVeToxaPy+u7XZeFnC+X13a/L0YtHb2h9X/7617/KzJkzRUTke9/7nuzdu1dERH7/+9/LjBkzRETkyy+/FH9/f9mzZ49cuHBBunTp4ln/vvvukx07dsihQ4fkwQcflMLCQhER+dGPfiSrV68WERFA1q9fLyIimZmZcuedd0pxcbGIiFy+fFlERC5duuQpe/PNN+XnP/+5iIg888wz0qdPH8nLyxMRka1bt8oDDzwgIiJZWVnidDpFROTTTz+VcePGlanjbevWrRIWFia9evXyPI4dOyYiIgsXLpQnn3xS5s2bJy+88IKIiDidTsnKyhIRkYyMDOnSpYsUFxfL8ePHxd/fX/bv3y8ul0v69OkjM2bMkOLiYtm0aZOMGTPG0/bY2FjJy8uTjIwMiY6OljNnzsjx48elR48eIiLyxhtvyNKlS0VEpKCgQPr27SvffPONvPTSS/Lcc8+JiEhRUZFkZ2dX+O84bdo06dSpk+e4Hn300QrbUN72b7/9dsnIyBARkRYtWoiIyIYNG2TkyJFSVFQk3377rdx2221y9uxZz+d5+vRpcblcMnDgQNmxY0eZtr344oulPvOSx4IFC8rUXbZsmefzEBF59tlnZdmyZRUe+/z580ut4+1Gv2Oq4QGSpZy4aothmbTMNFbuW0lESATRYdFkX81m5b6VzOo9C0fr6l/ZtHbtWn76058CMGnSJNauXUufPn3Yvn27Z2w7NjbWk9I3MjKSO+64g3/+8584HA6OHDnC4MGDee2119i7d68nX0t+fj5t27YFrGyQ48ePB6x88cHBwcyePZsHHnjA0wtPT09n4sSJnDt3jsLCwlJ3rB89ejQhISFl2p6VlcW0adNIS0vDGIPT6az0eMsbllm8eDH9+vUjODiYV155BbA6Bb/+9a/Zvn07fn5+nDlzxvMXRefOnYmJiQGs7JojRozAGENMTIwnrTHAmDFjCAkJISQkhGHDhrF7925PCmSALVu2sH//fk8Ss6ysLNLS0ujXrx8zZ87E6XQyduzYUuuUp7xhGV9tuJHt79y5k8mTJ+Pv70+7du34/ve/z549ewgLC6N///5ER0cDEBcXx4kTJxgyZEip9RMSEkhISKi0/eA722hF90J95513SE5O5vPPP6/S9pW92GJYJvGbRCJCImgV3Ao/40er4FZEhESQ+E31TwhmZmaSlJTE7Nmz6dSpE8uWLWP9+vWeL1h5X6qJEyfy3nvvsXHjRh5++GGMMYgI06ZNIyUlhZSUFI4cOeIZvw8ODvaMswcEBLB7927Gjx/Ppk2buO+++wBYsGABTzzxBAcOHOCNN94olYq3vLTDixYtYtiwYaSmpvLRRx/VKH3vpUuXyM3NJScnx7OdNWvWkJGRwd69e0lJSaFdu3ae97xTGfv5+Xle+/n5edIaQ9nP8PrXIsIf//hHz+d2/PhxRo0axdChQ9m+fTtRUVE89thjnsyd1eGrDTeyfV8Bt0R5KZ29LVu2rNRJ7JKHrxPj0dHRnD592vM6PT2dDuVkx0tMTOT5559n8+bNpdqhmg5bBPczOWcICyqd8zcsKIyzOdXP+bthwwamTp3KyZMnOXHiBKdPn6Zz587s3LmToUOHelLxpqamsn//fs9648aNY9OmTaxdu5aJEycCVvrgDRs2cOHCBcAKlr5mMOTm5pKVlcX999/PH/7wB1JSUgCrxxoVFQVQ7gyJ63mvs2rVqmp9BiXmzp3L0qVLmTJlCr/85S8922/bti2BgYFs3bq1WjMyPvzwQwoKCsjMzGTbtm1lMlHee++9rFixwvNXx9GjR7ly5QonT56kbdu2zJkzh1mzZvHFF18AMHXqVM+Yfk3aUN72fRk6dCjr16/H5XKRkZHB9u3b6d+/f5X3n5CQ4Pnx8n6U/IV0/eexZcsWLl++zOXLl9myZQv33ntvmXr79u3j8ccfZ/PmzZ6/EFXTY4thmajQKLKvZtMquJWnLPtqNh1Cq5/zd+3atTz99NOlysaPH8+7777L8uXLmTFjBrGxscTFxZX6MoeHh9O9e3cOHTrkKe/evTvPPfcco0aNori4mMDAQF577TVuv/32UtvPyclhzJgxFBQUICK8/PLLgHXib8KECURFRTFw4ECOHz9eafufeuoppk2bxvLlyxk+fHiVjnnHjh2lhiB+85vfkJeXR0BAAI8++igul4tBgwaRlJTElClTeOihh4iPjycuLq7KqYm99e/fnwceeIBTp06xaNEiOnToUGrYZvbs2Zw4cYI+ffogIkRGRrJp0ya2bdvGsmXLCAwMpGXLlp6e9f79+2nfvr3PfSUkJPDcc895Xpf8CPhqw+rVq31u35eHH36YXbt20atXL4wxvPjii9x6660VTp2troiICBYtWuT5EVy8eLHnTmCLFy8mPj6e0aNHk5CQQG5uLhMmTACgY8eObN68udbboxo2e6T89RpzDwsKI/tqNpfyL9V4zF3VnSVLltCyZUvPjKCays7OZtasWbz//vv11obGRlP+Nn4Vpfy1xbCMo7WDWb1nEdoslDPZZwhtFqqBvYkJCwu7ocCulN3ZoueulLpx+h1r/Gzfc1dKKVWaBnellLIhDe5KKWVDlQZ3Y8xtxpitxpivjDEHjTE/cZdHGGM+NcakuZfhXuv8yhhzzBhzxBhTdiKuUkqpOlWVnnsR8KSIdAMGAvONMd2Bp4HPRMQBfOZ+jfu9SUAP4D7gdWNM9VIdNgCa8vfm0pS/FatKyt9Vq1YRGRnpOd6//OUvN7mVqiGo9CImETkHnHM/zzHGfAVEAWOAu93VVgPbgF+6y9eJyFXguDHmGNAf2FXbjb8ZNOWvPTSllL9gpcF49dVX66GVqqG4oTF3Y0wnoDfwL6CdO/CX/ACUXOccBZz2Wi3dXdboaMpfTfnbGFP+KgVUPeUv0BLYC4xzv/7uuvcvu5evAf/Xq3wlMN7H9uYCyUByx44dy6SyvNF0pBePXpTdr++WzxZ+Jrtf3y0Xj168ofV90ZS/mvK3Mab8ffvtt+XWW2+VmJgYGT9+vJw6dcrn56Ipfxs/apry1xgTCGwE1ojIf7uLzxtj2ovIOWNMe+CCuzwduM1r9WigTAYvEfkz8GewLmKqSjvKk5mWyb6V+wiJCCEsOoyr2VfZt3IfvWf1prWjdbW3qyl/LZryt/ztN8SUvw899BCTJ08mKCiIP/3pT0ybNo2kpKQq7UPZR1Vmyxis3vdXIrLc663NwDT382nAh17lk4wxQcaYzoADuLFUfTfom8RvCIkIIbhVMMbPENwqmJCIEL5J/Kba29SUv9doyt/GlfK3devWnn3PmTOHvXv3lttGZV9VGXMfDDwGDDfGpLgf9wP/CdxjjEkD7nG/RkQOAu8Bh4BPgPki4qqT1rvlnMkhKKx0zuqgsCByzuZUe5ua8vcaTfnbuFL+njt3zvN88+bNmmKgiarKbJmdQHm3exlRzjrPA8/XoF03JDQqlKvZVwluFewpu5p9ldAOodXepqb81ZS/jTXl7yuvvMLmzZsJCAggIiKixj/uqnGyReIw7zH3oLAgrmZfJf9Sfo3H3FXd0ZS/9U8ThzV+tk8c1trRmt6zetMstBnZZ7JpFtpMA3sToyl/lSrNFj13pdSN0+9Y42f7nrtSSqnSNLgrpZQNaXBXSikb0uCulFI2pMG9Aunp6YwZMwaHw0GXLl34yU9+QmFhoc+6Z8+erVLWQe+0sTdqyZIlvPTSSz7Lo6KiiIuLw+FwMG7cOA4dOlTp9latWsXZs2UyQyilbECDezlEhHHjxjF27FjS0tI4evQoubm5LFy4sEzdoqIiOnTo4MmBUpGPP/6YVq1a1Xp7f/azn5GSkkJaWhoTJ05k+PDhZGRkVLiOBnel7EuDezmSkpIIDg5mxowZgJUb5OWXX+att94iLy+PVatWMWHCBB566CFGjRpV6iYTeXl5/OAHP/CkBB4wYIDnZh4laWNPnDhBt27dmDNnDj169GDUqFGe1MFvvvkm/fr1o1evXowfP568vLwbavvEiRMZNWoU7777LgDPPvss/fr1o2fPnsydOxcRYcOGDSQnJzNlyhTi4uLIz8/3WU8p1TjZJ7inpcGKFfCb31jLtLQabe7gwYP07du3VFlYWBgdO3bk2LFjAOzatYvVq1eXybj3+uuvEx4ezv79+1m0aFG5iZvS0tKYP38+Bw8epFWrVmzcuBGw8tPs2bOHL7/8km7durFy5cobbn+fPn08l8A/8cQT7Nmzh9TUVPLz8/nb3/7GI488Qnx8PGvWrCElJYWQkBCf9ZRSjZM9gntaGqxcCTk5EB1tLVeurFGAFxGfmR+9y0tunHC9nTt3MmnSJAB69uzpSQl8vZJbvwH07dvXk1clNTWVu+66i5iYGNasWcPBgwer1f4SW7duZcCAAcTExJCUlFTu9qpaTynV8NkjuCcmQkQEtGoFfn7WMiLCKq+mHj16lLkvanZ2NqdPn6ZLly5A+el2qzqcUV5K2OnTp/Pqq69y4MABnnnmmWql6923bx/dunWjoKCAefPmsWHDBg4cOMCcOXN8bq+q9ZRSjYM9gvuZMxAWVrosLAxqcLJwxIgR5OXleTICulwunnzySaZPn07z5s0rXHfIkCG89957ABw6dIgDBw7c0L5zcnJo3749TqfTk1r4RmzcuJEtW7YwefJkT4Bu06YNubm5pU76hoaGkpNjpUWuqJ5SqvGxR3CPioLs7NJl2dng40YGVWWM4YMPPuD999/H4XBw5513EhwczAsvvFDpuvPmzSMjI4PY2Fh+97vfERsbyy233FLlfS9dupQBAwZwzz33VDmV7ssvv+yZCvnOO++QlJREZGQkrVq1Ys6cOcTExDB27NhSOdOnT5/OD3/4Q+Li4ggKCiq3nlKq8bFH4rCSMfeICKvHnp0Nly7BrFngcNRBiyvmcrlwOp0EBwfz9ddfM2LECI4ePUqzZs1ueluUKo8mDmv8KkocVqV7qDZ4DocVyBMTrSGaDh3g4YfrJbCDNRVy2LBhOJ1ORIQVK1ZoYFdK3VT2CO5gBfJ6CubXCw0NLXMyVimlbiZ7jLkrpZQqRYO7UkrZkAZ3pZSyIQ3uSillQxrcK/HBBx9gjPHkafHl7rvvbhAnUJOTk/nxj39cYZ1t27bx4IMP+iy/5ZZbiIuL8zwSa3CFb3V5J2CrLdOnT/ekeoiLi2PQoEHV2o53uuaWLVvWYgurbvXq1TgcDhwOB6tXr/ZZZ/ny5XTv3p3Y2FhGjBjByZMnb3IrVUNgn9kydWTt2rUMGTKEdevWsWTJkjrdl8vlwt/fv1rrFhUVER8fT3y8zymvVXLXXXfZNlnYsmXLqpRvvyIff/xxLbWmei5dusRvf/tbkpOTMcbQt29fRo8eTXh4eKl6vXv3Jjk5mebNm7NixQqeeuop1q9fX0+tVvVFe+4VyM3N5R//+AcrV65k3bp1nvL8/HwmTZrkSelbkqq35ItUYtWqVSxYsACAd955h/79+xMXF8fjjz+Oy+UCrB7g4sWLGTBgALt27eLpp5/29Lp+8YtfAPDRRx8xYMAAevfuzciRIzl//jxg3aRj7ty5jBo1iqlTp5bqle/evZtBgwbRu3dvBg0axJEjR6r1GezZs4fY2FgKCgq4cuUKPXr0IDU1ldzcXEaMGEGfPn2IiYnhww8/BKyed9euXZk9ezY9e/ZkypQpJCYmMnjwYBwOB7t37/a0/bHHHmP48OE4HA7efPPNMvt2uVwkJCTQr18/YmNjeeONNwA4d+4cQ4cOJS4ujp49e7Jjx45qHVt5bShv+yXpmr2JCAkJCfTs2ZOYmBhPEN22bRt33303jzzyCF27dmXKlCk1TqH897//3ZOsLjw8nHvuuYdPPvmkTL1hw4Z5UmQMHDiQ9PT0Gu1XNVIiUu+Pvn37yvUOHTpUpqxCWUdFjrwukrLQWmYdvbH1ffjrX/8qM2fOFBGR733ve7J3714REfn9738vM2bMEBGRL7/8Uvz9/WXPnj1y4cIF6dKli2f9++67T3bs2CGHDh2SBx98UAoLC0VE5Ec/+pGsXr1aREQAWb9+vYiIZGZmyp133inFxcUiInL58mUREbl06ZKn7M0335Sf//znIiLyzDPPSJ8+fSQvL09ERLZu3SoPPPCA9XFkZYnT6RQRkU8//VTGjRtXpo63rVu3SlhYmPTq1cvzOHbsmIiILFy4UJ588kmZN2+evPDCCyIi4nQ6JSsrS0REMjIypEuXLlJcXCzHjx8Xf39/2b9/v7hcLunTp4/MmDFDiouLZdOmTTJmzBhP22NjYyUvL08yMjIkOjpazpw5I8ePH5cePXqIiMgbb7whS5cuFRGRgoIC6du3r3zzzTfy0ksvyXPPPSciIkVFRZKdnV3hv+O0adOkU6dOnuN69NFHK2xDedu//fbbJSMjQ0REWrRoISIiGzZskJEjR0pRUZF8++23ctttt8nZs2c9n+fp06fF5XLJwIEDZceOHWXa9uKLL5b6zEseCxYsKFN32bJlns9DROTZZ5+VZcuWVXjs8+fPL7WOtxv+jqkGB0iWcuKqPYZlstPg65UQFAHNo8GZbb3uMgvCqn9h09q1a/npT38KwKRJk1i7di19+vRh+/btnrHt2NhYT0rfyMhI7rjjDv75z3/icDg4cuQIgwcP5rXXXmPv3r2efC35+fm0bdsWsLJBjh8/HrDyxQcHBzN79mweeOABTy88PT2diRMncu7cOQoLC+ncubOnjaNHjyYkJKRM27Oyspg2bRppaWkYY3A6nZUeb3nDMosXL6Zfv34EBwfzyiuvAFan4Ne//jXbt2/Hz8+PM2fOeP6i6Ny5MzExMYCVXXPEiBEYY4iJifGkNQYYM2YMISEhhISEMGzYMHbv3u1JgQywZcsW9u/f70lilpWVRVpaGv369WPmzJk4nU7Gjh1bap3ylDcs46sNN7L9nTt3MnnyZPz9/WnXrh3f//732bNnD2FhYfTv35/o6GgA4uLiOHHiBEOGDCm1fkJCAgkJCZW2H3xnG/WVlrrEO++8Q3JyMp9//nmVtq/sxR7B/dtEK7A3a2W9Lll+m1jt4J6ZmUlSUhKpqakYY3C5XBhjePHFF4Hyv1QTJ07kvffeo2vXrjz88MMYYxARpk2bxn/8x3+UqR8cHOwZZw8ICGD37t189tlnrFu3jldffZWkpCQWLFjAz3/+c0aPHs22bdtKjf2Xl3Z40aJFDBs2jA8++IATJ05w9913V+tzAGusNzc3F6fTSUFBAS1atGDNmjVkZGSwd+9eAgMD6dSpkyezpHcqYz8/P89rPz8/T1pjKPsZXv9aRPjjH//IvffeW6ZN27dv53/+53947LHHSEhIYOrUqdU6Nl9tGDp0aJW37yvgligvpbO3ZcuW+cz8OXToUM8PaYno6Gi2bdvmeZ2enl7uv2tiYiLPP/88n3/+eal2qKbDHmPu+Wcg8LqUv4FhkF/9lL8bNmxg6tSpnDx5khMnTnD69Gk6d+7Mzp07GTp0qOcLmZqayv79+z3rjRs3jk2bNrF27VomTpwIWOmDN2zYwIULFwArWPqawZCbm0tWVhb3338/f/jDH0hJSQGsHmtUVBRAuTMkrue9zqpVq6r1GZSYO3cuS5cuZcqUKfzyl7/0bL9t27YEBgaydevWas3I+PDDDykoKCAzM5Nt27aVyUR57733smLFCs9fHUePHuXKlSucPHmStm3bMmfOHGbNmsUXX3wBwNSpUz1j+jVpQ3nb92Xo0KGsX78el8tFRkYG27dvp3///lXef0JCAikpKWUe1wf2ks9jy5YtXL58mcuXL7NlyxafP3z79u3j8ccfZ/PmzZ6/EFXTY4+ee0iUNRRT0mMH63VI9VP+rl27lqeffrpU2fjx43n33XdZvnw5M2bMIDY2lri4uFJf5vDwcLp3786hQ4c85d27d+e5555j1KhRFBcXExgYyGuvvcbtt99eavs5OTmMGTOGgoICRISXX34ZsE78TZgwgaioKAYOHMjx48crbf9TTz3FtGnTWL58OcOHD6/SMe/YsaPUEMRvfvMb8vLyCAgI4NFHH8XlcjFo0CCSkpKYMmUKDz30EPHx8cTFxVU5NbG3/v3788ADD3Dq1CkWLVpEhw4dSg3bzJ49mxMnTtCnTx9EhMjISDZt2sS2bdtYtmwZgYGBtGzZ0pNzf//+/bRv397nvhISEnjuuec8r0t+BHy1YfXq1T6378vDDz/Mrl276NWrl+cvu1tvvbXCqbPVFRERwaJFizw/gosXL/bcCWzx4sXEx8czevRoEhISyM3NZcKECQB07NiRzZs313p7VMNmj5S/3mPugWFWYL96qcZj7qruLFmyhJYtW3pmBNVUdnY2s2bN4v3336+3NjQ2mvK38aso5a89hmXCHFYgDwiFvDPWUgN7kxIWFnZDgV0pu6t0WMYY8xbwIHBBRHq6y5YAc4AMd7Vfi8jH7vd+BcwCXMCPReTvddDussIcGswbkbq+IKyxtEGpulKVnvsq4D4f5S+LSJz7URLYuwOTgB7udV43xlTvkkullFLVVmlwF5HtwKUqbm8MsE5ErorIceAYUPWpA0oppWpFTcbcnzDG7DfGvGWMKUluEQWc9qqT7i5TSil1E1U3uK8AugBxwDng9+5yX1f2+JyOY4yZa4xJNsYkZ2Rk+KqilFKqmqoV3EXkvIi4RKQYeJNrQy/pwG1eVaMBn1cSicifRSReROIjIyOr04ybQlP+3lya8rdiVUn5u2rVKiIjIz3H+5e//OUmt1I1BNW6iMkY015EzrlfPgykup9vBt41xiwHOgAO4MYuGWxgNOWvPTSllL9gpcF49dVX66GVqqGotOdujFkL7AL+3RiTboyZBbxojDlgjNkPDAN+BiAiB4H3gEPAJ8B8EXHVWevrmKb81ZS/jTHlr1KAjVL+ylEReV1EFrqXmvJXU/5amlrK37fffltuvfVWiYmJkfHjx8upU6d8fi6a8rfxw/Ypf0kDVgIRWMP82e7Xs7BGhqpHU/5aNOVv+dtviCl/H3roISZPnkxQUBB/+tOfmDZtGklJSVXah7IPe6QfIBErsLfCOqRW7tfVPyFYkvJ39uzZdOrUiWXLlrF+/XrPF6yylL8bN24sk/K3JOPfkSNHPOP3vlL+jh8/nk2bNnHffda1YwsWLOCJJ57gwIEDvPHGG57UulB5yt/U1FQ++uijUuvcqJKUvzk5OZ7teKf8TUlJoV27dnWW8rfkczt+/DijRo3ypOSNioriscceqzCxV2UqSvlble37Crglqpry1/skdsnD14nx6OhoTp++NtM4PT2dDh3KJsdr3bq1Z99z5sxh79695bZR2ZdNgvsZ4LqUv4RRzkSdKtGUv9doyt/GlfL33LlznuebN2/W5GBNlE2GZaKwhmJaeZVlY03YqR5N+aspfxtryt9XXnmFzZs3ExAQQERERI1/3FXjZI+Uv6XG3MOwAvslajrmruqOpvytf5ryt/Gzf8pfHFiBPBRriCYUDexNi6b8Vao0mwzLgBXINZg3Fg0h3W5DaINSdcUmPXellFLeGnRwbwjnA5SyI/1u2V+DDe7BwcFkZmbqf0KlapmIkJmZSXBwcH03RdWhBjvmHh0dTXp6OpoOWKnaFxwc7Ll6VtlTgw3ugYGBpS6zV0opVXUNdlhGKaVU9WlwV0opG9LgrpRSNqTBXSmlbEiDu1JK2ZAGd6WUsiEN7kopZUMa3JVSyoY0uCullA1pcFdKKRvS4K6UUjakwV0ppWxIg7tSStmQBnellLIhDe5KKWVDGtyVUsqGNLgrpZQNaXBXSikb0uCulFI2pMFdKaVsSIO7UkrZUKXB3RjzljHmgjEm1asswhjzqTEmzb0M93rvV8aYY8aYI8aYe+uq4UoppcpXlZ77KuC+68qeBj4TEQfwmfs1xpjuwCSgh3ud140x/rXWWqWUUlVSaXAXke3ApeuKxwCr3c9XA2O9yteJyFUROQ4cA/rXTlOVUkpVVXXH3NuJyDkA97KtuzwKOO1VL91dppRS6iaq7ROqxkeZ+KxozFxjTLIxJjkjI6OWm6GUUk1bdYP7eWNMewD38oK7PB24zateNHDW1wZE5M8iEi8i8ZGRkdVshlJKKV+qG9w3A9Pcz6cBH3qVTzLGBBljOgMOYHfNmqiUUupGBVRWwRizFrgbaGOMSQeeAf4TeM8YMws4BUwAEJGDxpj3gENAETBfRFx11HallFLlqDS4i8jkct4aUU7954Hna9IopZRSNaNXqCqllA1pcFdKKRvS4K6UUjakwV0ppWxIg7tSStmQBnel7Ozw27AzHv51m7U8/HZ9t0jdJJVOhVRKNVKH34aCn0LkVQhwQVEG5P8UDgNdZ9Rz41Rd0567UnZ1+bcQnms9L2hmLcNzrXJle9pzV8quIs9AQQAUub/mRQFQIFa5sj3tuStlVwaQ6xK1Xv9a2ZYGd6Xs6kIHCCmyxtsptpYhRZDRob5bpm4CDe5K2VX4ErjUHCiGIKe1vNTcKle2p2PuStlV1xnWzJiM1yDwAjjbQpv5OlOmidDgrpSddZ0BaDBvinRYRimlbEiDu1KNmV6BqsqhwV2pxurw25C9EPxywHmrtcxeqAFeARrclWq8Lr4GRaEgrcD4WcuiUKtcNXka3JVqrALPQ3FY6bLiMGtmjGryNLgr1Vg524Ffdukyv2xryqNq8jS4K9VYtZkPATlgvgMptpYBOVa5avI0uCvVWHWdAWHPQ3Goe4gm1HqtFykp9CImpRo3vUhJlUN77kopZUPac1eq3qQBiVBwANJPwr5sONkSQgfA8CngcNR3A1UjpsFdqXqRBqyEfBecPADnz4DjCrTPg4Ak2P8qnH0Svv+r+m6oaqR0WEapepEIRED2WbiYC35OaJUJkU7IDoHwApD/1KtNVbVpcFeqXpwBwqDwMuQ5IeISXPUHfz8QfygOgKwAvdpUVZsGd6XqRRSQDc3CoXkgBBUCfnDVgCkGp4GiFnq1qao2De5K1YuRwCUI6wBtWkKRHwQVwSU/wAVXAqG1v15tqqpNg7tS9cIBzIKQO6BTLMi/QxZQWAz5LaBNC2h+Va82VdWms2WUqjcO6xEMdMM6eRr0GrS8AM7WEKa3xFPVV6Pgbow5AeQALqBIROKNMRHAeqATcAL4gYhcrlkzlbKpw29bJ00Dz1uJwPQep6qW1MawzDARiRORePfrp4HPRMQBfOZ+rZS6nt5sQ9WhuhhzHwOsdj9fDYytg30o1fjpzTZUHappcBdgizFmrzFmrrusnYicA3Av9XS/Ut4Ovw27OkGPveA4Cm33gV+G9Z7ebEPVkpqeUB0sImeNMW2BT40xh6u6ovvHYC5Ax44da9gMpRqJw29DyEzoDPgDxUCIE4JPQDoggTr9UdWKGvXcReSse3kB+ADoD5w3xrQHcC99dkNE5M8iEi8i8ZGRkTVphlIN0+G3YWc8/Os2a1kS2G8DAgGD9Q0MBsIEIs7qzTZUral2cDfGtDDGhJY8B0YBqcBmYJq72jTgw5o2UqlGx9fJ0lB3YPfmhzW42Rzwd+nNNlStqcmwTDvgA2NMyXbeFZFPjDF7gPeMMbOAU8CEmjdTqUbm4mvgV3KyFGiWaX1jvBmswO6HNTzzbS8YooFd1Y5qB3cR+Qbo5aM8ExhRk0Yp1egFnrd67Mb9OuqS73ol7xeiwzGqVukVqkrVFu8LklpegitXwa/YCuxtxKpTTNnBUAG+6Ax3aa9d1R4N7krVBs8Ye6jVY88rBMcFaywdrB56ycOFFeBLeu1fBsNd39RDo5WdaXBXqjZc/q2Vkz3gIhQEQn6INQumZDzdxbWpj7iXAuyLgAGZ9dNmZWuaFVKpmjr8Ntx2GhArsAe4oFOm1XUqBvLdj6tWFQCONofkGRrYVZ3RnrtSNXH4bXA9ASHFEFhope11BoK5ar0vXnWdWEMxV4AeV256U1XTosFdqeo6/Dbk/wQ65llBOxQIK4T8QuubVRLY/bGGZQKw6p0MB71uT9UxHZZRqrouL4GIPCji2li6HxCEVVYI5GIF+SCswJ7uBy1/Xw+NVU2N9tyVqq62ZyE/AAr9oL3TfVcDrAD/XQBkBUPLgmuzYjKiIPwZvQJV3RQa3JWqroBiaFl0bVqjYA3BCHA+GlxtoPA8DDhl1f+3emupaoJ0WEapG1WSsje8GFpybe66P9ZJ03OBUNwW/LI1w6OqNxrclboRJRcrRXwLF/2sYB7CtTF3A2S0B/OdZnhU9UqHZZS6ERdfgyAXRLinOhZijbMLkBEAzYrAX8AZCmFP6/i6qjca3JWqjHfOmDvSrStPg7CGYgrddfL8IOMOKA6FIcn111al3DS4K1WR/50Jt/4VosUagomg9Bz2ZkAeEFxsDcOE6f3gVcOgwV2p65X01Ft8DXd+B1cM5IZAhzzrDkri9QjASg52Hr3RhmpQNLgr5c07u2PLHOtkaXOBkDwriBdz7fZ4JQnBioHTffVGG6pB0eCulLeSOygFXoQo17V5682xnsO1qY8l0x/90VkxqsHR4K6Ut8Dz0DX92vx17+EXvMq8b7qRiw7HqAZH57krVeLzwRCXbiUAK0kZUDIEI16vi7BOrhZipfE9F3KzW6pUpbTnrhRYgb3//1onTH3xwwrq+Vi99mKgwMD5FpD77zerlUpVmQZ3pQBi/9eau14eAxQAWQaK/SC9t5VeQK9CVQ2UDsuopq0kT0xYJfUEuAx82wauBFlj88WhOv1RNVjac1dNV8m0x387d22MvTxFYE2LCQD/V2GABnTVsGlwV02LdyqBlpcgKg9aV7KOC8jA6rFrT101Ejoso5oOzwVKOeB3BbrkQftK1nECmcDZ9laPXQO7aiS0566ajpILlFqchm75lf/vd2FNdcxsrj121ehocFdNR+B5cPmDowqBvUQW2mNXjZIOy6imo9gPup20UvZWpuQq1JyK5kcq1XBpcFf2dvht2BkP+8IpvvMUxbdcuwbJ18PDBRxpDt/dbg3nKNXINPJhmTTIfxcyPoPTmfBVBLhGwPAp4HDUd+NUfft8MPT4X4iG4iCuJf6qQDHgVwwcbg45PUGKreEcpRqZRhzc06DgJTiTAt9cghZB0O88HPwY/noWHkuwQYBPAxKh4ACkn4R92XCyJYQO0B+wyuy4A753HPys0ZgqE+CQO7CD3uRaNVqNeFgmEXIuwLeFEBgKtIDCFnB7IbTLgMTE+m5gDaUBKyH/GzhxAM58BXech45OkG3w12WQlnZd/RVQMA+OPQApHeG7ELjqD7n+kBIAm26HU+OBeOA29/Jtr/WXQEEM5LSEXAOFBoque1w1cMHAdwZcBooNVh9h/E37ZCpzcdNgigcdpzjgBgO7mzOgudVj15tcq0asznruxpj7gP8P64/hv4jIf9buHs5AQQHkOaF5C6uoMBCa50HIVTh+tnZ3d9MlAhGQ/SVczIXiVlAsEJUNudHXfsAcDq79ELjg5AEISIWeWeAUK3OhP9ANaH4KIk5BZjA0bwdXU8B/JuyeCSFQHIt1c4pmpVtScl8Kz4s21zW12AX8N6Uu8ywGUoDDIAJFRRA04y3gNazbFrUD5gNDKPevk3sHQ6dj1947+i0EFsDtftZt75qLu8EOYChkzcPVEiJGV/9TF4B2OZBz3uqx602uVSNVJ8HdGOOP9S2+B0gH9hhjNovIodrbSxQEH4TmgeB0QrNAaOaEvADID4IOHWpvV/XiDBANhZev/YAVCjTPB0Ig5LLXD9h1PwR9rli9asSKVkVYwbYzkAM4CyD7FLjECuTx7h5uc98tKfZdXDE/oI/1MFjJFouLZyIC/q5wcO0Fv5lW3QCsdnQGboPifCDgEzAg2WCKrXLuuLbp4lIN2wusg9DKswhUxhgo8gsjcMCpGm5JqfpVV8My/YFjIvKNiBQC64AxtbuLkRDaFm5tBs4c4Ao0uwInm8H5SBg5snZ3d9NFAdnQLNzrB8wJeSFA/nU/YGeAsGs/BIEuK1p7/+sWYb2+6l66xAqOTqxsiOUE9tpmDBQHXLb2648V9b1ypxcHAi2wgn0QmCDr0LxT8Vbrx6aKRKCosLJ8BEo1fHUV3KOA016v091ltcgBwb+AqPuhawdAILkd5Nxvk5OpI4FLENYB2rQEv++sS+bPhEHhpet+wK7/IfC3pnx4R8FArNcl07Zd7mUQVsCvaZf3Rnnfyeh6/ly749FNPuXvKgKK593cnSpVB+rqq+MrVEipCsbMBeYCdOzYsZq7cUDIM9DxGegIDK7mZhokBzALQhKhUx4EhMG+HDgdCC2HwGPes2VGAivdPwSn4GwL6JR1LbgHYAXME1i5VIRrveZAIBXrb62bHeCr4ia2yemEwkuvENp+wc3bqVJ1pK6CezrWKGmJaKDUGU4R+TPwZ4D4+PhSgV+VcFiPYODf3I9y63n9EKSHQeoB6JQBQVfhqsARYLefNQvkHqx/kVxgG3AOuBMIr+vj8VKVvC4lN6eu4wAvAs48CA4VgipLJKZUI1FXwX0P4DDGdMYaEJ4EPFpH+1JAmR8Cb3/4D/jTn+DyZQgPhys/hAm3wq7fQs4ZyCnC789QPJvS6W9Lhm5K7iMKpe8lWhMurDsb+UoF4OJaUC+i/Fvf1ZDTCbmnLtLa0Rr/0LrZh1L1xYjUTafZGHM/8Aes/tdbIvJ8eXXj4+MlOTm5TtqhbkQasAb4F1dznWQc7Mzp/72LgOBu3DHyDlo7WsOxt+HQTIrvpWzQLZmZ8x7wA6yug9ePgGdKZNb/hW83QXYudAdu4doYu9OaLSN+YJwgThAXmFvAhFj1/EzNT6o6nRAUpH8wqsbNGLNXROJ9vldXwf1GaHBX1eO+gjczFVIvwcB1uAKsGTnl8fzAaGBXNlBRcG/E6QeUcg9FtQa+D7C2Kulj8K9KJaUauUacfkAppVR5NLgrpZQNaXBXSikb0uCulFI2pMFdKaVsqEFMhTTGZAAn62n3bYCL9bTvm6UpHCPocdqNHmflbheRSF9vNIjgXp+MMcnlzRO1i6ZwjKDHaTd6nDWjwzJKKWVDGtyVUsqGNLi7M1PaXFM4RtDjtBs9zhpo8mPuSillR9pzV0opG2qSwd0Ys8wYc9gYs98Y84ExppXXe78yxhwzxhwxxtxbj82sMWPMBGPMQWNMsTEm/rr3bHOcAMaY+9zHcswY83R9t6e2GGPeMsZcMMakepVFGGM+NcakuZc38zYrdcIYc5sxZqsx5iv3/9mfuMttc6zGmGBjzG5jzJfuY/ytu7xOjrFJBnfgU6CniMQCR4FfARhjumPdWKQHcB/wujGmMecQTAXGAdu9C+12nO62vwb8H6wM8ZPdx2gHq7D+jbw9DXwmIg7gM/frxq4IeFJEugEDgfnuf0M7HetVYLiI9ALigPuMMQOpo2NsksFdRLaISJH75T+xbjoHMAZYJyJXReQ4cAzr7qKNkoh8JSJHfLxlq+PEavsxEflGRAqBdVjH2OiJyHbg0nXFY4DV7uergbE3s011QUTOicgX7uc5wFdYd363zbGKJdf9suQOxkIdHWOTDO7XmQn8/+7nUcBpr/fS3WV2Y7fjtNvxVKadiJwDKygCbeu5PbXKGNMJ6A38C5sdqzHG3xiTAlwAPhWROjtG296swxiTCNzq462FIvKhu85CrD8H15Ss5qN+g55OVJXj9LWaj7IGfZyVsNvxNFnGmJbARuCnIpJtKrqtViMkIi4gzn2e7wNjTM+62pdtg7uIjKzofWPMNOBBYIRcmw+aDtzmVS0aOFs3LawdlR1nORrdcVbCbsdTmfPGmPYics4Y0x6rF9joGWMCsQL7GhH5b3exLY9VRL4zxmzDOp9SJ8fYJIdljDH3Ab8ERotIntdbm4FJxpggY0xnrPu47a6PNtYxux3nHsBhjOlsjGmGdbJ4cz23qS5tBqa5n08DyvsLrdEwVhd9JfCViCz3ess2x2qMiSyZmWeMCQFGAoepq2MUkSb3wDqBeBpIcT/+5PXeQuBr4Ajwf+q7rTU8zoexerVXgfPA3+14nO7juR9r5tPXWENS9d6mWjqutcA5wOn+t5yFddfYz7DuEP4ZEFHf7ayF4xyCNZS23+t7eb+djhWIBfa5jzEVWOwur5Nj1CtUlVLKhprksIxSStmdBnellLIhDe5KKWVDGtyVUsqGNLgrpZQNaXBXSikb0uCulFI2pMFdKaVs6P8BhdjTnoS5Hx0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "print('Error Rate without adversarial examples:', error_rate(model.predict(X).T, Y.reshape(X.shape[0], 1)))\n",
    "Xadv, Y = adversial_samples(X, Y, model, epsilon=0.2)\n",
    "print('Error Rate with adversarial examples, epsilon = 0.2:', error_rate(model.predict(Xadv).T, Y.reshape(Xadv.shape[0], 1)))\n",
    "Xadv2, Y = adversial_samples(X, Y, model, epsilon=0.5)\n",
    "print('Error Rate with adversarial examples, epsilon = 0.5:', error_rate(model.predict(Xadv2).T, Y.reshape(Xadv2.shape[0], 1)))\n",
    "\n",
    "plt.scatter(X[:9999, 0], X[:9999, 1], color='blue', alpha=0.4, label='Original Data')\n",
    "plt.scatter(Xadv[:9999, 0], Xadv[:9999, 1], color='green', alpha=0.4, label='Adversarial Examples, Epsilon = 0.2')\n",
    "plt.scatter(Xadv2[:9999, 0], Xadv2[:9999, 1], color='purple', alpha=0.4, label='Adversarial Examples, Epsilon = 0.5')\n",
    "plt.scatter(X[10000:, 0], X[10000:, 1], color='red', alpha=0.4, label='Original Data')\n",
    "plt.scatter(Xadv[10000:, 0], Xadv[10000:, 1], color='orange', alpha=0.4, label='Adversarial Examples, Epsilon = 0.2')\n",
    "plt.scatter(Xadv2[10000:, 0], Xadv2[10000:, 1], color='yellow', alpha=0.4, label='Adversarial Examples, Epsilon = 0.5')\n",
    "plt.legend(loc='best')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}