Result.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Software Defect Prediction with Neural Network based SMOTE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Research Question 1\n",
    "Does SMOTE can handle the class imbalance in the software defect prediction based on neural networks?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "bal_SMOTE_NN = np.round(np.multiply([0.7053587718679937, #KC1\n",
    "                            0.754198067673243, #KC2\n",
    "                            0.6402985801295518, #JM1\n",
    "                            0.7147920626898701, #PC1\n",
    "                            0.766904211631598, #PC3\n",
    "                            0.808600457044966 #PC4\n",
    "                            ],100), 2)\n",
    "\n",
    "recall_SMOTE_NN = np.round(np.multiply([0.7362237762237761, #KC1\n",
    "                            0.7385281385281385, #KC2\n",
    "                            0.5757708404026693, #JM1\n",
    "                            0.7025, #PC1\n",
    "                            0.7977207977207976, #PC3\n",
    "                            0.78 #PC4\n",
    "                            ],100),2)\n",
    "\n",
    "bal_NN = np.round(np.multiply([0.4227649821574329, #KC1\n",
    "                            0.6186221083385607, #KC2\n",
    "                            0.3573839492652331, #JM1\n",
    "                            0.4317471248357726, #PC1\n",
    "                            0.41813456264739834, #PC3\n",
    "                            0.6130976246408557 #PC4\n",
    "                            ],100), 2)\n",
    "\n",
    "recall_NN = np.round(np.multiply([0.1839160839160839, #KC1\n",
    "                            0.4662337662337662, #KC2\n",
    "                            0.09131433095803641, #JM1\n",
    "                            0.19666666666666668, #PC1\n",
    "                            0.17834757834757836, #PC3\n",
    "                            0.45460317460317456 #PC4\n",
    "                            ],100), 2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Dataset</th>\n",
       "      <th>Bal NN + SMOTE</th>\n",
       "      <th>Recall NN + SMOTE</th>\n",
       "      <th>Bal NN</th>\n",
       "      <th>Recall NN</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>KC1</td>\n",
       "      <td>70.54</td>\n",
       "      <td>73.62</td>\n",
       "      <td>42.28</td>\n",
       "      <td>18.39</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>KC2</td>\n",
       "      <td>75.42</td>\n",
       "      <td>73.85</td>\n",
       "      <td>61.86</td>\n",
       "      <td>46.62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>JM1</td>\n",
       "      <td>64.03</td>\n",
       "      <td>57.58</td>\n",
       "      <td>35.74</td>\n",
       "      <td>9.13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>PC1</td>\n",
       "      <td>71.48</td>\n",
       "      <td>70.25</td>\n",
       "      <td>43.17</td>\n",
       "      <td>19.67</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>PC3</td>\n",
       "      <td>76.69</td>\n",
       "      <td>79.77</td>\n",
       "      <td>41.81</td>\n",
       "      <td>17.83</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>PC4</td>\n",
       "      <td>80.86</td>\n",
       "      <td>78.00</td>\n",
       "      <td>61.31</td>\n",
       "      <td>45.46</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Dataset  Bal NN + SMOTE  Recall NN + SMOTE  Bal NN  Recall NN\n",
       "0     KC1           70.54              73.62   42.28      18.39\n",
       "1     KC2           75.42              73.85   61.86      46.62\n",
       "2     JM1           64.03              57.58   35.74       9.13\n",
       "3     PC1           71.48              70.25   43.17      19.67\n",
       "4     PC3           76.69              79.77   41.81      17.83\n",
       "5     PC4           80.86              78.00   61.31      45.46"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "table = {\n",
    "    'Dataset' : ['KC1', 'KC2', 'JM1', 'PC1', 'PC3', 'PC4'],\n",
    "    'Bal NN + SMOTE': [70.54, 75.42, 64.03, 71.48, 76.69, 80.86],\n",
    "    'Recall NN + SMOTE': [73.62, 73.85, 57.58, 70.25, 79.77, 78.  ],\n",
    "    'Bal NN' : [42.28, 61.86, 35.74, 43.17, 41.81, 61.31],\n",
    "    'Recall NN' : [18.39, 46.62,  9.13, 19.67, 17.83, 45.46]\n",
    "}\n",
    "\n",
    "result = pd.DataFrame(data = table, columns = ['Dataset', 'Bal NN + SMOTE', 'Recall NN + SMOTE', 'Bal NN', 'Recall NN'])\n",
    "result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bal "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3df1zV5f3/8ccLf6X5KxMMRaMfKggCGlO3mbkcmulUTE2zjaXm6rP2WZY5vvZpWZvTmk0tW5/Zslg2LZcGMzMVs/pU5tDIac0stUSZmmmaaAhe3z84noGAgvKGc+B5v924nfO+3u/rfV6Xoq/3+3pf57rMOYeIiEigCanpAERERMqiBCUiIgFJCUpERAKSEpSIiAQkJSgREQlISlAiIhKQPE1QZvZLM9tiZlvN7G5fWSszW21m232vl3gZg4iIBCfPEpSZxQK3Az2AeGCwmXUEUoFM51xHINO3LSIiUoKXd1DRwHrnXJ5zrgB4E0gGhgJpvmPSgGEexiAiIkGqvofn3gJMN7NLgePAjUAW0MY5lwvgnMs1s7CyKpvZRGAiwMUXX3xNVFSUh6GKiEhN2bhx45fOudAzyz1LUM65j83sEWA18A3wIVBQifrzgfkAiYmJLisry5M4RUSkZpnZ52WVezpIwjn3jHOuu3OuD/AVsB3YZ2bhvqDCgf1exiAiIsHJ61F8Yb7XDsBwYBGQAaT4DkkB0r2MQUREgpOXz6AAXvY9gzoJ/Nw5d8jMZgIvmdl44AtgpMcxiIhIEPI0QTnnri2j7CDQz8vPFRGR4KeZJEREJCApQYmISEBSghIRkYCkBCUiIgFJCUpERAKSEpSIiAQkJSgREQlISlAiIgFg9uzZxMTEEBsby5gxYzhx4gRfffUVSUlJdOzYkaSkJA4dOlThuqc98cQTdO7cmZiYGKZMmVJdzakSSlAiIjVsz549PP7442RlZbFlyxYKCwtZvHgxM2fOpF+/fmzfvp1+/foxc+bMCtcFeOONN0hPT2fz5s1s3bqVyZMnV3fTLogSlIhIACgoKOD48eMUFBSQl5dH27ZtSU9PJyWlaOrSlJQUXnnllQrXBXjqqadITU2lUaNGAISFlbm6UcBSghIRqWHt2rVj8uTJdOjQgfDwcFq0aEH//v3Zt28f4eHhAISHh7N/f+nFH8qrC/DJJ5/w9ttv07NnT6677jr+8Y9/VGu7LpQSlIhIDTt06BDp6ens3LmTvXv3cuzYMRYuXHjBdQsKCjh06BDr16/n97//PaNGjcI552VTqpQSlIhIDVuzZg1XXHEFoaGhNGjQgOHDh/Puu+/Spk0bcnNzAcjNzS2zi668ugAREREMHz4cM6NHjx6EhITw5ZdfVmvbLoQSlIhIDevQoQPr168nLy8P5xyZmZlER0czZMgQ0tLSAEhLS2Po0KEVrgswbNgw1q5dCxR19+Xn59O6devqa9gF8no9KBEROYeePXsyYsQIunfvTv369enWrRsTJ07km2++YdSoUTzzzDN06NCBJUuWALB3714mTJjAihUryq0LMG7cOMaNG0dsbCwNGzYkLS0NM6vJplaKBUN/ZGJiosvKyqrpMERExANmttE5l3hmubr4REQkIClBiYhIQPI0QZnZJDPbamZbzGyRmV1kZq3MbLWZbfe9XuJlDCISnLZt20ZCQoL/p3nz5syZMweo2PQ9hw8fZsSIEURFRREdHc17770HwIcffsh3v/tdunbtyo9+9COOHDlSbW2SyvHsGZSZtQP+D+jinDtuZi8BK4AuwFfOuZlmlgpc4pz71dnOpWdQInVbYWEh7dq14/3332fHjh1Mnz6dV199lUaNGrF///4yh1+npKRw7bXXMmHCBPLz88nLy6Nly5Z85zvfYdasWVx33XUsWLCAnTt38pvf/KYGWlUkMvVVT8+/a+YgT89fFWrqGVR9oLGZ1QeaAHuBoUCab38aMMzjGEQkyGVmZnLVVVdx+eWXV2j6niNHjvDWW28xfvx4ABo2bEjLli2BojuzPn36AJCUlMTLL79cTa2QyvIsQTnn9gCzgC+AXOBr59wqoI1zLtd3TC4QXJNDiUi1W7x4MWPGjAEqNn3Pjh07CA0N5bbbbqNbt25MmDCBY8eOARAbG0tGRgYAS5YsYffu3dXXEKkUzxKU79nSUOAKoC1wsZndWon6E80sy8yyDhw44FWYtVZ5/ffTpk2jXbt2/vIVK1aUe47CwkK6devG4MGD/WX33XcfUVFRxMXFkZyczOHDh6ujOVKH5efnk5GRwciRI4GKTd9TUFDApk2buPPOO/nggw+4+OKL/TOBL1iwgCeffJJrrrmGo0eP0rBhw2pvk1SMl118PwR2OucOOOdOAkuB7wH7zCwcwPdaevZDwDk33zmX6JxLDA0N9TDM2qlz585kZ2eTnZ3Nxo0badKkCcnJyQBMmjTJv+/GG28s9xxz5871fyP9tKSkJLZs2cLmzZvp1KkTM2bM8LQdIq+99hrdu3enTZs2QMWm74mIiCAiIoKePXsCMGLECDZt2gRAVFQUq1atYuPGjYwZM4arrrqqehskFeZlgvoC6GVmTazoq8v9gI+BDCDFd0wKkO5hDELJ/vuKysnJ4dVXX2XChAklyvv370/9+kUTkPTq1YucnJwqjVXkTIsWLfJ370HFpu+57LLLaN++Pdu2bQOK/g106dIFwD8j+KlTp/jtb3/LHXfcUR3NkPPg5TOo94G/AZuAf/o+az4wE0gys+1Akm9bPFS8/x5g3rx5xMXFMW7cuHJX6Lz77rt59NFHCQkp/1dkwYIFDBw4sMrjFTktLy+P1atXM3z4cH/ZuHHj2LFjB7GxsYwePdo/fc/evXtL9Ag88cQTjB07lri4OLKzs5k6dSpQlPA6depEVFQUbdu25bbbbqv2dknFaKqjWi4/P5+2bduydetW2rRpw759+2jdujVmxgMPPEBubi4LFiwoUWf58uWsWLGCP/7xj6xbt45Zs2axfPnyEsdMnz6drKwsli5dGlRze4kEGg0zL3+YuSaLreXO7L8//Qpw++23lxgAcdo777xDRkYGK1as4MSJExw5coRbb73Vv8ZMWloay5cvJzMzU8lJRDyjqY5quTP770+vLQOwbNkyYmNjS9WZMWMGOTk57Nq1i8WLF3P99df7k9PKlSt55JFHyMjIoEmTJt43QETqLCWoWqys/vspU6bQtWtX4uLieOONN5g9ezZAqf778tx1110cPXqUpKQkEhIS9IBZRDyjZ1AiIjVIz6D0DEpEgoj+0xZQF5+IiAQoJSiRIFLeFFZLliwhJiaGkJAQztUdXtYUVtnZ2fTq1YuEhAQSExPZsGGD100ROSd18dVC6h6pvU5PYQX/WYIiOTmZvLw8li5dys9+9rNznuP0FFbF10GaMmUKDz74IAMHDmTFihVMmTKFdevWedUMkQqpU3dQ5V19fvXVVyQlJdGxY0eSkpLKnV0hMjKSrl27+q8yzzRr1izMrNS8YCJeKD6FVXR0NJ07dz5nnfKmsDIzf8L6+uuvadu2rScxi1RGnbqDKu/qc+bMmfTr14/U1FRmzpzJzJkzeeSRR8o8xxtvvFFq3i+A3bt3s3r1ajp06OBpG0ROO3MKq4o4PYXV0aNHS5TPmTOHAQMGMHnyZE6dOsW7775blaGKnJc6dQdVXPGrz/T0dFJSiuavTUlJ4ZVXXqn0+SZNmsSjjz6qmRWkWpy5BEVFLF++nLCwMK655ppS+5566ilmz57N7t27mT17tn+hP5GaVGcTVPGrz3379hEeHg5AeHi4f7bjM5kZ/fv355prrmH+/Pn+8oyMDNq1a0d8fLz3gYtQegqrijg9hVVkZCSjR49m7dq13Hpr0RJtaWlp/i90jxw5UoMkJCDUyQR1PlefUPQPfNOmTbz22ms8+eSTvPXWW+Tl5TF9+nQefvhhj6IVKe3MKawq4mxTWLVt25Y333wTgLVr19KxY8cqj1mksupkgiprAtXTc9Tl5uYSFlb2KvSnHxyHhYWRnJzMhg0b+Oyzz9i5cyfx8fFERkaSk5ND9+7d+fe//109jZE6p6wprJYtW0ZERATvvfcegwYNYsCAAUDFp7B6+umnuffee4mPj2fq1KkleghEakqdGiRx2plXn0OGDCEtLY3U1FTS0tIYOnRoqTrHjh3j1KlTNGvWjGPHjrFq1Sp+/etf07Vr1xJdgpGRkWRlZZU5kEKkKjRp0oSDBw+WKEtOTvavmFxc27ZtWbFiRanyvn370rdvX/9279692bhxY5XHKnIh6twdVFlXn6mpqaxevZqOHTuyevVqUlNTgZJXn/v27aN3797Ex8fTo0cPBg0axA033FAjbRARqQvq3B1UWVefl156KZmZmaWOLX71eeWVV/Lhhx+e8/y7du2qkjhFROq6OncHJSIiwaHO3UGJBCtNYSV1jWd3UGbW2cyyi/0cMbO7zayVma02s+2+10u8ikFERIKXZ3dQzrltQAKAmdUD9gDLgFQg0zk308xSfdu/8iqO03T1KSISXKrrGVQ/4DPn3OfAUCDNV54GDKumGEREJIhUV4IaDSzyvW/jnMsF8L2W+a1YM5toZllmlnXgwIFqClNERAKF5wnKzBoCQ4AllannnJvvnEt0ziWGhoZ6E5yIiASs6riDGghscs7t823vM7NwAN9r2TOziohInVYdCWoM/+neA8gAUnzvU4D0aohBRESCjKcJysyaAEnA0mLFM4EkM9vu2zfTyxikbjh8+DAjRowgKiqK6Oho3nvvPf++c610vHLlSjp37szVV1/NzJn/+XV84IEHiIuLIyEhgf79+7N3717P2yEi/+FpgnLO5TnnLnXOfV2s7KBzrp9zrqPv9SsvY5C64Ze//CU33HAD//rXv/jwww+Jjo4Gzr3ScWFhIT//+c957bXX+Oijj1i0aBEfffQRAPfddx+bN28mOzubwYMHa0kVkWqmqY4k6B05coS33nrLvwpsw4YNadmyJXDulY43bNjA1VdfzZVXXknDhg0ZPXo06elFvc7Nmzf3H3fs2DGtlixSzTTVkQS9HTt2EBoaym233caHH37INddcw9y5c8nMzDznSsd79uyhffv2/u2IiAjef/99//b999/PX/7yF1q0aMEbb7zhaTtEpCTdQUnQKygoYNOmTdx555188MEHXHzxxUybNq1CKx0750qVFb9Tmj59Ort372bs2LHMmzevymMXkfIpQUnQi4iIICIigp49ewIwYsQINm3aVKGVjiMiIti9e7d/Oycnx79ycnG33HILL7/8srcNEZESlKDkvJU1cm7JkiXExMQQEhJCVlZWuXVnz55NTEwMsbGxjBkzhhMnTvj3PfHEE3Tu3JmYmBimTJlyzjguu+wy2rdvz7Zt2wDIzMyke/fu7N+/n127drFr1y4iIiLYtGkTl112WYm63/nOd9i+fTs7d+4kPz+fxYsXM2TIEAC2b9/uPy4jI4OoqKhK/fmIyIVRgpLzVtbIudjYWJYuXUqfPn3Krbdnzx4ef/xxsrKy2LJlC4WFhSxevBiAN954g/T0dDZv3szWrVuZPHlyhWJ54oknGDt2LHFxcWRnZzN16tRyjy2+UnL9+vWZN28eAwYMIDo6mlGjRhETEwMUrbQcGxtLXFwcq1atYu7cuRX9oxEJOOd7QXnixAl69OhBfHw8MTExPPjgg/59Fb0gPV8aJCHn5fTIueeeew4oGjlXfPTcuRQUFHD8+HEaNGhAXl6ev1vtqaeeIjU1lUaNGgEQFlbmVI2lJCQknPUfSPGVjouvlAxw4403+hNWcerSk9rk9AXl3/72N/Lz88nLy6Nly5YsXbqUn/3sZ+XWa9SoEWvXrqVp06acPHmS3r17M3DgQHr16uW/ID1b/QuhOyg5L8VHznXr1o0JEyZw7NixCtVt164dkydPpkOHDoSHh9OiRQv69+8PwCeffMLbb79Nz549ue666/jHP/7hZTNE6oTyvooRHR1N586dz1rXzGjatCkAJ0+e5OTJk/6BRBWpfyGUoOS8lDVyrvgsDGdz6NAh0tPT2blzJ3v37uXYsWMsXLjQf95Dhw6xfv16fv/73zNq1KgyR9qJSMVdyAUlFH2hPSEhgbCwMJKSkvwDkrymBCXnpbyRcxWxZs0arrjiCkJDQ2nQoAHDhw/n3Xff9Z93+PDhmBk9evQgJCSk3CmKRKRiLuSCEqBevXpkZ2eTk5PDhg0b2LJli4fR/oeeQcl5KT5yrnPnzmRmZtKlS5cK1e3QoQPr168nLy+Pxo0bk5mZSWJiIgDDhg1j7dq19O3bl08++YT8/Hxat25d7rm0UrLIuZV1QVmZBHVay5Yt6du3LytXriQ2NraqwyxFd1By3soaObds2TIiIiJ47733GDRoEAMGDABKjpzr2bMnI0aMoHv37nTt2pVTp04xceJEAMaNG8eOHTuIjY1l9OjRpKWlaYohkQtU1lcxKnpBeeDAAQ4fPgzA8ePHWbNmTbV95UJ3UHLeyho5l5ycTHJycqljzxw599BDD/HQQw+VOq5hw4b+51EiUnVOX1Dm5+dz5ZVX8uyzz7Js2TJ+8YtfcODAAQYNGkRCQgKvv/46e/fuZcKECaxYsYLc3FxSUlIoLCzk1KlTjBo1isGDBwOUW7+qKEGJiNQB53tBGRcXxwcffFDmOcurX1XUxSciIgFJCUpERAKSuvik8qa18PDcX5/7GBGpE3QHJSIiAcnTOygzawn8GYgFHDAO2Aa8CEQCu4BRzrlDXsYhIlJnBXGPh9d3UHOBlc65KCAe+BhIBTKdcx2BTN+2iIhICZ4lKDNrDvQBngFwzuU75w4DQ4E032FpwDCvYhARkeDl5R3UlcAB4Fkz+8DM/mxmFwNtnHO5AL7XMtdTMLOJZpZlZlkHDhzwMEwREQlEXiao+kB34CnnXDfgGJXoznPOzXfOJTrnEkNDQ72KUUREApSXCSoHyHHOve/b/htFCWufmYUD+F73exiDiIgEKc8SlHPu38BuMzu9mlU/4CMgA0jxlaUA6V7FICIiwcvrL+r+AnjBzBoCO4DbKEqKL5nZeOALYKTHMYiISBDyNEE557KBxDJ29fPyc0VEJPhpJgkREQlISlAiIhKQlKBERCQgKUGJiEhAUoISEZGApAQlIiIBSQlKREQCkhKUiIgEJCUoEREJSEpQIiISkJSgREQkIClBiYhIQFKCEhGRgKQEJSIiAUkJSkREApISlIiIBCQlKBERCUhKUCIiEpA8XfLdzHYBR4FCoMA5l2hmrYAXgUhgFzDKOXfIyzhERCT4VMcd1A+ccwnOuUTfdiqQ6ZzrCGT6tkVEREqoiS6+oUCa730aMKwGYhARkQDndYJywCoz22hmE31lbZxzuQC+17CyKprZRDPLMrOsAwcOeBymiIgEGk+fQQHfd87tNbMwYLWZ/auiFZ1z84H5AImJic6rAEVEJDB5egflnNvre90PLAN6APvMLBzA97rfyxhERCQ4eZagzOxiM2t2+j3QH9gCZAApvsNSgHSvYhARkeDlZRdfG2CZmZ3+nL8651aa2T+Al8xsPPAFMNLDGEREJEh5lqCcczuA+DLKDwL9vPpcERGpHTSThIiIBCQlKBEppbCwkG7dujF48GAAHnjgAeLi4khISKB///7s3bu3zHrjxo0jLCyM2NjYEuU333wzCQkJJCQkEBkZSUJCgudtkOCnBCUipcydO5fo6Gj/9n333cfmzZvJzs5m8ODBPPzww2XW++lPf8rKlStLlb/44otkZ2eTnZ3NTTfdxPDhwz2LXWoPJSgPnHn1ed999xEVFUVcXBzJyckcPny4VJ3du3fzgx/8gOjoaGJiYpg7d65/X3Z2Nr169SIhIYHExEQ2bNhQbW2RuicnJ4dXX32VCRMm+MuaN2/uf3/s2DF8g59K6dOnD61atSr33M45XnrpJcaMGVN1AUutddYEZWZHzexIGT9HzexIdQUZbM68+kxKSmLLli1s3ryZTp06MWPGjFJ16tevz2OPPcbHH3/M+vXrefLJJ/noo48AmDJlCg8++CDZ2dk8/PDDTJkypdraInXP3XffzaOPPkpISMn/Hu6//37at2/PCy+8UO4d1Lm8/fbbtGnTho4dO1ZFqFLLnTVBOeeaOeeal/HTzDnX/Gx166qyrj779+9P/fpFAyZ79epFTk5OqXrh4eF0794dgGbNmhEdHc2ePXsAMDOOHCm6Hvj6669p27at182QOmr58uWEhYVxzTXXlNo3ffp0du/ezdixY5k3b955nX/RokUBd/d0Zo/HkiVLiImJISQkhKysrHLrzZ49m5iYGGJjYxkzZgwnTpyoVH05t0p18ZlZmJl1OP3jVVDBrLyrz9MWLFjAwIEDz3qOXbt28cEHH9CzZ08A5syZw3333Uf79u2ZPHlymXdgIlXhnXfeISMjg8jISEaPHs3atWu59dZbSxxzyy238PLLL1f63AUFBSxdupSbb765qsKtEmf2eMTGxrJ06VL69OlTbp09e/bw+OOPk5WVxZYtWygsLGTx4sUVri8VU6EEZWZDzGw7sBN4k6J1nF7zMK6gdLarTyi6Aq1fvz5jx44t9xzffPMNN910E3PmzPH3+z/11FPMnj2b3bt3M3v2bMaPH+9J/CIzZswgJyeHXbt2sXjxYq6//noWLlzI9u3b/cdkZGQQFRVV6XOvWbOGqKgoIiIiqjLkC1JWj0d0dDSdO3c+Z92CggKOHz9OQUEBeXl5/p6NitaXc6voHdRvgF7AJ865Kyj6ou07nkUVpM529ZmWlsby5ct54YUXyn3AfPLkSW666SbGjh1bYpRTWlqaf3vkyJEaJCHVLjU1ldjYWOLi4li1apV/EM/evXu58cYb/ceNGTOG7373u2zbto2IiAieeeYZ/77FixcHXPfeuXo8ytOuXTsmT55Mhw4dCA8Pp0WLFvTv39+jKOuuis4kcdI5d9DMQswsxDn3hpk94mlkQWjGjBn+7rd169Yxa9YsFi5cyMqVK3nkkUd48803adKkSZl1nXOMHz+e6Oho7rnnnhL72rZty5tvvknfvn1Zu3atHjBLtejbty99+/YFKLdLr23btqxYscK/vWjRonLP99xzz1VleBeseI/HunXrKlX30KFDpKens3PnTlq2bMnIkSNZuHBhqe5QuTAVvWw4bGZNgbeAF8xsLlDgXVi1y1133cXRo0dJSkoiISGBO+64Ayh59fnOO+/w/PPPs3btWv8XGk//w3/66ae59957iY+PZ+rUqcyfP7/G2iJSW1TkeVt51qxZwxVXXEFoaCgNGjRg+PDhvPvuux5HXPdU9A5qKHAcmASMBVoA5zfOtI4ofvX56aeflnlM8avP3r1741zZy1717t2bjRs3ehKnSF1VXo9HRXTo0IH169eTl5dH48aNyczMJDEx0ctw66QK3UE5544550455wqAV4EnfJO+iojUKsuWLSMiIoL33nuPQYMGMWDAAKBkj0fPnj0ZMWIE3bt3p2vXrpw6dYqJEyeetb5U3lnvoMysFzAT+IqigRLPA62BEDP7iXOu9JwmIhKcprXw8Nxfe3fuKlC8xyM5OZnk5ORSx5z5vO2hhx7ioYceKnVcefWl8s7VxTcPmEpRl95aYKBzbr2ZRQGLACUoERHxxLm6+Oo751Y555YA/3bOrQdwzv3L+9BERKQuO9cd1Kli74+fsa/sJ/p1VR3uHhER8cK5ElS8b1JYAxoXmyDWgIs8jUxExCu6oAwKZ01Qzrl61RWIiIhIcZ6vB2Vm9czsAzNb7ttuZWarzWy77/USr2MQEZHgUx0LFv4S+LjYdiqQ6ZzrCGT6tkVERErwNEGZWQQwCPhzseKhQJrvfRowzMsYRC7UiRMn6NGjB/Hx8cTExPDggw8CMG3aNNq1a1dqaqritm3b5t+fkJBA8+bNmTNnToljZs2ahZnx5ZdfVkt7RIJFRac6Ol9zgClAs2JlbZxzuQDOuVwzCyuroplNBCZC0bQiIjWlUaNGrF27lqZNm3Ly5El69+7tX9Nr0qRJTJ48udy6nTt3Jjs7GyhaGK9du3YlvsS5e/duVq9erd9xkTJ4dgdlZoOB/c6585pEzjk33zmX6JxLDA0NreLoRCrOzGjatClQtCTKyZMny10y5WwyMzO56qqruPzyy/1lkyZN4tFHHz2v84nUdl528X0fGGJmu4DFwPVmthDYZ2bhAL7X/R7GIFIlCgsLSUhIICwsjKSkJP9qx/PmzSMuLo5x48Zx6NChs57jzPWQMjIyaNeuHfHx8Z7GLhKsPEtQzrn/55yLcM5FAqOBtc65W4EMIMV3WAqQ7lUMIlWlXr16ZGdnk5OTw4YNG9iyZQt33nknn332GdnZ2YSHh3PvvfeWWz8/P5+MjAxGjhwJQF5eHtOnT+fhh7UogEh5qmMU35lmAkm+JeSTfNsiQaFly5b07duXlStX0qZNG+rVq0dISAi33377WVc6fu211+jevTtt2rQB4LPPPmPnzp3Ex8cTGRlJTk4O3bt359///nd1NUUk4Hk9SAIA59w6YJ3v/UGKlowXCQoHDhygQYMGtGzZkuPHj7NmzRp+9atfkZubS3h4OFC0xEJsbGy551i0aFGJ7r2uXbuyf/9/ercjIyPJysqidevW3jVEJMhUS4ISCWa5ubmkpKRQWFjIqVOnGDVqFIMHD+bHP/4x2dnZmBmRkZH86U9/AorWDZowYYJ/2HleXh6rV6/27xeRilGCEjmHuLg4Pvjgg1Llzz//fJnHn7luUJMmTTh48Ozre+7ateuCYhSpjWriGZSIiMg5KUGJiEhAUoISEZGApGdQImejdYNEaozuoEREJCApQYmISEBSghIRkYCkBCUiIgFJCUpERAKSEpSIiAQkJSgREQlISlAiIhKQlKBERCQgKUGJiEhAUoISEZGApAQlIiIBybMEZWYXmdkGM/vQzLaa2UO+8lZmttrMtvteL/EqBhERCV5e3kF9C1zvnIsHEoAbzKwXkApkOuc6Apm+bRERkRI8S1CuyDe+zQa+HwcMBdJ85WnAMK9iEBGR4OXpMygzq2dm2cB+YLVz7n2gjXMuF8D3GlZO3YlmlmVmWQcOHPAyTBERCUCeJijnXJtfVTgAAA3PSURBVKFzLgGIAHqYWWwl6s53ziU65xJDQ0O9C1JERAJStYzic84dBtYBNwD7zCwcwPe6vzpiEBGR4OLlKL5QM2vpe98Y+CHwLyADSPEdlgKkexWDiIgEr/oenjscSDOzehQlwpecc8vN7D3gJTMbD3wBjPQwBhERCVKeJSjn3GagWxnlB4F+Xn2uiIjUDppJQkREApISlIiIBCQlKBERCUhKUCIiEpCUoEREJCApQYmISEBSghIRkYCkBCUiIgFJCUpERAKSEpSIiAQkJSgREQlISlAiIhKQlKBERCQgKUGJiEhAUoISEZGApAQlIiIBSQlKREQCkhKUiIgEJM8SlJm1N7M3zOxjM9tqZr/0lbcys9Vmtt33eolXMYiISPDy8g6qALjXORcN9AJ+bmZdgFQg0znXEcj0bYuIiJTgWYJyzuU65zb53h8FPgbaAUOBNN9hacAwr2IQEZHgVS3PoMwsEugGvA+0cc7lQlESA8LKqTPRzLLMLOvAgQPVEaaIiAQQzxOUmTUFXgbuds4dqWg959x851yicy4xNDTUuwBFRCQgeZqgzKwBRcnpBefcUl/xPjML9+0PB/Z7GYOIiAQnL0fxGfAM8LFz7g/FdmUAKb73KUC6VzGIiEjwqu/hub8P/Bj4p5ll+8qmAjOBl8xsPPAFMNLDGEREJEh5lqCcc/8HWDm7+3n1uSIiUjtoJgkREQlISlAiIhKQlKBERCQgKUGJiEhAUoISEZGApAQlIiIBSQlKREQCkhKUiIgEJCUoEREJSEpQIiISkJSgREQkIClBiYhIQFKCEhGRgKQEJSIiAUkJSkREApISlIiIBCQvV9StMs2aNWPnzp2cOHHivM/x9JDwKoyotI/tJQ9P/nGlDr/Qtjocnx8+yRPvH+LIt6cu6FwiIucrKBLUT3/6U5o1a0ZkZCRm5S3Se3Yncw5XcVQlRYecX1wV0ja6UodfaFudc1x66RF+AUx/6+AFnUtE5Hx51sVnZgvMbL+ZbSlW1srMVpvZdt/rJRU5V0REBJdeeul5JyepHDOjfpPmXN6yQU2HIiJ1mJfPoJ4DbjijLBXIdM51BDJ92+dkZkpO1czMMPRnLiI1x7ME5Zx7C/jqjOKhQJrvfRowzKvPFxGR4Fbdz6DaOOdyAZxzuWYWdj4niUx9tUqDyrjr++c8Jr79Jfz49p8z+de/BSDtf58gL+8Yd96TylN/mEnaU3PZ9f5ywlq3AqBpx+/zzfZ3KvT56zdu5pcPzuLbb/P5Nj+fm4f0Z9q9d/Dcixncds801qxZQ79+/QBYtmwZw4cPZ8mSJYwYMYL8/HymTJnC3//+d0JCQujSpQt3/c/vaHRRYyaOHgrAlwf2ExJSj1aXXgrAC3/PpEfHcDpGdfHHMGDIcMb/fFLF/9BERDwWsIMkzGwiMBFg1apVNRwNNGzUiMyVf2f8XZO4pNWlpfa3btWSx/70PI/c/8tyz7Hu3Syee+nvPDfnoRLlKXc/yEv/+wjxMZ0oLCxk22ef+/d1jb6aRYsW+RPU4sWLiY+P9++fOnUqR48e5ZNPPqFevXo8++yzTLr9J7zw9zW89PrbADz1h5k0aXIxKXf8wl+v0UWN/ftFRAJRdX8Pap+ZhQP4XveXd6Bzbr5zLtE5l1ivXr1qC7A89erVZ8QtKSx8+o9l7h83eggvZqziq0NfV/rc+w9+RXib1r7PqUeXTlf6913boxsbNmzg5MmTfPPNN3z66ackJCQAkJeXx7PPPsvs2bM5/Wd022230bBhQza881al4xARCSTVnaAygBTf+xQgvZo//4LcnDKBFa8s4eiR0kmo6cVNGDd6KHOf+Wulzzvp9rF07pNM8vh7+dPzf+PEiW/9+8yMH/7wh7z++uukp6czZMgQ/75PP/2UDh060Lx58xLni4nrxqef/Ousn/ntieOMGnCt/2dlxtJKxy0i4iXPuvjMbBHQF2htZjnAg8BM4CUzGw98AYz06vO90LRZcwbfNJq/LpjPRRddVGr/f48bQ0L/0dz7sx+XKO85+Cd8+20+3+Tl8dXhIyQkjQbgkfv/mwF9v8evJ01kbPJAVr25nr++spJF6a+z7m9P++uPHj2axx9/nK+//prHHnuM3/3ud0DR95XKGt3oKLu8OHXxiUig8yxBOefGlLOrn1efWR1uHX8no2+8jqGjxpba17JFM24ZdgN/TFtSovz95X8Byn8GBXBVZHvujGzP7WOTCY3rx8Gv/vNl2x49erBlyxYaN25Mp06d/OVXX301n3/+OUePHqVZs2b+8o//+SHX/fDMEf4iIsFFc/FVUotLLqH/4GEsW/x8mfvv+dmt/GnhyxQUFlb4nK+ueRvnHADbd35BvZB6tGzRrMQxM2bM8N85nXbxxReTkpLCPffcQ6Hv8/7yl79w4vhxeny/T2WaJSIScAJ2FN/Z7Jo5qNJ1NlfhVEc/mXgXi5/7c5n7Wre6hOQbfsDsp1+o8Pmef/lVJk17jCaNL6J+/Xq8MO+3nDkwZODAgWXWnTFjBpMnT6ZTp06EhIQQFRXF7D8/f84uvtPPoE77Xt9+3P3/plU4ZhERr9npK/dAtnbtWnf99ddf0DmqMkGVJS5kp3cnb9utUodXVVv3fbGD2zNyS5XvuuiWKjl/maZVbhRkVX8n7kxqaxVRW8tVl9paHjPb6JxLPLNcXXwiIhKQlKBERCQgKUGJiEhAUoISEZGApAQlIiIBSQlKREQCUlB+D4ppLSpdJe4s+zZP+Pwse4t4udyGiIiUpjuoCjq93Mahrw6Wuf/0chsiIlI1lKAqyMvlNkREpDQlqErwarkNEREpTQmqEoovt1GW/x43hrQlyzly9JtqjkxEpPZRgqqkW8ffySsvPs/x43ml9pW33IaIiFSeElQlebHchoiIlBakw8wrPxAhkJfbEBGR0oIzQdWA9dty/O8vDQ3j/e17/dt33pNaYrmNP0y7lz9Mu7da4xMRqW3UxSciIgGpRhKUmd1gZtvM7FMzS62JGEREJLBVe4Iys3rAk8BAoAswxsy6nK2Oc45gWPm3NnHO4dCfuYjUnJq4g+oBfOqc2+GcywcWA0PPViEnJ4eDBw8qSVUT5xwFeUf4/PDJmg5FROowq+7/9M1sBHCDc26Cb/vHQE/n3F1nHDcRmAjQqlWrqJkzZxZERkY2NrNqjbeiTp06FRISEnKqpuOoCs45du3adfz+++/fdeDAgYIyDmkNfFndcdUQtbV2UlsDy+XOudAzC2tiFF9ZGaZUlnTOzQfKnrIhAJlZlnMusabjqEq33357meW1sa3lUVtrJ7U1ONREF18O0L7YdgSwt5xjRUSkjqqJBPUPoKOZXWFmDYHRQEYNxCEiIgGs2rv4nHMFZnYX8DpQD1jgnNta3XF4IGi6I6uA2lo7qa21U9C2tdoHSYiIiFSEZpIQEZGApAQlIiIBSQmqAszsm2LvbzSz7WbWwcwuM7PFZvaZmX1kZivMrJPvuJVmdtjMltdc5JVX2baaWYKZvWdmW81ss5ndXJPxV5aZfWNmkWbmzOw3xcpbm9lJM5vn2+5jZpvMrMD3Xb6gYmaFZpZtZlvMbImZNfGV18bf4Uq11cwuN7ONvjpbzeyOmm5DRZ3P36tvf3Mz23P69ztQKUFVgpn1A54AbgB2A8uAdc65q5xzXYCpQBvf4b8HflwjgVaBSrQ1D/iJcy7Gd+wcM2tZQ2FfiB3A4GLbI4Hig3e+AH4K/LUaY6pKx51zCc65WCAfuMOKvvVeG3+HK9vWXOB7zrkEoCeQamZtayr4Sjqfv1eA3wBvVn+4laPlNirIzK4FngZudM59ZmbXAyedc/97+hjnXHax95lm1rf6I71wlW1rsbK9ZrYfCAWqbgGu6nEc+NjMEp1zWcDNwEtAWwDn3C4AM6sNs4W8DcQBP6CW/g4XU6G2FtOI4L1wr1BbzewaipLVSiCgv8AbrH8R1a0RkA4Mc879y1cWC2ysuZA8c95tNbMeQEPgM+/C89RiYLSZRQCF1MIvkJtZfYomav4ntfd3GKhcW82svZltpqi34BHnXFD93Ve0rWYWAjwG3Fd90Z0/JaiKOQm8C4yv6UCqwXm11czCgeeB25xzwXqXsRJIAsYAL9ZwLFWtsZllA1kUdVc+U8PxeKnSbXXO7XbOxQFXAylm1uZcdQJEZdv6X8AK59xuzyOrAuriq5hTwChgjZlNdc79jqLnE0H3sLwCKt1WM2sOvAr8j3NuffWEWfWcc/lmthG4F4gBflTDIVWl475nLH5mVlt/h8+7rb5u6q3AtcDfPIqvKlW2rd8FrjWz/wKaAg3N7BvnXECuy6c7qApyzuVR9BB9rJmNB9YCjczMP6OqmX3HzK6rqRirSmXaakXTVS0D/uKcW1IzEVepx4BfOecO1nQg1aDW/g6X4Wy/wxFm1thXdgnwfWBbDcVZFcptq3NurHOug3MuEphM0b/bgExOoARVKc65rygaqfY/wBAgGUjyDeXcCkzD99zCzN4GlgD9zCzHzAbUTNTnpxJtHQX0AX7qG+6abWYJ5Zw2oPj67b8tXuac2+qcSyvj2O+YWQ5Fo/v+5PszCGquaBqZWvs7XNw52hoNvG9mH1I0sm2Wc+6fNRbsBTrX32sw0VRHUmeZWTzwtHOuR03HIiKl6Q5K6iTflzEXUXSHKCIBSHdQIiISkHQHJSIiAUkJSkREApISlIiIBCQlKBERCUhKUCIiEpD+PyDY/PScGDFLAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "labels = ['KC1', 'KC2', 'JM1', 'PC1', 'PC3', 'PC4']\n",
    "bal_NN_SMOTE = [70.54, 75.42, 64.03, 71.48, 76.69, 80.86]\n",
    "bal_NN = [42.28, 61.86, 35.74, 43.17, 41.81, 61.31]\n",
    "\n",
    "x = np.arange(len(labels))  # the label locations\n",
    "width = 0.40  # the width of the bars\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "rects1 = ax.bar(x - width/2, bal_NN_SMOTE, width, label='NN+SMOTE')\n",
    "rects2 = ax.bar(x + width/2, bal_NN, width, label='NN')\n",
    "\n",
    "# Add some text for labels, title and custom x-axis tick labels, etc.\n",
    "ax.set_ylabel('Bal')\n",
    "ax.set_title('')\n",
    "ax.set_xticks(x)\n",
    "ax.set_xticklabels(labels)\n",
    "ax.set_ylim(top=90)\n",
    "ax.legend(loc=(0,0))\n",
    "\n",
    "\n",
    "def autolabel(rects):\n",
    "    \"\"\"Attach a text label above each bar in *rects*, displaying its height.\"\"\"\n",
    "    for rect in rects:\n",
    "        height = rect.get_height()\n",
    "        ax.annotate('{}'.format(height),\n",
    "                    xy=(rect.get_x() + rect.get_width() / 2, height),\n",
    "                    xytext=(3, 3),  # 3 points vertical offset\n",
    "                    textcoords=\"offset points\",\n",
    "                    ha='center', va='bottom')\n",
    "\n",
    "\n",
    "autolabel(rects1)\n",
    "autolabel(rects2)\n",
    "\n",
    "fig.tight_layout()\n",
    "\n",
    "plt.savefig('bal_NN_SMOTE.png')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Recall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3dfXgV1bn38e8NEQR5V0BCQKqCQWIIGgFPLWJpECxVeQdpi0DkgaeeUgUpR48VbS14WgsqVgsVjegBsWpBpRQIoj5apCABUYwoUEE4ggiCvAiB+/kjm30SsgM7kEkmye9zXbn2njVrzb5XCLkza9asMXdHREQkbKqVdwAiIiKxKEGJiEgoKUGJiEgoKUGJiEgoKUGJiEgoKUGJiEgoBZqgzGyMma0zsw/M7BeRskZmttjMNkReGwYZg4iIVEyBJSgzSwFuBToC7YFeZtYamABku3trIDuyLSIiUkiQZ1BtgeXufsDd84A3gN7AjUBWpE4WcFOAMYiISAWVEOCx1wEPmNm5wEHgemAl0NTdtwO4+3YzaxKrsZmNBEYCnHPOOVckJycHGKqIiJSXVatWfenujU8sDyxBuft6M3sQWAx8A6wB8krQfjowHSA9Pd1XrlwZSJwiIlK+zOxfscoDnSTh7k+6++Xu3gX4CtgAfGFmzSJBNQN2BBmDiIhUTEHP4msSeW0J9AFmA/OBoZEqQ4F5QcYgIiIVU5DXoABejFyDOgL8zN13m9lkYK6ZjQA+A/oHHIOIiFRAgSYod/9ejLJdQLcgP1dERCo+rSQhIiKhpAQlIiKhpAQlIiKhpAQlIiKhpAQlIiKhpAQlIiKhpAQlIiKhpAQlIiKhpAQlIiKhpAQlIiKhpAQlIqGUm5tLWlpa9KtevXpMnTqVNWvWcNVVV3HZZZfxox/9iL1798bdFmDgwIHR8latWpGWllbWXZM4mbuXdwynpOdBiVRtR48epXnz5rz77rv069eP3//+91xzzTXMnDmTTZs28etf/zquthdccEGhfWPHjqV+/fr86le/CroLchJmtsrd008s1xmUiIRednY2F110ERdccAG5ubl06dIFgIyMDF588cW42xbk7sydO5fBgwcHFrecGSUoEQm9OXPmRBNJSkoK8+fPB+CFF15gy5Ytcbct6K233qJp06a0bt269AOWUqEEJSKhdvjwYebPn0///vmPjps5cyaPPfYYV1xxBfv27aNGjRpxty1o9uzZOnsKOSUoEQm1v/3tb1x++eU0bdoUgOTkZBYtWsSqVasYPHgwF110Udxtj8vLy+Oll15i4MCBgcZ+uoqb5JGTk0Pnzp1JS0sjPT2dFStWxGy/cOFCLrnkEi6++GImT55cxtGXIncP/dcVV1zhIlI1DRw40GfOnBnd/uKLL9zd/ejRo/6Tn/zEn3zyybjbHve3v/3Nu3TpUvrBBiAvL8+bNm3qmzdv9oyMDF+wYIG7u7/22mt+zTXXxKx/4YUX+qeffurffvutp6am+gcffFDGUZcMsNJj/O7XGZSIhNaBAwdYvHgxffr0iZbNnj2bNm3akJycTGJiIsOGDQNg27ZtXH/99Sdte1xx16XCqOAkDzOLTqv/+uuvSUxMLFJ/xYoVXHzxxVx44YXUqFGDQYMGMW/evLIOu1QEOs3czG4HMgEH3geGAbWB54FWwGZggLvvPtlxNM1cRKqq4cOHc/nll3Pbbbexfv16rrvuOtydY8eO8c477xSZnfiXv/yFhQsX8uc//xmAWbNm8e677zJt2rTyCD8uZT7N3MyaAz8H0t09BagODAImANnu3hrIjmyLiMgJTpzk8fjjjzNlyhS2bNnClClTGDFiRJE2sU46zCzwWIMQ9BBfAlDLzBLIP3PaBtwIZEX2ZwE3BRyDiEiFdOIkj6ysrOiQZf/+/WNOkkhKSio09X7r1q0xhwIrgsASlLt/Dvwe+AzYDnzt7ouApu6+PVJnO9AkqBhERCqyE6fCJyYm8sYbbwCwdOnSmPdwXXnllWzYsIFNmzZx+PBh5syZww033FBmMZemhKAObGYNyT9b+g6wB3jBzH5cgvYjgZEALVu2DCRGEQmnVhNeC/T4myf/MNDjl4bjkzz+9Kc/RctmzJjBmDFjyMvL4+yzz2b69OlA/gSRzMxMFixYQEJCAtOmTeO6667j6NGjDB8+nHbt2pVXN85IYJMkzKw/0MPdR0S2fwp0BroBXd19u5k1A5a5+yUnO5YmSYhULUpQVUt5rMX3GdDZzGpb/hW6bsB6YD4wNFJnKFAx5z+KiEigAhvic/d3zewvwHtAHrAamA7UAeaa2Qjyk1jRNUhERKTKC3QWn7vf6+7J7p7i7j9x92/dfZe7d3P31pHXr4KMoaoqbqmUe+65h9TUVNLS0ujevTvbtm2L2X7KlCm0a9eOlJQUBg8ezKFDhwCYOHEizZs3jx53wYIFZdktEalCqtTzoHJzcwutvbVx40buv/9+du3axbx586hWrRpNmjTh6aefjjktc8+ePWRmZrJu3TrMjJkzZ3LVVVdx55138sorr1CjRg0uuuginnrqKRo0aHDG8ZaWgs/DadiwIfXq1QPgkUce4cMPP+SJJ54oVP/zzz/n6quv5sMPP6RWrVoMGDCA66+/nltuuYWJEydSp04dxo0bVx5dkSqiKl2Dqkp9LY6eBwVccskl5OTkkJOTw6pVq6hduza9e/fmzjvvZO3ateTk5NCrVy/uv//+mO3HjBlDjx49+Oijj1izZg1t27YF8p9Js27dOtauXUubNm2YNGlSWXbrlAoulXI8OQHs37+/2Bv48vLyOHjwIHl5eRw4cKDC3kchIhVXlUpQBZX0l/bevXt58803o3du16hRI3qW1L17dxIS8i/nde7cma1bt5ZBD+J34rpjd999Ny1atOC5556LmYybN2/OuHHjaNmyJc2aNaN+/fp07949un/atGmkpqYyfPhwdu8+6SpVIiKnrcomqJL+0t64cSONGzdm2LBhdOjQgczMTPbv31+k3syZM+nZs2egsZdErOfhPPDAA2zZsoUhQ4bEXJ9r9+7dzJs3j02bNrFt2zb279/Ps88+C8Do0aP59NNPycnJoVmzZowdO7bM+iIiVUuVTFCn80s7Ly+P9957j9GjR7N69WrOOeecIs9ZeeCBB0hISGDIkCGB9yFexT0PB+Dmm2+O+bjsJUuW8J3vfIfGjRtz1lln0adPH9555x0AmjZtSvXq1alWrRq33nprsc+jERE5U1UyQZ3OL+2kpCSSkpLo1KkTAP369eO9996L7s/KyuLVV1/lueeeC9XCjCculbJhw4bo+/nz55OcnFykTcuWLVm+fDkHDhzA3cnOzo5eb9u+fXu03ssvv0xKSkqA0YtIVVYlE9Tp/NI+//zzadGiBbm5uUD+NaxLL70UyH965YMPPsj8+fOpXbt2wNHHL9bzcCZMmEBKSgqpqaksWrSIhx9+GCj8LJ1OnTrRr18/Lr/8ci677DKOHTvGyJEjARg/fjyXXXYZqampvP7660yZMqXsOyYiVUKVmmYO+b+0W7RowcaNG6lfvz4Affv2JTc3l2rVqnHBBRfwxBNP0Lx580LrWwHk5OSQmZnJ4cOHufDCC3nqqado2LAhF198Md9++y3nnnsukD9R4sSp2yISv6o09boq9bU4xU0zD2wlibCqXbs2u3btKlQWa0gP8lcOLngjalpaGrES5SeffFK6QYqISNUc4hOpqIpbIeSrr74iIyOD1q1bk5GREXP6/5YtW7j22mtp27Yt7dq1iw7vglYIkXBSghKpQIq72Xzy5Ml069aNDRs20K1btyIzTAESEhJ46KGHWL9+PcuXL+exxx7jww8/jO6//fbbo8c+fj1SpDxVuSG+qkBj2lVDwZvN582bx7JlywAYOnQoXbt25cEHHyxUv1mzZjRr1gyAunXr0rZtWz7//PPoZB+RsKkyCUq/tKWyKXiz+RdffBFNPs2aNWPHjh0nbbt582ZWr14dvW0C8lcIeeaZZ0hPT+ehhx6iYcOGwQUvEgcN8YlUQLFuNo/XN998Q9++fZk6dWp0mS+tECJhpAQlUgGdeLN506ZNozdRb9++nSZNmsRsd+TIEfr27cuQIUMK3R+nFUIkjJSgRCqgE282v+GGG8jKygLyVzW58cYbi7Rxd0aMGEHbtm254447Cu3TCiESRkpQIhVMcSuELF68mNatW7N48WImTJgAFF4h5O2332bWrFksXbq0yHRyrRAiYVRlJkmIVBaxbjY/99xzyc7OLlK34M3mV199NcWtHDNr1qzSD1TkDOkMSkREQimwBGVml5hZToGvvWb2CzNrZGaLzWxD5FVzWUVEpIjAEpS757p7mrunAVcAB4CXgQlAtru3BrIj2yIiIoWU1TWobsCn7v4vM7sR6BopzwKWAb8sozhEKizdbC5VTVldgxoEzI68b+ru2wEirzFv2DCzkWa20sxW7ty5s4zCFBGRsAg8QZlZDeAG4IWStHP36e6e7u7pjRs3DiY4EREJrbI4g+oJvOfuX0S2vzCzZgCR15MvGiYiIlVSWSSowfzv8B7AfGBo5P1QYF4ZxCAiIhVMoAnKzGoDGcBLBYonAxlmtiGyr+iDa0REpMoLdBafux8Azj2hbBf5s/pERESKpZUkREQklJSgREQklJSgREQklJSgREQklJSgREQklJSgREQklJSgREQklJSgREQklJSgREQklJSgREQklJSgREQklJSgREQklJSgREQklJSgREQklJSgREQklJSgREQklAJ9YKFIWWnVqhV169alevXqJCQksHLlSgYOHEhubi4Ae/bsoUGDBuTk5MTVFiAnJ4dRo0Zx6NAhEhIS+OMf/0jHjh3LtF8iVZkSlFQar7/+Ouedd150+/nnn4++Hzt2LPXr14+7LcD48eO599576dmzJwsWLGD8+PEsW7as1OMWkdgCHeIzswZm9hcz+8jM1pvZVWbWyMwWm9mGyGvDIGMQcXfmzp3L4MGDS9TOzNi7dy8AX3/9NYmJiUGEJyLFCPoa1MPAQndPBtoD64EJQLa7twayI9siZ8TM6N69O1dccQXTp08vtO+tt96iadOmtG7dukRtp06dyp133kmLFi0YN24ckyZNCrQPIlJYYEN8ZlYP6ALcAuDuh4HDZnYj0DVSLQtYBvwyqDikanj77bdJTExkx44dZGRkkJycTJcuXQCYPXv2Sc+eimv7+OOPM2XKFPr27cvcuXMZMWIES5YsKasuiVR5QZ5BXQjsBJ4ys9Vm9mczOwdo6u7bASKvTWI1NrORZrbSzFbu3LkzwDClMjg+/NakSRN69+7NihUrAMjLy+Oll15i4MCBJW6blZVFnz59AOjfv3+0XETKRpAJKgG4HHjc3TsA+ynBcJ67T3f3dHdPb9y4cVAxSiWwf/9+9u3bF32/aNEiUlJSAFiyZAnJyckkJSWVuG1iYiJvvPEGAEuXLi12iFBEghHkLL6twFZ3fzey/RfyE9QXZtbM3bebWTNgR4AxSBXwxRdf0Lt3byD/jOnmm2+mR48eAMyZM6fI8N62bdvIzMxkwYIFJ207Y8YMxowZQ15eHmeffXaRa1siEqzAEpS7/4+ZbTGzS9w9F+gGfBj5GgpMjrzOCyoGqRouvPBC1qxZE3Pf008/XaQsMTGRBQsWnLLt1VdfzapVq0otThEpmaDvg/p34DkzqwFsBIaRP6w418xGAJ8B/QOOQUREKqBAE5S75wDpMXZ1C/JzRUSk4tNafCIiEkpa6kgqtFYTXgv0+Jsn/zDQ44tI8XQGJSIioaQEJSIioaQEJSIioaQEJSIioaQEJSIioaQEJSIioaQEJSIioaQEJSIioaQEJSIioaQEJSIioaQEJSIioaQEJSIioaQEJSIioaQEJSIioaQEJSIioaQEJSIioRRogjKzzWb2vpnlmNnKSFkjM1tsZhsirw2DjEGCdfToUTp06ECvXr2iZY8++iiXXHIJ7dq1Y/z48THb7dmzh379+pGcnEzbtm35xz/+AcCdd95JcnIyqamp9O7dmz179pRJP0QkfMriDOpad09z9/TI9gQg291bA9mRbamgHn74Ydq2bRvdfv3115k3bx5r167lgw8+YNy4cTHbjRkzhh49evDRRx+xZs2a6DEyMjJYt24da9eupU2bNkyaNKlM+iEi4VMeQ3w3AlmR91nATeUQg5SCrVu38tprr5GZmRkte/zxx5kwYQI1a9YEoEmTJkXa7d27lzfffJMRI0YAUKNGDRo0aABA9+7dSUhIAKBz585s3bo16G6ISEgFnaAcWGRmq8xsZKSsqbtvB4i8Fv0NBpjZSDNbaWYrd+7cGXCYcjp+8Ytf8F//9V9Uq/a/P0Yff/wxb731Fp06deKaa67hn//8Z5F2GzdupHHjxgwbNowOHTqQmZnJ/v37i9SbOXMmPXv2DLQPIhJeQSeo77r75UBP4Gdm1iXehu4+3d3T3T29cePGwUUop+XVV1+lSZMmXHHFFYXK8/Ly2L17N8uXL+d3v/sdAwYMwN2L1HnvvfcYPXo0q1ev5pxzzmHy5MmF6jzwwAMkJCQwZMiQwPsiUlWceM144sSJNG/enLS0NNLS0liwYEHcbY+L55rz6Uoo1aOdwN23RV53mNnLQEfgCzNr5u7bzawZsCPIGCQYb7/9NvPnz2fBggUcOnSIvXv38uMf/5ikpCT69OmDmdGxY0eqVavGl19+ScE/MpKSkkhKSqJTp04A9OvXr1CCysrK4tVXXyU7OxszK/O+iVRWx68Z7927N1p2++23F3ut+FRtC15zrlmzJjt2lO6v85OeQZnZPjPbG+Nrn5ntPUXbc8ys7vH3QHdgHTAfGBqpNhSYd+bdkLI2adIktm7dyubNm5kzZw7f//73efbZZ7nppptYunQpkD/cd/jwYc4777xCbc8//3xatGhBbm4uANnZ2Vx66aUALFy4kAcffJD58+dTu3btsu2USCUW65rxmbaN55rzmThpgnL3uu5eL8ZXXXevd4pjNwX+n5mtAVYAr7n7QmAykGFmG4CMyLZUEsOHD2fjxo2kpKQwaNAgsrKyMDO2bdvG9ddfH6336KOPMmTIEFJTU8nJyeGuu+4C4LbbbmPfvn1kZGSQlpbGqFGjyqsrIpVKrGvGANOmTSM1NZXhw4eze/fuErWN55rzmTjpEJ+ZNTrZfnf/6iT7NgLtY5TvArrFG6CEX9euXenatSuQPyPv2WefLVInMTGx0Ph2WloaK1euLFLvk08+CSxOkaqq4DXjZcuWRctHjx7NPffcg5lxzz33MHbsWGbOnBlXWyh8zfmf//wnAwYMYOPGjaU2NH+qa1CryJ+JF+vTHLiwVKIQEZHAFHfNuOAfk7feemuRCRCnahvPNeczcaohvu+4+4WR1xO/lJxERCqA4q4Zb9++PVrn5ZdfJiUlJe62QFzXnM9E3LP4IksStQbOPl7m7m+WWiQiIlKmxo8fT05ODmZGq1at+NOf/gTAtm3byMzMPOm0c8i/5jx8+HBSUlKoUaNG9JpzaYkrQZlZJjAGSAJygM7AP4Dvl1okIiISuILXjGfNmhWzzonXjGO1heKvOZeWeM+gxgBXAsvd/VozSwbuCywqCbeJ9QM89tfBHVtEKpR4V5I45O6HAMysprt/BFwSXFgiIlLVxXsGtdXMGgB/BRab2W5gW3BhiYhIVRdXgnL33pG3E83sdaA+sDCwqEREpHRU4CH5uIb4zKzz8WWL3P0N4HWgQ5CBiYhI1RbvNajHgW8KbO+PlImIiAQi3gRlXuCZCe5+jIBXQhcRkaot3gS10cx+bmZnRb7GABuDDExERKq2eBPUKODfgM+BrUAnYORJW4iIiJyBeGfx7QAGBRyLiIhIVLyz+NqYWbaZrYtsp5rZfwYbmoiIVGXxDvHNAP4DOALg7mvRGZWIiAQo3gRV291XnFCWV9rBiIiIHBdvgvrSzC4i/yGFmFk/YPvJm4iIiJy+eO9l+hkwHUg2s8+BTcCQwKISEZEqL64zKHff6O4/ABoDyUBX4Op42ppZdTNbbWavRrYbmdliM9sQeW14mrGLiEgldtIEZWb1zOw/zGyamWUAB4ChwCfAgDg/YwywvsD2BCDb3VsD2ZFtERGRQk51BjWL/Oc+vQ/cCiwC+gM3ufuNpzq4mSUBPwT+XKD4RiAr8j4LuKmEMYuISBVwqmtQF7r7ZQBm9mfgS6Clu++L8/hTgfFA3QJlTd19O4C7bzezJrEamtlIIqtVtGzZMs6PExGRyuJUZ1BHjr9x96PApniTk5n1Ana4+6rTCczdp7t7urunN27c+HQOISIiFdipzqDam9neyHsDakW2DXB3r3eStt8FbjCz64GzgXpm9izwhZk1i5w9NQN2nGEfRESkEjrpGZS7V3f3epGvuu6eUOD9yZIT7v4f7p7k7q3IX3Viqbv/GJhP/kQLIq/zSqEfIiJSycR7o25pmgxkmNkGICOyLSIiUkiZPHTQ3ZcByyLvdwHdyuJzRUSk4iqPMygREZFTUoISEZFQUoISEZFQUoISEZFQUoISEZFQUoISEZFQUoISEZFQUoISEZFQUoISEZFQUoISEZFQUoISEZFQUoISEZFQUoISEZFQUoISEZFQUoISEZFQUoISEZFQUoISEZFQUoISEZFQCixBmdnZZrbCzNaY2Qdmdl+kvJGZLTazDZHXhkHFICIiFVeQZ1DfAt939/ZAGtDDzDoDE4Bsd28NZEe2RURECgksQXm+byKbZ0W+HLgRyIqUZwE3BRWDiIhUXIFegzKz6maWA+wAFrv7u0BTd98OEHltUkzbkWa20sxW7ty5M8gwRUQkhAJNUO5+1N3TgCSgo5mllKDtdHdPd/f0xo0bBxekiIiEUpnM4nP3PcAyoAfwhZk1A4i87iiLGEREpGIJchZfYzNrEHlfC/gB8BEwHxgaqTYUmBdUDCIiUnElBHjsZkCWmVUnPxHOdfdXzewfwFwzGwF8BvQPMAYREamgAktQ7r4W6BCjfBfQLajPFRGRykErSYiISCgpQYmISCgpQYmISCgpQYmISCgpQYmISCgpQYmISCgpQYmISCgpQYmISCgpQYmISCgpQYmISCgpQYlI1PDhw2nSpAkpKf/7ZJw1a9Zw1VVXcdlll/GjH/2IvXv3xmy7Z88e+vXrR3JyMm3btuUf//gHAAMHDiQtLY20tDRatWpFWlpamfRFKj4lKBGJuuWWW1i4cGGhsszMTCZPnsz7779P7969+d3vfhez7ZgxY+jRowcfffQRa9asoW3btgA8//zz5OTkkJOTQ9++fenTp0/g/ZDKQQlKRKK6dOlCo0aNCpXl5ubSpUsXADIyMnjxxReLtNu7dy9vvvkmI0aMAKBGjRo0aNCgUB13Z+7cuQwePDig6KWyUYIqRbGGR3JycujcuTNpaWmkp6ezYsWKIu0OHTpEx44dad++Pe3atePee++N7ot3eEUkKCkpKcyfPx+AF154gS1bthSps3HjRho3bsywYcPo0KEDmZmZ7N+/v1Cdt956i6ZNm9K6desyiVsqPiWoUhRreGT8+PHce++95OTkcP/99zN+/Pgi7WrWrMnSpUtZs2YNOTk5LFy4kOXLlwPxD6+IBGXmzJk89thjXHHFFezbt48aNWoUqZOXl8d7773H6NGjWb16Neeccw6TJ08uVGf27Nk6e5ISUYIqRbGGR8wsetbz9ddfk5iYWKSdmVGnTh0Ajhw5wpEjRzAzIL7hFZEgJScns2jRIlatWsXgwYO56KKLitRJSkoiKSmJTp06AdCvXz/ee++96P68vDxeeuklBg4cWGZxn0qsEY94J3RMmTKFdu3akZKSwuDBgzl06BAA99xzD6mpqaSlpdG9e3e2bdtWJn2prJSgAjZ16lTuvPNOWrRowbhx45g0aVLMekePHiUtLY0mTZqQkZER/Y8ez/CKSJB27NgBwLFjx/jNb37DqFGjitQ5//zzadGiBbm5uQBkZ2dz6aWXRvcvWbKE5ORkkpKSyiboOMQa8YhnQsfnn3/OI488wsqVK1m3bh1Hjx5lzpw5ANx5552sXbuWnJwcevXqxf33318mfamslKAC9vjjjzNlyhS2bNnClClToheRT1S9enVycnLYunUrK1asYN26dUB8wysipWXw4MFcddVV5ObmkpSUxJNPPsns2bNp06YNycnJJCYmMmzYMAC2bdvG9ddfH2376KOPMmTIEFJTU8nJyeGuu+6K7pszZ07ohvdijXgcd6oJHXl5eRw8eJC8vDwOHDgQHRmpV69etM7+/fujIyFyegJ75LuZtQCeAc4HjgHT3f1hM2sEPA+0AjYDA9x9d1BxlLesrCwefvhhAPr3709mZuZJ6zdo0ICuXbuycOFCUlJSosMrAB9//DGvvfZa4DFL1TV79uyY5WPGjClSlpiYyIIFC6LbaWlprFy5Mmb7p59+ulTiKysnm9DRvHlzxo0bR8uWLalVqxbdu3ene/fu0f133303zzzzDPXr1+f1118vy7ArnSDPoPKAse7eFugM/MzMLgUmANnu3hrIjmxXWomJibzxxhsALF26NOYP/M6dO9mzZw8ABw8ejA6HQHzDKyJSuk42oWP37t3MmzePTZs2sW3bNvbv38+zzz4b3f/AAw+wZcsWhgwZwrRp08oq5EopsATl7tvd/b3I+33AeqA5cCOQFamWBdwUVAxlLdbwyIwZMxg7dizt27fnrrvuYvr06UDh4ZHt27dz7bXXkpqaypVXXklGRga9evUCKHZ4RUSCcaoJHUuWLOE73/kOjRs35qyzzqJPnz688847RerdfPPNmtR0hgIb4ivIzFoBHYB3gabuvh3yk5iZNSmmzUhgJEDLli3LIswzVtzwyKpVq4qUFRweSU1NZfXq1THbjhkzJubwiogE41QTOlq2bMny5cs5cOAAtWrVIjs7m/T0dAA2bNgQHSWZP39+dCRETk/gCcrM6gAvAr9w973xXjR09+nAdID09HQPLkIRAWBi/QCP/XVwxz5NgwcPZtmyZXz55ZckJSVx3333MWLEiJgTOrZt20ZmZiYLFiygU6dO9OvXj8svv5yEhAQ6dOjAyJEjAZgwYQK5ublUq8RZDBYAAA5ESURBVFaNCy64gCeeeKI8ulZpBJqgzOws8pPTc+7+UqT4CzNrFjl7agbsCDIGEZFYihvxiDWh48QJIffddx/33XdfkXoa0itdgV2DsvxTpSeB9e7+hwK75gNDI++HAvOCikFERCquIM+gvgv8BHjfzHIiZXcBk4G5ZjYC+AzoH2AMIiJSQQWWoNz9/wHFXXDqFtTnlpsqNn4vIhK0MpnFJyISKvqDskLQUkciIhJKSlAiIhJKSlAiIhJKSlAiIhJKSlAiIhJKSlAiIhJKSlAiIhJKSlAiIhJKSlAiIhJKSlAiIhJKSlAiIhJKSlAiJfTwww+TkpJCu3btmDp1apH9H330EVdddRU1a9bk97//fbT80KFDdOzYkfbt29OuXTvuvffesgxbpMLRYrEiJbBu3TpmzJjBihUrqFGjBj169OCHP/xh9DHfAI0aNeKRRx7hr3/9a6G2NWvWZOnSpdSpU4cjR45w9dVX07NnTzp37lzW3RCpEHQGJVIC69evp3PnztSuXZuEhASuueYaXn755UJ1mjRpwpVXXslZZ51VqNzMqFOnDgBHjhzhyJEj5D/XU0RiUYISKYGUlBTefPNNdu3axYEDB1iwYAFbtmyJu/3Ro0dJS0ujSZMmZGRk0KlTpwCjFanYKsQQX926ddm0aROHDh067WPMuKFZKUZU1HqbG+DB15eo+pn21XH+tecIj767m73fHjujY1U2bdu25Ze//CUZGRnUqVOH9u3bk5AQ/3+j6tWrk5OTw549e+jduzfr1q0jJSUlwIhFKq4KkaBuueUW6tatS6tWrU57SOTI1j2lHFVhbasFOFST2LZE1c+0r+7Ouefu5d+BB97cdUbHqoxGjBjBiBEjALjrrrtISkoq8TEaNGhA165dWbhwoRKUSDECG+Izs5lmtsPM1hUoa2Rmi81sQ+S1YTzHSkpK4txzz9V4fRkxMxJq1+OCBmedunIVtGPHDgA+++wzXnrpJQYPHhxXu507d7JnT/4fDwcPHmTJkiUkJycHFqdIRRfkGdTTwDTgmQJlE4Bsd59sZhMi27881YHMTMmpjJkZhr7nsfTt25ddu3Zx1lln8dhjj9GwYUOeeOIJAEaNGsX//M//kJ6ezt69e6lWrRpTp07lww8/ZPv27QwdOpSjR49y7NgxBgwYQK9evcq5NyLhFViCcvc3zazVCcU3Al0j77OAZcSRoETC5K233ipSNmrUqOj7888/n61btxapk5qayurVqwONTaQyKetrUE3dfTuAu283syanc5BWE14r1aDm3/bdU9Zp36IhP7n1Z4z71W8AyHriUQ4c2M/oOybw+B8mk/X4w2x+91WanNcIgDqtv8s3G96O6/OXr1rLmHt/z7ffHubbw4cZeEN3Jo4dxdPPz2fYHRNZsmQJ3bp1A+Dll1+mT58+vPDCC/Tr14/Dhw8zfvx4XnnlFapVq8all17Kbf/5W2qeXYuRg24E4MudO6hWrTqNzj0XgOdeyaZj62a0Tr40GsN1N/RhxM9uj/+bJiISsNBOkjCzkcBIgEWLFpVzNFCjZk2yF77CiNtup2Gjc4vsP69RAx760ywevHtMscdY9s5Knp77Ck9Pva9Q+dBf3MvcJx6kfbs2HD16lNxP/xXdd1nbi5k9e3Y0Qc2ZM4f27dtH9991113s27ePjz/+mOrVq/PUU09x+60/5blXljD37/l/6T/+h8nUrn0OQ0f9e7RdzbNrRfeLiIRRWd8H9YWZNQOIvO4orqK7T3f3dHdPr169epkFWJzq1RPod/NQnp3xx5j7hw+6gefnL+Kr3V+X+Ng7dn1Fs6bnRT6nOpe2uTC673sdO7BixQqOHDnCN998wyeffEJaWhoABw4c4KmnnmLKlCkc/x4NGzaMGjVqsOLtN0sch4hImJT1GdR8YCgwOfI6r4w//4wMHJpJ/+5Xc8vonxfZV+ec2gwfdCMPP/nf3DdudImOe/utQ7ikS2+6XpVOj65XMbT/jzj77JpA/mSFH/zgB/z973/n66+/5oYbbmDTpk0AfPLJJ7Rs2ZJ69eoVOl671A588vFHdLr6mmI/89tDBxlw3fei28N/djs9buhTorirhIn1Azx2yf+YEalKAktQZjab/AkR55nZVuBe8hPTXDMbAXwG9A/q84NQp249evUdxH/PnM7ZZ59dZP/Phw8mrfsgxv6fnxQq79Trp3z77WG+OXCAr/bsJS1jEAAP3v1zruv6b/zq9pEM6d2TRW8s57//upDZ8/7Osr/MiLYfNGgQjzzyCF9//TUPPfQQv/3tb4H8+5VizW50YpcXpCE+EQm7IGfxFXdzSLegPrMs/HjEaAZdfw03DhhSZF+D+nW5+aYe/DHrhULl776aP9O+uGtQABe1asHoVi24dUhvGqd2Y9dX/3uzbceOHVm3bh21atWiTZs20fKLL76Yf/3rX+zbt4+6detGy9e/v4ZrftDjjPsqIlKetBZfCdVv2JDuvW7i5TmzYu6/4//8mD89+yJ5R4/GfczXlryFuwOwYdNnVK9WnQb16xaqM2nSpOiZ03HnnHMOQ4cO5Y477uBo5POeeeYZDh08SMfvdilJt0REQie0s/hOZvPkH5a4zdpSXOropyNvY87Tf46577xGDend41qmzHgu7uPNevE1bp/4ELVrnU1CQnWem/YbTpwY0rNnz5htJ02axLhx42jTpg3VqlUjOTmZKX+edcohvhOvQf1b12784j8mxh2ziEjQ7Phf7mG2dOlS//73v39GxyjNBBVLarVNwR08sUOJqpdWX7/4bCO3zt9epHzz2TeXyvFjKuHEgdK+J+5E6mspUV+LVZX6WhwzW+Xu6SeWa4hPRERCSQlKRERCSQlKRERCSQlKRERCSQlKRERCSQlKRERCqULeB3U666OlnmTf2sx/nWRvviAftyEiIkXpDCpOxx+3sfurXTH3H3/choiIlA4lqDgF+bgNEREpSgmqBAYOzWTBX19g396iSajg4zZEROTMKUGVQMHHbcTy8+GDyXrhVfbu+6aMIxMRqXyUoEroxyNG89fnZ3Hw4IEi+4p73IaIiJScElQJBfG4DRERKaqCTjMv+USEMD9uQ0REiqqYCaocLM/dGn1/buMmvLthW3R79B0TCj1u4w8Tx/KHiWPLND4RkcpGQ3wiIhJK5ZKgzKyHmeWa2SdmNqE8YhARkXAr8wRlZtWBx4CewKXAYDO79GRt3J2K8OTfysTdcfQ9F5HyUx5nUB2BT9x9o7sfBuYAN56swdatW9m1a5eSVBlxd/IO7OVfe46UdygiUoVZWf/SN7N+QA93z4xs/wTo5O63nVBvJDASoFGjRsmTJ0/Oa9WqVS0zK9N443Xs2LFq1apVO1becZQGd2fz5s0H77777s07d+7Mi1HlPODLso6rnKivlZP6Gi4XuHvjEwvLYxZfrAxTJEu6+3Qg9pINIWRmK909vbzjKE233nprzPLK2NfiqK+Vk/paMZTHEN9WoEWB7SRgWzF1RUSkiiqPBPVPoLWZfcfMagCDgPnlEIeIiIRYmQ/xuXuemd0G/B2oDsx09w/KOo4AVJjhyFKgvlZO6mvlVGH7WuaTJEREROKhlSRERCSUlKBERCSUlKDiYGbfFHh/vZltMLOWZna+mc0xs0/N7EMzW2BmbSL1FprZHjN7tfwiL7mS9tXM0szsH2b2gZmtNbOB5Rl/SZnZN2bWyszczH5doPw8MztiZtMi213M7D0zy4vcy1ehmNlRM8sxs3Vm9oKZ1Y6UV8af4RL11cwuMLNVkTYfmNmo8u5DvE7n3zWyv56ZfX785zuslKBKwMy6AY8CPYAtwMvAMne/yN0vBe4Cmkaq/w74SbkEWgpK0NcDwE/dvV2k7lQza1BOYZ+JjUCvAtv9gYKTdz4DbgH+uwxjKk0H3T3N3VOAw8Aoy7/rvTL+DJe0r9uBf3P3NKATMMHMEssr+BI6nX9XgF8Db5R9uCWjx23Eycy+B8wArnf3T83s+8ARd3/ieB13zynwPtvMupZ9pGeupH0tULbNzHYAjYHSewBX2TgIrDezdHdfCQwE5gKJAO6+GcDMKsNqIW8BqcC1VNKf4QLi6msBNam4f7jH1Vczu4L8ZLUQCPUNvBX1H6Ks1QTmATe5+0eRshRgVfmFFJjT7quZdQRqAJ8GF16g5gCDzCwJOEolvIHczBLIX6j5fSrvzzBQsr6aWQszW0v+aMGD7l6h/u3j7auZVQMeAu4su+hOnxJUfI4A7wAjyjuQMnBafTWzZsAsYJi7V9SzjIVABjAYeL6cYylttcwsB1hJ/nDlk+UcT5BK3Fd33+LuqcDFwFAza3qqNiFR0r7+X2CBu28JPLJSoCG++BwDBgBLzOwud/8t+dcnKtzF8jiUuK9mVg94DfhPd19eNmGWPnc/bGargLFAO+BH5RxSaToYucYSZWaV9Wf4tPsaGab+APge8JeA4itNJe3rVcD3zOz/AnWAGmb2jbuH8rl8OoOKk7sfIP8i+hAzGwEsBWqaWXRFVTO70syuKa8YS0tJ+mr5y1W9DDzj7i+UT8Sl6iHgl+6+q7wDKQOV9mc4hpP9DCeZWa1IWUPgu0BuOcVZGortq7sPcfeW7t4KGEf+/9tQJidQgioRd/+K/Jlq/wncAPQGMiJTOT8AJhK5bmFmbwEvAN3MbKuZXVc+UZ+eEvR1ANAFuCUy3TXHzNKKOWyoRMbtvy1Y5u4fuHtWjLpXmtlW8mf3/SnyPajQPH8ZmUr7M1zQKfraFnjXzNaQP7Pt9+7+frkFe4ZO9e9akWipI6myzKw9MMPdO5Z3LCJSlM6gpEqK3Iw5m/wzRBEJIZ1BiYhIKOkMSkREQkkJSkREQkkJSkREQkkJSkREQkkJSkREQun/A9DbVhXfrribAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "recall_NN_SMOTE = [73.62, 73.85, 57.58, 70.25, 79.77, 78.  ]\n",
    "recall_NN = [18.39, 46.62,  9.13, 19.67, 17.83, 45.46]\n",
    "\n",
    "x = np.arange(len(labels))  # the label locations\n",
    "width = 0.40  # the width of the bars\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "rects1 = ax.bar(x - width/2, recall_NN_SMOTE, width, label='NN+SMOTE')\n",
    "rects2 = ax.bar(x + width/2, recall_NN, width, label='NN')\n",
    "\n",
    "# Add some text for labels, title and custom x-axis tick labels, etc.\n",
    "ax.set_ylabel('Recall')\n",
    "ax.set_title('')\n",
    "ax.set_xticks(x)\n",
    "ax.set_xticklabels(labels)\n",
    "ax.set_ylim(top=90)\n",
    "ax.legend(loc=(0,0))\n",
    "\n",
    "\n",
    "def autolabel(rects):\n",
    "    \"\"\"Attach a text label above each bar in *rects*, displaying its height.\"\"\"\n",
    "    for rect in rects:\n",
    "        height = rect.get_height()\n",
    "        ax.annotate('{}'.format(height),\n",
    "                    xy=(rect.get_x() + rect.get_width() / 2, height),\n",
    "                    xytext=(3, 3),  # 3 points vertical offset\n",
    "                    textcoords=\"offset points\",\n",
    "                    ha='center', va='bottom')\n",
    "\n",
    "\n",
    "autolabel(rects1)\n",
    "autolabel(rects2)\n",
    "\n",
    "fig.tight_layout()\n",
    "\n",
    "plt.savefig('recall_NN_SMOTE.png')\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Research Question 2\n",
    "How does the performance of SMOTE and Neural Network compare to other models in software defect prediction?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "bal_RF = np.round(np.multiply([0.7151147989539443, #KC1\n",
    "                            0.7226893138628165, #KC2\n",
    "                            0.6478701796965811, #JM1\n",
    "                            0.6950286280334226, #PC1\n",
    "                            0.7232577110085563, #PC3\n",
    "                            0.8315200834593393 #PC4\n",
    "                            ],100), 2)\n",
    "\n",
    "bal_KNN = np.round(np.multiply([0.7075084373908205, #KC1\n",
    "                            0.7568941854460896, #KC2\n",
    "                            0.6564886253800352, #JM1\n",
    "                            0.7736206371976065, #PC1\n",
    "                            0.7324845009703858, #PC3\n",
    "                            0.7810912967921397 #PC4\n",
    "                            ],100), 2)\n",
    "\n",
    "bal_DT = np.round(np.multiply([0.6652404657500962, #KC1\n",
    "                            0.7057545297576305, #KC2\n",
    "                            0.6511349355186618, #JM1\n",
    "                            0.6911950652094997, #PC1\n",
    "                            0.7007449983957051, #PC3\n",
    "                            0.8006948681399209 #PC4\n",
    "                            ],100), 2)\n",
    "\n",
    "bal_LR = np.round(np.multiply([0.6990750606076016, #KC1\n",
    "                            0.7057545297576305, #KC2\n",
    "                            0.6487161434823655, #JM1\n",
    "                            0.6771427981867418, #PC1\n",
    "                            0.7329179938165378, #PC3\n",
    "                            0.8057663968439721 #PC4\n",
    "                            ],100), 2)\n",
    "\n",
    "bal_NB = np.round(np.multiply([0.5768472797235854, #KC1\n",
    "                            0.6247230023550923, #KC2\n",
    "                            0.44248594840171346, #JM1\n",
    "                            0.5192222953087149, #PC1\n",
    "                            0.4346563424302782, #PC3\n",
    "                            0.7513783894954688 #PC4\n",
    "                            ],100), 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "recall_RF = np.round(np.multiply([0.6839160839160839, #KC1\n",
    "                            0.6623376623376623, #KC2\n",
    "                            0.5649010292953286, #JM1\n",
    "                            0.5974999999999999, #PC1\n",
    "                            0.6786324786324787, #PC3\n",
    "                            0.8138095238095238 #PC4\n",
    "                            ],100), 2)\n",
    "\n",
    "recall_KNN = np.round(np.multiply([0.7238228438228439, #KC1\n",
    "                            0.7471861471861472, #KC2\n",
    "                            0.6352901255514081, #JM1\n",
    "                            0.7925, #PC1\n",
    "                            0.7227920227920228, #PC3\n",
    "                            0.7923809523809524 #PC4\n",
    "                            ],100), 2)\n",
    "\n",
    "recall_DT = np.round(np.multiply([0.5765967365967366, #KC1\n",
    "                            0.6346320346320347, #KC2\n",
    "                            0.6390906006107906, #JM1\n",
    "                            0.61, #PC1\n",
    "                            0.6481481481481481, #PC3\n",
    "                            0.7628571428571428 #PC4\n",
    "                            ],100), 2)\n",
    "\n",
    "recall_LR = np.round(np.multiply([0.6655944055944055, #KC1\n",
    "                            0.7467532467532467, #KC2\n",
    "                            0.5886766202918222, #JM1\n",
    "                            0.5850000000000001, #PC1\n",
    "                            0.72991452991453, #PC3\n",
    "                            0.7973015873015873 #PC4\n",
    "                            ],100), 2)\n",
    "\n",
    "recall_NB = np.round(np.multiply([0.6655944055944055, #KC1\n",
    "                            0.47489177489177486, #KC2\n",
    "                            0.21351430833616108, #JM1\n",
    "                            0.3258333333333333, #PC1\n",
    "                            0.8720797720797719, #PC3\n",
    "                            0.7020634920634921 #PC4\n",
    "                            ],100), 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Highlight the maximum value in each column:\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Dataset</th>\n",
       "      <th>NN</th>\n",
       "      <th>RF</th>\n",
       "      <th>KNN</th>\n",
       "      <th>DT</th>\n",
       "      <th>LR</th>\n",
       "      <th>NB</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>KC1</td>\n",
       "      <td>70.54</td>\n",
       "      <td>71.51</td>\n",
       "      <td>70.75</td>\n",
       "      <td>66.52</td>\n",
       "      <td>69.91</td>\n",
       "      <td>57.68</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>KC2</td>\n",
       "      <td>75.42</td>\n",
       "      <td>72.27</td>\n",
       "      <td>75.69</td>\n",
       "      <td>70.58</td>\n",
       "      <td>70.58</td>\n",
       "      <td>62.47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>JM1</td>\n",
       "      <td>64.03</td>\n",
       "      <td>64.79</td>\n",
       "      <td>65.65</td>\n",
       "      <td>65.11</td>\n",
       "      <td>64.87</td>\n",
       "      <td>44.25</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>PC1</td>\n",
       "      <td>71.48</td>\n",
       "      <td>69.50</td>\n",
       "      <td>77.36</td>\n",
       "      <td>69.12</td>\n",
       "      <td>67.71</td>\n",
       "      <td>51.92</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>PC3</td>\n",
       "      <td>76.69</td>\n",
       "      <td>72.33</td>\n",
       "      <td>73.25</td>\n",
       "      <td>70.07</td>\n",
       "      <td>73.29</td>\n",
       "      <td>43.47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>PC4</td>\n",
       "      <td>80.86</td>\n",
       "      <td>83.15</td>\n",
       "      <td>78.11</td>\n",
       "      <td>80.07</td>\n",
       "      <td>80.58</td>\n",
       "      <td>75.14</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Dataset     NN     RF    KNN     DT     LR     NB\n",
       "0     KC1  70.54  71.51  70.75  66.52  69.91  57.68\n",
       "1     KC2  75.42  72.27  75.69  70.58  70.58  62.47\n",
       "2     JM1  64.03  64.79  65.65  65.11  64.87  44.25\n",
       "3     PC1  71.48  69.50  77.36  69.12  67.71  51.92\n",
       "4     PC3  76.69  72.33  73.25  70.07  73.29  43.47\n",
       "5     PC4  80.86  83.15  78.11  80.07  80.58  75.14"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# BAL\n",
    "table = {\n",
    "    'Dataset' : ['KC1', 'KC2', 'JM1', 'PC1', 'PC3', 'PC4'],\n",
    "    'NN': [70.54, 75.42, 64.03, 71.48, 76.69, 80.86],\n",
    "#     'Recall NN + SMOTE': [73.62, 73.85, 57.58, 70.25, 79.77, 78.  ],\n",
    "    'RF' : bal_RF.tolist(),\n",
    "    'KNN' : bal_KNN.tolist(),\n",
    "    'DT' : bal_DT.tolist(),\n",
    "    'LR' : bal_LR.tolist(),\n",
    "    'NB' : bal_NB.tolist()\n",
    "}\n",
    "\n",
    "result = pd.DataFrame(data = table, columns = ['Dataset', 'NN', 'RF', 'KNN', 'DT', 'LR', 'NB'])\n",
    "\n",
    "def highlight_max(s):\n",
    "    '''\n",
    "    highlight the maximum in a Series green.\n",
    "    '''\n",
    "    is_max = s == s.max()\n",
    "    return ['background-color: green' if v else '' for v in is_max]\n",
    "\n",
    "print(\"\\nHighlight the maximum value in each column:\")\n",
    "result.style.apply(highlight_max,subset=pd.IndexSlice[:, ['NN', 'RF', 'KNN', 'DT', 'LR', 'NB']])\n",
    "result"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}