digit_recognizer.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Digit Recognizer Kaggle Competition"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Trained on MNIST Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2024-02-15 14:22:29.229036: I tensorflow/core/util/port.cc:113] oneDNN custom operations are on. You may see slightly different numerical results due to floating-point round-off errors from different computation orders. To turn them off, set the environment variable `TF_ENABLE_ONEDNN_OPTS=0`.\n",
      "2024-02-15 14:22:29.281141: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
      "2024-02-15 14:22:29.281182: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
      "2024-02-15 14:22:29.282751: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
      "2024-02-15 14:22:29.292276: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
      "To enable the following instructions: AVX2 AVX512F AVX512_VNNI FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2024-02-15 14:22:34.367691: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Could not find TensorRT\n"
     ]
    }
   ],
   "source": [
    "# import necessary libraries\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Case 1 - Train and test on entire set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the data\n",
    "train1 = pd.read_csv('train.csv')\n",
    "test1 = pd.read_csv('test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((42000, 784), (42000,), (28000, 784))"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train Data\n",
    "y_train = train1['label'].astype('float32')\n",
    "X_train = train1.drop('label', axis=1).astype('int32')\n",
    "X_test = test1.astype('int32')\n",
    "X_train.shape, y_train.shape, X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Axes: xlabel='label', ylabel='count'>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkQAAAGwCAYAAABIC3rIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAoP0lEQVR4nO3df1TUdb7H8deADpD8MFRA4keYXRXzR2Lq1GZmrOSynjq5ZS1bFNZuXSyRe9Xlbmmr22Lumj9ZzTJpW71p7WqppRImrgmpJIVaZq138aRAdxNGKQFh7h/3OKdJM0DgO/B5Ps75nuN8vx+G98fW9enMF7C5XC6XAAAADOZj9QAAAABWI4gAAIDxCCIAAGA8gggAABiPIAIAAMYjiAAAgPEIIgAAYLwuVg/QETQ2NurEiRMKCgqSzWazehwAANAELpdLp0+fVmRkpHx8Lv0aEEHUBCdOnFB0dLTVYwAAgBY4fvy4oqKiLrmGIGqCoKAgSf//GxocHGzxNAAAoCmcTqeio6Pdf49fCkHUBOffJgsODiaIAADoYJpyuws3VQMAAOMRRAAAwHgEEQAAMB5BBAAAjEcQAQAA4xFEAADAeAQRAAAwHkEEAACMRxABAADjEUQAAMB4BBEAADAeQQQAAIxHEAEAAOMRRAAAwHgEEQAAMF4XqweA9RKm/9nqES6p+A8PWD0CAKCT4xUiAABgPIIIAAAYjyACAADGI4gAAIDxCCIAAGA8gggAABiPIAIAAMYjiAAAgPEIIgAAYDyCCAAAGI8gAgAAxiOIAACA8QgiAABgPIIIAAAYjyACAADGI4gAAIDxCCIAAGA8gggAABiPIAIAAMYjiAAAgPG6WD0AAAA/JGH6n60e4XsV/+EBq0dAK+AVIgAAYDyCCAAAGI8gAgAAxuMeIsCLePN9EhL3SgDovHiFCAAAGI8gAgAAxiOIAACA8QgiAABgPIIIAAAYjyACAADGI4gAAIDxCCIAAGA8gggAABiPIAIAAMYjiAAAgPH4WWYAAKDJvPlnLl7Oz1skiNBpdNY/pACAtsdbZgAAwHgEEQAAMB5vmV0Gb36LRuJtGgD8/xTQVLxCBAAAjEcQAQAA4xFEAADAeAQRAAAwHjdVA2h13nwjLzfxwire/OdC4s8GrxABAADjeU0QzZs3TzabTRkZGe5zZ8+eVXp6unr06KHAwEBNnDhRFRUVHh9XVlam5ORkXXHFFQoLC9P06dN17tw5jzU7d+7UsGHD5Ofnp759+yo3N7cddgQAADoKrwiiffv26fnnn9fgwYM9zk+bNk2bNm3Sa6+9poKCAp04cUJ33XWX+3pDQ4OSk5NVV1enPXv26OWXX1Zubq5mzZrlXnPs2DElJyfr1ltvVUlJiTIyMvTwww9r27Zt7bY/AADg3SwPojNnziglJUUvvPCCrrzySvf56upqrVq1Ss8995zGjh2rhIQErV69Wnv27FFRUZEkafv27Tp8+LD+8pe/aOjQoRo/frzmzp2rnJwc1dXVSZJWrFihuLg4LViwQAMGDNCUKVP0s5/9TAsXLvzemWpra+V0Oj0OAADQeVkeROnp6UpOTlZiYqLH+eLiYtXX13uc79+/v2JiYlRYWChJKiws1KBBgxQeHu5ek5SUJKfTqUOHDrnXfPe5k5KS3M9xMdnZ2QoJCXEf0dHRl71PAADgvSwNoldffVUffPCBsrOzL7hWXl4uu92u7t27e5wPDw9XeXm5e823Y+j89fPXLrXG6XTqm2++uehcWVlZqq6udh/Hjx9v0f4AAEDHYNmX3R8/flxTp05VXl6e/P39rRrjovz8/OTn52f1GAAAoJ1Y9gpRcXGxKisrNWzYMHXp0kVdunRRQUGBlixZoi5duig8PFx1dXWqqqry+LiKigpFRERIkiIiIi74qrPzj39oTXBwsAICAtpodwAAoCOxLIhuu+02lZaWqqSkxH0MHz5cKSkp7l937dpV+fn57o85cuSIysrK5HA4JEkOh0OlpaWqrKx0r8nLy1NwcLDi4+Pda779HOfXnH8OAAAAy94yCwoK0nXXXedxrlu3burRo4f7/OTJk5WZmanQ0FAFBwfr8ccfl8Ph0KhRoyRJ48aNU3x8vO6//37Nnz9f5eXlevLJJ5Wenu5+y+vRRx/VsmXLNGPGDKWlpWnHjh1av369tmzZ0r4bBgAAXsurf3THwoUL5ePjo4kTJ6q2tlZJSUn605/+5L7u6+urzZs367HHHpPD4VC3bt2UmpqqOXPmuNfExcVpy5YtmjZtmhYvXqyoqCi9+OKLSkpKsmJLAADAC3lVEO3cudPjsb+/v3JycpSTk/O9HxMbG6u33nrrks87ZswYHThwoDVGBAAAnZDl34cIAADAagQRAAAwHkEEAACMRxABAADjEUQAAMB4BBEAADAeQQQAAIxHEAEAAON51TdmBABvkTD9z1aPcEnFf3jA6hGAToVXiAAAgPEIIgAAYDyCCAAAGI8gAgAAxiOIAACA8QgiAABgPIIIAAAYjyACAADGI4gAAIDxCCIAAGA8gggAABiPIAIAAMYjiAAAgPEIIgAAYDyCCAAAGI8gAgAAxiOIAACA8QgiAABgPIIIAAAYjyACAADGI4gAAIDxCCIAAGA8gggAABiPIAIAAMYjiAAAgPEIIgAAYDyCCAAAGI8gAgAAxiOIAACA8QgiAABgPIIIAAAYjyACAADGI4gAAIDxCCIAAGA8gggAABiPIAIAAMYjiAAAgPEIIgAAYDyCCAAAGI8gAgAAxiOIAACA8QgiAABgPIIIAAAYjyACAADGI4gAAIDxCCIAAGA8gggAABiPIAIAAMYjiAAAgPEIIgAAYDyCCAAAGI8gAgAAxiOIAACA8QgiAABgPIIIAAAYz9IgWr58uQYPHqzg4GAFBwfL4XDo7bffdl8/e/as0tPT1aNHDwUGBmrixImqqKjweI6ysjIlJyfriiuuUFhYmKZPn65z5855rNm5c6eGDRsmPz8/9e3bV7m5ue2xPQAA0EFYGkRRUVGaN2+eiouLtX//fo0dO1Z33HGHDh06JEmaNm2aNm3apNdee00FBQU6ceKE7rrrLvfHNzQ0KDk5WXV1ddqzZ49efvll5ebmatasWe41x44dU3Jysm699VaVlJQoIyNDDz/8sLZt29bu+wUAAN6pi5WffMKECR6Pn3nmGS1fvlxFRUWKiorSqlWrtHbtWo0dO1aStHr1ag0YMEBFRUUaNWqUtm/frsOHD+udd95ReHi4hg4dqrlz52rmzJl6+umnZbfbtWLFCsXFxWnBggWSpAEDBmj37t1auHChkpKSLjpXbW2tamtr3Y+dTmcb/Q4AAABv4DX3EDU0NOjVV19VTU2NHA6HiouLVV9fr8TERPea/v37KyYmRoWFhZKkwsJCDRo0SOHh4e41SUlJcjqd7leZCgsLPZ7j/Jrzz3Ex2dnZCgkJcR/R0dGtuVUAAOBlLA+i0tJSBQYGys/PT48++qg2bNig+Ph4lZeXy263q3v37h7rw8PDVV5eLkkqLy/3iKHz189fu9Qap9Opb7755qIzZWVlqbq62n0cP368NbYKAAC8lKVvmUlSv379VFJSourqar3++utKTU1VQUGBpTP5+fnJz8/P0hkAAED7sTyI7Ha7+vbtK0lKSEjQvn37tHjxYk2aNEl1dXWqqqryeJWooqJCERERkqSIiAjt3bvX4/nOfxXat9d89yvTKioqFBwcrICAgLbaFgAA6EAsf8vsuxobG1VbW6uEhAR17dpV+fn57mtHjhxRWVmZHA6HJMnhcKi0tFSVlZXuNXl5eQoODlZ8fLx7zbef4/ya888BAABg6StEWVlZGj9+vGJiYnT69GmtXbtWO3fu1LZt2xQSEqLJkycrMzNToaGhCg4O1uOPPy6Hw6FRo0ZJksaNG6f4+Hjdf//9mj9/vsrLy/Xkk08qPT3d/ZbXo48+qmXLlmnGjBlKS0vTjh07tH79em3ZssXKrQMAAC9iaRBVVlbqgQce0MmTJxUSEqLBgwdr27Zt+vGPfyxJWrhwoXx8fDRx4kTV1tYqKSlJf/rTn9wf7+vrq82bN+uxxx6Tw+FQt27dlJqaqjlz5rjXxMXFacuWLZo2bZoWL16sqKgovfjii9/7JfcAAMA8lgbRqlWrLnnd399fOTk5ysnJ+d41sbGxeuutty75PGPGjNGBAwdaNCMAAOj8vO4eIgAAgPZGEAEAAOMRRAAAwHgEEQAAMB5BBAAAjEcQAQAA4xFEAADAeAQRAAAwHkEEAACMRxABAADjEUQAAMB4BBEAADAeQQQAAIxHEAEAAOMRRAAAwHgEEQAAMF6Lgmjs2LGqqqq64LzT6dTYsWMvdyYAAIB21aIg2rlzp+rq6i44f/bsWf3973+/7KEAAADaU5fmLP7oo4/cvz58+LDKy8vdjxsaGrR161ZdddVVrTcdAABAO2hWEA0dOlQ2m002m+2ib40FBARo6dKlrTYcAABAe2hWEB07dkwul0t9+vTR3r171atXL/c1u92usLAw+fr6tvqQAAAAbalZQRQbGytJamxsbJNhAAAArNCsIPq2o0eP6t1331VlZeUFgTRr1qzLHgwAAKC9tCiIXnjhBT322GPq2bOnIiIiZLPZ3NdsNhtBBAAAOpQWBdHvfvc7PfPMM5o5c2ZrzwMAANDuWvR9iE6dOqW77767tWcBAACwRIuC6O6779b27dtbexYAAABLtOgts759++qpp55SUVGRBg0apK5du3pcf+KJJ1plOAAAgPbQoiBauXKlAgMDVVBQoIKCAo9rNpuNIAIAAB1Ki4Lo2LFjrT0HAACAZVp0DxEAAEBn0qJXiNLS0i55/aWXXmrRMAAAAFZoURCdOnXK43F9fb0OHjyoqqqqi/7QVwAAAG/WoiDasGHDBecaGxv12GOP6ZprrrnsoQAAANpTq91D5OPjo8zMTC1cuLC1nhIAAKBdtOpN1Z9//rnOnTvXmk8JAADQ5lr0lllmZqbHY5fLpZMnT2rLli1KTU1tlcEAAADaS4uC6MCBAx6PfXx81KtXLy1YsOAHvwINAADA27QoiN59993WngMAAMAyLQqi87788ksdOXJEktSvXz/16tWrVYYCAABoTy26qbqmpkZpaWnq3bu3Ro8erdGjRysyMlKTJ0/W119/3dozAgAAtKkWBVFmZqYKCgq0adMmVVVVqaqqSm+88YYKCgr0H//xH609IwAAQJtq0Vtmf/3rX/X6669rzJgx7nM/+clPFBAQoHvuuUfLly9vrfkAAADaXIteIfr6668VHh5+wfmwsDDeMgMAAB1Oi4LI4XBo9uzZOnv2rPvcN998o9/+9rdyOBytNhwAAEB7aNFbZosWLdLtt9+uqKgoDRkyRJL04Ycfys/PT9u3b2/VAQEAANpai4Jo0KBBOnr0qNasWaNPPvlEknTfffcpJSVFAQEBrTogAABAW2tREGVnZys8PFyPPPKIx/mXXnpJX375pWbOnNkqwwEAALSHFt1D9Pzzz6t///4XnB84cKBWrFhx2UMBAAC0pxYFUXl5uXr37n3B+V69eunkyZOXPRQAAEB7alEQRUdH67333rvg/HvvvafIyMjLHgoAAKA9tegeokceeUQZGRmqr6/X2LFjJUn5+fmaMWMG36kaAAB0OC0KounTp+tf//qX/v3f/111dXWSJH9/f82cOVNZWVmtOiAAAEBba1EQ2Ww2Pfvss3rqqaf08ccfKyAgQNdee638/Pxaez4AAIA216IgOi8wMFA33HBDa80CAABgiRbdVA0AANCZEEQAAMB4BBEAADAeQQQAAIxHEAEAAOMRRAAAwHgEEQAAMB5BBAAAjEcQAQAA41kaRNnZ2brhhhsUFBSksLAw3XnnnTpy5IjHmrNnzyo9PV09evRQYGCgJk6cqIqKCo81ZWVlSk5O1hVXXKGwsDBNnz5d586d81izc+dODRs2TH5+furbt69yc3PbensAAKCDsDSICgoKlJ6erqKiIuXl5am+vl7jxo1TTU2Ne820adO0adMmvfbaayooKNCJEyd01113ua83NDQoOTlZdXV12rNnj15++WXl5uZq1qxZ7jXHjh1TcnKybr31VpWUlCgjI0MPP/ywtm3b1q77BQAA3umyfpbZ5dq6davH49zcXIWFham4uFijR49WdXW1Vq1apbVr12rs2LGSpNWrV2vAgAEqKirSqFGjtH37dh0+fFjvvPOOwsPDNXToUM2dO1czZ87U008/LbvdrhUrViguLk4LFiyQJA0YMEC7d+/WwoULlZSUdMFctbW1qq2tdT92Op1t+LsAAACs5lX3EFVXV0uSQkNDJUnFxcWqr69XYmKie03//v0VExOjwsJCSVJhYaEGDRqk8PBw95qkpCQ5nU4dOnTIvebbz3F+zfnn+K7s7GyFhIS4j+jo6NbbJAAA8DpeE0SNjY3KyMjQTTfdpOuuu06SVF5eLrvdru7du3usDQ8PV3l5uXvNt2Po/PXz1y61xul06ptvvrlglqysLFVXV7uP48ePt8oeAQCAd7L0LbNvS09P18GDB7V7926rR5Gfn5/8/PysHgMAALQTr3iFaMqUKdq8ebPeffddRUVFuc9HRESorq5OVVVVHusrKioUERHhXvPdrzo7//iH1gQHBysgIKC1twMAADoYS4PI5XJpypQp2rBhg3bs2KG4uDiP6wkJCeratavy8/Pd544cOaKysjI5HA5JksPhUGlpqSorK91r8vLyFBwcrPj4ePeabz/H+TXnnwMAAJjN0rfM0tPTtXbtWr3xxhsKCgpy3/MTEhKigIAAhYSEaPLkycrMzFRoaKiCg4P1+OOPy+FwaNSoUZKkcePGKT4+Xvfff7/mz5+v8vJyPfnkk0pPT3e/7fXoo49q2bJlmjFjhtLS0rRjxw6tX79eW7ZssWzvAADAe1j6CtHy5ctVXV2tMWPGqHfv3u5j3bp17jULFy7UT3/6U02cOFGjR49WRESE/va3v7mv+/r6avPmzfL19ZXD4dAvfvELPfDAA5ozZ457TVxcnLZs2aK8vDwNGTJECxYs0IsvvnjRL7kHAADmsfQVIpfL9YNr/P39lZOTo5ycnO9dExsbq7feeuuSzzNmzBgdOHCg2TMCAIDOzytuqgYAALASQQQAAIxHEAEAAOMRRAAAwHgEEQAAMB5BBAAAjEcQAQAA4xFEAADAeAQRAAAwHkEEAACMRxABAADjEUQAAMB4BBEAADAeQQQAAIxHEAEAAOMRRAAAwHgEEQAAMB5BBAAAjEcQAQAA4xFEAADAeAQRAAAwHkEEAACMRxABAADjEUQAAMB4BBEAADAeQQQAAIxHEAEAAOMRRAAAwHgEEQAAMB5BBAAAjEcQAQAA4xFEAADAeAQRAAAwHkEEAACMRxABAADjEUQAAMB4BBEAADAeQQQAAIxHEAEAAOMRRAAAwHgEEQAAMB5BBAAAjEcQAQAA4xFEAADAeAQRAAAwHkEEAACMRxABAADjEUQAAMB4BBEAADAeQQQAAIxHEAEAAOMRRAAAwHgEEQAAMB5BBAAAjEcQAQAA4xFEAADAeAQRAAAwHkEEAACMRxABAADjEUQAAMB4BBEAADAeQQQAAIxHEAEAAOMRRAAAwHiWBtGuXbs0YcIERUZGymazaePGjR7XXS6XZs2apd69eysgIECJiYk6evSox5qvvvpKKSkpCg4OVvfu3TV58mSdOXPGY81HH32km2++Wf7+/oqOjtb8+fPbemsAAKADsTSIampqNGTIEOXk5Fz0+vz587VkyRKtWLFC77//vrp166akpCSdPXvWvSYlJUWHDh1SXl6eNm/erF27dumXv/yl+7rT6dS4ceMUGxur4uJi/eEPf9DTTz+tlStXtvn+AABAx9DFyk8+fvx4jR8//qLXXC6XFi1apCeffFJ33HGHJOnPf/6zwsPDtXHjRt177736+OOPtXXrVu3bt0/Dhw+XJC1dulQ/+clP9Mc//lGRkZFas2aN6urq9NJLL8lut2vgwIEqKSnRc8895xFOAADAXF57D9GxY8dUXl6uxMRE97mQkBCNHDlShYWFkqTCwkJ1797dHUOSlJiYKB8fH73//vvuNaNHj5bdbnevSUpK0pEjR3Tq1KmLfu7a2lo5nU6PAwAAdF5eG0Tl5eWSpPDwcI/z4eHh7mvl5eUKCwvzuN6lSxeFhoZ6rLnYc3z7c3xXdna2QkJC3Ed0dPTlbwgAAHgtrw0iK2VlZam6utp9HD9+3OqRAABAG/LaIIqIiJAkVVRUeJyvqKhwX4uIiFBlZaXH9XPnzumrr77yWHOx5/j25/guPz8/BQcHexwAAKDz8togiouLU0REhPLz893nnE6n3n//fTkcDkmSw+FQVVWViouL3Wt27NihxsZGjRw50r1m165dqq+vd6/Jy8tTv379dOWVV7bTbgAAgDezNIjOnDmjkpISlZSUSPr/G6lLSkpUVlYmm82mjIwM/e53v9Obb76p0tJSPfDAA4qMjNSdd94pSRowYIBuv/12PfLII9q7d6/ee+89TZkyRffee68iIyMlST//+c9lt9s1efJkHTp0SOvWrdPixYuVmZlp0a4BAIC3sfTL7vfv369bb73V/fh8pKSmpio3N1czZsxQTU2NfvnLX6qqqko/+tGPtHXrVvn7+7s/Zs2aNZoyZYpuu+02+fj4aOLEiVqyZIn7ekhIiLZv36709HQlJCSoZ8+emjVrFl9yDwAA3CwNojFjxsjlcn3vdZvNpjlz5mjOnDnfuyY0NFRr16695OcZPHiw/v73v7d4TgAA0Ll57T1EAAAA7YUgAgAAxiOIAACA8QgiAABgPIIIAAAYjyACAADGI4gAAIDxCCIAAGA8gggAABiPIAIAAMYjiAAAgPEIIgAAYDyCCAAAGI8gAgAAxiOIAACA8QgiAABgPIIIAAAYjyACAADGI4gAAIDxCCIAAGA8gggAABiPIAIAAMYjiAAAgPEIIgAAYDyCCAAAGI8gAgAAxiOIAACA8QgiAABgPIIIAAAYjyACAADGI4gAAIDxCCIAAGA8gggAABiPIAIAAMYjiAAAgPEIIgAAYDyCCAAAGI8gAgAAxiOIAACA8QgiAABgPIIIAAAYjyACAADGI4gAAIDxCCIAAGA8gggAABiPIAIAAMYjiAAAgPEIIgAAYDyCCAAAGI8gAgAAxiOIAACA8QgiAABgPIIIAAAYjyACAADGI4gAAIDxCCIAAGA8gggAABiPIAIAAMYjiAAAgPEIIgAAYDyCCAAAGI8gAgAAxiOIAACA8QgiAABgPKOCKCcnR1dffbX8/f01cuRI7d271+qRAACAFzAmiNatW6fMzEzNnj1bH3zwgYYMGaKkpCRVVlZaPRoAALCYMUH03HPP6ZFHHtFDDz2k+Ph4rVixQldccYVeeuklq0cDAAAW62L1AO2hrq5OxcXFysrKcp/z8fFRYmKiCgsLL1hfW1ur2tpa9+Pq6mpJktPp9FjXUPtNG03cOr477/dhH22vM+xB6hz76Ax7kNiHN+kMe5A6xz6+u4fzj10u1w9/sMsAX3zxhUuSa8+ePR7np0+f7hoxYsQF62fPnu2SxMHBwcHBwdEJjuPHj/9gKxjxClFzZWVlKTMz0/24sbFRX331lXr06CGbzdYmn9PpdCo6OlrHjx9XcHBwm3yO9tAZ9tEZ9iCxD2/SGfYgdY59dIY9SOyjqVwul06fPq3IyMgfXGtEEPXs2VO+vr6qqKjwOF9RUaGIiIgL1vv5+cnPz8/jXPfu3dtyRLfg4OAO/T/u8zrDPjrDHiT24U06wx6kzrGPzrAHiX00RUhISJPWGXFTtd1uV0JCgvLz893nGhsblZ+fL4fDYeFkAADAGxjxCpEkZWZmKjU1VcOHD9eIESO0aNEi1dTU6KGHHrJ6NAAAYDFjgmjSpEn68ssvNWvWLJWXl2vo0KHaunWrwsPDrR5N0v+/TTd79uwL3qrraDrDPjrDHiT24U06wx6kzrGPzrAHiX20BZvL1ZSvRQMAAOi8jLiHCAAA4FIIIgAAYDyCCAAAGI8gAgAAxiOIvEROTo6uvvpq+fv7a+TIkdq7d6/VIzXLrl27NGHCBEVGRspms2njxo1Wj9Rs2dnZuuGGGxQUFKSwsDDdeeedOnLkiNVjNdvy5cs1ePBg9zc6czgcevvtt60e67LMmzdPNptNGRkZVo/SLE8//bRsNpvH0b9/f6vHarYvvvhCv/jFL9SjRw8FBARo0KBB2r9/v9VjNcvVV199wX8Lm82m9PR0q0drloaGBj311FOKi4tTQECArrnmGs2dO7dpP6vLi5w+fVoZGRmKjY1VQECAbrzxRu3bt8/SmQgiL7Bu3TplZmZq9uzZ+uCDDzRkyBAlJSWpsrLS6tGarKamRkOGDFFOTo7Vo7RYQUGB0tPTVVRUpLy8PNXX12vcuHGqqamxerRmiYqK0rx581RcXKz9+/dr7NixuuOOO3To0CGrR2uRffv26fnnn9fgwYOtHqVFBg4cqJMnT7qP3bt3Wz1Ss5w6dUo33XSTunbtqrfffluHDx/WggULdOWVV1o9WrPs27fP479DXl6eJOnuu++2eLLmefbZZ7V8+XItW7ZMH3/8sZ599lnNnz9fS5cutXq0Znn44YeVl5enV155RaWlpRo3bpwSExP1xRdfWDdUq/z0VFyWESNGuNLT092PGxoaXJGRka7s7GwLp2o5Sa4NGzZYPcZlq6ysdElyFRQUWD3KZbvyyitdL774otVjNNvp06dd1157rSsvL891yy23uKZOnWr1SM0ye/Zs15AhQ6we47LMnDnT9aMf/cjqMVrd1KlTXddcc42rsbHR6lGaJTk52ZWWluZx7q677nKlpKRYNFHzff311y5fX1/X5s2bPc4PGzbM9Zvf/MaiqVwuXiGyWF1dnYqLi5WYmOg+5+Pjo8TERBUWFlo4GaqrqyVJoaGhFk/Scg0NDXr11VdVU1PTIX9MTXp6upKTkz3+fHQ0R48eVWRkpPr06aOUlBSVlZVZPVKzvPnmmxo+fLjuvvtuhYWF6frrr9cLL7xg9ViXpa6uTn/5y1+UlpbWZj+wu63ceOONys/P16effipJ+vDDD7V7926NHz/e4sma7ty5c2poaJC/v7/H+YCAAEtfQTXmO1V7q//93/9VQ0PDBd8xOzw8XJ988olFU6GxsVEZGRm66aabdN1111k9TrOVlpbK4XDo7NmzCgwM1IYNGxQfH2/1WM3y6quv6oMPPrD8voLLMXLkSOXm5qpfv346efKkfvvb3+rmm2/WwYMHFRQUZPV4TfKPf/xDy5cvV2Zmpv7rv/5L+/bt0xNPPCG73a7U1FSrx2uRjRs3qqqqSg8++KDVozTbr3/9azmdTvXv31++vr5qaGjQM888o5SUFKtHa7KgoCA5HA7NnTtXAwYMUHh4uP77v/9bhYWF6tu3r2VzEUTARaSnp+vgwYMd7n6P8/r166eSkhJVV1fr9ddfV2pqqgoKCjpMFB0/flxTp05VXl7eBf+K7Ei+/a/2wYMHa+TIkYqNjdX69es1efJkCydrusbGRg0fPly///3vJUnXX3+9Dh48qBUrVnTYIFq1apXGjx+vyMhIq0dptvXr12vNmjVau3atBg4cqJKSEmVkZCgyMrJD/fd45ZVXlJaWpquuukq+vr4aNmyY7rvvPhUXF1s2E0FksZ49e8rX11cVFRUe5ysqKhQREWHRVGabMmWKNm/erF27dikqKsrqcVrEbre7/6WVkJCgffv2afHixXr++ectnqxpiouLVVlZqWHDhrnPNTQ0aNeuXVq2bJlqa2vl6+tr4YQt0717d/3bv/2bPvvsM6tHabLevXtfENIDBgzQX//6V4smujz//Oc/9c477+hvf/ub1aO0yPTp0/XrX/9a9957ryRp0KBB+uc//6ns7OwOFUTXXHONCgoKVFNTI6fTqd69e2vSpEnq06ePZTNxD5HF7Ha7EhISlJ+f7z7X2Nio/Pz8DnnPR0fmcrk0ZcoUbdiwQTt27FBcXJzVI7WaxsZG1dbWWj1Gk912220qLS1VSUmJ+xg+fLhSUlJUUlLSIWNIks6cOaPPP/9cvXv3tnqUJrvpppsu+PYTn376qWJjYy2a6PKsXr1aYWFhSk5OtnqUFvn666/l4+P5V7evr68aGxstmujydOvWTb1799apU6e0bds23XHHHZbNwitEXiAzM1OpqakaPny4RowYoUWLFqmmpkYPPfSQ1aM12ZkzZzz+1Xvs2DGVlJQoNDRUMTExFk7WdOnp6Vq7dq3eeOMNBQUFqby8XJIUEhKigIAAi6druqysLI0fP14xMTE6ffq01q5dq507d2rbtm1Wj9ZkQUFBF9y71a1bN/Xo0aND3dP1n//5n5owYYJiY2N14sQJzZ49W76+vrrvvvusHq3Jpk2bphtvvFG///3vdc8992jv3r1auXKlVq5cafVozdbY2KjVq1crNTVVXbp0zL/+JkyYoGeeeUYxMTEaOHCgDhw4oOeee05paWlWj9Ys27Ztk8vlUr9+/fTZZ59p+vTp6t+/v7V/71n29W3wsHTpUldMTIzLbre7RowY4SoqKrJ6pGZ59913XZIuOFJTU60erckuNr8k1+rVq60erVnS0tJcsbGxLrvd7urVq5frtttuc23fvt3qsS5bR/yy+0mTJrl69+7tstvtrquuuso1adIk12effWb1WM22adMm13XXXefy8/Nz9e/f37Vy5UqrR2qRbdu2uSS5jhw5YvUoLeZ0Ol1Tp051xcTEuPz9/V19+vRx/eY3v3HV1tZaPVqzrFu3ztWnTx+X3W53RUREuNLT011VVVWWzmRzuTrYt7cEAABoZdxDBAAAjEcQAQAA4xFEAADAeAQRAAAwHkEEAACMRxABAADjEUQAAMB4BBEAADAeQQSgUxgzZowyMjKatHbnzp2y2Wyqqqq6rM959dVXa9GiRZf1HAC8A0EEAACMRxABAADjEUQAOp1XXnlFw4cPV1BQkCIiIvTzn/9clZWVF6x77733NHjwYPn7+2vUqFE6ePCgx/Xdu3fr5ptvVkBAgKKjo/XEE0+opqamvbYBoB0RRAA6nfr6es2dO1cffvihNm7cqP/5n//Rgw8+eMG66dOna8GCBdq3b5969eqlCRMmqL6+XpL0+eef6/bbb9fEiRP10Ucfad26ddq9e7emTJnSzrsB0B66WD0AALS2tLQ096/79OmjJUuW6IYbbtCZM2cUGBjovjZ79mz9+Mc/liS9/PLLioqK0oYNG3TPPfcoOztbKSkp7hu1r732Wi1ZskS33HKLli9fLn9//3bdE4C2xStEADqd4uJiTZgwQTExMQoKCtItt9wiSSorK/NY53A43L8ODQ1Vv3799PHHH0uSPvzwQ+Xm5iowMNB9JCUlqbGxUceOHWu/zQBoF7xCBKBTqampUVJSkpKSkrRmzRr16tVLZWVlSkpKUl1dXZOf58yZM/rVr36lJ5544oJrMTExrTkyAC9AEAHoVD755BP961//0rx58xQdHS1J2r9//0XXFhUVuePm1KlT+vTTTzVgwABJ0rBhw3T48GH17du3fQYHYCneMgPQqcTExMhut2vp0qX6xz/+oTfffFNz58696No5c+YoPz9fBw8e1IMPPqiePXvqzjvvlCTNnDlTe/bs0ZQpU1RSUqKjR4/qjTfe4KZqoJMiiAB0Kr169VJubq5ee+01xcfHa968efrjH/940bXz5s3T1KlTlZCQoPLycm3atEl2u12SNHjwYBUUFOjTTz/VzTffrOuvv16zZs1SZGRke24HQDuxuVwul9VDAAAAWIlXiAAAgPEIIgAAYDyCCAAAGI8gAgAAxiOIAACA8QgiAABgPIIIAAAYjyACAADGI4gAAIDxCCIAAGA8gggAABjv/wCEeyEW1DeP+wAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.countplot(x='label', data = train1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Data normalization\n",
    "X_train = X_train/255\n",
    "X_test = X_test/255"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((42000, 28, 28, 1), (28000, 28, 28, 1))"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train = X_train.values.reshape(-1,28,28,1)\n",
    "X_test = X_test.values.reshape(-1,28,28,1)\n",
    "X_train.shape, X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(42000, 10)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# one-hot encoding\n",
    "from keras.utils import to_categorical\n",
    "y_train = to_categorical(y_train, num_classes = 10)\n",
    "y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0    1\n",
      "1    0\n",
      "2    1\n",
      "3    4\n",
      "4    0\n",
      "Name: label, dtype: int64\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[0., 1., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
       "       [1., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
       "       [0., 1., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
       "       [0., 0., 0., 0., 1., 0., 0., 0., 0., 0.],\n",
       "       [1., 0., 0., 0., 0., 0., 0., 0., 0., 0.]], dtype=float32)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(train1['label'].head())\n",
    "y_train[0:5,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_cv, y_train, y_cv = train_test_split(X_train, y_train, test_size = 0.1, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaAAAAGzCAYAAABpdMNsAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAeSklEQVR4nO3df3BU9b3/8dcGkgUkWRpCfkmgAVRUIG0pxIhQlAwh3jqgtNdfvQOMAyMNtkCtTnpVtO1802Kv9auXwnRuC3Uu+Ot7BSq13Gow4WtNsCCUS9GU0FTCQIJyv+yGACEkn+8fXLddCeBZd/NONs/HzJlhzznvfN4cTnjl7Dn5rM855wQAQDdLsm4AANA3EUAAABMEEADABAEEADBBAAEATBBAAAATBBAAwAQBBAAwQQABAEwQQAAAEwQQ0A2qqqrk8/m6XGpra63bA0z0t24A6Eu+9a1vadKkSRHrxowZY9QNYIsAArrR1KlT9bWvfc26DaBH4C04oJu1tLTo3Llz1m0A5gggoBstWLBAaWlpGjBggG6++Wbt3LnTuiXADG/BAd0gJSVFc+fO1a233qqMjAzt379fP/nJTzR16lS9/fbb+uIXv2jdItDtfHwgHWCjvr5eEyZM0LRp07R161brdoBux1twgJExY8Zo9uzZevPNN9XR0WHdDtDtCCDAUF5ens6ePavW1lbrVoBuRwABhv7yl79owIABGjx4sHUrQLcjgIBu8OGHH16w7o9//KN+/etfa+bMmUpK4lsRfQ8PIQDd4JZbbtHAgQN14403KjMzU/v379fPf/5zJScnq6amRtdee611i0C3I4CAbvDMM89o/fr1qq+vVygU0rBhwzRjxgytWLGCqXjQZxFAAAATvPEMADBBAAEATBBAAAATBBAAwAQBBAAwQQABAEz0uI9j6Ozs1JEjR5Samiqfz2fdDgDAI+ecWlpalJube8lZPnpcAB05ckR5eXnWbQAAPqPGxkYNHz78ott7XAClpqZKkm7SreqvZONuAABenVO73tJr4f/PLyZuAbRq1So9+eSTampqUkFBgZ599llNnjz5snUfv+3WX8nq7yOAAKDX+Z/5dS53GyUuDyG8+OKLWr58uVasWKF3331XBQUFKikp0bFjx+IxHACgF4pLAD311FNauHChFixYoOuuu05r1qzRoEGD9Mtf/jIewwEAeqGYB9DZs2e1a9cuFRcX/22QpCQVFxerpqbmgv3b2toUCoUiFgBA4ot5AH300Ufq6OhQVlZWxPqsrCw1NTVdsH9FRYUCgUB44Qk4AOgbzH8Rtby8XMFgMLw0NjZatwQA6AYxfwouIyND/fr1U3Nzc8T65uZmZWdnX7C/3++X3++PdRsAgB4u5ldAKSkpmjhxoiorK8PrOjs7VVlZqaKiolgPBwDopeLye0DLly/XvHnz9OUvf1mTJ0/W008/rdbWVi1YsCAewwEAeqG4BNCdd96pDz/8UI899piampr0hS98QVu3br3gwQQAQN/lc8456yb+XigUUiAQ0HTNZiYEAOiFzrl2VWmzgsGg0tLSLrqf+VNwAIC+iQACAJgggAAAJgggAIAJAggAYIIAAgCYIIAAACYIIACACQIIAGCCAAIAmCCAAAAmCCAAgAkCCABgggACAJgggAAAJgggAIAJAggAYIIAAgCYIIAAACYIIACACQIIAGCCAAIAmCCAAAAmCCAAgAkCCABgggACAJgggAAAJgggAIAJAggAYIIAAgCYIIAAACYIIACACQIIAGCCAAIAmCCAAAAmCCAAgAkCCABgggACAJgggAAAJgggAIAJAggAYIIAAgCYIIAAACYIIACACQIIAGCCAAIAmCCAAAAmCCAAgAkCCABgggACAJgggAAAJgggAICJ/tYNoG/pmP4lzzV//WpKHDqJnQGjWjzXPHL9a3Ho5EJfH3w8qrp+Pu8/m3a4Ts8177e3ea5ZOu+bnmuSqnd7rkH8cQUEADBBAAEATMQ8gB5//HH5fL6IZezYsbEeBgDQy8XlHtD111+vN95442+D9OdWEwAgUlySoX///srOzo7HlwYAJIi43AM6cOCAcnNzNWrUKN177706dOjQRfdta2tTKBSKWAAAiS/mAVRYWKh169Zp69atWr16tRoaGjR16lS1tHT9qGpFRYUCgUB4ycvLi3VLAIAeKOYBVFpaqq9//euaMGGCSkpK9Nprr+nEiRN66aWXuty/vLxcwWAwvDQ2Nsa6JQBADxT3pwOGDBmiq6++WvX19V1u9/v98vv98W4DANDDxP33gE6ePKmDBw8qJycn3kMBAHqRmAfQgw8+qOrqav31r3/V22+/rdtvv139+vXT3XffHeuhAAC9WMzfgjt8+LDuvvtuHT9+XMOGDdNNN92k2tpaDRs2LNZDAQB6sZgH0AsvvBDrL4kEsuzfNniumTmwNQ6d9A3epwf9nzrXEdM+LubqZO8TzTZPHOi5Jqfacwm6AXPBAQBMEEAAABMEEADABAEEADBBAAEATBBAAAATBBAAwAQBBAAwQQABAEwQQAAAEwQQAMAEAQQAMBH3D6RD4vrgpfGea4oH/iGKkfg5CUhEfGcDAEwQQAAAEwQQAMAEAQQAMEEAAQBMEEAAABMEEADABAEEADBBAAEATBBAAAATBBAAwAQBBAAwQQABAEwwGzaimtVakv445Zeea5J6+M88/3Eyw3PNLw7f5LnmVHuy55oPd2d5rhn5m9OeaySpPeC9vwHN3sdqHXGF55ort+7xXNPpuQLdoWf/bwAASFgEEADABAEEADBBAAEATBBAAAATBBAAwAQBBAAwQQABAEwQQAAAEwQQAMAEAQQAMEEAAQBMMBlpgmn85xs91/xxyv+Oaqzumlj0hnfv9lwz9EeDohor+S9Nnmv6h/7bc01aR4fnmsFn/uK5Jlr+KGpcFDWDdnqvYWLRxMEVEADABAEEADBBAAEATBBAAAATBBAAwAQBBAAwQQABAEwQQAAAEwQQAMAEAQQAMEEAAQBMEEAAABNMRppgzl3hfUrI7ppUVJLWt+R4rsn+5inPNeca/+y5RpLcNWM81xz92ijPNVduOuS5JlQ03HNNa1Z0/7a5/9nsuabjzwejGgt9F1dAAAATBBAAwITnANq+fbtuu+025ebmyufzadOmTRHbnXN67LHHlJOTo4EDB6q4uFgHDhyIVb8AgAThOYBaW1tVUFCgVatWdbl95cqVeuaZZ7RmzRrt2LFDV1xxhUpKSnTmzJnP3CwAIHF4fgihtLRUpaWlXW5zzunpp5/WI488otmzZ0uSnnvuOWVlZWnTpk266667Plu3AICEEdN7QA0NDWpqalJxcXF4XSAQUGFhoWpqarqsaWtrUygUilgAAIkvpgHU1NQkScrKyopYn5WVFd72SRUVFQoEAuElLy8vli0BAHoo86fgysvLFQwGw0tjY6N1SwCAbhDTAMrOzpYkNTdH/hJbc3NzeNsn+f1+paWlRSwAgMQX0wDKz89Xdna2Kisrw+tCoZB27NihoqKiWA4FAOjlPD8Fd/LkSdXX14dfNzQ0aM+ePUpPT9eIESO0dOlS/fCHP9RVV12l/Px8Pfroo8rNzdWcOXNi2TcAoJfzHEA7d+7UzTffHH69fPlySdK8efO0bt06PfTQQ2ptbdWiRYt04sQJ3XTTTdq6dasGDBgQu64BAL2ezznnffbKOAqFQgoEApqu2ervS7Zup9dp+F/e3+r807x/jUMnXatvb/NcU33qKs81v/rgBs81ktT0wVDPNX++bbXnmt+cCniu+bK/6ydJLyWr30DPNZJUeXqQ55qN//0lzzWN870/9dqxP7qJZtF9zrl2VWmzgsHgJe/rmz8FBwDomwggAIAJAggAYIIAAgCYIIAAACYIIACACQIIAGCCAAIAmCCAAAAmCCAAgAkCCABgggACAJgggAAAJpgNO8GUHfA+U3DpoJY4dAJc3pPHr/Nc89adEzzXdLx3wHONJAW/4X1Wdf+JTs81A7a847mmJ2M2bABAj0YAAQBMEEAAABMEEADABAEEADBBAAEATBBAAAATBBAAwAQBBAAwQQABAEwQQAAAEwQQAMBEf+sGEFtrDk/3XFN69auxbwT4FL47dL/3ohe9l0QzgakkBf69Nqo6fDpcAQEATBBAAAATBBAAwAQBBAAwQQABAEwQQAAAEwQQAMAEAQQAMEEAAQBMEEAAABMEEADABAEEADDBZKQJ5h+y/su6hR7hO0dviKpuy5/Ge64Z8WK/qMbqyQ7d1eG55qvXeT/3/iXH+2Sf0Uxg2u+lTs81klT9tQLPNR119VGN1RdxBQQAMEEAAQBMEEAAABMEEADABAEEADBBAAEATBBAAAATBBAAwAQBBAAwQQABAEwQQAAAEwQQAMCEzznnrJv4e6FQSIFAQNM1W/19ydbt9Dr9rr3Kc82B+RlRjTV9+l7PNW/86VrPNcO3eJ/sM/WN9zzXSFJHKBRVHaR+aWmea97/1zGea9bd9EvPNUV+75OrStLVv7nfe82iP0Q1ViI559pVpc0KBoNKu8R5wRUQAMAEAQQAMOE5gLZv367bbrtNubm58vl82rRpU8T2+fPny+fzRSyzZs2KVb8AgAThOYBaW1tVUFCgVatWXXSfWbNm6ejRo+Hl+eef/0xNAgASj+dPRC0tLVVpaekl9/H7/crOzo66KQBA4ovLPaCqqiplZmbqmmuu0eLFi3X8+PGL7tvW1qZQKBSxAAASX8wDaNasWXruuedUWVmpH//4x6qurlZpaak6Orp+DLKiokKBQCC85OXlxbolAEAP5PktuMu56667wn8eP368JkyYoNGjR6uqqkozZsy4YP/y8nItX748/DoUChFCANAHxP0x7FGjRikjI0P19fVdbvf7/UpLS4tYAACJL+4BdPjwYR0/flw5OTnxHgoA0It4fgvu5MmTEVczDQ0N2rNnj9LT05Wenq4nnnhCc+fOVXZ2tg4ePKiHHnpIY8aMUUlJSUwbBwD0bp4DaOfOnbr55pvDrz++fzNv3jytXr1ae/fu1a9+9SudOHFCubm5mjlzpn7wgx/I7/fHrmsAQK/HZKQAzCQNGuS55szmYZ5rfnfdK55rJKnh3BnPNQ+MnBLVWImEyUgBAD0aAQQAMEEAAQBMEEAAABMEEADABAEEADBBAAEATBBAAAATBBAAwAQBBAAwQQABAEwQQAAAEwQQAMBEzD+SGwA+tVEjPJf87rr1nmuOdpz2XCNJ//gvD3muydLbUY3VF3EFBAAwQQABAEwQQAAAEwQQAMAEAQQAMEEAAQBMEEAAABMEEADABAEEADBBAAEATBBAAAATBBAAwASTkQKIiaRBgzzXtIwNxKGTC51xvqjqsv9v0HONi2qkvokrIACACQIIAGCCAAIAmCCAAAAmCCAAgAkCCABgggACAJgggAAAJgggAIAJAggAYIIAAgCYIIAAACaYjBTdqt/QdM81LifTc01SS6vnGkk690FjVHWJJmnAAM81LbeO91zT+k/eJ/ucXPGA55p+bdFNETp0d01Udfh0uAICAJgggAAAJgggAIAJAggAYIIAAgCYIIAAACYIIACACQIIAGCCAAIAmCCAAAAmCCAAgAkCCABggslI0a0aF4z1XPPusmc91/zHyQzPNZK09p++6r3onf+KaiyvkgYN8lzjuzI7qrHeXzrMc03dnJ95rjl07rTnmuLmZZ5rrl70B881iD+ugAAAJgggAIAJTwFUUVGhSZMmKTU1VZmZmZozZ47q6uoi9jlz5ozKyso0dOhQDR48WHPnzlVzc3NMmwYA9H6eAqi6ulplZWWqra3V66+/rvb2ds2cOVOtrX/78K9ly5bp1Vdf1csvv6zq6modOXJEd9xxR8wbBwD0bp4eQti6dWvE63Xr1ikzM1O7du3StGnTFAwG9Ytf/EIbNmzQLbfcIklau3atrr32WtXW1uqGG26IXecAgF7tM90DCgbPf5xuevr5j1netWuX2tvbVVxcHN5n7NixGjFihGpquv5o27a2NoVCoYgFAJD4og6gzs5OLV26VFOmTNG4ceMkSU1NTUpJSdGQIUMi9s3KylJTU1OXX6eiokKBQCC85OXlRdsSAKAXiTqAysrKtG/fPr3wwgufqYHy8nIFg8Hw0tjY+Jm+HgCgd4jqF1GXLFmiLVu2aPv27Ro+fHh4fXZ2ts6ePasTJ05EXAU1NzcrO7vrX4jz+/3y+/3RtAEA6MU8XQE557RkyRJt3LhR27ZtU35+fsT2iRMnKjk5WZWVleF1dXV1OnTokIqKimLTMQAgIXi6AiorK9OGDRu0efNmpaamhu/rBAIBDRw4UIFAQPfdd5+WL1+u9PR0paWl6YEHHlBRURFPwAEAIngKoNWrV0uSpk+fHrF+7dq1mj9/viTppz/9qZKSkjR37ly1tbWppKREP/uZ9zmiAACJzeecc9ZN/L1QKKRAIKDpmq3+vmTrdhBjRx680XNNNJORRiuaSUz/+Q9zPNeMXu392+7sEO/fD6//fI3nmp6u8rT3SVl/OubaOHSCiznn2lWlzQoGg0pLS7vofswFBwAwQQABAEwQQAAAEwQQAMAEAQQAMEEAAQBMEEAAABMEEADABAEEADBBAAEATBBAAAATBBAAwAQBBAAwEdUnogKJau7gj7zX3Pxvnmt+W5jqueaHf/4HzzU93az3bvdc03DE+4zlV+ldzzWIP66AAAAmCCAAgAkCCABgggACAJgggAAAJgggAIAJAggAYIIAAgCYIIAAACYIIACACQIIAGCCAAIAmGAyUnSrnJpTnmt+vzjZc03RgDbPNZKU1E0/k5UOavFe84UX4tBJ7Gw/k+K5pv8jQzzXXFXLxKKJgisgAIAJAggAYIIAAgCYIIAAACYIIACACQIIAGCCAAIAmCCAAAAmCCAAgAkCCABgggACAJgggAAAJpiMFN3K9/s9nmsqRk/wXHO4/EbPNZLUr/D/ea7ZNenfoxqrO/zu9BVR1S19ZYHnmlH/56T3gd7Z670GCYMrIACACQIIAGCCAAIAmCCAAAAmCCAAgAkCCABgggACAJgggAAAJgggAIAJAggAYIIAAgCYIIAAACaYjBQJaXjF29021lc1sdvG6i6jVGPdAvoAroAAACYIIACACU8BVFFRoUmTJik1NVWZmZmaM2eO6urqIvaZPn26fD5fxHL//ffHtGkAQO/nKYCqq6tVVlam2tpavf7662pvb9fMmTPV2toasd/ChQt19OjR8LJy5cqYNg0A6P08PYSwdevWiNfr1q1TZmamdu3apWnTpoXXDxo0SNnZ2bHpEACQkD7TPaBgMChJSk9Pj1i/fv16ZWRkaNy4cSovL9epU6cu+jXa2toUCoUiFgBA4ov6MezOzk4tXbpUU6ZM0bhx48Lr77nnHo0cOVK5ubnau3evHn74YdXV1emVV17p8utUVFToiSeeiLYNAEAv5XPOuWgKFy9erN/+9rd66623NHz48Ivut23bNs2YMUP19fUaPXr0Bdvb2trU1tYWfh0KhZSXl6fpmq3+vuRoWgMAGDrn2lWlzQoGg0pLS7voflFdAS1ZskRbtmzR9u3bLxk+klRYWChJFw0gv98vv98fTRsAgF7MUwA55/TAAw9o48aNqqqqUn5+/mVr9uzZI0nKycmJqkEAQGLyFEBlZWXasGGDNm/erNTUVDU1NUmSAoGABg4cqIMHD2rDhg269dZbNXToUO3du1fLli3TtGnTNGHChLj8BQAAvZOne0A+n6/L9WvXrtX8+fPV2Niob3zjG9q3b59aW1uVl5en22+/XY888sgl3wf8e6FQSIFAgHtAANBLxeUe0OWyKi8vT9XV1V6+JACgj2IuOACACQIIAGCCAAIAmCCAAAAmCCAAgAkCCABgggACAJgggAAAJgggAIAJAggAYIIAAgCYIIAAACYIIACACQIIAGCCAAIAmCCAAAAmCCAAgAkCCABgggACAJgggAAAJgggAIAJAggAYIIAAgCYIIAAACYIIACAif7WDXySc06SdE7tkjNuBgDg2Tm1S/rb/+cX0+MCqKWlRZL0ll4z7gQA8Fm0tLQoEAhcdLvPXS6iullnZ6eOHDmi1NRU+Xy+iG2hUEh5eXlqbGxUWlqaUYf2OA7ncRzO4zicx3E4ryccB+ecWlpalJubq6Ski9/p6XFXQElJSRo+fPgl90lLS+vTJ9jHOA7ncRzO4zicx3E4z/o4XOrK52M8hAAAMEEAAQBM9KoA8vv9WrFihfx+v3UrpjgO53EczuM4nMdxOK83HYce9xACAKBv6FVXQACAxEEAAQBMEEAAABMEEADABAEEADDRawJo1apV+vznP68BAwaosLBQ77zzjnVL3e7xxx+Xz+eLWMaOHWvdVtxt375dt912m3Jzc+Xz+bRp06aI7c45PfbYY8rJydHAgQNVXFysAwcO2DQbR5c7DvPnz7/g/Jg1a5ZNs3FSUVGhSZMmKTU1VZmZmZozZ47q6uoi9jlz5ozKyso0dOhQDR48WHPnzlVzc7NRx/HxaY7D9OnTLzgf7r//fqOOu9YrAujFF1/U8uXLtWLFCr377rsqKChQSUmJjh07Zt1at7v++ut19OjR8PLWW29ZtxR3ra2tKigo0KpVq7rcvnLlSj3zzDNas2aNduzYoSuuuEIlJSU6c+ZMN3caX5c7DpI0a9asiPPj+eef78YO46+6ulplZWWqra3V66+/rvb2ds2cOVOtra3hfZYtW6ZXX31VL7/8sqqrq3XkyBHdcccdhl3H3qc5DpK0cOHCiPNh5cqVRh1fhOsFJk+e7MrKysKvOzo6XG5urquoqDDsqvutWLHCFRQUWLdhSpLbuHFj+HVnZ6fLzs52Tz75ZHjdiRMnnN/vd88//7xBh93jk8fBOefmzZvnZs+ebdKPlWPHjjlJrrq62jl3/t8+OTnZvfzyy+F93nvvPSfJ1dTUWLUZd588Ds4595WvfMV9+9vftmvqU+jxV0Bnz57Vrl27VFxcHF6XlJSk4uJi1dTUGHZm48CBA8rNzdWoUaN077336tChQ9YtmWpoaFBTU1PE+REIBFRYWNgnz4+qqiplZmbqmmuu0eLFi3X8+HHrluIqGAxKktLT0yVJu3btUnt7e8T5MHbsWI0YMSKhz4dPHoePrV+/XhkZGRo3bpzKy8t16tQpi/YuqsfNhv1JH330kTo6OpSVlRWxPisrS++//75RVzYKCwu1bt06XXPNNTp69KieeOIJTZ06Vfv27VNqaqp1eyaampokqcvz4+NtfcWsWbN0xx13KD8/XwcPHtT3vvc9lZaWqqamRv369bNuL+Y6Ozu1dOlSTZkyRePGjZN0/nxISUnRkCFDIvZN5POhq+MgSffcc49Gjhyp3Nxc7d27Vw8//LDq6ur0yiuvGHYbqccHEP6mtLQ0/OcJEyaosLBQI0eO1EsvvaT77rvPsDP0BHfddVf4z+PHj9eECRM0evRoVVVVacaMGYadxUdZWZn27dvXJ+6DXsrFjsOiRYvCfx4/frxycnI0Y8YMHTx4UKNHj+7uNrvU49+Cy8jIUL9+/S54iqW5uVnZ2dlGXfUMQ4YM0dVXX636+nrrVsx8fA5wflxo1KhRysjISMjzY8mSJdqyZYvefPPNiM8Py87O1tmzZ3XixImI/RP1fLjYcehKYWGhJPWo86HHB1BKSoomTpyoysrK8LrOzk5VVlaqqKjIsDN7J0+e1MGDB5WTk2Pdipn8/HxlZ2dHnB+hUEg7duzo8+fH4cOHdfz48YQ6P5xzWrJkiTZu3Kht27YpPz8/YvvEiROVnJwccT7U1dXp0KFDCXU+XO44dGXPnj2S1LPOB+unID6NF154wfn9frdu3Tq3f/9+t2jRIjdkyBDX1NRk3Vq3+s53vuOqqqpcQ0OD+/3vf++Ki4tdRkaGO3bsmHVrcdXS0uJ2797tdu/e7SS5p556yu3evdt98MEHzjnnfvSjH7khQ4a4zZs3u71797rZs2e7/Px8d/r0aePOY+tSx6GlpcU9+OCDrqamxjU0NLg33njDfelLX3JXXXWVO3PmjHXrMbN48WIXCARcVVWVO3r0aHg5depUeJ/777/fjRgxwm3bts3t3LnTFRUVuaKiIsOuY+9yx6G+vt59//vfdzt37nQNDQ1u8+bNbtSoUW7atGnGnUfqFQHknHPPPvusGzFihEtJSXGTJ092tbW11i11uzvvvNPl5OS4lJQUd+WVV7o777zT1dfXW7cVd2+++aaTdMEyb94859z5R7EfffRRl5WV5fx+v5sxY4arq6uzbToOLnUcTp065WbOnOmGDRvmkpOT3ciRI93ChQsT7oe0rv7+ktzatWvD+5w+fdp985vfdJ/73OfcoEGD3O233+6OHj1q13QcXO44HDp0yE2bNs2lp6c7v9/vxowZ47773e+6YDBo2/gn8HlAAAATPf4eEAAgMRFAAAATBBAAwAQBBAAwQQABAEwQQAAAEwQQAMAEAQQAMEEAAQBMEEAAABMEEADAxP8HJs5xFNAKNQQAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(X_train[1][:,:,0])\n",
    "plt.title(y_train[1].argmax());"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import Input,InputLayer, Dense, Activation, ZeroPadding2D, BatchNormalization, Flatten, Conv2D\n",
    "from keras.layers import AveragePooling2D, MaxPooling2D, Dropout\n",
    "from keras.models import Sequential,Model\n",
    "from keras.optimizers import SGD\n",
    "from keras.callbacks import ModelCheckpoint,LearningRateScheduler\n",
    "import keras\n",
    "from keras import backend as K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"Predict\"\n",
      "_________________________________________________________________\n",
      " Layer (type)                Output Shape              Param #   \n",
      "=================================================================\n",
      " input_1 (InputLayer)        [(None, 28, 28, 1)]       0         \n",
      "                                                                 \n",
      " layer_conv1 (Conv2D)        (None, 28, 28, 64)        640       \n",
      "                                                                 \n",
      " batch_normalization (Batch  (None, 28, 28, 64)        256       \n",
      " Normalization)                                                  \n",
      "                                                                 \n",
      " activation (Activation)     (None, 28, 28, 64)        0         \n",
      "                                                                 \n",
      " maxPool1 (MaxPooling2D)     (None, 14, 14, 64)        0         \n",
      "                                                                 \n",
      " layer_conv2 (Conv2D)        (None, 14, 14, 32)        18464     \n",
      "                                                                 \n",
      " batch_normalization_1 (Bat  (None, 14, 14, 32)        128       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " activation_1 (Activation)   (None, 14, 14, 32)        0         \n",
      "                                                                 \n",
      " maxPool2 (MaxPooling2D)     (None, 7, 7, 32)          0         \n",
      "                                                                 \n",
      " conv3 (Conv2D)              (None, 7, 7, 32)          9248      \n",
      "                                                                 \n",
      " batch_normalization_2 (Bat  (None, 7, 7, 32)          128       \n",
      " chNormalization)                                                \n",
      "                                                                 \n",
      " activation_2 (Activation)   (None, 7, 7, 32)          0         \n",
      "                                                                 \n",
      " maxPool3 (MaxPooling2D)     (None, 3, 3, 32)          0         \n",
      "                                                                 \n",
      " flatten (Flatten)           (None, 288)               0         \n",
      "                                                                 \n",
      " fc0 (Dense)                 (None, 64)                18496     \n",
      "                                                                 \n",
      " dropout (Dropout)           (None, 64)                0         \n",
      "                                                                 \n",
      " fc1 (Dense)                 (None, 32)                2080      \n",
      "                                                                 \n",
      " dropout_1 (Dropout)         (None, 32)                0         \n",
      "                                                                 \n",
      " fc2 (Dense)                 (None, 10)                330       \n",
      "                                                                 \n",
      "=================================================================\n",
      "Total params: 49770 (194.41 KB)\n",
      "Trainable params: 49514 (193.41 KB)\n",
      "Non-trainable params: 256 (1.00 KB)\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Building a CNN model\n",
    "input_shape = (28,28,1)\n",
    "X_input = Input(input_shape)\n",
    "\n",
    "# layer 1\n",
    "x = Conv2D(64,(3,3),strides=(1,1),name='layer_conv1',padding='same')(X_input)\n",
    "x = BatchNormalization()(x)\n",
    "x = Activation('relu')(x)\n",
    "x = MaxPooling2D((2,2),name='maxPool1')(x)\n",
    "# layer 2\n",
    "x = Conv2D(32,(3,3),strides=(1,1),name='layer_conv2',padding='same')(x)\n",
    "x = BatchNormalization()(x)\n",
    "x = Activation('relu')(x)\n",
    "x = MaxPooling2D((2,2),name='maxPool2')(x)\n",
    "# layer 3\n",
    "x = Conv2D(32,(3,3),strides=(1,1),name='conv3',padding='same')(x)\n",
    "x = BatchNormalization()(x)\n",
    "x = Activation('relu')(x)\n",
    "x = MaxPooling2D((2,2), name='maxPool3')(x)\n",
    "# fc\n",
    "x = Flatten()(x)\n",
    "x = Dense(64,activation ='relu',name='fc0')(x)\n",
    "x = Dropout(0.25)(x)\n",
    "x = Dense(32,activation ='relu',name='fc1')(x)\n",
    "x = Dropout(0.25)(x)\n",
    "x = Dense(10,activation ='softmax',name='fc2')(x)\n",
    "\n",
    "conv_model = Model(inputs=X_input, outputs=x, name='Predict')\n",
    "conv_model.summary()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "378/378 [==============================] - 8s 16ms/step - loss: 0.5794 - accuracy: 0.8144 - val_loss: 1.3297 - val_accuracy: 0.5229\n",
      "Epoch 2/10\n",
      "378/378 [==============================] - 6s 16ms/step - loss: 0.1322 - accuracy: 0.9632 - val_loss: 0.0625 - val_accuracy: 0.9831\n",
      "Epoch 3/10\n",
      "378/378 [==============================] - 6s 15ms/step - loss: 0.0945 - accuracy: 0.9736 - val_loss: 0.0551 - val_accuracy: 0.9821\n",
      "Epoch 4/10\n",
      "378/378 [==============================] - 6s 15ms/step - loss: 0.0754 - accuracy: 0.9797 - val_loss: 0.0532 - val_accuracy: 0.9836\n",
      "Epoch 5/10\n",
      "378/378 [==============================] - 6s 15ms/step - loss: 0.0627 - accuracy: 0.9822 - val_loss: 0.0755 - val_accuracy: 0.9783\n",
      "Epoch 6/10\n",
      "378/378 [==============================] - 6s 15ms/step - loss: 0.0547 - accuracy: 0.9847 - val_loss: 0.0419 - val_accuracy: 0.9883\n",
      "Epoch 7/10\n",
      "378/378 [==============================] - 6s 16ms/step - loss: 0.0516 - accuracy: 0.9853 - val_loss: 0.0556 - val_accuracy: 0.9869\n",
      "Epoch 8/10\n",
      "378/378 [==============================] - 6s 16ms/step - loss: 0.0445 - accuracy: 0.9873 - val_loss: 0.0532 - val_accuracy: 0.9867\n",
      "Epoch 9/10\n",
      "378/378 [==============================] - 6s 16ms/step - loss: 0.0399 - accuracy: 0.9889 - val_loss: 0.0559 - val_accuracy: 0.9848\n",
      "Epoch 10/10\n",
      "378/378 [==============================] - 6s 16ms/step - loss: 0.0370 - accuracy: 0.9894 - val_loss: 0.0516 - val_accuracy: 0.9871\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.src.callbacks.History at 0x7f01f8c42d60>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Adam optimizer\n",
    "conv_model.compile(optimizer='adam',loss='categorical_crossentropy',metrics=['accuracy'])\n",
    "conv_model.fit(X_train, y_train, epochs=10, batch_size=100, validation_data=(X_cv,y_cv))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "1182/1182 [==============================] - 13s 10ms/step - loss: 0.0289 - accuracy: 0.9923 - val_loss: 0.0370 - val_accuracy: 0.9900 - lr: 5.0000e-04\n",
      "Epoch 2/30\n",
      "1182/1182 [==============================] - 12s 10ms/step - loss: 0.0252 - accuracy: 0.9931 - val_loss: 0.0358 - val_accuracy: 0.9907 - lr: 5.0000e-04\n",
      "Epoch 3/30\n",
      "1182/1182 [==============================] - 11s 10ms/step - loss: 0.0229 - accuracy: 0.9937 - val_loss: 0.0361 - val_accuracy: 0.9914 - lr: 5.0000e-04\n",
      "Epoch 4/30\n",
      "1182/1182 [==============================] - 11s 10ms/step - loss: 0.0212 - accuracy: 0.9945 - val_loss: 0.0352 - val_accuracy: 0.9914 - lr: 5.0000e-04\n",
      "Epoch 5/30\n",
      "1182/1182 [==============================] - 12s 10ms/step - loss: 0.0214 - accuracy: 0.9937 - val_loss: 0.0350 - val_accuracy: 0.9907 - lr: 5.0000e-04\n",
      "Epoch 6/30\n",
      "1182/1182 [==============================] - 12s 10ms/step - loss: 0.0208 - accuracy: 0.9944 - val_loss: 0.0348 - val_accuracy: 0.9910 - lr: 5.0000e-04\n",
      "Epoch 7/30\n",
      "1182/1182 [==============================] - 12s 10ms/step - loss: 0.0192 - accuracy: 0.9954 - val_loss: 0.0345 - val_accuracy: 0.9910 - lr: 5.0000e-04\n",
      "Epoch 8/30\n",
      "1182/1182 [==============================] - 12s 10ms/step - loss: 0.0196 - accuracy: 0.9951 - val_loss: 0.0346 - val_accuracy: 0.9907 - lr: 5.0000e-04\n",
      "Epoch 9/30\n",
      "1182/1182 [==============================] - 12s 10ms/step - loss: 0.0194 - accuracy: 0.9947 - val_loss: 0.0343 - val_accuracy: 0.9898 - lr: 5.0000e-04\n",
      "Epoch 10/30\n",
      "1182/1182 [==============================] - 11s 10ms/step - loss: 0.0191 - accuracy: 0.9945 - val_loss: 0.0344 - val_accuracy: 0.9905 - lr: 5.0000e-04\n",
      "Epoch 11/30\n",
      "1182/1182 [==============================] - 11s 10ms/step - loss: 0.0190 - accuracy: 0.9945 - val_loss: 0.0344 - val_accuracy: 0.9910 - lr: 5.0000e-05\n",
      "Epoch 12/30\n",
      "1182/1182 [==============================] - 12s 10ms/step - loss: 0.0172 - accuracy: 0.9950 - val_loss: 0.0345 - val_accuracy: 0.9910 - lr: 5.0000e-05\n",
      "Epoch 13/30\n",
      "1182/1182 [==============================] - 12s 10ms/step - loss: 0.0180 - accuracy: 0.9957 - val_loss: 0.0344 - val_accuracy: 0.9907 - lr: 5.0000e-05\n",
      "Epoch 14/30\n",
      "1182/1182 [==============================] - 11s 10ms/step - loss: 0.0172 - accuracy: 0.9953 - val_loss: 0.0345 - val_accuracy: 0.9910 - lr: 5.0000e-05\n",
      "Epoch 15/30\n",
      "1182/1182 [==============================] - 11s 10ms/step - loss: 0.0198 - accuracy: 0.9945 - val_loss: 0.0344 - val_accuracy: 0.9905 - lr: 5.0000e-05\n",
      "Epoch 16/30\n",
      "1182/1182 [==============================] - 11s 10ms/step - loss: 0.0185 - accuracy: 0.9943 - val_loss: 0.0343 - val_accuracy: 0.9905 - lr: 5.0000e-05\n",
      "Epoch 17/30\n",
      "1182/1182 [==============================] - 11s 9ms/step - loss: 0.0190 - accuracy: 0.9948 - val_loss: 0.0343 - val_accuracy: 0.9905 - lr: 5.0000e-05\n",
      "Epoch 18/30\n",
      "1182/1182 [==============================] - 11s 9ms/step - loss: 0.0176 - accuracy: 0.9954 - val_loss: 0.0344 - val_accuracy: 0.9905 - lr: 5.0000e-05\n",
      "Epoch 19/30\n",
      "1182/1182 [==============================] - 11s 10ms/step - loss: 0.0183 - accuracy: 0.9948 - val_loss: 0.0343 - val_accuracy: 0.9905 - lr: 5.0000e-05\n",
      "Epoch 20/30\n",
      "1182/1182 [==============================] - 11s 10ms/step - loss: 0.0177 - accuracy: 0.9953 - val_loss: 0.0342 - val_accuracy: 0.9907 - lr: 5.0000e-05\n",
      "Epoch 21/30\n",
      "1182/1182 [==============================] - 11s 10ms/step - loss: 0.0187 - accuracy: 0.9950 - val_loss: 0.0343 - val_accuracy: 0.9907 - lr: 5.0000e-05\n",
      "Epoch 22/30\n",
      "1182/1182 [==============================] - 11s 10ms/step - loss: 0.0185 - accuracy: 0.9950 - val_loss: 0.0343 - val_accuracy: 0.9907 - lr: 5.0000e-05\n",
      "Epoch 23/30\n",
      "1182/1182 [==============================] - 11s 9ms/step - loss: 0.0185 - accuracy: 0.9951 - val_loss: 0.0344 - val_accuracy: 0.9910 - lr: 5.0000e-05\n",
      "Epoch 24/30\n",
      "1182/1182 [==============================] - 12s 10ms/step - loss: 0.0173 - accuracy: 0.9950 - val_loss: 0.0344 - val_accuracy: 0.9910 - lr: 5.0000e-05\n",
      "Epoch 25/30\n",
      "1182/1182 [==============================] - 11s 10ms/step - loss: 0.0186 - accuracy: 0.9948 - val_loss: 0.0344 - val_accuracy: 0.9902 - lr: 5.0000e-05\n",
      "Epoch 26/30\n",
      "1182/1182 [==============================] - 11s 10ms/step - loss: 0.0161 - accuracy: 0.9955 - val_loss: 0.0346 - val_accuracy: 0.9905 - lr: 5.0000e-05\n",
      "Epoch 27/30\n",
      "1182/1182 [==============================] - 11s 10ms/step - loss: 0.0178 - accuracy: 0.9955 - val_loss: 0.0343 - val_accuracy: 0.9900 - lr: 5.0000e-05\n",
      "Epoch 28/30\n",
      "1182/1182 [==============================] - 11s 10ms/step - loss: 0.0175 - accuracy: 0.9955 - val_loss: 0.0344 - val_accuracy: 0.9902 - lr: 5.0000e-05\n",
      "Epoch 29/30\n",
      "1182/1182 [==============================] - 11s 10ms/step - loss: 0.0162 - accuracy: 0.9955 - val_loss: 0.0345 - val_accuracy: 0.9907 - lr: 5.0000e-05\n",
      "Epoch 30/30\n",
      "1182/1182 [==============================] - 11s 9ms/step - loss: 0.0188 - accuracy: 0.9952 - val_loss: 0.0344 - val_accuracy: 0.9907 - lr: 5.0000e-05\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.src.callbacks.History at 0x7f02dc281d90>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from keras.optimizers import SGD\n",
    "from keras.callbacks import LearningRateScheduler\n",
    "\n",
    "def learning_rate_schedule(epoch):\n",
    "    initial_lr = 0.0005\n",
    "    decay = 0.0\n",
    "    if epoch < 10:\n",
    "        return initial_lr\n",
    "    else:\n",
    "        return initial_lr * 0.1  # You can adjust the decay rate as needed\n",
    "\n",
    "# SGD optimizer with momentum\n",
    "sgd = SGD(learning_rate=0.0005, momentum=0.5, nesterov=False) \n",
    "\n",
    "# Compile model with SGD optimizer and categorical crossentropy loss\n",
    "conv_model.compile(optimizer=sgd, loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "# Learning rate schedule callback\n",
    "lr_schedule = LearningRateScheduler(learning_rate_schedule)\n",
    "\n",
    "# Train the model with learning rate schedule\n",
    "conv_model.fit(X_train, y_train, epochs=30, validation_data=(X_cv, y_cv), callbacks=[lr_schedule])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "875/875 [==============================] - 3s 4ms/step\n"
     ]
    }
   ],
   "source": [
    "y_pred = conv_model.predict(X_test)\n",
    "y_pred = np.argmax(y_pred,axis=1)\n",
    "my_submission = pd.DataFrame({'ImageId': list(range(1, len(y_pred)+1)), 'Label': y_pred})\n",
    "my_submission.to_csv('submission.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Case 2 - Train on half of training set and test on the other half of training set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load Data\n",
    "train2 = pd.read_csv('train_halfImages.csv')\n",
    "test2 = pd.read_csv('test_halfImages.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train Data\n",
    "y_train = train1['label'].astype('float32')\n",
    "X_train = train1.drop('label', axis=1).astype('int32')\n",
    "X_test = test1.astype('int32')\n",
    "X_train.shape, y_train.shape, X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.countplot(x='label', data = train1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Data normalization\n",
    "X_train = X_train/255\n",
    "X_test = X_test/255"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = X_train.values.reshape(-1,28,28,1)\n",
    "X_test = X_test.values.reshape(-1,28,28,1)\n",
    "X_train.shape, X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# one-hot encoding\n",
    "from keras.utils import to_categorical\n",
    "y_train = to_categorical(y_train, num_classes = 10)\n",
    "y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(train1['label'].head())\n",
    "y_train[0:5,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_cv, y_train, y_cv = train_test_split(X_train, y_train, test_size = 0.1, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.imshow(X_train[1][:,:,0])\n",
    "plt.title(y_train[1].argmax());"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Building a CNN model\n",
    "input_shape = (28,28,1)\n",
    "X_input = Input(input_shape)\n",
    "\n",
    "# layer 1\n",
    "x = Conv2D(64,(3,3),strides=(1,1),name='layer_conv1',padding='same')(X_input)\n",
    "x = BatchNormalization()(x)\n",
    "x = Activation('relu')(x)\n",
    "x = MaxPooling2D((2,2),name='maxPool1')(x)\n",
    "# layer 2\n",
    "x = Conv2D(32,(3,3),strides=(1,1),name='layer_conv2',padding='same')(x)\n",
    "x = BatchNormalization()(x)\n",
    "x = Activation('relu')(x)\n",
    "x = MaxPooling2D((2,2),name='maxPool2')(x)\n",
    "# layer 3\n",
    "x = Conv2D(32,(3,3),strides=(1,1),name='conv3',padding='same')(x)\n",
    "x = BatchNormalization()(x)\n",
    "x = Activation('relu')(x)\n",
    "x = MaxPooling2D((2,2), name='maxPool3')(x)\n",
    "# fc\n",
    "x = Flatten()(x)\n",
    "x = Dense(64,activation ='relu',name='fc0')(x)\n",
    "x = Dropout(0.25)(x)\n",
    "x = Dense(32,activation ='relu',name='fc1')(x)\n",
    "x = Dropout(0.25)(x)\n",
    "x = Dense(10,activation ='softmax',name='fc2')(x)\n",
    "\n",
    "conv_model = Model(inputs=X_input, outputs=x, name='Predict')\n",
    "conv_model.summary()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adam optimizer\n",
    "conv_model.compile(optimizer='adam',loss='categorical_crossentropy',metrics=['accuracy'])\n",
    "conv_model.fit(X_train, y_train, epochs=10, batch_size=100, validation_data=(X_cv,y_cv))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.optimizers import SGD\n",
    "from keras.callbacks import LearningRateScheduler\n",
    "\n",
    "def learning_rate_schedule(epoch):\n",
    "    initial_lr = 0.0005\n",
    "    decay = 0.0\n",
    "    if epoch < 10:\n",
    "        return initial_lr\n",
    "    else:\n",
    "        return initial_lr * 0.1  # You can adjust the decay rate as needed\n",
    "\n",
    "# SGD optimizer with momentum\n",
    "sgd = SGD(learning_rate=0.0005, momentum=0.5, nesterov=False) \n",
    "\n",
    "# Compile model with SGD optimizer and categorical crossentropy loss\n",
    "conv_model.compile(optimizer=sgd, loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "# Learning rate schedule callback\n",
    "lr_schedule = LearningRateScheduler(learning_rate_schedule)\n",
    "\n",
    "# Train the model with learning rate schedule\n",
    "conv_model.fit(X_train, y_train, epochs=30, validation_data=(X_cv, y_cv), callbacks=[lr_schedule])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = conv_model.predict(X_test)\n",
    "y_pred = np.argmax(y_pred,axis=1)\n",
    "my_submission = pd.DataFrame({'ImageId': list(range(1, len(y_pred)+1)), 'Label': y_pred})\n",
    "my_submission.to_csv('submission2.csv', index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Case 3 - Train on lables 0-4 and test on other half of labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load Data\n",
    "train3 = pd.read_csv('train_halfDigits.csv')\n",
    "test3 = pd.read_csv('test_halfDigits.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Train Data\n",
    "y_train = train1['label'].astype('float32')\n",
    "X_train = train1.drop('label', axis=1).astype('int32')\n",
    "X_test = test1.astype('int32')\n",
    "X_train.shape, y_train.shape, X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.countplot(x='label', data = train1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Data normalization\n",
    "X_train = X_train/255\n",
    "X_test = X_test/255"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = X_train.values.reshape(-1,28,28,1)\n",
    "X_test = X_test.values.reshape(-1,28,28,1)\n",
    "X_train.shape, X_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# one-hot encoding\n",
    "from keras.utils import to_categorical\n",
    "y_train = to_categorical(y_train, num_classes = 10)\n",
    "y_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(train1['label'].head())\n",
    "y_train[0:5,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "X_train, X_cv, y_train, y_cv = train_test_split(X_train, y_train, test_size = 0.1, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.imshow(X_train[1][:,:,0])\n",
    "plt.title(y_train[1].argmax());"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Building a CNN model\n",
    "input_shape = (28,28,1)\n",
    "X_input = Input(input_shape)\n",
    "\n",
    "# layer 1\n",
    "x = Conv2D(64,(3,3),strides=(1,1),name='layer_conv1',padding='same')(X_input)\n",
    "x = BatchNormalization()(x)\n",
    "x = Activation('relu')(x)\n",
    "x = MaxPooling2D((2,2),name='maxPool1')(x)\n",
    "# layer 2\n",
    "x = Conv2D(32,(3,3),strides=(1,1),name='layer_conv2',padding='same')(x)\n",
    "x = BatchNormalization()(x)\n",
    "x = Activation('relu')(x)\n",
    "x = MaxPooling2D((2,2),name='maxPool2')(x)\n",
    "# layer 3\n",
    "x = Conv2D(32,(3,3),strides=(1,1),name='conv3',padding='same')(x)\n",
    "x = BatchNormalization()(x)\n",
    "x = Activation('relu')(x)\n",
    "x = MaxPooling2D((2,2), name='maxPool3')(x)\n",
    "# fc\n",
    "x = Flatten()(x)\n",
    "x = Dense(64,activation ='relu',name='fc0')(x)\n",
    "x = Dropout(0.25)(x)\n",
    "x = Dense(32,activation ='relu',name='fc1')(x)\n",
    "x = Dropout(0.25)(x)\n",
    "x = Dense(10,activation ='softmax',name='fc2')(x)\n",
    "\n",
    "conv_model = Model(inputs=X_input, outputs=x, name='Predict')\n",
    "conv_model.summary()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adam optimizer\n",
    "conv_model.compile(optimizer='adam',loss='categorical_crossentropy',metrics=['accuracy'])\n",
    "conv_model.fit(X_train, y_train, epochs=10, batch_size=100, validation_data=(X_cv,y_cv))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.optimizers import SGD\n",
    "from keras.callbacks import LearningRateScheduler\n",
    "\n",
    "def learning_rate_schedule(epoch):\n",
    "    initial_lr = 0.0005\n",
    "    decay = 0.0\n",
    "    if epoch < 10:\n",
    "        return initial_lr\n",
    "    else:\n",
    "        return initial_lr * 0.1  # You can adjust the decay rate as needed\n",
    "\n",
    "# SGD optimizer with momentum\n",
    "sgd = SGD(learning_rate=0.0005, momentum=0.5, nesterov=False) \n",
    "\n",
    "# Compile model with SGD optimizer and categorical crossentropy loss\n",
    "conv_model.compile(optimizer=sgd, loss='categorical_crossentropy', metrics=['accuracy'])\n",
    "\n",
    "# Learning rate schedule callback\n",
    "lr_schedule = LearningRateScheduler(learning_rate_schedule)\n",
    "\n",
    "# Train the model with learning rate schedule\n",
    "conv_model.fit(X_train, y_train, epochs=30, validation_data=(X_cv, y_cv), callbacks=[lr_schedule])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_pred = conv_model.predict(X_test)\n",
    "y_pred = np.argmax(y_pred,axis=1)\n",
    "my_submission = pd.DataFrame({'ImageId': list(range(1, len(y_pred)+1)), 'Label': y_pred})\n",
    "my_submission.to_csv('submission3.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}