IOT23_Compilation.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "IOT23_Compilation",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/AAnirudh07/Aposemat-IoT23-Network-Classification/blob/main/IOT23_Compilation.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6QvARz5qOcTR",
        "outputId": "2b8e826f-9ff6-49f7-e877-176ae2b6d913"
      },
      "source": [
        "\n",
        "#File Read\n",
        "import pandas as pd\n",
        "from sklearn.svm import SVC\n",
        "from sklearn.model_selection import GridSearchCV\n",
        "from sklearn.metrics import classification_report, confusion_matrix\n",
        "from sklearn.model_selection import train_test_split\n",
        "import time\n",
        "import numpy as np\n",
        "import csv\n",
        "import os\n",
        " \n",
        "main=\"/content/drive/MyDrive/IoTScenarios\"\n",
        "fols=[main+\"/\"+i for i in os.listdir(main)]\n",
        "files=[]\n",
        "for i in fols:\n",
        "    if(\"points\" in i or \"csv\" in i):\n",
        "      continue\n",
        "    F=os.path.join(i,os.listdir(i)[0])\n",
        "    while(F.find(\"conn.log.labeled\")==-1):\n",
        "        F=F+\"//\"+os.listdir(F)[0]\n",
        "    files.append(F)\n",
        " \n",
        "label='tunnel_parents   label   detailed-label'\n",
        "j=0\n",
        "Dict=dict()\n",
        "li=[]\n",
        "li2=[]\n",
        "for fi in files:\n",
        "    j=0\n",
        "    tsv_file = open(fi)\n",
        "    read_tsv = csv.reader(tsv_file, delimiter=\"\\t\")\n",
        "    for i in read_tsv:\n",
        "        j=j+1\n",
        "        if(j==7):\n",
        "            li=i\n",
        "        if(j==8):\n",
        "            li2=i\n",
        "            for key in range(1,len(li)):\n",
        "                Dict[li[key]]=li2[key]\n",
        "        if(j==9):\n",
        "            break\n",
        "#open(os.path.join(main,'my_csv.csv'),\"w\").close()\n",
        "#CONCATTING CHANGE\n",
        " \n",
        "frames=[]\n",
        "j=0\n",
        "print(\"All cols:\",Dict.keys())\n",
        "cols=['duration', 'orig_bytes', 'resp_bytes', 'missed_bytes', 'orig_pkts', 'orig_ip_bytes', 'resp_pkts', 'resp_ip_bytes', 'label']\n",
        "cols12 = [\"ts\",\"uid\",\"id.orig_h\",\"id.orig_p\",\"id.resp_h\",\"id.resp_p\",\"proto\",\"service\",\"duration\",\"orig_bytes\",\"resp_bytes\",\"conn_state\",\"local_orig\",\"local_resp\",\"missed_bytes\",\"history\",\"orig_pkts\",\"orig_ip_bytes\",\"resp_pkts\",\"resp_ip_bytes\",\"label\"]\n",
        "print(cols)\n",
        "for fi in files:\n",
        "    df = pd.read_csv(fi, sep=\"\\t\",skiprows=10,names=cols12,nrows=10000)\n",
        "    df.rename(columns={label:'label'},inplace=True)\n",
        "    df.drop(df.tail(1).index,inplace=True)\n",
        "    df.label = df.label.apply(lambda x: 'B' if ('benign' in x or 'Benign' in x) else 'A')\n",
        "    #df.to_csv(os.path.join(main,'my_csv.csv'), mode='a', header=False)\n",
        "    frames.append(df)\n",
        "  \n",
        "#CONCATTING CHANGE\n",
        "df=pd.concat(frames)\n",
        "print(len(df))# nrows takes min(nrows,actual)\n",
        "# df = pd.get_dummies(df, columns=['proto'])\n",
        "# df = pd.get_dummies(df, columns=['conn_state'])\n",
        "# df = pd.get_dummies(df, columns=['service'])\n",
        "\n",
        "\n",
        "main_train=['duration', 'orig_bytes', 'resp_bytes', 'missed_bytes', 'orig_pkts', 'orig_ip_bytes', 'resp_pkts', 'resp_ip_bytes', 'proto_icmp', 'proto_tcp', 'proto_udp', 'conn_state_OTH', 'conn_state_REJ', 'conn_state_RSTO', 'conn_state_RSTOS0', 'conn_state_RSTR', 'conn_state_RSTRH', 'conn_state_S0', 'conn_state_S1', 'conn_state_S2', 'conn_state_S3', 'conn_state_SF', 'conn_state_SH', 'conn_state_SHR']\n",
        "#actual_col=Dict.keys()\n",
        "actual_col=df.columns\n",
        "X_cols=['label']\n",
        "for i in main_train:\n",
        "    if(i in actual_col):\n",
        "        X_cols.append(i)\n",
        " \n",
        "print(\"X_cols\",X_cols)\n",
        "df.drop_duplicates(keep=\"first\",inplace=True)\n",
        " \n",
        "X_train=df[X_cols]\n",
        " \n",
        " \n",
        "drop_cols=[]\n",
        "#Directly read from \n",
        "output=\"label\" \n",
        "if(False):\n",
        "  X_train=pd.read_csv(\"/content/drive/MyDrive/IOT23/iot_traindata.csv\")\n",
        "#comment to include from main folder\n",
        " \n",
        " \n",
        " \n",
        "########################################################################\n",
        "# finding mean of duration\n",
        "temp_duration  = X_train.loc[X_train['duration']!='-','duration'].astype('float16')\n",
        "index_inf_duration = temp_duration[temp_duration.isin([-np.inf, np.inf])]\n",
        "#print(\"Hello\",index_inf_duration)\n",
        "temp_duration.drop([448,449], inplace = True)\n",
        "avg_duration = np.asarray(temp_duration).mean()\n",
        "X_train.loc[(X_train['duration']=='-'), 'duration'] = str(avg_duration )\n",
        "# drop missed bytes column\n",
        "X_train.drop([448,449], inplace = True)\n",
        "#X_train.drop(columns=['missed_bytes'], inplace=True)\n",
        "X_train['duration'] = X_train['duration'].astype('float16')\n",
        " \n",
        "# detecting outliers\n",
        "# #   w.r.t duration\n",
        "# Q1 = np.percentile(X_train['duration'], 25, interpolation = 'midpoint')   \n",
        "# Q3 = np.percentile(X_train['duration'], 75, interpolation = 'midpoint') \n",
        "# IQR = Q3 - Q1 \n",
        "# upper = np.where(X_train['duration'] >= (Q3+1.5*IQR))\n",
        "# lower = np.where(X_train['duration'] <= (Q1-1.5*IQR))\n",
        "# print(type(upper[0]))\n",
        "# X_train.drop(upper[0][1], inplace=True)\n",
        "# X_train.drop(lower, inplace=True)\n",
        "X_train.reset_index(drop=True)\n",
        "# Change in #rows after removing outliers : 31613 -> 28700\n",
        "#########################################################################\n",
        "X_train=X_train.replace(to_replace =\"-\",value =\"0\")\n",
        " \n",
        "#output=\"output\"\n",
        "Y = X_train[[output]]\n",
        "Y_train=Y\n",
        "X_train.drop(columns=[output], inplace=True)\n",
        "X_train['duration'] = X_train['duration'].astype('float64')\n",
        "X_train['orig_bytes'] = X_train['orig_bytes'].astype('float64')\n",
        "X_train['resp_bytes'] = X_train['resp_bytes'].astype('float64')\n",
        "X=X_train"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "All cols: dict_keys(['ts', 'uid', 'id.orig_h', 'id.orig_p', 'id.resp_h', 'id.resp_p', 'proto', 'service', 'duration', 'orig_bytes', 'resp_bytes', 'conn_state', 'local_orig', 'local_resp', 'missed_bytes', 'history', 'orig_pkts', 'orig_ip_bytes', 'resp_pkts', 'resp_ip_bytes', 'tunnel_parents   label   detailed-label'])\n",
            "['duration', 'orig_bytes', 'resp_bytes', 'missed_bytes', 'orig_pkts', 'orig_ip_bytes', 'resp_pkts', 'resp_ip_bytes', 'label']\n",
            "173084\n",
            "X_cols ['label', 'duration', 'orig_bytes', 'resp_bytes', 'missed_bytes', 'orig_pkts', 'orig_ip_bytes', 'resp_pkts', 'resp_ip_bytes']\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/pandas/core/indexing.py:1763: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  isetter(loc, value)\n",
            "/usr/local/lib/python3.7/dist-packages/pandas/core/frame.py:4174: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  errors=errors,\n",
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:102: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5eqjx4IXB05J",
        "outputId": "1bad435f-c404-424e-a48f-d9020c6496c2"
      },
      "source": [
        "import pandas as pd\n",
        "import os\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive',force_remount=True)\n",
        "os.chdir('/content/drive/My Drive/IoTScenarios')\n",
        "print(\"Change successful.\")\n",
        "import glob\n",
        "file_names = [i for i in glob.glob('**/**/*')]\n",
        "file_names[-3]+='/conn.log.labeled'\n",
        "#file_names+='/conn.log.labeled'\n",
        "file_names\n",
        "#now combine all the files into 1 single dataframe\n",
        "cols = [\"ts\",\"uid\",\"id.orig_h\",\"id.orig_p\",\"id.resp_h\",\"id.resp_p\",\"proto\",\"service\",\"duration\",\"orig_bytes\",\"resp_bytes\",\"conn_state\",\"local_orig\",\"local_resp\",\"missed_bytes\",\"history\",\"orig_pkts\",\"orig_ip_bytes\",\"resp_pkts\",\"resp_ip_bytes\",\"label\"]\n",
        "iotdf = pd.DataFrame(columns=cols)\n",
        "# for file in file_names:\n",
        "\n",
        "for file in file_names: \n",
        "  currdf = pd.read_csv(file,sep='\\t',nrows=10000,names=cols,skiprows=10)\n",
        "  currdf.drop(currdf.tail(1).index,inplace=True)\n",
        "  iotdf = pd.concat([iotdf,currdf])\n",
        "print(\"Concatenation complete.\")\n",
        "iotdf['service'].replace('-','new_service',inplace=True)\n",
        "niotdf = iotdf.drop(columns=['ts','uid','local_orig','local_resp','missed_bytes'])\n",
        "\n",
        "import numpy as np\n",
        "niotdf.replace('-', np.nan,inplace=True)\n",
        "ans = niotdf.groupby('label')\n",
        "clean_niotdf = niotdf.apply(lambda x: x.fillna(x.value_counts().index[0]))\n",
        "iotcp = clean_niotdf.copy()\n",
        "iotcp = iotcp.replace(to_replace ='.*[bB]enign.*', value = 'Benign', regex = True)\n",
        "iotcp = iotcp.replace(to_replace ='.*Malicious.*', value = 'Malicious', regex = True)\n",
        "iotcp['duration'] = iotcp['duration'].astype('float64')\n",
        "iotcp['orig_bytes'] = iotcp['orig_bytes'].astype('float64')\n",
        "iotcp['resp_bytes'] = iotcp['resp_bytes'].astype('float64')\n",
        "#import the necessary libraries\n",
        "import seaborn as sns\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "import numpy as np\n",
        "np.random.seed(0)\n",
        "#train_iot.shape\n",
        "print(iotcp.columns)\n",
        "iotcp = iotcp.drop(columns=['id.orig_h','id.resp_h'])\n",
        "iotcp= pd.get_dummies(iotcp, columns=['proto'])\n",
        "iotcp = pd.get_dummies(iotcp, columns=['conn_state'])\n",
        "iotcp = pd.get_dummies(iotcp, columns=['service'])\n",
        "from sklearn.metrics import classification_report, confusion_matrix\n",
        "from sklearn.model_selection import train_test_split\n",
        "y = iotcp['label']\n",
        "x = iotcp.drop(columns=['label'])\n",
        "print(x.head())\n",
        "Y=y\n",
        "X=x"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n",
            "Change successful.\n",
            "Concatenation complete.\n",
            "Index(['id.orig_h', 'id.orig_p', 'id.resp_h', 'id.resp_p', 'proto', 'service',\n",
            "       'duration', 'orig_bytes', 'resp_bytes', 'conn_state', 'history',\n",
            "       'orig_pkts', 'orig_ip_bytes', 'resp_pkts', 'resp_ip_bytes', 'label'],\n",
            "      dtype='object')\n",
            "   id.orig_p  id.resp_p  ...  service_ssh  service_ssl\n",
            "0    41101.0       23.0  ...            0            0\n",
            "1    60905.0       23.0  ...            0            0\n",
            "2    44301.0       23.0  ...            0            0\n",
            "3    50244.0       23.0  ...            0            0\n",
            "4    34243.0    49560.0  ...            0            0\n",
            "\n",
            "[5 rows x 33 columns]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3b82CT3HgHwO",
        "outputId": "d36210e8-fd7c-42ee-e71d-b7b34dfe04c9"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-Ar2pdb9oW1A"
      },
      "source": [
        "import pandas as pd\n",
        "\n",
        "import numpy as np\n",
        "\n",
        "import seaborn as sns\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "from sklearn.metrics import classification_report, confusion_matrix, f1_score, precision_score, recall_score, accuracy_score\n",
        "\n",
        "from sklearn.tree import DecisionTreeClassifier\n",
        "\n",
        "from scipy import stats\n",
        "\n",
        "import time\n",
        "\n",
        "from sklearn.preprocessing import RobustScaler"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GkvBg1neOjIv",
        "outputId": "25020b42-c168-4edd-d8fe-a7dd66b4bd95"
      },
      "source": [
        "#Decision Tree\n",
        "\n",
        "#X_train, X_test, Y_train, Y_test = train_test_split(X, Y, random_state=10, shuffle=True ,test_size=0.2)\n",
        "DT = DecisionTreeClassifier()\n",
        "start = time.time()\n",
        "print('program start...')\n",
        "print()\n",
        "import seaborn as sns\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "import numpy as np\n",
        "from sklearn.metrics import classification_report, confusion_matrix, f1_score, precision_score, recall_score, accuracy_score\n",
        "np.random.seed(0)\n",
        "xtrain, xtest, ytrain, ytest = train_test_split(X,Y,test_size =0.3,stratify=Y,random_state=10)\n",
        "ydf = pd.DataFrame(ytrain)\n",
        "from sklearn import preprocessing\n",
        "le = preprocessing.LabelEncoder()\n",
        "for column_name in xtrain.columns:\n",
        "        if xtrain[column_name].dtype == object:\n",
        "            print(column_name)\n",
        "            xtrain[column_name] = le.fit_transform(xtrain[column_name])\n",
        "\n",
        "        else:\n",
        "            pass\n",
        "for column_name in xtest.columns:\n",
        "        if xtest[column_name].dtype == object:\n",
        "            print(column_name)\n",
        "            xtest[column_name] = le.fit_transform(xtest[column_name])\n",
        "\n",
        "        else:\n",
        "            pass\n",
        "\n",
        "ydf['label'] = le.fit_transform(ydf['label'])\n",
        "ytest=le.fit_transform(ytest)\n",
        "ytrain=ydf\n",
        "\n",
        "Y_test=ytest\n",
        "X_test=xtest\n",
        "X_train=xtrain\n",
        "Y_train=ytrain\n",
        "\n",
        "DT.fit(X_train, Y_train)\n",
        "print()\n",
        "\n",
        "print('prediction:')\n",
        "y_pred = DT.predict(X_test)\n",
        "train_pred=DT.predict(X_train)\n",
        "print(train_pred[train_pred=='1'])\n",
        "print(len(y_pred))\n",
        "print(\"size of Y_test\",len(Y_test))\n",
        "\n",
        "\n",
        "print('Score:')\n",
        "score = DT.score(X_test,Y_test)\n",
        "print(score)\n",
        "\n",
        "# y_pred = y_pred.astype('int16')\n",
        "# Y_test = Y_test.astype('int16')\n",
        "\n",
        "\n",
        "end = time.time()\n",
        "print('program end...')\n",
        "print()\n",
        "print('time cost: ')\n",
        "print(end - start, 'seconds')\n",
        "\n",
        "print(\"----Accuracy-----  \")\n",
        "print(accuracy_score( Y_test , y_pred ))\n",
        "print('----F1 score-----  ')\n",
        "print(f1_score( Y_test , y_pred ))\n",
        "print('----Recall------- ')\n",
        "print(recall_score(Y_test , y_pred ))\n",
        "print('----Precision----')\n",
        "print(precision_score( Y_test , y_pred))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "program start...\n",
            "\n",
            "history\n",
            "history\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:20: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:27: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "prediction:\n",
            "[]\n",
            "51926\n",
            "size of Y_test 51926\n",
            "Score:\n",
            "0.9879251242152294\n",
            "program end...\n",
            "\n",
            "time cost: \n",
            "0.746208667755127 seconds\n",
            "----Accuracy-----  \n",
            "0.9879251242152294\n",
            "----F1 score-----  \n",
            "0.992368456285982\n",
            "----Recall------- \n",
            "0.9953365725028689\n",
            "----Precision----\n",
            "0.9894179894179894\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:47: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GhdThl4dDOk8"
      },
      "source": [
        "ydf = pd.DataFrame(ytrain)\n",
        "from sklearn import preprocessing\n",
        "le = preprocessing.LabelEncoder()\n",
        "for column_name in xtrain.columns:\n",
        "        if xtrain[column_name].dtype == object:\n",
        "            print(column_name)\n",
        "            xtrain[column_name] = le.fit_transform(xtrain[column_name])\n",
        "\n",
        "        else:\n",
        "            pass\n",
        "\n",
        "ydf['label'] = le.fit_transform(ydf['label'])\n",
        "ytrain=ydf"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9FQZQf6P17Dw",
        "outputId": "d8aac7be-9ca5-492d-b256-d5e2db67dd87"
      },
      "source": [
        "path = DT.cost_complexity_pruning_path(X_train, Y_train)\n",
        "path\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'ccp_alphas': array([0.        , 0.32387924]),\n",
              " 'impurities': array([0.        , 0.32387924])}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7Pltn0PjpVuQ",
        "outputId": "021fc6a4-0888-4c0d-b435-03559aba6285"
      },
      "source": [
        "#SVM\n",
        "import sklearn.metrics as skm\n",
        "print(df.shape)\n",
        "print(X_cols)\n",
        "#SVM_classifier\n",
        "drop_cols=[]\n",
        "SVM_classifier = SVC()\n",
        "import seaborn as sns\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "import numpy as np\n",
        "from sklearn.metrics import classification_report, confusion_matrix, f1_score, precision_score, recall_score, accuracy_score\n",
        "np.random.seed(0)\n",
        "xtrain, xtest, ytrain, ytest = train_test_split(X,Y,test_size =0.3,stratify=Y,random_state=10)\n",
        "ydf = pd.DataFrame(ytrain)\n",
        "from sklearn import preprocessing\n",
        "le = preprocessing.LabelEncoder()\n",
        "for column_name in xtrain.columns:\n",
        "        if xtrain[column_name].dtype == object:\n",
        "            print(column_name)\n",
        "            xtrain[column_name] = le.fit_transform(xtrain[column_name])\n",
        "\n",
        "        else:\n",
        "            pass\n",
        "for column_name in xtest.columns:\n",
        "        if xtest[column_name].dtype == object:\n",
        "            print(column_name)\n",
        "            xtest[column_name] = le.fit_transform(xtest[column_name])\n",
        "\n",
        "        else:\n",
        "            pass\n",
        "\n",
        "ydf['label'] = le.fit_transform(ydf['label'])\n",
        "ytest=le.fit_transform(ytest)\n",
        "ytrain=ydf\n",
        "\n",
        "Y_test=ytest\n",
        "X_test=xtest\n",
        "X_train=xtrain\n",
        "Y_train=ytrain\n",
        "start = time.time()\n",
        "print('program start...')\n",
        "print()\n",
        "\n",
        "SVM_classifier = SVC(C=1.0, cache_size=700, verbose=True).fit(X_train, Y_train.values.ravel())\n",
        "print()\n",
        "print(SVM_classifier.score(X_test, Y_test))\n",
        "\n",
        "y_pred = SVM_classifier.predict(X_test)\n",
        "wee=skm.accuracy_score(Y_test,y_pred,normalize=False)\n",
        "print(\"Acuracy\",wee)\n",
        "end = time.time()\n",
        "print('program end...')\n",
        "print()\n",
        "print('time cost: ')\n",
        "print(end - start, 'seconds')\n",
        "\n",
        "print(\"Classifiction Report :\")\n",
        "print(classification_report(Y_test, y_pred))\n",
        "print(\"----Accuracy-----  \")\n",
        "print(accuracy_score( Y_test , y_pred ))\n",
        "print('----F1 score-----  ')\n",
        "print(f1_score( Y_test , y_pred))\n",
        "print('----Recall------- ')\n",
        "print(recall_score(Y_test , y_pred))\n",
        "print('----Precision----')\n",
        "print(precision_score( Y_test , y_pred))\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(173084, 21)\n",
            "['label', 'duration', 'orig_bytes', 'resp_bytes', 'missed_bytes', 'orig_pkts', 'orig_ip_bytes', 'resp_pkts', 'resp_ip_bytes']\n",
            "history\n",
            "history\n",
            "program start...\n",
            "\n",
            "[LibSVM]"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:20: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:27: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "0.7887570773793475\n",
            "Acuracy 40957\n",
            "program end...\n",
            "\n",
            "time cost: \n",
            "2653.947418689728 seconds\n",
            "Classifiction Report :\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.00      0.00      0.00     10969\n",
            "           1       0.79      1.00      0.88     40957\n",
            "\n",
            "    accuracy                           0.79     51926\n",
            "   macro avg       0.39      0.50      0.44     51926\n",
            "weighted avg       0.62      0.79      0.70     51926\n",
            "\n",
            "----Accuracy-----  \n",
            "0.7887570773793475\n",
            "----F1 score-----  \n",
            "0.8819051925540734\n",
            "----Recall------- \n",
            "1.0\n",
            "----Precision----\n",
            "0.7887570773793475\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/metrics/_classification.py:1272: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
            "  _warn_prf(average, modifier, msg_start, len(result))\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EtJpuqweOmzE",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8e08993d-bc1c-4a46-c28d-3bb7ca86f56d"
      },
      "source": [
        "#Rain Forest\n",
        "#Rain forest is here\n",
        "#import the necessary libraries\n",
        "import seaborn as sns\n",
        "from sklearn.ensemble import RandomForestClassifier\n",
        "import numpy as np\n",
        "from sklearn.metrics import classification_report, confusion_matrix, f1_score, precision_score, recall_score, accuracy_score\n",
        "np.random.seed(0)\n",
        "xtrain, xtest, ytrain, ytest = train_test_split(X,Y,test_size =0.3,stratify=Y,random_state=10)\n",
        "clf = RandomForestClassifier(n_jobs=2,random_state=0)\n",
        "ydf = pd.DataFrame(ytrain)\n",
        "from sklearn import preprocessing\n",
        "le = preprocessing.LabelEncoder()\n",
        "for column_name in xtrain.columns:\n",
        "        if xtrain[column_name].dtype == object:\n",
        "            #print(column_name)\n",
        "            xtrain[column_name] = le.fit_transform(xtrain[column_name])\n",
        "\n",
        "        else:\n",
        "            pass\n",
        "for column_name in xtest.columns:\n",
        "        if xtest[column_name].dtype == object:\n",
        "          xtest[column_name] = le.fit_transform(xtest[column_name])\n",
        "\n",
        "        else:\n",
        "            pass\n",
        "\n",
        "ydf['label'] = le.fit_transform(ydf['label'])\n",
        "ytest=le.fit_transform(ytest)\n",
        "ytrain=ydf\n",
        "clf.fit(xtrain,ytrain)\n",
        "print(xtrain.shape)\n",
        "predic_res = clf.predict(xtest)\n",
        "ytestdf=ytest\n",
        "print(\"----Accuracy-----  \")\n",
        "print(accuracy_score( ytestdf , predic_res ))\n",
        "print('----F1 score-----  ')\n",
        "print(f1_score( ytestdf , predic_res))\n",
        "print('----Recall------- ')\n",
        "print(recall_score(ytestdf , predic_res))\n",
        "print('----Precision----')\n",
        "print(precision_score( ytestdf , predic_res))\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:17: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:23: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:31: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "(121158, 33)\n",
            "----Accuracy-----  \n",
            "0.9882910295420406\n",
            "----F1 score-----  \n",
            "0.9925672371638141\n",
            "----Recall------- \n",
            "0.9911858778719145\n",
            "----Precision----\n",
            "0.9939524520725705\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N4cAjR0HBzJK"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "N3djhmPjOyNo",
        "outputId": "29c7a88a-c96f-4698-f0b9-fd4b6c06f268"
      },
      "source": [
        "#1D convnet\n",
        "\n",
        "#Cnn\n",
        "from sklearn import preprocessing\n",
        "le = preprocessing.LabelEncoder()\n",
        "Y=pd.DataFrame(Y)\n",
        "Y=le.fit_transform(Y)\n",
        "Y=np.asarray(Y).astype(np.int32)\n",
        "# Y=pd.DataFrame(Y)\n",
        "X=pd.DataFrame(X)\n",
        "\n",
        "#Y.astype('int32').dtypes\n",
        "# print(\"weee\",Y.dtypes,\"hello\", Y.shape)\n",
        "\n",
        "X.history=le.fit_transform(X.history)\n",
        "# print(Y.dtypes)\n",
        "# print(X.history.head())\n",
        "# print(X.head())\n",
        "X_train, X_test, Y_train, Y_test = train_test_split(X, Y,stratify=Y, test_size=0.2)\n",
        "\n",
        "# cnn model\n",
        "model = Sequential()\n",
        "model.add(Conv1D(filters=64, kernel_size=3, activation='relu', input_shape=(X_train.shape[1],1)))\n",
        "model.add(MaxPooling1D(pool_size=2))\n",
        "model.add(Flatten())\n",
        "model.add(Dense(500, activation='relu'))\n",
        "model.add(Dense(1, activation='sigmoid'))\n",
        "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
        "import time\n",
        "import tensorflow as tf\n",
        "\n",
        "start = time.time()\n",
        "tf.config.experimental_run_functions_eagerly(True)\n",
        "print('program start...')\n",
        "print()\n",
        "siz_train=[X_train.shape[0],X_train.shape[1],1]\n",
        "siz_test=[X_test.shape[0],X_test.shape[1],1]\n",
        "for col in X_train:\n",
        "  X_train[col] = np.asarray(X_train[col]).astype('float32')\n",
        "  column = X_train[col]\n",
        "  max_value = column.max()\n",
        "  if(max_value>0):\n",
        "    X_train[col] = X_train[col].apply(lambda x: x/max_value)\n",
        "for col in X_test:\n",
        "  X_test[col] = np.asarray(X_test[col]).astype('float32')\n",
        "  column = X_test[col]\n",
        "  max_value = column.max()\n",
        "  if(max_value>0):\n",
        "    X_test[col] = X_test[col].apply(lambda x: x/max_value)\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "x_train, x_test = X_train.values.reshape(siz_train), X_test.values.reshape(siz_test)\n",
        "history = model.fit(x_train, Y_train, epochs = 20, batch_size=256, validation_data=(x_test,Y_test),verbose=1)\n",
        "end = time.time()\n",
        "print('program end...')\n",
        "print()\n",
        "print('time cost: ')\n",
        "print(end - start, 'seconds')\n",
        "\n",
        "\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/sklearn/preprocessing/_label.py:251: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
            "  y = column_or_1d(y, warn=True)\n",
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:39: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "program start...\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:43: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:45: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:49: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/20\n",
            "  2/541 [..............................] - ETA: 28s - loss: 0.6360 - accuracy: 0.8184"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/tensorflow/python/data/ops/dataset_ops.py:4212: UserWarning: Even though the `tf.config.experimental_run_functions_eagerly` option is set, this option does not apply to tf.data functions. To force eager execution of tf.data functions, please use `tf.data.experimental.enable_debug_mode()`.\n",
            "  \"Even though the `tf.config.experimental_run_functions_eagerly` \"\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "541/541 [==============================] - 26s 49ms/step - loss: 0.2588 - accuracy: 0.9101 - val_loss: 0.2366 - val_accuracy: 0.9170\n",
            "Epoch 2/20\n",
            "541/541 [==============================] - 26s 48ms/step - loss: 0.2433 - accuracy: 0.9138 - val_loss: 0.2326 - val_accuracy: 0.9175\n",
            "Epoch 3/20\n",
            "541/541 [==============================] - 27s 50ms/step - loss: 0.2399 - accuracy: 0.9142 - val_loss: 0.2294 - val_accuracy: 0.9174\n",
            "Epoch 4/20\n",
            "541/541 [==============================] - 27s 50ms/step - loss: 0.2343 - accuracy: 0.9142 - val_loss: 0.2320 - val_accuracy: 0.9172\n",
            "Epoch 5/20\n",
            "541/541 [==============================] - 26s 49ms/step - loss: 0.2280 - accuracy: 0.9143 - val_loss: 0.2183 - val_accuracy: 0.9175\n",
            "Epoch 6/20\n",
            "541/541 [==============================] - 27s 49ms/step - loss: 0.2237 - accuracy: 0.9144 - val_loss: 0.2164 - val_accuracy: 0.9180\n",
            "Epoch 7/20\n",
            "541/541 [==============================] - 26s 49ms/step - loss: 0.2192 - accuracy: 0.9146 - val_loss: 0.2098 - val_accuracy: 0.9178\n",
            "Epoch 8/20\n",
            "541/541 [==============================] - 27s 50ms/step - loss: 0.2157 - accuracy: 0.9149 - val_loss: 0.2075 - val_accuracy: 0.9181\n",
            "Epoch 9/20\n",
            "541/541 [==============================] - 27s 50ms/step - loss: 0.2140 - accuracy: 0.9160 - val_loss: 0.2103 - val_accuracy: 0.9333\n",
            "Epoch 10/20\n",
            "541/541 [==============================] - 28s 51ms/step - loss: 0.2089 - accuracy: 0.9200 - val_loss: 0.1984 - val_accuracy: 0.9336\n",
            "Epoch 11/20\n",
            "541/541 [==============================] - 27s 50ms/step - loss: 0.2028 - accuracy: 0.9268 - val_loss: 0.1936 - val_accuracy: 0.9323\n",
            "Epoch 12/20\n",
            "541/541 [==============================] - 27s 50ms/step - loss: 0.1982 - accuracy: 0.9284 - val_loss: 0.1895 - val_accuracy: 0.9321\n",
            "Epoch 13/20\n",
            "541/541 [==============================] - 26s 49ms/step - loss: 0.1944 - accuracy: 0.9294 - val_loss: 0.1901 - val_accuracy: 0.9316\n",
            "Epoch 14/20\n",
            "541/541 [==============================] - 27s 50ms/step - loss: 0.1935 - accuracy: 0.9299 - val_loss: 0.1867 - val_accuracy: 0.9352\n",
            "Epoch 15/20\n",
            "541/541 [==============================] - 26s 48ms/step - loss: 0.1912 - accuracy: 0.9313 - val_loss: 0.1882 - val_accuracy: 0.9324\n",
            "Epoch 16/20\n",
            "541/541 [==============================] - 26s 49ms/step - loss: 0.1881 - accuracy: 0.9328 - val_loss: 0.1840 - val_accuracy: 0.9373\n",
            "Epoch 17/20\n",
            "541/541 [==============================] - 26s 48ms/step - loss: 0.1879 - accuracy: 0.9335 - val_loss: 0.1839 - val_accuracy: 0.9381\n",
            "Epoch 18/20\n",
            "541/541 [==============================] - 26s 49ms/step - loss: 0.1865 - accuracy: 0.9340 - val_loss: 0.1827 - val_accuracy: 0.9363\n",
            "Epoch 19/20\n",
            "541/541 [==============================] - 28s 51ms/step - loss: 0.1859 - accuracy: 0.9341 - val_loss: 0.1800 - val_accuracy: 0.9375\n",
            "Epoch 20/20\n",
            "541/541 [==============================] - 27s 50ms/step - loss: 0.1846 - accuracy: 0.9346 - val_loss: 0.1804 - val_accuracy: 0.9367\n",
            "program end...\n",
            "\n",
            "time cost: \n",
            "535.9161953926086 seconds\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lfHz2EuNkYAq",
        "outputId": "ae03f5fe-9a90-436a-e2cc-a483fd28e9d0"
      },
      "source": [
        "model.save('/content/drive/MyDrive/IOT231dcnn')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Assets written to: /content/drive/MyDrive/IOT231dcnn/assets\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VRynY01LlSv3",
        "outputId": "4b72671b-e415-4b83-a80b-4e71c2f94182"
      },
      "source": [
        "print(history.history['accuracy'][-1],history.history['accuracy'][-2])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.9375374913215637 0.9373857975006104\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "PAjLOgKIlK2R",
        "outputId": "da12f426-46bb-4089-9016-8108309f24fb"
      },
      "source": [
        "while(history.history['accuracy'][-1]-history.history['accuracy'][-2]>0.001):\n",
        "  history=model.fit(x_train, Y_train, epochs = 20, batch_size=256, validation_data=(x_test,Y_test),verbose=1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/20\n",
            "  2/541 [..............................] - ETA: 27s - loss: 0.1826 - accuracy: 0.9355"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/tensorflow/python/data/ops/dataset_ops.py:4212: UserWarning: Even though the `tf.config.experimental_run_functions_eagerly` option is set, this option does not apply to tf.data functions. To force eager execution of tf.data functions, please use `tf.data.experimental.enable_debug_mode()`.\n",
            "  \"Even though the `tf.config.experimental_run_functions_eagerly` \"\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "541/541 [==============================] - 27s 50ms/step - loss: 0.1838 - accuracy: 0.9344 - val_loss: 0.1973 - val_accuracy: 0.9358\n",
            "Epoch 2/20\n",
            "541/541 [==============================] - 27s 50ms/step - loss: 0.1834 - accuracy: 0.9346 - val_loss: 0.1791 - val_accuracy: 0.9380\n",
            "Epoch 3/20\n",
            "541/541 [==============================] - 26s 49ms/step - loss: 0.1812 - accuracy: 0.9349 - val_loss: 0.1789 - val_accuracy: 0.9378\n",
            "Epoch 4/20\n",
            "541/541 [==============================] - 27s 50ms/step - loss: 0.1814 - accuracy: 0.9349 - val_loss: 0.1768 - val_accuracy: 0.9385\n",
            "Epoch 5/20\n",
            "541/541 [==============================] - 27s 49ms/step - loss: 0.1811 - accuracy: 0.9349 - val_loss: 0.1781 - val_accuracy: 0.9376\n",
            "Epoch 6/20\n",
            "541/541 [==============================] - 26s 48ms/step - loss: 0.1799 - accuracy: 0.9351 - val_loss: 0.1850 - val_accuracy: 0.9375\n",
            "Epoch 7/20\n",
            "541/541 [==============================] - 26s 49ms/step - loss: 0.1794 - accuracy: 0.9352 - val_loss: 0.1772 - val_accuracy: 0.9378\n",
            "Epoch 8/20\n",
            "541/541 [==============================] - 27s 50ms/step - loss: 0.1788 - accuracy: 0.9352 - val_loss: 0.1790 - val_accuracy: 0.9383\n",
            "Epoch 9/20\n",
            "541/541 [==============================] - 27s 50ms/step - loss: 0.1785 - accuracy: 0.9353 - val_loss: 0.1758 - val_accuracy: 0.9376\n",
            "Epoch 10/20\n",
            "541/541 [==============================] - 27s 50ms/step - loss: 0.1772 - accuracy: 0.9355 - val_loss: 0.1780 - val_accuracy: 0.9379\n",
            "Epoch 11/20\n",
            "541/541 [==============================] - 27s 50ms/step - loss: 0.1780 - accuracy: 0.9353 - val_loss: 0.1766 - val_accuracy: 0.9388\n",
            "Epoch 12/20\n",
            "541/541 [==============================] - 27s 50ms/step - loss: 0.1771 - accuracy: 0.9355 - val_loss: 0.1780 - val_accuracy: 0.9381\n",
            "Epoch 13/20\n",
            "541/541 [==============================] - 26s 49ms/step - loss: 0.1769 - accuracy: 0.9355 - val_loss: 0.1758 - val_accuracy: 0.9382\n",
            "Epoch 14/20\n",
            "541/541 [==============================] - 26s 48ms/step - loss: 0.1763 - accuracy: 0.9355 - val_loss: 0.1924 - val_accuracy: 0.9377\n",
            "Epoch 15/20\n",
            "541/541 [==============================] - 27s 49ms/step - loss: 0.1754 - accuracy: 0.9357 - val_loss: 0.1752 - val_accuracy: 0.9375\n",
            "Epoch 16/20\n",
            "541/541 [==============================] - 27s 50ms/step - loss: 0.1749 - accuracy: 0.9357 - val_loss: 0.1782 - val_accuracy: 0.9375\n",
            "Epoch 17/20\n",
            "541/541 [==============================] - 26s 49ms/step - loss: 0.1748 - accuracy: 0.9358 - val_loss: 0.1744 - val_accuracy: 0.9383\n",
            "Epoch 18/20\n",
            "541/541 [==============================] - 27s 49ms/step - loss: 0.1741 - accuracy: 0.9359 - val_loss: 0.1793 - val_accuracy: 0.9379\n",
            "Epoch 19/20\n",
            "541/541 [==============================] - 27s 50ms/step - loss: 0.1743 - accuracy: 0.9359 - val_loss: 0.1751 - val_accuracy: 0.9383\n",
            "Epoch 20/20\n",
            "541/541 [==============================] - 26s 49ms/step - loss: 0.1733 - accuracy: 0.9358 - val_loss: 0.1744 - val_accuracy: 0.9384\n",
            "Epoch 1/20\n",
            "541/541 [==============================] - 27s 50ms/step - loss: 0.1727 - accuracy: 0.9361 - val_loss: 0.1766 - val_accuracy: 0.9384\n",
            "Epoch 2/20\n",
            "541/541 [==============================] - 27s 50ms/step - loss: 0.1717 - accuracy: 0.9361 - val_loss: 0.1717 - val_accuracy: 0.9384\n",
            "Epoch 3/20\n",
            "541/541 [==============================] - 26s 49ms/step - loss: 0.1706 - accuracy: 0.9362 - val_loss: 0.1725 - val_accuracy: 0.9383\n",
            "Epoch 4/20\n",
            "541/541 [==============================] - 26s 48ms/step - loss: 0.1701 - accuracy: 0.9363 - val_loss: 0.1693 - val_accuracy: 0.9381\n",
            "Epoch 5/20\n",
            "541/541 [==============================] - 27s 50ms/step - loss: 0.1693 - accuracy: 0.9364 - val_loss: 0.1723 - val_accuracy: 0.9388\n",
            "Epoch 6/20\n",
            "541/541 [==============================] - 27s 50ms/step - loss: 0.1689 - accuracy: 0.9366 - val_loss: 0.1785 - val_accuracy: 0.9392\n",
            "Epoch 7/20\n",
            "541/541 [==============================] - 27s 50ms/step - loss: 0.1683 - accuracy: 0.9365 - val_loss: 0.1822 - val_accuracy: 0.9386\n",
            "Epoch 8/20\n",
            "541/541 [==============================] - 27s 50ms/step - loss: 0.1683 - accuracy: 0.9366 - val_loss: 0.1716 - val_accuracy: 0.9388\n",
            "Epoch 9/20\n",
            "541/541 [==============================] - 27s 50ms/step - loss: 0.1675 - accuracy: 0.9365 - val_loss: 0.1693 - val_accuracy: 0.9395\n",
            "Epoch 10/20\n",
            "541/541 [==============================] - 26s 49ms/step - loss: 0.1661 - accuracy: 0.9368 - val_loss: 0.1662 - val_accuracy: 0.9390\n",
            "Epoch 11/20\n",
            "541/541 [==============================] - 27s 50ms/step - loss: 0.1652 - accuracy: 0.9369 - val_loss: 0.1677 - val_accuracy: 0.9395\n",
            "Epoch 12/20\n",
            "541/541 [==============================] - 27s 50ms/step - loss: 0.1658 - accuracy: 0.9368 - val_loss: 0.1716 - val_accuracy: 0.9393\n",
            "Epoch 13/20\n",
            "541/541 [==============================] - 27s 49ms/step - loss: 0.1641 - accuracy: 0.9369 - val_loss: 0.1647 - val_accuracy: 0.9389\n",
            "Epoch 14/20\n",
            "541/541 [==============================] - 27s 50ms/step - loss: 0.1632 - accuracy: 0.9370 - val_loss: 0.1733 - val_accuracy: 0.9392\n",
            "Epoch 15/20\n",
            "541/541 [==============================] - 26s 49ms/step - loss: 0.1644 - accuracy: 0.9368 - val_loss: 0.1691 - val_accuracy: 0.9388\n",
            "Epoch 16/20\n",
            "541/541 [==============================] - 26s 49ms/step - loss: 0.1630 - accuracy: 0.9370 - val_loss: 0.1636 - val_accuracy: 0.9392\n",
            "Epoch 17/20\n",
            "541/541 [==============================] - 27s 49ms/step - loss: 0.1631 - accuracy: 0.9368 - val_loss: 0.1649 - val_accuracy: 0.9394\n",
            "Epoch 18/20\n",
            "541/541 [==============================] - 27s 50ms/step - loss: 0.1624 - accuracy: 0.9372 - val_loss: 0.1691 - val_accuracy: 0.9388\n",
            "Epoch 19/20\n",
            "541/541 [==============================] - 26s 49ms/step - loss: 0.1626 - accuracy: 0.9367 - val_loss: 0.1697 - val_accuracy: 0.9394\n",
            "Epoch 20/20\n",
            "541/541 [==============================] - 26s 49ms/step - loss: 0.1634 - accuracy: 0.9368 - val_loss: 0.1621 - val_accuracy: 0.9402\n",
            "Epoch 1/20\n",
            "541/541 [==============================] - 26s 48ms/step - loss: 0.1619 - accuracy: 0.9372 - val_loss: 0.1629 - val_accuracy: 0.9397\n",
            "Epoch 2/20\n",
            "541/541 [==============================] - 26s 47ms/step - loss: 0.1624 - accuracy: 0.9370 - val_loss: 0.1633 - val_accuracy: 0.9394\n",
            "Epoch 3/20\n",
            "541/541 [==============================] - 26s 48ms/step - loss: 0.1613 - accuracy: 0.9371 - val_loss: 0.1631 - val_accuracy: 0.9393\n",
            "Epoch 4/20\n",
            "541/541 [==============================] - 26s 48ms/step - loss: 0.1605 - accuracy: 0.9372 - val_loss: 0.1628 - val_accuracy: 0.9383\n",
            "Epoch 5/20\n",
            "541/541 [==============================] - 26s 48ms/step - loss: 0.1613 - accuracy: 0.9370 - val_loss: 0.1672 - val_accuracy: 0.9398\n",
            "Epoch 6/20\n",
            "541/541 [==============================] - 26s 48ms/step - loss: 0.1608 - accuracy: 0.9372 - val_loss: 0.1609 - val_accuracy: 0.9392\n",
            "Epoch 7/20\n",
            "541/541 [==============================] - 26s 48ms/step - loss: 0.1604 - accuracy: 0.9371 - val_loss: 0.1644 - val_accuracy: 0.9392\n",
            "Epoch 8/20\n",
            "541/541 [==============================] - 26s 48ms/step - loss: 0.1616 - accuracy: 0.9371 - val_loss: 0.1681 - val_accuracy: 0.9390\n",
            "Epoch 9/20\n",
            "541/541 [==============================] - 26s 47ms/step - loss: 0.1597 - accuracy: 0.9373 - val_loss: 0.1642 - val_accuracy: 0.9390\n",
            "Epoch 10/20\n",
            "541/541 [==============================] - 26s 47ms/step - loss: 0.1603 - accuracy: 0.9372 - val_loss: 0.1625 - val_accuracy: 0.9390\n",
            "Epoch 11/20\n",
            "541/541 [==============================] - 26s 47ms/step - loss: 0.1600 - accuracy: 0.9372 - val_loss: 0.1617 - val_accuracy: 0.9393\n",
            "Epoch 12/20\n",
            "541/541 [==============================] - 26s 48ms/step - loss: 0.1603 - accuracy: 0.9373 - val_loss: 0.1601 - val_accuracy: 0.9391\n",
            "Epoch 13/20\n",
            "541/541 [==============================] - 26s 47ms/step - loss: 0.1609 - accuracy: 0.9369 - val_loss: 0.1612 - val_accuracy: 0.9394\n",
            "Epoch 14/20\n",
            "541/541 [==============================] - 26s 48ms/step - loss: 0.1593 - accuracy: 0.9375 - val_loss: 0.1640 - val_accuracy: 0.9393\n",
            "Epoch 15/20\n",
            "541/541 [==============================] - 26s 48ms/step - loss: 0.1596 - accuracy: 0.9373 - val_loss: 0.1624 - val_accuracy: 0.9391\n",
            "Epoch 16/20\n",
            "541/541 [==============================] - 26s 48ms/step - loss: 0.1598 - accuracy: 0.9373 - val_loss: 0.1649 - val_accuracy: 0.9397\n",
            "Epoch 17/20\n",
            "541/541 [==============================] - 26s 48ms/step - loss: 0.1600 - accuracy: 0.9371 - val_loss: 0.1597 - val_accuracy: 0.9402\n",
            "Epoch 18/20\n",
            "541/541 [==============================] - 26s 47ms/step - loss: 0.1586 - accuracy: 0.9375 - val_loss: 0.1607 - val_accuracy: 0.9394\n",
            "Epoch 19/20\n",
            "541/541 [==============================] - 26s 47ms/step - loss: 0.1597 - accuracy: 0.9374 - val_loss: 0.1595 - val_accuracy: 0.9392\n",
            "Epoch 20/20\n",
            "541/541 [==============================] - 26s 47ms/step - loss: 0.1590 - accuracy: 0.9375 - val_loss: 0.1625 - val_accuracy: 0.9396\n",
            "Epoch 1/20\n",
            "541/541 [==============================] - ETA: 0s - loss: 0.1589 - accuracy: 0.9376"
          ],
          "name": "stdout"
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-122-6ab4c040deb2>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mwhile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'accuracy'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m<=\u001b[0m\u001b[0;36m0.001\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m   \u001b[0mhistory\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mY_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m20\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m256\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_test\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mY_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m   1224\u001b[0m               \u001b[0muse_multiprocessing\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0muse_multiprocessing\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1225\u001b[0m               \u001b[0mreturn_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1226\u001b[0;31m               _use_cached_eval_dataset=True)\n\u001b[0m\u001b[1;32m   1227\u001b[0m           \u001b[0mval_logs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m'val_'\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mval\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mval_logs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitems\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1228\u001b[0m           \u001b[0mepoch_logs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mval_logs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mevaluate\u001b[0;34m(self, x, y, batch_size, verbose, sample_weight, steps, callbacks, max_queue_size, workers, use_multiprocessing, return_dict, **kwargs)\u001b[0m\n\u001b[1;32m   1499\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprofiler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperimental\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTrace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'test'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstep_num\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0m_r\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1500\u001b[0m               \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_test_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1501\u001b[0;31m               \u001b[0mtmp_logs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtest_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1502\u001b[0m               \u001b[0;32mif\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_sync\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1503\u001b[0m                 \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masync_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mtest_function\u001b[0;34m(iterator)\u001b[0m\n\u001b[1;32m   1328\u001b[0m       \u001b[0;32mdef\u001b[0m \u001b[0mtest_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1329\u001b[0m         \u001b[0;34m\"\"\"Runs an evaluation execution with one step.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1330\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mstep_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1331\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1332\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mstep_function\u001b[0;34m(model, iterator)\u001b[0m\n\u001b[1;32m   1318\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1319\u001b[0m       \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1320\u001b[0;31m       \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdistribute_strategy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_step\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1321\u001b[0m       outputs = reduce_per_replica(\n\u001b[1;32m   1322\u001b[0m           outputs, self.distribute_strategy, reduction='first')\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/distribute/distribute_lib.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(***failed resolving arguments***)\u001b[0m\n\u001b[1;32m   1284\u001b[0m       fn = autograph.tf_convert(\n\u001b[1;32m   1285\u001b[0m           fn, autograph_ctx.control_status_ctx(), convert_by_default=False)\n\u001b[0;32m-> 1286\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_extended\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcall_for_each_replica\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1287\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1288\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mreduce\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreduce_op\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/distribute/distribute_lib.py\u001b[0m in \u001b[0;36mcall_for_each_replica\u001b[0;34m(self, fn, args, kwargs)\u001b[0m\n\u001b[1;32m   2847\u001b[0m       \u001b[0mkwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2848\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_container_strategy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mscope\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2849\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_for_each_replica\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2850\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2851\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_call_for_each_replica\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/distribute/distribute_lib.py\u001b[0m in \u001b[0;36m_call_for_each_replica\u001b[0;34m(self, fn, args, kwargs)\u001b[0m\n\u001b[1;32m   3630\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_call_for_each_replica\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3631\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mReplicaContext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_container_strategy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreplica_id_in_sync_group\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3632\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3633\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3634\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_reduce_to\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreduce_op\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdestinations\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/autograph/impl/api.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    595\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    596\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mag_ctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mControlStatusCtx\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstatus\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mag_ctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mStatus\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mUNSPECIFIED\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 597\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    598\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    599\u001b[0m   \u001b[0;32mif\u001b[0m \u001b[0minspect\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misfunction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0minspect\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mismethod\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mrun_step\u001b[0;34m(data)\u001b[0m\n\u001b[1;32m   1311\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1312\u001b[0m       \u001b[0;32mdef\u001b[0m \u001b[0mrun_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1313\u001b[0;31m         \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtest_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1314\u001b[0m         \u001b[0;31m# Ensure counter is updated only if `test_step` succeeds.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1315\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontrol_dependencies\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_minimum_control_deps\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/engine/training.py\u001b[0m in \u001b[0;36mtest_step\u001b[0;34m(self, data)\u001b[0m\n\u001b[1;32m   1265\u001b[0m     \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata_adapter\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munpack_x_y_sample_weight\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1266\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1267\u001b[0;31m     \u001b[0my_pred\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtraining\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1268\u001b[0m     \u001b[0;31m# Updates stateful loss metrics.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1269\u001b[0m     self.compiled_loss(\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/engine/base_layer.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1035\u001b[0m         with autocast_variable.enable_auto_cast_variables(\n\u001b[1;32m   1036\u001b[0m             self._compute_dtype_object):\n\u001b[0;32m-> 1037\u001b[0;31m           \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcall_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1038\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1039\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_activity_regularizer\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/engine/sequential.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, inputs, training, mask)\u001b[0m\n\u001b[1;32m    367\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuilt\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    368\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_init_graph_network\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 369\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mSequential\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcall\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtraining\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtraining\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmask\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmask\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    370\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    371\u001b[0m     \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minputs\u001b[0m  \u001b[0;31m# handle the corner case where self.layers is empty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/engine/functional.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, inputs, training, mask)\u001b[0m\n\u001b[1;32m    413\u001b[0m     \"\"\"\n\u001b[1;32m    414\u001b[0m     return self._run_internal_graph(\n\u001b[0;32m--> 415\u001b[0;31m         inputs, training=training, mask=mask)\n\u001b[0m\u001b[1;32m    416\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    417\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0mcompute_output_shape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput_shape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/engine/functional.py\u001b[0m in \u001b[0;36m_run_internal_graph\u001b[0;34m(self, inputs, training, mask)\u001b[0m\n\u001b[1;32m    548\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    549\u001b[0m         \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwargs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmap_arguments\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtensor_dict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 550\u001b[0;31m         \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    551\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    552\u001b[0m         \u001b[0;31m# Update tensor_dict.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/engine/base_layer.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1035\u001b[0m         with autocast_variable.enable_auto_cast_variables(\n\u001b[1;32m   1036\u001b[0m             self._compute_dtype_object):\n\u001b[0;32m-> 1037\u001b[0;31m           \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcall_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1038\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1039\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_activity_regularizer\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/keras/layers/core.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   1227\u001b[0m             self.kernel, ids, weights, combiner='sum')\n\u001b[1;32m   1228\u001b[0m       \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1229\u001b[0;31m         \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mraw_ops\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mMatMul\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mb\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkernel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1230\u001b[0m     \u001b[0;31m# Broadcast kernel to inputs.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1231\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/util/tf_export.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    402\u001b[0m           \u001b[0;34m'Please pass these args as kwargs instead.'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    403\u001b[0m           .format(f=f.__name__, kwargs=f_argspec.args))\n\u001b[0;32m--> 404\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    405\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    406\u001b[0m   \u001b[0;32mreturn\u001b[0m \u001b[0mtf_decorator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmake_decorator\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdecorator_argspec\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mf_argspec\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/ops/gen_math_ops.py\u001b[0m in \u001b[0;36mmat_mul\u001b[0;34m(a, b, transpose_a, transpose_b, name)\u001b[0m\n\u001b[1;32m   5691\u001b[0m       _result = pywrap_tfe.TFE_Py_FastPathExecute(\n\u001b[1;32m   5692\u001b[0m         \u001b[0m_ctx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"MatMul\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mb\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"transpose_a\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtranspose_a\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"transpose_b\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 5693\u001b[0;31m         transpose_b)\n\u001b[0m\u001b[1;32m   5694\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0m_result\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   5695\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0m_core\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N8noB8peGh7V",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9e78a848-4f09-4f7d-92de-91600aef5d42"
      },
      "source": [
        "from sklearn.metrics import accuracy_score\n",
        "from sklearn.metrics import precision_score\n",
        "from sklearn.metrics import recall_score\n",
        "from sklearn.metrics import f1_score\n",
        "from sklearn.metrics import cohen_kappa_score\n",
        "from sklearn.metrics import roc_auc_score\n",
        "from sklearn.metrics import confusion_matrix\n",
        "# predict probabilities for test set\n",
        "#yhat_probs = model.predict(X_test)\n",
        "# predict crisp classes for test set\n",
        "X_test=x_test\n",
        "yhat_classes = model.predict(X_test)\n",
        "yhat_classes = np.where(yhat_classes > 0.5, 1, 0)\n",
        "# reduce to 1d array\n",
        "#yhat_probs = yhat_probs[:, 0]\n",
        "print(yhat_classes)\n",
        "yhat_classes = yhat_classes\n",
        "print(len(yhat_classes))\n",
        "# accuracy: (tp + tn) / (p + n)\n",
        "accuracy = accuracy_score(Y_test, yhat_classes)\n",
        "print('Accuracy: %f' % accuracy)\n",
        "# precision tp / (tp + fp)\n",
        "precision = precision_score(Y_test, yhat_classes)\n",
        "print('Precision: %f' % precision)\n",
        "# recall: tp / (tp + fn)\n",
        "recall = recall_score(Y_test, yhat_classes)\n",
        "print('Recall: %f' % recall)\n",
        "# f1: 2 tp / (2 tp + fp + fn)\n",
        "f1 = f1_score(Y_test, yhat_classes)\n",
        "print('F1 score: %f' % f1)\n",
        "#https://machinelearningmastery.com/how-to-calculate-precision-recall-f1-and-more-for-deep-learning-models/\n",
        "#https://stats.stackexchange.com/questions/258166/good-accuracy-despite-high-loss-value\n",
        "#https://stats.stackexchange.com/questions/272754/how-do-you-interpret-the-cross-entropy-value"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/tensorflow/python/data/ops/dataset_ops.py:4212: UserWarning: Even though the `tf.config.experimental_run_functions_eagerly` option is set, this option does not apply to tf.data functions. To force eager execution of tf.data functions, please use `tf.data.experimental.enable_debug_mode()`.\n",
            "  \"Even though the `tf.config.experimental_run_functions_eagerly` \"\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "[[1]\n",
            " [1]\n",
            " [1]\n",
            " ...\n",
            " [1]\n",
            " [1]\n",
            " [1]]\n",
            "34617\n",
            "Accuracy: 0.939018\n",
            "Precision: 0.930164\n",
            "Recall: 0.997583\n",
            "F1 score: 0.962695\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}