Hybrid_Modeling.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "view-in-github"
   },
   "source": [
    "<a href=\"https://colab.research.google.com/github/EdWangLoDaSc/DVI_BNN/blob/main/nW%20oNE.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "Ex-EcKipVXiN",
    "outputId": "440caede-1ca9-41bb-e3f0-e2a10d73bc9e"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/drive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "kbL7PE8HY3fh",
    "outputId": "106f45e9-3873-4101-c3dd-84f7e554b6f6"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['LICENSE',\n",
       " 'readme.md',\n",
       " 'experiment.py',\n",
       " 'Hy_Modeling_Experiments.ipynb',\n",
       " 'UCI_Datasets',\n",
       " '.vs',\n",
       " '.ipynb_checkpoints',\n",
       " 'EndUserData2021P2CWPState.csv',\n",
       " 'EndUserData2020P2CWPState.csv',\n",
       " 'EndUserData2020P2.csv',\n",
       " 'EndUserData2021P2_fix.csv',\n",
       " '__pycache__',\n",
       " 'net.py.gdoc',\n",
       " 'Hydric_Modeling',\n",
       " 'DataProcessing.py',\n",
       " 'net.py']"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "path=\"/content/drive/MyDrive/DropoutUncertaintyExps-master\"\n",
    "os.chdir(path)\n",
    "os.listdir(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "2cCnSoG-lpIE",
    "outputId": "d59c7c4e-4229-4ea6-9622-34eadd9faa83"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: miceforest in c:\\users\\ssyhw7\\.conda\\envs\\mc_dropout\\lib\\site-packages (5.5.4)\n",
      "Requirement already satisfied: blosc in c:\\users\\ssyhw7\\.conda\\envs\\mc_dropout\\lib\\site-packages (from miceforest) (1.10.6)\n",
      "Requirement already satisfied: numpy in c:\\users\\ssyhw7\\.conda\\envs\\mc_dropout\\lib\\site-packages (from miceforest) (1.21.5)\n",
      "Requirement already satisfied: dill in c:\\users\\ssyhw7\\.conda\\envs\\mc_dropout\\lib\\site-packages (from miceforest) (0.3.5.1)\n",
      "Requirement already satisfied: lightgbm>=3.3.1 in c:\\users\\ssyhw7\\.conda\\envs\\mc_dropout\\lib\\site-packages (from miceforest) (3.3.2)\n",
      "Requirement already satisfied: wheel in c:\\users\\ssyhw7\\.conda\\envs\\mc_dropout\\lib\\site-packages (from lightgbm>=3.3.1->miceforest) (0.37.1)\n",
      "Requirement already satisfied: scipy in c:\\users\\ssyhw7\\.conda\\envs\\mc_dropout\\lib\\site-packages (from lightgbm>=3.3.1->miceforest) (1.7.3)\n",
      "Requirement already satisfied: scikit-learn!=0.22.0 in c:\\users\\ssyhw7\\.conda\\envs\\mc_dropout\\lib\\site-packages (from lightgbm>=3.3.1->miceforest) (1.0.2)\n",
      "Requirement already satisfied: joblib>=0.11 in c:\\users\\ssyhw7\\.conda\\envs\\mc_dropout\\lib\\site-packages (from scikit-learn!=0.22.0->lightgbm>=3.3.1->miceforest) (1.1.0)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in c:\\users\\ssyhw7\\.conda\\envs\\mc_dropout\\lib\\site-packages (from scikit-learn!=0.22.0->lightgbm>=3.3.1->miceforest) (3.1.0)\n",
      "Requirement already satisfied: scipy in c:\\users\\ssyhw7\\.conda\\envs\\mc_dropout\\lib\\site-packages (1.7.3)\n",
      "Requirement already satisfied: numpy<1.23.0,>=1.16.5 in c:\\users\\ssyhw7\\.conda\\envs\\mc_dropout\\lib\\site-packages (from scipy) (1.21.5)\n",
      "Requirement already satisfied: pandas in c:\\users\\ssyhw7\\.conda\\envs\\mc_dropout\\lib\\site-packages (1.3.5)\n",
      "Requirement already satisfied: numpy>=1.17.3 in c:\\users\\ssyhw7\\.conda\\envs\\mc_dropout\\lib\\site-packages (from pandas) (1.21.5)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in c:\\users\\ssyhw7\\.conda\\envs\\mc_dropout\\lib\\site-packages (from pandas) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2017.3 in c:\\users\\ssyhw7\\.conda\\envs\\mc_dropout\\lib\\site-packages (from pandas) (2022.1)\n",
      "Requirement already satisfied: six>=1.5 in c:\\users\\ssyhw7\\.conda\\envs\\mc_dropout\\lib\\site-packages (from python-dateutil>=2.7.3->pandas) (1.16.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install miceforest\n",
    "!pip install scipy --upgrade\n",
    "!pip install pandas\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "tL00fbapZcrC"
   },
   "outputs": [],
   "source": [
    "import math\n",
    "import numpy as np\n",
    "import argparse\n",
    "import sys\n",
    "import pandas as pd\n",
    "from DataProcessing import DataProcessing as dp\n",
    "\n",
    "from subprocess import call\n",
    "import net"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "uebW7cppY80Z"
   },
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: 'EndUserData2020P2.csv'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m~\\AppData\\Local\\Temp\\ipykernel_22968\\3217605056.py\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0mdoc_2020\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'EndUserData2020P2.csv'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mencoding\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'ISO-8859-1'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mskiprows\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'float'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      2\u001b[0m \u001b[0mdoc_2021\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'EndUserData2021P2_fix.csv'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mencoding\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'ISO-8859-1'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mskiprows\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'float'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;31m#print(doc_2020)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m \u001b[0mdoc_2020_value\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mdp\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdoc_2020\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdoc_2021\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdata_process_20\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mastype\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'float64'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\.conda\\envs\\mc_DROPOUT\\lib\\site-packages\\pandas\\util\\_decorators.py\u001b[0m in \u001b[0;36mwrapper\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m    309\u001b[0m                     \u001b[0mstacklevel\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mstacklevel\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    310\u001b[0m                 )\n\u001b[1;32m--> 311\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    312\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    313\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mwrapper\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\.conda\\envs\\mc_DROPOUT\\lib\\site-packages\\pandas\\io\\parsers\\readers.py\u001b[0m in \u001b[0;36mread_csv\u001b[1;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, encoding_errors, dialect, error_bad_lines, warn_bad_lines, on_bad_lines, delim_whitespace, low_memory, memory_map, float_precision, storage_options)\u001b[0m\n\u001b[0;32m    584\u001b[0m     \u001b[0mkwds\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mkwds_defaults\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    585\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 586\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    587\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    588\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\.conda\\envs\\mc_DROPOUT\\lib\\site-packages\\pandas\\io\\parsers\\readers.py\u001b[0m in \u001b[0;36m_read\u001b[1;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[0;32m    480\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    481\u001b[0m     \u001b[1;31m# Create the parser.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 482\u001b[1;33m     \u001b[0mparser\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    483\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    484\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\.conda\\envs\\mc_DROPOUT\\lib\\site-packages\\pandas\\io\\parsers\\readers.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[0;32m    809\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"has_index_names\"\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mkwds\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;34m\"has_index_names\"\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    810\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 811\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    812\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    813\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\.conda\\envs\\mc_DROPOUT\\lib\\site-packages\\pandas\\io\\parsers\\readers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[1;34m(self, engine)\u001b[0m\n\u001b[0;32m   1038\u001b[0m             )\n\u001b[0;32m   1039\u001b[0m         \u001b[1;31m# error: Too many arguments for \"ParserBase\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1040\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mmapping\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mengine\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[1;33m)\u001b[0m  \u001b[1;31m# type: ignore[call-arg]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1041\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1042\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_failover_to_python\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\.conda\\envs\\mc_DROPOUT\\lib\\site-packages\\pandas\\io\\parsers\\c_parser_wrapper.py\u001b[0m in \u001b[0;36m__init__\u001b[1;34m(self, src, **kwds)\u001b[0m\n\u001b[0;32m     49\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     50\u001b[0m         \u001b[1;31m# open handles\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 51\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_open_handles\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     52\u001b[0m         \u001b[1;32massert\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhandles\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     53\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\.conda\\envs\\mc_DROPOUT\\lib\\site-packages\\pandas\\io\\parsers\\base_parser.py\u001b[0m in \u001b[0;36m_open_handles\u001b[1;34m(self, src, kwds)\u001b[0m\n\u001b[0;32m    227\u001b[0m             \u001b[0mmemory_map\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"memory_map\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    228\u001b[0m             \u001b[0mstorage_options\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"storage_options\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 229\u001b[1;33m             \u001b[0merrors\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mkwds\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"encoding_errors\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m\"strict\"\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    230\u001b[0m         )\n\u001b[0;32m    231\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\.conda\\envs\\mc_DROPOUT\\lib\\site-packages\\pandas\\io\\common.py\u001b[0m in \u001b[0;36mget_handle\u001b[1;34m(path_or_buf, mode, encoding, compression, memory_map, is_text, errors, storage_options)\u001b[0m\n\u001b[0;32m    705\u001b[0m                 \u001b[0mencoding\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mioargs\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mencoding\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    706\u001b[0m                 \u001b[0merrors\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0merrors\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 707\u001b[1;33m                 \u001b[0mnewline\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;34m\"\"\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    708\u001b[0m             )\n\u001b[0;32m    709\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: 'EndUserData2020P2.csv'"
     ]
    }
   ],
   "source": [
    "doc_2020 = pd.read_csv('EndUserData2020P2.csv', encoding = 'ISO-8859-1', skiprows = [0,2]).astype('float')\n",
    "doc_2021 = pd.read_csv('EndUserData2021P2_fix.csv', encoding = 'ISO-8859-1', skiprows = [0,2]).astype('float')\n",
    "#print(doc_2020)\n",
    "\n",
    "doc_2020_value = dp(doc_2020, doc_2021).data_process_20().astype('float64')\n",
    "dco_2021_value = dp(doc_2020, doc_2021).data_process_21().astype('float64')\n",
    "\n",
    "BNN_20 = pd.DataFrame(np.c_[doc_2020_value['NB2_S_1_NYZ_sys_x_PcwOut_x'], doc_2020_value['NB2_S_1_NYZ_sys_x_PcwIn_x'], doc_2020_value['NB2_S_1_NYZ_cwp_9_HzSPR_x'], doc_2020_value['NB2_S_1_NYZ_cwp_10_HzSPR_x'], doc_2020_value['NB2_S_1_NYZ_cwp_11_HzSPR_x'], doc_2020_value['NB2_S_1_NYZ_cwp_12_HzSPR_x'], doc_2020_value['p_diff'], doc_2020_value['NB2_S_x_NYZ_x_x_Fcw_x']], columns = ['NB2_S_1_NYZ_sys_x_PcwOut_x', 'NB2_S_1_NYZ_sys_x_PcwIn_x', 'NB2_S_1_NYZ_cwp_9_HzSPR_x', 'NB2_S_1_NYZ_cwp_10_HzSPR_x', 'NB2_S_1_NYZ_cwp_11_HzSPR_x', 'NB2_S_1_NYZ_cwp_12_HzSPR_x', 'diff', 'NB2_S_x_NYZ_x_x_Fcw_x'])\n",
    "BNN_21 = pd.DataFrame(np.c_[dco_2021_value['NB2_S_1_NYZ_sys_x_PcwOut_x'], dco_2021_value['NB2_S_1_NYZ_sys_x_PcwIn_x'], dco_2021_value['NB2_S_1_NYZ_cwp_9_HzSPR_x'], dco_2021_value['NB2_S_1_NYZ_cwp_10_HzSPR_x'], dco_2021_value['NB2_S_1_NYZ_cwp_11_HzSPR_x'],  dco_2021_value['NB2_S_1_NYZ_cwp_12_HzSPR_x'],  dco_2021_value['p_diff'], dco_2021_value['NB2_S_x_NYZ_x_x_Fcw_x']], columns = ['NB2_S_1_NYZ_sys_x_PcwOut_x', 'NB2_S_1_NYZ_sys_x_PcwIn_x', 'NB2_S_1_NYZ_cwp_9_HzSPR_x', 'NB2_S_1_NYZ_cwp_10_HzSPR_x', 'NB2_S_1_NYZ_cwp_11_HzSPR_x', 'NB2_S_1_NYZ_cwp_12_HzSPR_x', 'diff', 'NB2_S_x_NYZ_x_x_Fcw_x'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "BcOfJU7wo-A-"
   },
   "outputs": [],
   "source": [
    "dp.dataframe_to_txt(BNN_20)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "N3qauXcy3KN3",
    "outputId": "1c106ac1-3943-404d-bbb8-2397b5694e93"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0\n",
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n",
      "11\n",
      "12\n",
      "13\n",
      "14\n",
      "15\n",
      "16\n",
      "17\n",
      "18\n",
      "19\n"
     ]
    }
   ],
   "source": [
    "dp.split_data_train_test('data_file.txt')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "z0WYDm6TP3m-",
    "outputId": "7a30ae0d-f74f-44dc-c167-633689563a0c"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['data_file.txt',\n",
       " 'dropout_rates.txt',\n",
       " 'index_features.txt',\n",
       " 'index_target.txt',\n",
       " 'index_test_0.txt',\n",
       " 'index_test_1.txt',\n",
       " 'index_test_10.txt',\n",
       " 'index_test_11.txt',\n",
       " 'index_test_12.txt',\n",
       " 'index_test_13.txt',\n",
       " 'index_test_14.txt',\n",
       " 'index_test_15.txt',\n",
       " 'index_test_16.txt',\n",
       " 'index_test_17.txt',\n",
       " 'index_test_18.txt',\n",
       " 'index_test_19.txt',\n",
       " 'index_test_2.txt',\n",
       " 'index_test_3.txt',\n",
       " 'index_test_4.txt',\n",
       " 'index_test_5.txt',\n",
       " 'index_test_6.txt',\n",
       " 'index_test_7.txt',\n",
       " 'index_test_8.txt',\n",
       " 'index_test_9.txt',\n",
       " 'index_train_0.txt',\n",
       " 'index_train_1.txt',\n",
       " 'index_train_10.txt',\n",
       " 'index_train_11.txt',\n",
       " 'index_train_12.txt',\n",
       " 'index_train_13.txt',\n",
       " 'index_train_14.txt',\n",
       " 'index_train_15.txt',\n",
       " 'index_train_16.txt',\n",
       " 'index_train_17.txt',\n",
       " 'index_train_18.txt',\n",
       " 'index_train_19.txt',\n",
       " 'index_train_2.txt',\n",
       " 'index_train_3.txt',\n",
       " 'index_train_4.txt',\n",
       " 'index_train_5.txt',\n",
       " 'index_train_6.txt',\n",
       " 'index_train_7.txt',\n",
       " 'index_train_8.txt',\n",
       " 'index_train_9.txt',\n",
       " 'n_epochs.txt',\n",
       " 'n_hidden.txt',\n",
       " 'n_splits.txt',\n",
       " 'tau_values.txt']"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import os\n",
    "path=\"C:/Users\\ssyhw7\\Desktop\\DropoutUncertaintyExps-master\\Hydric_Modeling\\First Version\\Data\"\n",
    "os.chdir(path)\n",
    "os.listdir(path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[4.43000e+00 3.81940e+00 0.00000e+00 ... 2.68020e+01 6.10600e-01\n",
      "  2.03837e+03]\n",
      " [4.42000e+00 3.82810e+00 0.00000e+00 ... 2.67771e+01 5.91900e-01\n",
      "  2.03837e+03]\n",
      " [4.43000e+00 3.81940e+00 0.00000e+00 ... 2.67873e+01 6.10600e-01\n",
      "  2.03837e+03]\n",
      " ...\n",
      " [5.16000e+00 4.31420e+00 2.89000e+01 ... 0.00000e+00 8.45800e-01\n",
      "  3.06741e+03]\n",
      " [5.16000e+00 4.30560e+00 2.89000e+01 ... 0.00000e+00 8.54400e-01\n",
      "  3.06741e+03]\n",
      " [5.16000e+00 4.29690e+00 2.89000e+01 ... 0.00000e+00 8.63100e-01\n",
      "  3.05044e+03]]\n"
     ]
    }
   ],
   "source": [
    "_DATA_DIRECTORY_PATH = \"C:/Users\\ssyhw7\\Desktop\\DropoutUncertaintyExps-master\\Hydric_Modeling\\First Version\" + \"\\Data\\data_file.txt\"\n",
    "data_directory = 'First Version'\n",
    "data = np.loadtxt(_DATA_DIRECTORY_PATH)\n",
    "print(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "smWU291RVS9s",
    "outputId": "ff926413-0c13-42a3-bba5-eb5dfb7d78e1"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Removing existing result files...\n",
      "Result files removed.\n",
      "Loading data and other hyperparameters...\n",
      "Done.\n",
      "Loading file: index_train_0.txt\n",
      "Loading file: index_test_0.txt\n",
      "Number of training examples: 56294\n",
      "Number of validation examples: 14074\n",
      "Number of test examples: 7819\n",
      "Number of train_original examples: 70368\n",
      "Grid search step: Tau: 0.025 Dropout rate: 0.005\n",
      "29/29 [==============================] - 0s 3ms/step\n",
      "Best log_likelihood changed to: -13.39414133147633\n",
      "Best tau changed to: 0.025\n",
      "Best dropout rate changed to: 0.005\n",
      "Grid search step: Tau: 0.05 Dropout rate: 0.005\n",
      "29/29 [==============================] - 0s 2ms/step\n",
      "Grid search step: Tau: 0.075 Dropout rate: 0.005\n",
      "29/29 [==============================] - 0s 1ms/step\n",
      "Grid search step: Tau: 0.025 Dropout rate: 0.01\n",
      "29/29 [==============================] - 0s 2ms/step\n",
      "Best log_likelihood changed to: -11.075371888602486\n",
      "Best tau changed to: 0.025\n",
      "Best dropout rate changed to: 0.01\n",
      "Grid search step: Tau: 0.05 Dropout rate: 0.01\n",
      "29/29 [==============================] - 0s 1ms/step\n",
      "Grid search step: Tau: 0.075 Dropout rate: 0.01\n",
      "29/29 [==============================] - 0s 2ms/step\n",
      "Grid search step: Tau: 0.025 Dropout rate: 0.05\n",
      "29/29 [==============================] - 0s 2ms/step\n",
      "Best log_likelihood changed to: -7.151936182374809\n",
      "Best tau changed to: 0.025\n",
      "Best dropout rate changed to: 0.05\n",
      "Grid search step: Tau: 0.05 Dropout rate: 0.05\n",
      "29/29 [==============================] - 0s 2ms/step\n",
      "Grid search step: Tau: 0.075 Dropout rate: 0.05\n",
      "29/29 [==============================] - 0s 2ms/step\n",
      "Grid search step: Tau: 0.025 Dropout rate: 0.1\n",
      "29/29 [==============================] - 0s 1ms/step\n",
      "Best log_likelihood changed to: -5.99034646229791\n",
      "Best tau changed to: 0.025\n",
      "Best dropout rate changed to: 0.1\n",
      "Grid search step: Tau: 0.05 Dropout rate: 0.1\n",
      "29/29 [==============================] - 0s 2ms/step\n",
      "Grid search step: Tau: 0.075 Dropout rate: 0.1\n",
      "29/29 [==============================] - 0s 2ms/step\n",
      "16/16 [==============================] - 0s 3ms/step\n",
      "Tests on split 0 complete.\n",
      "Loading file: index_train_1.txt\n",
      "Loading file: index_test_1.txt\n",
      "Number of training examples: 56294\n",
      "Number of validation examples: 14074\n",
      "Number of test examples: 7819\n",
      "Number of train_original examples: 70368\n",
      "Grid search step: Tau: 0.025 Dropout rate: 0.005\n",
      "29/29 [==============================] - 0s 2ms/step\n"
     ]
    }
   ],
   "source": [
    "# This code is based on the code by Jose Miguel Hernandez-Lobato used for his \n",
    "# paper \"Probabilistic Backpropagation for Scalable Learning of Bayesian Neural Networks\".\n",
    "\n",
    "# This file contains code to train dropout networks on the UCI datasets using the following algorithm:\n",
    "# 1. Create 20 random splits of the training-test dataset.\n",
    "# 2. For each split:\n",
    "# 3.   Create a validation (val) set taking 20% of the training set.\n",
    "# 4.   Get best hyperparameters: dropout_rate and tau by training on (train-val) set and testing on val set.\n",
    "# 5.   Train a network on the entire training set with the best pair of hyperparameters.\n",
    "# 6.   Get the performance (MC RMSE and log-likelihood) on the test set.\n",
    "# 7. Report the averaged performance (Monte Carlo RMSE and log-likelihood) on all 20 splits.\n",
    "\n",
    "import math\n",
    "import numpy as np\n",
    "import argparse\n",
    "import sys\n",
    "\n",
    "import pandas as pd\n",
    "from DataProcessing import DataProcessing as dp\n",
    "\n",
    "from subprocess import call\n",
    "import net\n",
    "\n",
    "parser=argparse.ArgumentParser()\n",
    "\n",
    "parser.add_argument('--dir', '-d', required=True, help='Name of the UCI Dataset directory. Eg')\n",
    "parser.add_argument('--epochx','-e', default=500, type=int, help='Multiplier for the number of epochs for training.')\n",
    "parser.add_argument('--hidden', '-nh', default=2, type=int, help='Number of hidden layers for the neural net')\n",
    "\n",
    "args = parser.parse_args(args=['--dir','First Version','--epochx','10','--hidden','3'])\n",
    "\n",
    "data_directory = args.dir\n",
    "epochs_multiplier = args.epochx\n",
    "num_hidden_layers = args.hidden\n",
    "\n",
    "sys.path.append('net/')\n",
    "#C:/Users\\ssyhw7\\Desktop\\DropoutUncertaintyExps-master\\Hydric_Modeling\\First Version\" + \"\\Data\\data_file.txt\"\n",
    "_RESULTS_VALIDATION_LL = \"C:/Users/ssyhw7/Desktop/DropoutUncertaintyExps-master/Hydric_Modeling/\" + data_directory + \"/results/validation_ll_\" + str(epochs_multiplier) + \"_xepochs_\" + str(num_hidden_layers) + \"_hidden_layers.txt\"\n",
    "_RESULTS_VALIDATION_RMSE = \"C:/Users/ssyhw7/Desktop/DropoutUncertaintyExps-master/Hydric_Modeling/\" + data_directory + \"/results/validation_rmse_\" + str(epochs_multiplier) + \"_xepochs_\" + str(num_hidden_layers) + \"_hidden_layers.txt\"\n",
    "_RESULTS_VALIDATION_MC_RMSE = \"C:/Users/ssyhw7/Desktop/DropoutUncertaintyExps-master/Hydric_Modeling/\" + data_directory + \"/results/validation_MC_rmse_\" + str(epochs_multiplier) + \"_xepochs_\" + str(num_hidden_layers) + \"_hidden_layers.txt\"\n",
    "\n",
    "_RESULTS_TEST_LL = \"C:/Users/ssyhw7/Desktop/DropoutUncertaintyExps-master/Hydric_Modeling/\" + data_directory + \"/results/test_ll_\" + str(epochs_multiplier) + \"_xepochs_\" + str(num_hidden_layers) + \"_hidden_layers.txt\"\n",
    "_RESULTS_TEST_TAU = \"C:/Users/ssyhw7/Desktop/DropoutUncertaintyExps-master/Hydric_Modeling/\" + data_directory + \"/results/test_tau_\" + str(epochs_multiplier) + \"_xepochs_\" + str(num_hidden_layers) + \"_hidden_layers.txt\"\n",
    "_RESULTS_TEST_RMSE = \"C:/Users/ssyhw7/Desktop/DropoutUncertaintyExps-master/Hydric_Modeling/\" + data_directory + \"/results/test_rmse_\" + str(epochs_multiplier) + \"_xepochs_\" + str(num_hidden_layers) + \"_hidden_layers.txt\"\n",
    "_RESULTS_TEST_MC_RMSE = \"C:/Users/ssyhw7/Desktop/DropoutUncertaintyExps-master/Hydric_Modeling/\" + data_directory + \"/results/test_MC_rmse_\" + str(epochs_multiplier) + \"_xepochs_\" + str(num_hidden_layers) + \"_hidden_layers.txt\"\n",
    "_RESULTS_TEST_LOG = \"C:/Users/ssyhw7/Desktop/DropoutUncertaintyExps-master/Hydric_Modeling/\" + data_directory + \"/results/log_\" + str(epochs_multiplier) + \"_xepochs_\" + str(num_hidden_layers) + \"_hidden_layers.txt\"\n",
    "\n",
    "_DATA_DIRECTORY_PATH = \"C:/Users/ssyhw7/Desktop/DropoutUncertaintyExps-master/Hydric_Modeling/\" + data_directory + \"/Data/\"\n",
    "_DROPOUT_RATES_FILE = _DATA_DIRECTORY_PATH + \"dropout_rates.txt\"\n",
    "_TAU_VALUES_FILE = _DATA_DIRECTORY_PATH + \"tau_values.txt\"\n",
    "_DATA_FILE = _DATA_DIRECTORY_PATH + \"data_file.txt\"\n",
    "_HIDDEN_UNITS_FILE = _DATA_DIRECTORY_PATH + \"n_hidden.txt\"\n",
    "_EPOCHS_FILE = _DATA_DIRECTORY_PATH + \"n_epochs.txt\"\n",
    "_INDEX_FEATURES_FILE = _DATA_DIRECTORY_PATH + \"index_features.txt\"\n",
    "_INDEX_TARGET_FILE = _DATA_DIRECTORY_PATH + \"index_target.txt\"\n",
    "_N_SPLITS_FILE = _DATA_DIRECTORY_PATH + \"n_splits.txt\"\n",
    "\n",
    "def _get_index_train_test_path(split_num, train = True):\n",
    "    \"\"\"\n",
    "       Method to generate the path containing the training/test split for the given\n",
    "       split number (generally from 1 to 20).\n",
    "       @param split_num      Split number for which the data has to be generated\n",
    "       @param train          Is true if the data is training data. Else false.\n",
    "       @return path          Path of the file containing the requried data\n",
    "    \"\"\"\n",
    "    if train:\n",
    "        return \"index_train_\" + str(split_num) + \".txt\"\n",
    "    else:\n",
    "        return \"index_test_\" + str(split_num) + \".txt\" \n",
    "\n",
    "\n",
    "print (\"Removing existing result files...\")\n",
    "call([\"rm\", _RESULTS_VALIDATION_LL])\n",
    "call([\"rm\", _RESULTS_VALIDATION_RMSE])\n",
    "call([\"rm\", _RESULTS_VALIDATION_MC_RMSE])\n",
    "call([\"rm\", _RESULTS_TEST_LL])\n",
    "call([\"rm\", _RESULTS_TEST_TAU])\n",
    "call([\"rm\", _RESULTS_TEST_RMSE])\n",
    "call([\"rm\", _RESULTS_TEST_MC_RMSE])\n",
    "call([\"rm\", _RESULTS_TEST_LOG])\n",
    "print (\"Result files removed.\")\n",
    "\n",
    "# We fix the random seed\n",
    "\n",
    "np.random.seed(1)\n",
    "\n",
    "print (\"Loading data and other hyperparameters...\")\n",
    "# We load the data\n",
    "\n",
    "data = np.loadtxt(_DATA_FILE)\n",
    "\n",
    "# We load the number of hidden units\n",
    "\n",
    "n_hidden = np.loadtxt(_HIDDEN_UNITS_FILE).tolist()\n",
    "\n",
    "# We load the number of training epocs\n",
    "\n",
    "n_epochs = np.loadtxt(_EPOCHS_FILE).tolist()\n",
    "\n",
    "# We load the indexes for the features and for the target\n",
    "\n",
    "index_features = np.loadtxt(_INDEX_FEATURES_FILE)\n",
    "index_target = np.loadtxt(_INDEX_TARGET_FILE)\n",
    "\n",
    "X = data[ : , [int(i) for i in index_features.tolist()] ]\n",
    "y = data[ : , int(index_target.tolist()) ]\n",
    "\n",
    "# We iterate over the training test splits\n",
    "\n",
    "n_splits = np.loadtxt(_N_SPLITS_FILE)\n",
    "print (\"Done.\")\n",
    "\n",
    "errors, MC_errors, lls = [], [], []\n",
    "for split in range(int(n_splits)):\n",
    "\n",
    "    # We load the indexes of the training and test sets\n",
    "    print ('Loading file: ' + _get_index_train_test_path(split, train=True))\n",
    "    print ('Loading file: ' + _get_index_train_test_path(split, train=False))\n",
    "    index_train = np.loadtxt(_get_index_train_test_path(split, train=True))\n",
    "    index_test = np.loadtxt(_get_index_train_test_path(split, train=False))\n",
    "\n",
    "    X_train = X[ [int(i) for i in index_train.tolist()] ]\n",
    "    y_train = y[ [int(i) for i in index_train.tolist()] ]\n",
    "    \n",
    "    X_test = X[ [int(i) for i in index_test.tolist()] ]\n",
    "    y_test = y[ [int(i) for i in index_test.tolist()] ]\n",
    "\n",
    "    X_train_original = X_train\n",
    "    y_train_original = y_train\n",
    "    num_training_examples = int(0.8 * X_train.shape[0])\n",
    "    X_validation = X_train[num_training_examples:, :]\n",
    "    y_validation = y_train[num_training_examples:]\n",
    "    X_train = X_train[0:num_training_examples, :]\n",
    "    y_train = y_train[0:num_training_examples]\n",
    "    \n",
    "    # Printing the size of the training, validation and test sets\n",
    "    print ('Number of training examples: ' + str(X_train.shape[0]))\n",
    "    print ('Number of validation examples: ' + str(X_validation.shape[0]))\n",
    "    print ('Number of test examples: ' + str(X_test.shape[0]))\n",
    "    print ('Number of train_original examples: ' + str(X_train_original.shape[0]))\n",
    "\n",
    "    # List of hyperparameters which we will try out using grid-search\n",
    "    dropout_rates = np.loadtxt(_DROPOUT_RATES_FILE).tolist()\n",
    "    tau_values = np.loadtxt(_TAU_VALUES_FILE).tolist()\n",
    "\n",
    "    # We perform grid-search to select the best hyperparameters based on the highest log-likelihood value\n",
    "    best_network = None\n",
    "    best_ll = -float('inf')\n",
    "    best_tau = 0\n",
    "    best_dropout = 0\n",
    "    for dropout_rate in dropout_rates:\n",
    "        for tau in tau_values:\n",
    "            print ('Grid search step: Tau: ' + str(tau) + ' Dropout rate: ' + str(dropout_rate))\n",
    "            network = net.net(X_train, y_train, ([ int(n_hidden) ] * num_hidden_layers),\n",
    "                    normalize = True, n_epochs = int(n_epochs * epochs_multiplier), tau = tau,\n",
    "                    dropout = dropout_rate)\n",
    "\n",
    "            # We obtain the test RMSE and the test ll from the validation sets\n",
    "\n",
    "            error, MC_error, ll = network.predict(X_validation, y_validation)\n",
    "            if (ll > best_ll):\n",
    "                best_ll = ll\n",
    "                best_network = network\n",
    "                best_tau = tau\n",
    "                best_dropout = dropout_rate\n",
    "                print ('Best log_likelihood changed to: ' + str(best_ll))\n",
    "                print ('Best tau changed to: ' + str(best_tau))\n",
    "                print ('Best dropout rate changed to: ' + str(best_dropout))\n",
    "            \n",
    "            # Storing validation results\n",
    "            with open(_RESULTS_VALIDATION_RMSE, \"a\") as myfile:\n",
    "                myfile.write('Dropout_Rate: ' + repr(dropout_rate) + ' Tau: ' + repr(tau) + ' :: ')\n",
    "                myfile.write(repr(error) + '\\n')\n",
    "\n",
    "            with open(_RESULTS_VALIDATION_MC_RMSE, \"a\") as myfile:\n",
    "                myfile.write('Dropout_Rate: ' + repr(dropout_rate) + ' Tau: ' + repr(tau) + ' :: ')\n",
    "                myfile.write(repr(MC_error) + '\\n')\n",
    "\n",
    "            with open(_RESULTS_VALIDATION_LL, \"a\") as myfile:\n",
    "                myfile.write('Dropout_Rate: ' + repr(dropout_rate) + ' Tau: ' + repr(tau) + ' :: ')\n",
    "                myfile.write(repr(ll) + '\\n')\n",
    "\n",
    "    # Storing test results\n",
    "    best_network = net.net(X_train_original, y_train_original, ([ int(n_hidden) ] * num_hidden_layers),\n",
    "                    normalize = True, n_epochs = int(n_epochs * epochs_multiplier), tau = best_tau,\n",
    "                    dropout = best_dropout)\n",
    "    error, MC_error, ll = best_network.predict(X_test, y_test)\n",
    "    \n",
    "    with open(_RESULTS_TEST_RMSE, \"a\") as myfile:\n",
    "        myfile.write(repr(error) + '\\n')\n",
    "\n",
    "    with open(_RESULTS_TEST_MC_RMSE, \"a\") as myfile:\n",
    "        myfile.write(repr(MC_error) + '\\n')\n",
    "\n",
    "    with open(_RESULTS_TEST_LL, \"a\") as myfile:\n",
    "        myfile.write(repr(ll) + '\\n')\n",
    "\n",
    "    with open(_RESULTS_TEST_TAU, \"a\") as myfile:\n",
    "        myfile.write(repr(best_network.tau) + '\\n')\n",
    "\n",
    "    print (\"Tests on split \" + str(split) + \" complete.\")\n",
    "    errors += [error]\n",
    "    MC_errors += [MC_error]\n",
    "    lls += [ll]\n",
    "\n",
    "with open(_RESULTS_TEST_LOG, \"a\") as myfile:\n",
    "    myfile.write('errors %f +- %f (stddev) +- %f (std error), median %f 25p %f 75p %f \\n' % (\n",
    "        np.mean(errors), np.std(errors), np.std(errors)/math.sqrt(n_splits),\n",
    "        np.percentile(errors, 50), np.percentile(errors, 25), np.percentile(errors, 75)))\n",
    "    myfile.write('MC errors %f +- %f (stddev) +- %f (std error), median %f 25p %f 75p %f \\n' % (\n",
    "        np.mean(MC_errors), np.std(MC_errors), np.std(MC_errors)/math.sqrt(n_splits),\n",
    "        np.percentile(MC_errors, 50), np.percentile(MC_errors, 25), np.percentile(MC_errors, 75)))\n",
    "    myfile.write('lls %f +- %f (stddev) +- %f (std error), median %f 25p %f 75p %f \\n' % (\n",
    "        np.mean(lls), np.std(lls), np.std(lls)/math.sqrt(n_splits), \n",
    "        np.percentile(lls, 50), np.percentile(lls, 25), np.percentile(lls, 75)))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "authorship_tag": "ABX9TyPjI8PAW3DgREmo9TWh4I+2",
   "include_colab_link": true,
   "name": "Untitled5.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}