bbp_hetero_Hybrid.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/EdWangLoDaSc/Dropout-as-a-Grid-Search_Representing-Model-Uncertainty-in-Deep-Learning/blob/main/bbp_hetero_Hybrid.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pVmDXZ3MUJfB",
        "outputId": "a4dfe86b-f36d-4a75-c5e0-2fb448d35ff4"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xAYPKSFrG8AF",
        "outputId": "4c810d23-ca3e-43e9-fb42-79721e7319f9"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "\u001b[31mERROR: torch-0.4.1-cp36-cp36m-linux_x86_64.whl is not a supported wheel on this platform.\u001b[0m\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.7/dist-packages (0.13.0+cu113)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from torchvision) (1.21.6)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from torchvision) (2.23.0)\n",
            "Requirement already satisfied: torch==1.12.0 in /usr/local/lib/python3.7/dist-packages (from torchvision) (1.12.0+cu113)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torchvision) (4.1.1)\n",
            "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/local/lib/python3.7/dist-packages (from torchvision) (7.1.2)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->torchvision) (2022.6.15)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->torchvision) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->torchvision) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->torchvision) (1.24.3)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting GPy\n",
            "  Downloading GPy-1.10.0.tar.gz (959 kB)\n",
            "\u001b[K     |████████████████████████████████| 959 kB 14.3 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.7 in /usr/local/lib/python3.7/dist-packages (from GPy) (1.21.6)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from GPy) (1.15.0)\n",
            "Collecting paramz>=0.9.0\n",
            "  Downloading paramz-0.9.5.tar.gz (71 kB)\n",
            "\u001b[K     |████████████████████████████████| 71 kB 10.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: cython>=0.29 in /usr/local/lib/python3.7/dist-packages (from GPy) (0.29.30)\n",
            "Requirement already satisfied: scipy>=1.3.0 in /usr/local/lib/python3.7/dist-packages (from GPy) (1.7.3)\n",
            "Requirement already satisfied: decorator>=4.0.10 in /usr/local/lib/python3.7/dist-packages (from paramz>=0.9.0->GPy) (4.4.2)\n",
            "Building wheels for collected packages: GPy, paramz\n",
            "  Building wheel for GPy (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for GPy: filename=GPy-1.10.0-cp37-cp37m-linux_x86_64.whl size=2565125 sha256=8e5a1be8e78323e3cc7e367319d36b44d168ab410d1871be5057228f738bd679\n",
            "  Stored in directory: /root/.cache/pip/wheels/f7/18/28/dd1ce0192a81b71a3b086fd952511d088b21e8359ea496860a\n",
            "  Building wheel for paramz (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for paramz: filename=paramz-0.9.5-py3-none-any.whl size=102566 sha256=6ee7d57cc53ec73e3206aef4eb24de83afa2908146b94648a7f33f59a1dcc455\n",
            "  Stored in directory: /root/.cache/pip/wheels/c8/95/f5/ce28482da28162e6028c4b3a32c41d147395825b3cd62bc810\n",
            "Successfully built GPy paramz\n",
            "Installing collected packages: paramz, GPy\n",
            "Successfully installed GPy-1.10.0 paramz-0.9.5\n"
          ]
        }
      ],
      "source": [
        "!pip3 install http://download.pytorch.org/whl/cu92/torch-0.4.1-cp36-cp36m-linux_x86_64.whl\n",
        "!pip3 install torchvision\n",
        "!pip install GPy\n",
        "import GPy\n",
        "import pandas as pd\n",
        "import zipfile\n",
        "import urllib.request\n",
        "import time\n",
        "import copy\n",
        "import math\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import seaborn as sns\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torch.autograd import Variable\n",
        "from torch.optim import Optimizer\n",
        "from sklearn.model_selection import KFold\n",
        "\n",
        "from torchvision import datasets, transforms\n",
        "from torchvision.utils import make_grid\n",
        "from tqdm import tqdm, trange\n",
        "from google.colab import files\n",
        "%config InlineBackend.figure_format = 'svg'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "1-7dNcVmHA3I",
        "outputId": "5ebc802e-0b44-42b7-d9e9-5475933190fb"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Tesla P100-PCIE-16GB'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 3
        }
      ],
      "source": [
        "torch.cuda.device(0)\n",
        "torch.cuda.get_device_name(torch.cuda.current_device())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "_BzTB5WsP9Kx"
      },
      "outputs": [],
      "source": [
        "def to_variable(var=(), cuda=True, volatile=False):\n",
        "    out = []\n",
        "    for v in var:\n",
        "        \n",
        "        if isinstance(v, np.ndarray):\n",
        "            v = torch.from_numpy(v).type(torch.FloatTensor)\n",
        "\n",
        "        if not v.is_cuda and cuda:\n",
        "            v = v.cuda()\n",
        "\n",
        "        if not isinstance(v, Variable):\n",
        "            v = Variable(v, volatile=volatile)\n",
        "\n",
        "        out.append(v)\n",
        "    return out"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "Va8V78eFFsc9"
      },
      "outputs": [],
      "source": [
        "def log_gaussian_loss(output, target, sigma, no_dim, sum_reduce=True):\n",
        "    exponent = -0.5*(target - output)**2/sigma**2\n",
        "    log_coeff = -no_dim*torch.log(sigma) - 0.5*no_dim*np.log(2*np.pi)\n",
        "    \n",
        "    if sum_reduce:\n",
        "        return -(log_coeff + exponent).sum()\n",
        "    else:\n",
        "        return -(log_coeff + exponent)\n",
        "\n",
        "\n",
        "def get_kl_divergence(weights, prior, varpost):\n",
        "    prior_loglik = prior.loglik(weights)\n",
        "    \n",
        "    varpost_loglik = varpost.loglik(weights)\n",
        "    varpost_lik = varpost_loglik.exp()\n",
        "    \n",
        "    return (varpost_lik*(varpost_loglik - prior_loglik)).sum()\n",
        "\n",
        "\n",
        "class gaussian:\n",
        "    def __init__(self, mu, sigma):\n",
        "        self.mu = mu\n",
        "        self.sigma = sigma\n",
        "        \n",
        "    def loglik(self, weights):\n",
        "        exponent = -0.5*(weights - self.mu)**2/self.sigma**2\n",
        "        log_coeff = -0.5*(np.log(2*np.pi) + 2*np.log(self.sigma))\n",
        "        \n",
        "        return (exponent + log_coeff).sum()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "ASGi2Ecx5G-F"
      },
      "outputs": [],
      "source": [
        "class BayesLinear_Normalq(nn.Module):\n",
        "    def __init__(self, input_dim, output_dim, prior):\n",
        "        super(BayesLinear_Normalq, self).__init__()\n",
        "        self.input_dim = input_dim\n",
        "        self.output_dim = output_dim\n",
        "        self.prior = prior\n",
        "        \n",
        "        scale = (2/self.input_dim)**0.5\n",
        "        rho_init = np.log(np.exp((2/self.input_dim)**0.5) - 1)\n",
        "        self.weight_mus = nn.Parameter(torch.Tensor(self.input_dim, self.output_dim).uniform_(-0.01, 0.01))\n",
        "        self.weight_rhos = nn.Parameter(torch.Tensor(self.input_dim, self.output_dim).uniform_(-3, -3))\n",
        "        \n",
        "        self.bias_mus = nn.Parameter(torch.Tensor(self.output_dim).uniform_(-0.01, 0.01))\n",
        "        self.bias_rhos = nn.Parameter(torch.Tensor(self.output_dim).uniform_(-4, -3))\n",
        "        \n",
        "    def forward(self, x, sample = True):\n",
        "        \n",
        "        if sample:\n",
        "            # sample gaussian noise for each weight and each bias\n",
        "            weight_epsilons = Variable(self.weight_mus.data.new(self.weight_mus.size()).normal_())\n",
        "            bias_epsilons =  Variable(self.bias_mus.data.new(self.bias_mus.size()).normal_())\n",
        "            \n",
        "            # calculate the weight and bias stds from the rho parameters\n",
        "            weight_stds = torch.log(1 + torch.exp(self.weight_rhos))\n",
        "            bias_stds = torch.log(1 + torch.exp(self.bias_rhos))\n",
        "            \n",
        "            # calculate samples from the posterior from the sampled noise and mus/stds\n",
        "            weight_sample = self.weight_mus + weight_epsilons*weight_stds\n",
        "            bias_sample = self.bias_mus + bias_epsilons*bias_stds\n",
        "            \n",
        "            output = torch.mm(x, weight_sample) + bias_sample\n",
        "            \n",
        "            # computing the KL loss term\n",
        "            prior_cov, varpost_cov = self.prior.sigma**2, weight_stds**2\n",
        "            KL_loss = 0.5*(torch.log(prior_cov/varpost_cov)).sum() - 0.5*weight_stds.numel()\n",
        "            KL_loss = KL_loss + 0.5*(varpost_cov/prior_cov).sum()\n",
        "            KL_loss = KL_loss + 0.5*((self.weight_mus - self.prior.mu)**2/prior_cov).sum()\n",
        "            \n",
        "            prior_cov, varpost_cov = self.prior.sigma**2, bias_stds**2\n",
        "            KL_loss = KL_loss + 0.5*(torch.log(prior_cov/varpost_cov)).sum() - 0.5*bias_stds.numel()\n",
        "            KL_loss = KL_loss + 0.5*(varpost_cov/prior_cov).sum()\n",
        "            KL_loss = KL_loss + 0.5*((self.bias_mus - self.prior.mu)**2/prior_cov).sum()\n",
        "            \n",
        "            return output, KL_loss\n",
        "        \n",
        "        else:\n",
        "            output = torch.mm(x, self.weight_mus) + self.bias_mus\n",
        "            return output, KL_loss\n",
        "        \n",
        "    def sample_layer(self, no_samples):\n",
        "        all_samples = []\n",
        "        for i in range(no_samples):\n",
        "            # sample gaussian noise for each weight and each bias\n",
        "            weight_epsilons = Variable(self.weight_mus.data.new(self.weight_mus.size()).normal_())\n",
        "            \n",
        "            # calculate the weight and bias stds from the rho parameters\n",
        "            weight_stds = torch.log(1 + torch.exp(self.weight_rhos))\n",
        "            \n",
        "            # calculate samples from the posterior from the sampled noise and mus/stds\n",
        "            weight_sample = self.weight_mus + weight_epsilons*weight_stds\n",
        "            \n",
        "            all_samples += weight_sample.view(-1).cpu().data.numpy().tolist()\n",
        "            \n",
        "        return all_samples"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "_8dV-QIq5G-I"
      },
      "outputs": [],
      "source": [
        "class BBP_Heteroscedastic_Model(nn.Module):\n",
        "    def __init__(self, input_dim, output_dim, num_units):\n",
        "        super(BBP_Heteroscedastic_Model, self).__init__()\n",
        "        \n",
        "        self.input_dim = input_dim\n",
        "        self.output_dim = output_dim\n",
        "        \n",
        "        # network with two hidden and one output layer\n",
        "        self.layer1 = BayesLinear_Normalq(input_dim, num_units, gaussian(0, 1))\n",
        "        self.layer2 = BayesLinear_Normalq(num_units, 2*output_dim, gaussian(0, 1))\n",
        "        \n",
        "        # activation to be used between hidden layers\n",
        "        self.activation = nn.ReLU(inplace = True)\n",
        "    \n",
        "    def forward(self, x):\n",
        "        \n",
        "        KL_loss_total = 0\n",
        "        x = x.view(-1, self.input_dim)\n",
        "        \n",
        "        x, KL_loss = self.layer1(x)\n",
        "        KL_loss_total = KL_loss_total + KL_loss\n",
        "        x = self.activation(x)\n",
        "        \n",
        "        x, KL_loss = self.layer2(x)\n",
        "        KL_loss_total = KL_loss_total + KL_loss\n",
        "        \n",
        "        return x, KL_loss_total"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "oAYelw3B5G-K"
      },
      "outputs": [],
      "source": [
        "class BBP_Heteroscedastic_Model_Wrapper:\n",
        "    def __init__(self, network, learn_rate, batch_size, no_batches):\n",
        "        \n",
        "        self.learn_rate = learn_rate\n",
        "        self.batch_size = batch_size\n",
        "        self.no_batches = no_batches\n",
        "        \n",
        "        self.network = network\n",
        "        self.network.cuda()\n",
        "        \n",
        "        self.optimizer = torch.optim.Adam(self.network.parameters(), lr = self.learn_rate)\n",
        "        self.loss_func = log_gaussian_loss\n",
        "    \n",
        "    def fit(self, x, y, no_samples):\n",
        "        x, y = to_variable(var=(x, y), cuda=True)\n",
        "        \n",
        "        # reset gradient and total loss\n",
        "        self.optimizer.zero_grad()\n",
        "        fit_loss_total = 0\n",
        "        \n",
        "        for i in range(no_samples):\n",
        "            output, KL_loss_total = self.network(x)\n",
        "\n",
        "            # calculate fit loss based on mean and standard deviation of output\n",
        "            fit_loss = self.loss_func(output[:, :1], y, output[:, 1:].exp(), 1)\n",
        "            fit_loss_total = fit_loss_total + fit_loss\n",
        "        \n",
        "        KL_loss_total = KL_loss_total/self.no_batches\n",
        "        total_loss = (fit_loss_total + KL_loss_total)/(no_samples*x.shape[0])\n",
        "        total_loss.backward()\n",
        "        self.optimizer.step()\n",
        "\n",
        "        return fit_loss_total/no_samples, KL_loss_total\n",
        "    \n",
        "    def get_loss_and_rmse(self, x, y, no_samples):\n",
        "        x, y = to_variable(var=(x, y), cuda=True)\n",
        "        \n",
        "        means, stds = [], []\n",
        "        for i in range(no_samples):\n",
        "            output, KL_loss_total = self.network(x)\n",
        "            means.append(output[:, :1, None])\n",
        "            stds.append(output[:, 1:, None].exp())\n",
        "            \n",
        "        means, stds = torch.cat(means, 2), torch.cat(stds, 2)\n",
        "        mean = means.mean(dim=2)\n",
        "        std = (means.var(dim=2) + stds.mean(dim=2)**2)**0.5\n",
        "            \n",
        "        # calculate fit loss based on mean and standard deviation of output\n",
        "        logliks = self.loss_func(output[:, :1], y, output[:, 1:].exp(), 1, sum_reduce=False)\n",
        "        rmse = float((((mean - y)**2).mean()**0.5).cpu().data)\n",
        "\n",
        "        return logliks, rmse"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "oN9rvBjyVKxt"
      },
      "outputs": [],
      "source": [
        "class BBP_Heteroscedastic_Model_UCI(nn.Module):\n",
        "    def __init__(self, input_dim, output_dim, num_units):\n",
        "        super(BBP_Heteroscedastic_Model_UCI, self).__init__()\n",
        "        \n",
        "        self.input_dim = input_dim\n",
        "        self.output_dim = output_dim\n",
        "        \n",
        "        # network with two hidden and one output layer\n",
        "        self.layer1 = BayesLinear_Normalq(input_dim, num_units, gaussian(0, 1))\n",
        "        self.layer2 = BayesLinear_Normalq(num_units, num_units, gaussian(0, 1))\n",
        "        self.layer3 = BayesLinear_Normalq(num_units, 2*output_dim, gaussian(0, 1))\n",
        "        \n",
        "        # activation to be used between hidden layers\n",
        "        self.activation = nn.ReLU(inplace = True)\n",
        "    \n",
        "    def forward(self, x):\n",
        "        \n",
        "        KL_loss_total = 0\n",
        "        x = x.view(-1, self.input_dim)\n",
        "        \n",
        "        x, KL_loss = self.layer1(x)\n",
        "        KL_loss_total = KL_loss_total + KL_loss\n",
        "        x = self.activation(x)\n",
        "        \n",
        "        x, KL_loss = self.layer2(x)\n",
        "        KL_loss_total = KL_loss_total + KL_loss\n",
        "        \n",
        "        return x, KL_loss_total\n",
        "\n",
        "def train_BBP(data, n_splits, num_epochs, num_units, learn_rate, log_every):\n",
        "    kf = KFold(n_splits=n_splits)\n",
        "    in_dim = data.shape[1] - 1\n",
        "    train_logliks, test_logliks = [], []\n",
        "    train_rmses, test_rmses = [], []\n",
        "\n",
        "    for i, idx in enumerate(kf.split(data)):\n",
        "        print('FOLD %d:' % i)\n",
        "\n",
        "        train_index, test_index = idx\n",
        "\n",
        "        x_train, y_train = data[train_index, :in_dim], data[train_index, in_dim:]\n",
        "        x_test, y_test = data[test_index, :in_dim], data[test_index, in_dim:]\n",
        "\n",
        "        x_means, x_stds = x_train.mean(axis = 0), x_train.var(axis = 0)**0.5\n",
        "        y_means, y_stds = y_train.mean(axis = 0), y_train.var(axis = 0)**0.5\n",
        "\n",
        "        x_train = (x_train - x_means)/x_stds\n",
        "        y_train = (y_train - y_means)/y_stds\n",
        "\n",
        "        x_test = (x_test - x_means)/x_stds\n",
        "        y_test = (y_test - y_means)/y_stds\n",
        "\n",
        "        batch_size, nb_train = len(x_train), len(x_train)\n",
        "\n",
        "        net = BBP_Heteroscedastic_Model_Wrapper(network=BBP_Heteroscedastic_Model_UCI(input_dim=x_test.shape[-1], output_dim=1, num_units=num_units),\n",
        "                                                learn_rate=1e-2, batch_size=batch_size, no_batches=1)\n",
        "\n",
        "        fit_loss_train = np.zeros(num_epochs)\n",
        "        KL_loss_train = np.zeros(num_epochs)\n",
        "        total_loss = np.zeros(num_epochs)\n",
        "\n",
        "        best_net, best_loss = None, float('inf')\n",
        "\n",
        "        for i in range(num_epochs):\n",
        "\n",
        "            fit_loss, KL_loss = net.fit(x_train, y_train, no_samples = 20)\n",
        "            fit_loss_train[i] += fit_loss.cpu().data.numpy()\n",
        "            KL_loss_train[i] += KL_loss.cpu().data.numpy()\n",
        "\n",
        "            total_loss[i] = fit_loss_train[i] + KL_loss_train[i]\n",
        "\n",
        "            if fit_loss < best_loss:\n",
        "                best_loss = fit_loss\n",
        "                best_net = copy.deepcopy(net.network)\n",
        "\n",
        "            if i % log_every == 0 or i == num_epochs - 1:\n",
        "\n",
        "                train_losses, train_rmse = net.get_loss_and_rmse(x_train, y_train, 20)\n",
        "                test_losses, test_rmse = net.get_loss_and_rmse(x_test, y_test, 20)\n",
        "\n",
        "                print('Epoch: %s/%d, Train loglik = %.3f, Test loglik = %.3f, Train RMSE = %.3f, Test RMSE = %.3f' %\\\n",
        "                      (str(i+1).zfill(3), num_epochs, -train_losses.mean() - np.log(y_stds)[0],\n",
        "                       -test_losses.mean() - np.log(y_stds)[0], y_stds*train_rmse, y_stds*test_rmse))\n",
        "\n",
        "\n",
        "        train_losses, train_rmse = net.get_loss_and_rmse(x_train, y_train, 20)\n",
        "        test_losses, test_rmse = net.get_loss_and_rmse(x_test, y_test, 20)\n",
        "\n",
        "        train_logliks.append((train_losses.cpu().data.numpy().mean() + np.log(y_stds)[0]))\n",
        "        test_logliks.append((test_losses.cpu().data.numpy().mean() + np.log(y_stds)[0]))\n",
        "\n",
        "        train_rmses.append(y_stds*train_rmse)\n",
        "        test_rmses.append(y_stds*test_rmse)\n",
        "\n",
        "    print('Train log. lik. = %6.3f +/- %6.3f' % (-np.array(train_logliks).mean(), np.array(train_logliks).var()**0.5))\n",
        "    print('Test  log. lik. = %6.3f +/- %6.3f' % (-np.array(test_logliks).mean(), np.array(test_logliks).var()**0.5))\n",
        "    print('Train RMSE      = %6.3f +/- %6.3f' % (np.array(train_rmses).mean(), np.array(train_rmses).var()**0.5))\n",
        "    print('Test  RMSE      = %6.3f +/- %6.3f' % (np.array(test_rmses).mean(), np.array(test_rmses).var()**0.5))\n",
        "\n",
        "    return best_net"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "os.chdir('/content/drive/MyDrive')"
      ],
      "metadata": {
        "id": "CEj1b3Pefkqh"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pip install miceforest"
      ],
      "metadata": {
        "id": "oUrVAe19frQL",
        "outputId": "3a534b95-e718-4c4f-d3ca-4c8eb8d993b8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting miceforest\n",
            "  Downloading miceforest-5.5.4-py3-none-any.whl (49 kB)\n",
            "\u001b[K     |████████████████████████████████| 49 kB 5.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: dill in /usr/local/lib/python3.7/dist-packages (from miceforest) (0.3.5.1)\n",
            "Collecting lightgbm>=3.3.1\n",
            "  Downloading lightgbm-3.3.2-py3-none-manylinux1_x86_64.whl (2.0 MB)\n",
            "\u001b[K     |████████████████████████████████| 2.0 MB 27.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from miceforest) (1.21.6)\n",
            "Collecting blosc\n",
            "  Downloading blosc-1.10.6-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (2.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 2.6 MB 57.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from lightgbm>=3.3.1->miceforest) (1.7.3)\n",
            "Requirement already satisfied: wheel in /usr/local/lib/python3.7/dist-packages (from lightgbm>=3.3.1->miceforest) (0.37.1)\n",
            "Requirement already satisfied: scikit-learn!=0.22.0 in /usr/local/lib/python3.7/dist-packages (from lightgbm>=3.3.1->miceforest) (1.0.2)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn!=0.22.0->lightgbm>=3.3.1->miceforest) (3.1.0)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn!=0.22.0->lightgbm>=3.3.1->miceforest) (1.1.0)\n",
            "Installing collected packages: lightgbm, blosc, miceforest\n",
            "  Attempting uninstall: lightgbm\n",
            "    Found existing installation: lightgbm 2.2.3\n",
            "    Uninstalling lightgbm-2.2.3:\n",
            "      Successfully uninstalled lightgbm-2.2.3\n",
            "Successfully installed blosc-1.10.6 lightgbm-3.3.2 miceforest-5.5.4\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "os.chdir('/content/drive/MyDrive/DropoutUncertaintyExps-master/Hydrid_modeling')"
      ],
      "metadata": {
        "id": "re2BT7I9fU6X"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cOq4tM_PLRUV",
        "outputId": "64144571-dbc3-4013-968e-e2a106b393cd"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            " /content/drive/MyDrive/DataProcessing.py:19: SettingWithCopyWarning:\n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            " /content/drive/MyDrive/DataProcessing.py:21: SettingWithCopyWarning:\n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            " /content/drive/MyDrive/DataProcessing.py:23: SettingWithCopyWarning:\n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            " /content/drive/MyDrive/DataProcessing.py:25: SettingWithCopyWarning:\n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            " /content/drive/MyDrive/DataProcessing.py:26: SettingWithCopyWarning:\n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "FOLD 0:\n",
            "Epoch: 001/500, Train loglik = -8.228, Test loglik = -8.091, Train RMSE = 929.159, Test RMSE = 729.341\n",
            "Epoch: 031/500, Train loglik = -6.759, Test loglik = -7.273, Train RMSE = 170.985, Test RMSE = 182.911\n",
            "Epoch: 061/500, Train loglik = -6.953, Test loglik = -6.720, Train RMSE = 183.431, Test RMSE = 150.203\n",
            "Epoch: 091/500, Train loglik = -6.578, Test loglik = -6.592, Train RMSE = 174.506, Test RMSE = 111.421\n",
            "Epoch: 121/500, Train loglik = -6.216, Test loglik = -6.076, Train RMSE = 158.928, Test RMSE = 100.325\n",
            "Epoch: 151/500, Train loglik = -6.285, Test loglik = -6.252, Train RMSE = 147.642, Test RMSE = 96.925\n",
            "Epoch: 181/500, Train loglik = -6.208, Test loglik = -6.065, Train RMSE = 119.790, Test RMSE = 90.959\n",
            "Epoch: 211/500, Train loglik = -6.072, Test loglik = -5.990, Train RMSE = 102.884, Test RMSE = 86.281\n",
            "Epoch: 241/500, Train loglik = -6.011, Test loglik = -5.909, Train RMSE = 109.087, Test RMSE = 81.286\n",
            "Epoch: 271/500, Train loglik = -5.825, Test loglik = -5.841, Train RMSE = 102.231, Test RMSE = 81.188\n",
            "Epoch: 301/500, Train loglik = -5.825, Test loglik = -5.801, Train RMSE = 101.478, Test RMSE = 83.507\n",
            "Epoch: 331/500, Train loglik = -5.928, Test loglik = -5.648, Train RMSE = 102.447, Test RMSE = 83.943\n",
            "Epoch: 361/500, Train loglik = -5.978, Test loglik = -5.762, Train RMSE = 102.886, Test RMSE = 88.637\n",
            "Epoch: 391/500, Train loglik = -5.804, Test loglik = -5.646, Train RMSE = 103.197, Test RMSE = 86.956\n",
            "Epoch: 421/500, Train loglik = -5.839, Test loglik = -5.741, Train RMSE = 104.028, Test RMSE = 83.834\n",
            "Epoch: 451/500, Train loglik = -5.798, Test loglik = -5.707, Train RMSE = 99.545, Test RMSE = 79.908\n",
            "Epoch: 481/500, Train loglik = -5.728, Test loglik = -5.653, Train RMSE = 104.466, Test RMSE = 80.007\n",
            "Epoch: 500/500, Train loglik = -5.869, Test loglik = -5.672, Train RMSE = 103.114, Test RMSE = 86.611\n",
            "FOLD 1:\n",
            "Epoch: 001/500, Train loglik = -8.227, Test loglik = -8.372, Train RMSE = 902.447, Test RMSE = 989.700\n",
            "Epoch: 031/500, Train loglik = -6.687, Test loglik = -6.649, Train RMSE = 179.792, Test RMSE = 138.320\n",
            "Epoch: 061/500, Train loglik = -6.611, Test loglik = -6.426, Train RMSE = 163.108, Test RMSE = 111.896\n",
            "Epoch: 091/500, Train loglik = -6.517, Test loglik = -6.483, Train RMSE = 172.243, Test RMSE = 93.530\n",
            "Epoch: 121/500, Train loglik = -6.288, Test loglik = -6.348, Train RMSE = 167.733, Test RMSE = 98.032\n",
            "Epoch: 151/500, Train loglik = -6.094, Test loglik = -5.954, Train RMSE = 141.054, Test RMSE = 88.960\n",
            "Epoch: 181/500, Train loglik = -6.530, Test loglik = -5.919, Train RMSE = 130.406, Test RMSE = 89.348\n",
            "Epoch: 211/500, Train loglik = -6.095, Test loglik = -5.838, Train RMSE = 113.152, Test RMSE = 89.132\n",
            "Epoch: 241/500, Train loglik = -5.910, Test loglik = -6.039, Train RMSE = 102.607, Test RMSE = 89.424\n",
            "Epoch: 271/500, Train loglik = -5.839, Test loglik = -5.889, Train RMSE = 103.945, Test RMSE = 84.109\n",
            "Epoch: 301/500, Train loglik = -5.913, Test loglik = -5.841, Train RMSE = 103.657, Test RMSE = 84.649\n",
            "Epoch: 331/500, Train loglik = -6.142, Test loglik = -5.877, Train RMSE = 103.478, Test RMSE = 93.548\n",
            "Epoch: 361/500, Train loglik = -5.786, Test loglik = -5.715, Train RMSE = 99.342, Test RMSE = 85.032\n",
            "Epoch: 391/500, Train loglik = -5.758, Test loglik = -5.860, Train RMSE = 99.099, Test RMSE = 83.968\n",
            "Epoch: 421/500, Train loglik = -5.723, Test loglik = -5.836, Train RMSE = 99.352, Test RMSE = 83.955\n",
            "Epoch: 451/500, Train loglik = -5.689, Test loglik = -5.820, Train RMSE = 98.518, Test RMSE = 83.431\n",
            "Epoch: 481/500, Train loglik = -5.761, Test loglik = -5.998, Train RMSE = 100.384, Test RMSE = 85.162\n",
            "Epoch: 500/500, Train loglik = -5.651, Test loglik = -5.805, Train RMSE = 99.319, Test RMSE = 81.199\n",
            "FOLD 2:\n",
            "Epoch: 001/500, Train loglik = -8.226, Test loglik = -8.349, Train RMSE = 885.445, Test RMSE = 1026.723\n",
            "Epoch: 031/500, Train loglik = -6.843, Test loglik = -7.094, Train RMSE = 173.602, Test RMSE = 146.080\n",
            "Epoch: 061/500, Train loglik = -6.574, Test loglik = -6.681, Train RMSE = 163.419, Test RMSE = 127.785\n",
            "Epoch: 091/500, Train loglik = -6.587, Test loglik = -6.619, Train RMSE = 160.622, Test RMSE = 118.342\n",
            "Epoch: 121/500, Train loglik = -6.302, Test loglik = -6.122, Train RMSE = 171.282, Test RMSE = 92.429\n",
            "Epoch: 151/500, Train loglik = -6.171, Test loglik = -6.265, Train RMSE = 129.725, Test RMSE = 110.232\n",
            "Epoch: 181/500, Train loglik = -6.007, Test loglik = -5.993, Train RMSE = 115.596, Test RMSE = 90.772\n",
            "Epoch: 211/500, Train loglik = -5.940, Test loglik = -5.978, Train RMSE = 100.213, Test RMSE = 82.764\n",
            "Epoch: 241/500, Train loglik = -6.087, Test loglik = -6.043, Train RMSE = 101.590, Test RMSE = 82.443\n",
            "Epoch: 271/500, Train loglik = -5.862, Test loglik = -6.022, Train RMSE = 100.710, Test RMSE = 84.398\n",
            "Epoch: 301/500, Train loglik = -5.874, Test loglik = -5.871, Train RMSE = 101.762, Test RMSE = 87.520\n",
            "Epoch: 331/500, Train loglik = -6.130, Test loglik = -5.818, Train RMSE = 109.008, Test RMSE = 85.598\n",
            "Epoch: 361/500, Train loglik = -5.778, Test loglik = -5.802, Train RMSE = 101.017, Test RMSE = 84.391\n",
            "Epoch: 391/500, Train loglik = -5.772, Test loglik = -6.093, Train RMSE = 102.780, Test RMSE = 84.498\n",
            "Epoch: 421/500, Train loglik = -5.709, Test loglik = -5.798, Train RMSE = 99.704, Test RMSE = 83.373\n",
            "Epoch: 451/500, Train loglik = -5.725, Test loglik = -5.846, Train RMSE = 101.227, Test RMSE = 81.373\n",
            "Epoch: 481/500, Train loglik = -5.724, Test loglik = -5.881, Train RMSE = 99.104, Test RMSE = 83.112\n",
            "Epoch: 500/500, Train loglik = -5.722, Test loglik = -5.760, Train RMSE = 99.047, Test RMSE = 85.191\n",
            "FOLD 3:\n",
            "Epoch: 001/500, Train loglik = -8.220, Test loglik = -8.224, Train RMSE = 906.663, Test RMSE = 878.237\n",
            "Epoch: 031/500, Train loglik = -7.036, Test loglik = -6.576, Train RMSE = 188.511, Test RMSE = 145.077\n",
            "Epoch: 061/500, Train loglik = -6.745, Test loglik = -6.390, Train RMSE = 164.249, Test RMSE = 109.504\n",
            "Epoch: 091/500, Train loglik = -6.698, Test loglik = -6.240, Train RMSE = 170.171, Test RMSE = 94.595\n",
            "Epoch: 121/500, Train loglik = -6.512, Test loglik = -6.213, Train RMSE = 166.549, Test RMSE = 91.274\n",
            "Epoch: 151/500, Train loglik = -6.232, Test loglik = -6.257, Train RMSE = 164.923, Test RMSE = 92.049\n",
            "Epoch: 181/500, Train loglik = -6.353, Test loglik = -6.079, Train RMSE = 155.690, Test RMSE = 104.829\n",
            "Epoch: 211/500, Train loglik = -6.162, Test loglik = -6.028, Train RMSE = 168.336, Test RMSE = 84.923\n",
            "Epoch: 241/500, Train loglik = -5.954, Test loglik = -6.094, Train RMSE = 143.124, Test RMSE = 83.575\n",
            "Epoch: 271/500, Train loglik = -5.922, Test loglik = -5.835, Train RMSE = 134.608, Test RMSE = 84.192\n",
            "Epoch: 301/500, Train loglik = -6.110, Test loglik = -5.966, Train RMSE = 125.998, Test RMSE = 99.211\n",
            "Epoch: 331/500, Train loglik = -6.411, Test loglik = -5.907, Train RMSE = 105.080, Test RMSE = 79.376\n",
            "Epoch: 361/500, Train loglik = -5.845, Test loglik = -5.867, Train RMSE = 100.971, Test RMSE = 82.037\n",
            "Epoch: 391/500, Train loglik = -5.692, Test loglik = -5.731, Train RMSE = 99.461, Test RMSE = 84.753\n",
            "Epoch: 421/500, Train loglik = -5.719, Test loglik = -5.737, Train RMSE = 102.412, Test RMSE = 84.940\n",
            "Epoch: 451/500, Train loglik = -5.825, Test loglik = -5.806, Train RMSE = 99.561, Test RMSE = 79.673\n",
            "Epoch: 481/500, Train loglik = -5.782, Test loglik = -5.843, Train RMSE = 100.460, Test RMSE = 84.767\n",
            "Epoch: 500/500, Train loglik = -5.705, Test loglik = -5.757, Train RMSE = 98.691, Test RMSE = 79.549\n",
            "FOLD 4:\n",
            "Epoch: 001/500, Train loglik = -8.206, Test loglik = -8.498, Train RMSE = 891.576, Test RMSE = 1065.971\n",
            "Epoch: 031/500, Train loglik = -6.700, Test loglik = -6.796, Train RMSE = 178.419, Test RMSE = 170.008\n",
            "Epoch: 061/500, Train loglik = -6.587, Test loglik = -6.893, Train RMSE = 165.018, Test RMSE = 148.632\n",
            "Epoch: 091/500, Train loglik = -6.373, Test loglik = -6.328, Train RMSE = 161.885, Test RMSE = 134.324\n",
            "Epoch: 121/500, Train loglik = -6.410, Test loglik = -6.362, Train RMSE = 166.169, Test RMSE = 121.957\n",
            "Epoch: 151/500, Train loglik = -6.242, Test loglik = -6.191, Train RMSE = 168.896, Test RMSE = 117.724\n",
            "Epoch: 181/500, Train loglik = -6.127, Test loglik = -6.291, Train RMSE = 155.968, Test RMSE = 111.188\n",
            "Epoch: 211/500, Train loglik = -6.059, Test loglik = -6.041, Train RMSE = 138.135, Test RMSE = 114.601\n",
            "Epoch: 241/500, Train loglik = -5.913, Test loglik = -6.023, Train RMSE = 120.415, Test RMSE = 122.201\n",
            "Epoch: 271/500, Train loglik = -5.984, Test loglik = -6.023, Train RMSE = 107.734, Test RMSE = 132.828\n",
            "Epoch: 301/500, Train loglik = -5.816, Test loglik = -6.007, Train RMSE = 101.772, Test RMSE = 137.425\n",
            "Epoch: 331/500, Train loglik = -5.772, Test loglik = -6.005, Train RMSE = 93.020, Test RMSE = 142.882\n",
            "Epoch: 361/500, Train loglik = -5.769, Test loglik = -5.949, Train RMSE = 93.121, Test RMSE = 149.057\n",
            "Epoch: 391/500, Train loglik = -6.096, Test loglik = -6.051, Train RMSE = 93.588, Test RMSE = 148.728\n",
            "Epoch: 421/500, Train loglik = -5.699, Test loglik = -5.887, Train RMSE = 92.598, Test RMSE = 146.338\n",
            "Epoch: 451/500, Train loglik = -5.833, Test loglik = -5.894, Train RMSE = 91.617, Test RMSE = 146.554\n",
            "Epoch: 481/500, Train loglik = -5.810, Test loglik = -6.103, Train RMSE = 93.608, Test RMSE = 148.671\n",
            "Epoch: 500/500, Train loglik = -5.662, Test loglik = -6.016, Train RMSE = 92.412, Test RMSE = 143.463\n",
            "FOLD 5:\n",
            "Epoch: 001/500, Train loglik = -8.179, Test loglik = -8.387, Train RMSE = 895.074, Test RMSE = 1037.470\n",
            "Epoch: 031/500, Train loglik = -7.004, Test loglik = -6.998, Train RMSE = 181.925, Test RMSE = 140.030\n",
            "Epoch: 061/500, Train loglik = -6.716, Test loglik = -6.556, Train RMSE = 161.050, Test RMSE = 144.455\n",
            "Epoch: 091/500, Train loglik = -6.533, Test loglik = -6.272, Train RMSE = 175.879, Test RMSE = 143.947\n",
            "Epoch: 121/500, Train loglik = -6.324, Test loglik = -6.439, Train RMSE = 167.284, Test RMSE = 111.604\n",
            "Epoch: 151/500, Train loglik = -6.508, Test loglik = -6.082, Train RMSE = 155.143, Test RMSE = 111.415\n",
            "Epoch: 181/500, Train loglik = -6.068, Test loglik = -6.057, Train RMSE = 153.733, Test RMSE = 102.440\n",
            "Epoch: 211/500, Train loglik = -6.007, Test loglik = -6.040, Train RMSE = 141.295, Test RMSE = 98.706\n",
            "Epoch: 241/500, Train loglik = -5.900, Test loglik = -5.891, Train RMSE = 131.395, Test RMSE = 98.915\n",
            "Epoch: 271/500, Train loglik = -6.094, Test loglik = -5.855, Train RMSE = 105.415, Test RMSE = 97.332\n",
            "Epoch: 301/500, Train loglik = -5.866, Test loglik = -5.756, Train RMSE = 100.231, Test RMSE = 94.276\n",
            "Epoch: 331/500, Train loglik = -5.763, Test loglik = -5.745, Train RMSE = 102.152, Test RMSE = 96.161\n",
            "Epoch: 361/500, Train loglik = -5.827, Test loglik = -5.862, Train RMSE = 101.182, Test RMSE = 102.663\n",
            "Epoch: 391/500, Train loglik = -5.797, Test loglik = -5.814, Train RMSE = 99.170, Test RMSE = 93.386\n",
            "Epoch: 421/500, Train loglik = -5.701, Test loglik = -5.782, Train RMSE = 97.334, Test RMSE = 93.976\n",
            "Epoch: 451/500, Train loglik = -5.705, Test loglik = -5.744, Train RMSE = 98.149, Test RMSE = 94.995\n",
            "Epoch: 481/500, Train loglik = -6.080, Test loglik = -5.737, Train RMSE = 98.256, Test RMSE = 93.572\n",
            "Epoch: 500/500, Train loglik = -5.719, Test loglik = -5.668, Train RMSE = 96.686, Test RMSE = 94.896\n",
            "FOLD 6:\n",
            "Epoch: 001/500, Train loglik = -8.255, Test loglik = -8.175, Train RMSE = 916.169, Test RMSE = 883.462\n",
            "Epoch: 031/500, Train loglik = -6.740, Test loglik = -7.165, Train RMSE = 168.527, Test RMSE = 189.091\n",
            "Epoch: 061/500, Train loglik = -6.624, Test loglik = -6.678, Train RMSE = 166.336, Test RMSE = 160.210\n",
            "Epoch: 091/500, Train loglik = -6.428, Test loglik = -6.768, Train RMSE = 163.741, Test RMSE = 144.738\n",
            "Epoch: 121/500, Train loglik = -6.284, Test loglik = -6.607, Train RMSE = 159.756, Test RMSE = 141.505\n",
            "Epoch: 151/500, Train loglik = -6.120, Test loglik = -6.507, Train RMSE = 171.393, Test RMSE = 131.732\n",
            "Epoch: 181/500, Train loglik = -6.103, Test loglik = -6.325, Train RMSE = 143.440, Test RMSE = 121.931\n",
            "Epoch: 211/500, Train loglik = -6.271, Test loglik = -6.422, Train RMSE = 133.511, Test RMSE = 115.140\n",
            "Epoch: 241/500, Train loglik = -5.891, Test loglik = -6.111, Train RMSE = 103.842, Test RMSE = 107.989\n",
            "Epoch: 271/500, Train loglik = -5.988, Test loglik = -5.946, Train RMSE = 100.429, Test RMSE = 103.986\n",
            "Epoch: 301/500, Train loglik = -5.847, Test loglik = -6.028, Train RMSE = 102.676, Test RMSE = 98.956\n",
            "Epoch: 331/500, Train loglik = -5.807, Test loglik = -5.932, Train RMSE = 100.676, Test RMSE = 101.553\n",
            "Epoch: 361/500, Train loglik = -5.755, Test loglik = -5.887, Train RMSE = 99.484, Test RMSE = 100.490\n",
            "Epoch: 391/500, Train loglik = -5.799, Test loglik = -5.870, Train RMSE = 102.055, Test RMSE = 98.191\n",
            "Epoch: 421/500, Train loglik = -5.759, Test loglik = -5.962, Train RMSE = 99.297, Test RMSE = 98.539\n",
            "Epoch: 451/500, Train loglik = -5.765, Test loglik = -5.881, Train RMSE = 98.650, Test RMSE = 97.779\n",
            "Epoch: 481/500, Train loglik = -5.706, Test loglik = -5.841, Train RMSE = 99.032, Test RMSE = 97.470\n",
            "Epoch: 500/500, Train loglik = -5.635, Test loglik = -5.892, Train RMSE = 97.489, Test RMSE = 99.242\n",
            "FOLD 7:\n",
            "Epoch: 001/500, Train loglik = -8.224, Test loglik = -8.142, Train RMSE = 919.070, Test RMSE = 815.456\n",
            "Epoch: 031/500, Train loglik = -6.965, Test loglik = -6.896, Train RMSE = 170.119, Test RMSE = 166.965\n",
            "Epoch: 061/500, Train loglik = -6.450, Test loglik = -6.674, Train RMSE = 156.765, Test RMSE = 139.653\n",
            "Epoch: 091/500, Train loglik = -6.496, Test loglik = -6.478, Train RMSE = 159.844, Test RMSE = 132.176\n",
            "Epoch: 121/500, Train loglik = -6.572, Test loglik = -6.386, Train RMSE = 141.929, Test RMSE = 126.934\n",
            "Epoch: 151/500, Train loglik = -6.199, Test loglik = -6.765, Train RMSE = 124.984, Test RMSE = 127.604\n",
            "Epoch: 181/500, Train loglik = -6.042, Test loglik = -6.505, Train RMSE = 106.990, Test RMSE = 122.724\n",
            "Epoch: 211/500, Train loglik = -5.890, Test loglik = -6.000, Train RMSE = 103.011, Test RMSE = 115.314\n",
            "Epoch: 241/500, Train loglik = -5.903, Test loglik = -6.026, Train RMSE = 100.160, Test RMSE = 110.196\n",
            "Epoch: 271/500, Train loglik = -5.914, Test loglik = -6.393, Train RMSE = 98.206, Test RMSE = 110.078\n",
            "Epoch: 301/500, Train loglik = -5.912, Test loglik = -6.543, Train RMSE = 98.848, Test RMSE = 109.735\n",
            "Epoch: 331/500, Train loglik = -5.798, Test loglik = -5.970, Train RMSE = 99.364, Test RMSE = 104.256\n",
            "Epoch: 361/500, Train loglik = -5.877, Test loglik = -6.109, Train RMSE = 98.523, Test RMSE = 106.987\n",
            "Epoch: 391/500, Train loglik = -5.848, Test loglik = -6.212, Train RMSE = 99.243, Test RMSE = 103.624\n",
            "Epoch: 421/500, Train loglik = -5.677, Test loglik = -5.990, Train RMSE = 97.953, Test RMSE = 102.042\n",
            "Epoch: 451/500, Train loglik = -5.778, Test loglik = -5.986, Train RMSE = 96.510, Test RMSE = 102.761\n",
            "Epoch: 481/500, Train loglik = -5.695, Test loglik = -6.113, Train RMSE = 97.479, Test RMSE = 103.508\n",
            "Epoch: 500/500, Train loglik = -5.598, Test loglik = -6.144, Train RMSE = 96.487, Test RMSE = 101.401\n",
            "FOLD 8:\n",
            "Epoch: 001/500, Train loglik = -8.213, Test loglik = -8.272, Train RMSE = 899.216, Test RMSE = 927.821\n",
            "Epoch: 031/500, Train loglik = -6.716, Test loglik = -8.551, Train RMSE = 145.429, Test RMSE = 446.416\n",
            "Epoch: 061/500, Train loglik = -6.646, Test loglik = -7.428, Train RMSE = 119.384, Test RMSE = 480.417\n",
            "Epoch: 091/500, Train loglik = -6.488, Test loglik = -7.941, Train RMSE = 120.384, Test RMSE = 446.625\n",
            "Epoch: 121/500, Train loglik = -6.183, Test loglik = -9.588, Train RMSE = 111.198, Test RMSE = 475.539\n",
            "Epoch: 151/500, Train loglik = -6.147, Test loglik = -8.155, Train RMSE = 104.910, Test RMSE = 470.354\n",
            "Epoch: 181/500, Train loglik = -6.252, Test loglik = -9.696, Train RMSE = 108.111, Test RMSE = 485.651\n",
            "Epoch: 211/500, Train loglik = -6.040, Test loglik = -7.763, Train RMSE = 101.176, Test RMSE = 473.682\n",
            "Epoch: 241/500, Train loglik = -5.826, Test loglik = -8.640, Train RMSE = 99.063, Test RMSE = 463.234\n",
            "Epoch: 271/500, Train loglik = -6.003, Test loglik = -7.336, Train RMSE = 102.549, Test RMSE = 449.556\n",
            "Epoch: 301/500, Train loglik = -5.821, Test loglik = -8.155, Train RMSE = 98.360, Test RMSE = 440.320\n",
            "Epoch: 331/500, Train loglik = -5.794, Test loglik = -6.910, Train RMSE = 99.492, Test RMSE = 425.931\n",
            "Epoch: 361/500, Train loglik = -5.856, Test loglik = -7.865, Train RMSE = 97.089, Test RMSE = 420.675\n",
            "Epoch: 391/500, Train loglik = -5.704, Test loglik = -6.990, Train RMSE = 97.283, Test RMSE = 397.720\n",
            "Epoch: 421/500, Train loglik = -5.752, Test loglik = -7.293, Train RMSE = 96.138, Test RMSE = 411.286\n",
            "Epoch: 451/500, Train loglik = -5.756, Test loglik = -7.170, Train RMSE = 96.109, Test RMSE = 391.318\n",
            "Epoch: 481/500, Train loglik = -5.651, Test loglik = -6.548, Train RMSE = 96.377, Test RMSE = 378.663\n",
            "Epoch: 500/500, Train loglik = -5.776, Test loglik = -6.638, Train RMSE = 95.596, Test RMSE = 377.798\n",
            "FOLD 9:\n",
            "Epoch: 001/500, Train loglik = -8.248, Test loglik = -8.121, Train RMSE = 926.137, Test RMSE = 764.453\n",
            "Epoch: 031/500, Train loglik = -6.704, Test loglik = -6.768, Train RMSE = 184.767, Test RMSE = 181.196\n",
            "Epoch: 061/500, Train loglik = -6.584, Test loglik = -7.079, Train RMSE = 168.797, Test RMSE = 188.361\n",
            "Epoch: 091/500, Train loglik = -6.867, Test loglik = -6.695, Train RMSE = 163.762, Test RMSE = 153.758\n",
            "Epoch: 121/500, Train loglik = -6.301, Test loglik = -6.480, Train RMSE = 155.541, Test RMSE = 137.002\n",
            "Epoch: 151/500, Train loglik = -6.450, Test loglik = -6.429, Train RMSE = 143.839, Test RMSE = 133.179\n",
            "Epoch: 181/500, Train loglik = -6.074, Test loglik = -6.768, Train RMSE = 138.428, Test RMSE = 128.134\n",
            "Epoch: 211/500, Train loglik = -5.977, Test loglik = -6.167, Train RMSE = 113.825, Test RMSE = 124.171\n",
            "Epoch: 241/500, Train loglik = -5.930, Test loglik = -6.226, Train RMSE = 100.331, Test RMSE = 120.707\n",
            "Epoch: 271/500, Train loglik = -6.110, Test loglik = -6.250, Train RMSE = 95.103, Test RMSE = 120.173\n",
            "Epoch: 301/500, Train loglik = -5.881, Test loglik = -6.066, Train RMSE = 95.436, Test RMSE = 118.832\n",
            "Epoch: 331/500, Train loglik = -5.998, Test loglik = -6.118, Train RMSE = 105.885, Test RMSE = 125.775\n",
            "Epoch: 361/500, Train loglik = -5.796, Test loglik = -6.144, Train RMSE = 98.193, Test RMSE = 119.861\n",
            "Epoch: 391/500, Train loglik = -5.799, Test loglik = -6.372, Train RMSE = 95.026, Test RMSE = 121.294\n",
            "Epoch: 421/500, Train loglik = -5.735, Test loglik = -6.117, Train RMSE = 94.370, Test RMSE = 117.846\n",
            "Epoch: 451/500, Train loglik = -5.688, Test loglik = -6.249, Train RMSE = 95.281, Test RMSE = 119.361\n",
            "Epoch: 481/500, Train loglik = -5.725, Test loglik = -6.167, Train RMSE = 95.038, Test RMSE = 117.322\n",
            "Epoch: 500/500, Train loglik = -5.635, Test loglik = -6.208, Train RMSE = 93.836, Test RMSE = 117.143\n",
            "Train log. lik. = -5.679 +/-  0.061\n",
            "Test  log. lik. = -5.998 +/-  0.316\n",
            "Train RMSE      = 97.284 +/-  2.834\n",
            "Test  RMSE      = 127.065 +/- 85.969\n"
          ]
        }
      ],
      "source": [
        "from DataProcessing import DataProcessing as dp\n",
        "doc_2020 = pd.read_csv('/content/drive/MyDrive/DropoutUncertaintyExps-master/Hydrid_modeling/EndUserData2020P2.csv', encoding = 'ISO-8859-1', skiprows = [0,2]).astype('float')\n",
        "doc_2021 = pd.read_csv('/content/drive/MyDrive/DropoutUncertaintyExps-master/Hydrid_modeling/EndUserData2021P2_fix.csv', encoding = 'ISO-8859-1', skiprows = [0,2]).astype('float')\n",
        "#print(doc_2020)\n",
        "\n",
        "doc_2020_value = dp(doc_2020, doc_2021).data_process_20().astype('float64')\n",
        "#dco_2021_value = dp(doc_2020, doc_2021).data_process_21().astype('float64')\n",
        "\n",
        "BNN_20 = pd.DataFrame(np.c_[doc_2020_value['NB2_S_1_NYZ_sys_x_PcwOut_x'], doc_2020_value['NB2_S_1_NYZ_sys_x_PcwIn_x'], doc_2020_value['NB2_S_1_NYZ_cwp_9_HzSPR_x'], doc_2020_value['NB2_S_1_NYZ_cwp_10_HzSPR_x'], doc_2020_value['NB2_S_1_NYZ_cwp_11_HzSPR_x'], doc_2020_value['NB2_S_1_NYZ_cwp_12_HzSPR_x'], doc_2020_value['p_diff'], doc_2020_value['NB2_S_x_NYZ_x_x_Fcw_x']], columns = ['NB2_S_1_NYZ_sys_x_PcwOut_x', 'NB2_S_1_NYZ_sys_x_PcwIn_x', 'NB2_S_1_NYZ_cwp_9_HzSPR_x', 'NB2_S_1_NYZ_cwp_10_HzSPR_x', 'NB2_S_1_NYZ_cwp_11_HzSPR_x', 'NB2_S_1_NYZ_cwp_12_HzSPR_x', 'diff', 'NB2_S_x_NYZ_x_x_Fcw_x']).values\n",
        "data=BNN_20\n",
        "batch_size = 2048\n",
        "model = train_BBP(data, n_splits=10, num_epochs=500, num_units=120, learn_rate=1e-2, log_every=30)"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model = train_BBP(data, n_splits=10, num_epochs=500, num_units=100, learn_rate=1e-2, log_every=50)"
      ],
      "metadata": {
        "id": "tMrXKW9ni4qm",
        "outputId": "905136d4-11fe-4005-f37a-13ad310ec9a8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "FOLD 0:\n",
            "Epoch: 001/500, Train loglik = -8.254, Test loglik = -8.080, Train RMSE = 923.442, Test RMSE = 747.700\n",
            "Epoch: 051/500, Train loglik = -6.517, Test loglik = -6.658, Train RMSE = 173.514, Test RMSE = 163.639\n",
            "Epoch: 101/500, Train loglik = -6.530, Test loglik = -6.667, Train RMSE = 165.940, Test RMSE = 143.203\n",
            "Epoch: 151/500, Train loglik = -6.448, Test loglik = -6.164, Train RMSE = 163.357, Test RMSE = 126.150\n",
            "Epoch: 201/500, Train loglik = -6.407, Test loglik = -5.887, Train RMSE = 146.029, Test RMSE = 94.495\n",
            "Epoch: 251/500, Train loglik = -6.029, Test loglik = -5.800, Train RMSE = 130.439, Test RMSE = 79.874\n",
            "Epoch: 301/500, Train loglik = -5.892, Test loglik = -5.801, Train RMSE = 100.825, Test RMSE = 86.627\n",
            "Epoch: 351/500, Train loglik = -6.018, Test loglik = -5.681, Train RMSE = 102.315, Test RMSE = 76.159\n",
            "Epoch: 401/500, Train loglik = -5.721, Test loglik = -5.794, Train RMSE = 99.767, Test RMSE = 76.565\n",
            "Epoch: 451/500, Train loglik = -5.718, Test loglik = -5.673, Train RMSE = 100.954, Test RMSE = 75.313\n",
            "Epoch: 500/500, Train loglik = -5.690, Test loglik = -5.682, Train RMSE = 101.587, Test RMSE = 77.222\n",
            "FOLD 1:\n",
            "Epoch: 001/500, Train loglik = -8.231, Test loglik = -8.290, Train RMSE = 900.269, Test RMSE = 987.187\n",
            "Epoch: 051/500, Train loglik = -6.588, Test loglik = -6.659, Train RMSE = 169.433, Test RMSE = 126.661\n",
            "Epoch: 101/500, Train loglik = -6.470, Test loglik = -6.305, Train RMSE = 161.394, Test RMSE = 111.113\n",
            "Epoch: 151/500, Train loglik = -6.507, Test loglik = -6.347, Train RMSE = 154.788, Test RMSE = 92.371\n",
            "Epoch: 201/500, Train loglik = -6.044, Test loglik = -6.403, Train RMSE = 117.530, Test RMSE = 85.911\n",
            "Epoch: 251/500, Train loglik = -6.051, Test loglik = -5.765, Train RMSE = 100.023, Test RMSE = 82.096\n",
            "Epoch: 301/500, Train loglik = -5.938, Test loglik = -6.074, Train RMSE = 99.482, Test RMSE = 81.333\n",
            "Epoch: 351/500, Train loglik = -5.733, Test loglik = -5.862, Train RMSE = 96.979, Test RMSE = 81.885\n",
            "Epoch: 401/500, Train loglik = -5.706, Test loglik = -5.742, Train RMSE = 99.130, Test RMSE = 83.050\n",
            "Epoch: 451/500, Train loglik = -5.798, Test loglik = -6.389, Train RMSE = 102.317, Test RMSE = 91.656\n",
            "Epoch: 500/500, Train loglik = -5.776, Test loglik = -6.026, Train RMSE = 96.318, Test RMSE = 82.662\n",
            "FOLD 2:\n",
            "Epoch: 001/500, Train loglik = -8.213, Test loglik = -8.352, Train RMSE = 897.026, Test RMSE = 1046.351\n",
            "Epoch: 051/500, Train loglik = -6.620, Test loglik = -6.726, Train RMSE = 171.022, Test RMSE = 139.304\n",
            "Epoch: 101/500, Train loglik = -6.604, Test loglik = -6.546, Train RMSE = 173.556, Test RMSE = 124.133\n",
            "Epoch: 151/500, Train loglik = -6.134, Test loglik = -6.224, Train RMSE = 145.588, Test RMSE = 115.066\n",
            "Epoch: 201/500, Train loglik = -5.986, Test loglik = -6.085, Train RMSE = 125.164, Test RMSE = 84.401\n",
            "Epoch: 251/500, Train loglik = -5.900, Test loglik = -5.845, Train RMSE = 101.735, Test RMSE = 82.663\n",
            "Epoch: 301/500, Train loglik = -5.800, Test loglik = -5.772, Train RMSE = 100.917, Test RMSE = 84.628\n",
            "Epoch: 351/500, Train loglik = -5.809, Test loglik = -5.843, Train RMSE = 103.976, Test RMSE = 83.541\n",
            "Epoch: 401/500, Train loglik = -5.735, Test loglik = -6.009, Train RMSE = 101.754, Test RMSE = 87.211\n",
            "Epoch: 451/500, Train loglik = -5.693, Test loglik = -5.771, Train RMSE = 100.178, Test RMSE = 83.548\n",
            "Epoch: 500/500, Train loglik = -5.660, Test loglik = -6.058, Train RMSE = 99.956, Test RMSE = 82.680\n",
            "FOLD 3:\n",
            "Epoch: 001/500, Train loglik = -8.267, Test loglik = -8.202, Train RMSE = 917.246, Test RMSE = 871.088\n",
            "Epoch: 051/500, Train loglik = -6.560, Test loglik = -6.615, Train RMSE = 173.591, Test RMSE = 141.231\n",
            "Epoch: 101/500, Train loglik = -6.484, Test loglik = -6.257, Train RMSE = 196.327, Test RMSE = 115.604\n",
            "Epoch: 151/500, Train loglik = -6.212, Test loglik = -6.099, Train RMSE = 132.861, Test RMSE = 86.960\n",
            "Epoch: 201/500, Train loglik = -5.950, Test loglik = -6.023, Train RMSE = 110.765, Test RMSE = 85.999\n",
            "Epoch: 251/500, Train loglik = -5.966, Test loglik = -5.938, Train RMSE = 101.149, Test RMSE = 80.843\n",
            "Epoch: 301/500, Train loglik = -5.950, Test loglik = -5.861, Train RMSE = 104.250, Test RMSE = 80.566\n",
            "Epoch: 351/500, Train loglik = -5.781, Test loglik = -5.855, Train RMSE = 99.920, Test RMSE = 78.498\n",
            "Epoch: 401/500, Train loglik = -5.960, Test loglik = -5.867, Train RMSE = 99.979, Test RMSE = 78.529\n",
            "Epoch: 451/500, Train loglik = -5.708, Test loglik = -5.791, Train RMSE = 100.693, Test RMSE = 82.908\n",
            "Epoch: 500/500, Train loglik = -5.674, Test loglik = -5.791, Train RMSE = 99.631, Test RMSE = 79.228\n",
            "FOLD 4:\n",
            "Epoch: 001/500, Train loglik = -8.207, Test loglik = -8.370, Train RMSE = 905.461, Test RMSE = 1063.844\n",
            "Epoch: 051/500, Train loglik = -6.691, Test loglik = -6.450, Train RMSE = 164.890, Test RMSE = 146.307\n",
            "Epoch: 101/500, Train loglik = -6.417, Test loglik = -6.421, Train RMSE = 170.604, Test RMSE = 134.147\n",
            "Epoch: 151/500, Train loglik = -6.408, Test loglik = -6.268, Train RMSE = 156.801, Test RMSE = 123.846\n",
            "Epoch: 201/500, Train loglik = -6.170, Test loglik = -6.191, Train RMSE = 140.139, Test RMSE = 115.820\n",
            "Epoch: 251/500, Train loglik = -5.956, Test loglik = -6.109, Train RMSE = 106.612, Test RMSE = 131.335\n",
            "Epoch: 301/500, Train loglik = -5.983, Test loglik = -6.036, Train RMSE = 94.396, Test RMSE = 143.967\n",
            "Epoch: 351/500, Train loglik = -5.755, Test loglik = -5.997, Train RMSE = 93.080, Test RMSE = 146.494\n",
            "Epoch: 401/500, Train loglik = -5.743, Test loglik = -5.983, Train RMSE = 95.313, Test RMSE = 143.914\n",
            "Epoch: 451/500, Train loglik = -5.656, Test loglik = -5.968, Train RMSE = 91.269, Test RMSE = 145.507\n",
            "Epoch: 500/500, Train loglik = -5.716, Test loglik = -5.929, Train RMSE = 91.828, Test RMSE = 145.037\n",
            "FOLD 5:\n",
            "Epoch: 001/500, Train loglik = -8.210, Test loglik = -8.446, Train RMSE = 897.627, Test RMSE = 1056.072\n",
            "Epoch: 051/500, Train loglik = -6.685, Test loglik = -6.565, Train RMSE = 187.014, Test RMSE = 148.444\n",
            "Epoch: 101/500, Train loglik = -6.412, Test loglik = -6.220, Train RMSE = 166.658, Test RMSE = 111.718\n",
            "Epoch: 151/500, Train loglik = -6.113, Test loglik = -6.317, Train RMSE = 146.396, Test RMSE = 98.413\n",
            "Epoch: 201/500, Train loglik = -5.941, Test loglik = -6.281, Train RMSE = 123.796, Test RMSE = 93.352\n",
            "Epoch: 251/500, Train loglik = -6.226, Test loglik = -5.981, Train RMSE = 120.327, Test RMSE = 102.892\n",
            "Epoch: 301/500, Train loglik = -5.908, Test loglik = -5.813, Train RMSE = 100.608, Test RMSE = 98.493\n",
            "Epoch: 351/500, Train loglik = -5.850, Test loglik = -5.731, Train RMSE = 99.688, Test RMSE = 96.135\n",
            "Epoch: 401/500, Train loglik = -5.762, Test loglik = -5.866, Train RMSE = 99.517, Test RMSE = 101.996\n",
            "Epoch: 451/500, Train loglik = -5.713, Test loglik = -5.654, Train RMSE = 98.339, Test RMSE = 95.813\n",
            "Epoch: 500/500, Train loglik = -5.668, Test loglik = -5.821, Train RMSE = 98.458, Test RMSE = 96.592\n",
            "FOLD 6:\n",
            "Epoch: 001/500, Train loglik = -8.238, Test loglik = -8.205, Train RMSE = 912.988, Test RMSE = 884.238\n",
            "Epoch: 051/500, Train loglik = -6.666, Test loglik = -6.901, Train RMSE = 160.205, Test RMSE = 175.442\n",
            "Epoch: 101/500, Train loglik = -6.366, Test loglik = -6.500, Train RMSE = 157.871, Test RMSE = 138.616\n",
            "Epoch: 151/500, Train loglik = -6.337, Test loglik = -6.346, Train RMSE = 141.005, Test RMSE = 139.423\n",
            "Epoch: 201/500, Train loglik = -5.980, Test loglik = -6.323, Train RMSE = 123.491, Test RMSE = 112.387\n",
            "Epoch: 251/500, Train loglik = -5.943, Test loglik = -6.746, Train RMSE = 99.237, Test RMSE = 110.084\n",
            "Epoch: 301/500, Train loglik = -5.800, Test loglik = -6.100, Train RMSE = 97.249, Test RMSE = 101.224\n",
            "Epoch: 351/500, Train loglik = -5.889, Test loglik = -5.974, Train RMSE = 100.842, Test RMSE = 100.534\n",
            "Epoch: 401/500, Train loglik = -5.724, Test loglik = -5.925, Train RMSE = 100.557, Test RMSE = 111.708\n",
            "Epoch: 451/500, Train loglik = -5.746, Test loglik = -5.891, Train RMSE = 99.960, Test RMSE = 98.210\n",
            "Epoch: 500/500, Train loglik = -5.762, Test loglik = -5.889, Train RMSE = 96.399, Test RMSE = 103.091\n",
            "FOLD 7:\n",
            "Epoch: 001/500, Train loglik = -8.257, Test loglik = -8.117, Train RMSE = 923.876, Test RMSE = 805.175\n",
            "Epoch: 051/500, Train loglik = -6.840, Test loglik = -6.652, Train RMSE = 174.812, Test RMSE = 191.346\n",
            "Epoch: 101/500, Train loglik = -6.397, Test loglik = -6.720, Train RMSE = 165.312, Test RMSE = 153.995\n",
            "Epoch: 151/500, Train loglik = -6.210, Test loglik = -6.620, Train RMSE = 133.710, Test RMSE = 124.798\n",
            "Epoch: 201/500, Train loglik = -6.462, Test loglik = -6.675, Train RMSE = 116.563, Test RMSE = 133.470\n",
            "Epoch: 251/500, Train loglik = -5.856, Test loglik = -6.036, Train RMSE = 101.145, Test RMSE = 105.379\n",
            "Epoch: 301/500, Train loglik = -6.081, Test loglik = -6.577, Train RMSE = 98.925, Test RMSE = 103.910\n",
            "Epoch: 351/500, Train loglik = -5.792, Test loglik = -6.064, Train RMSE = 98.271, Test RMSE = 103.812\n",
            "Epoch: 401/500, Train loglik = -5.787, Test loglik = -5.993, Train RMSE = 95.780, Test RMSE = 104.047\n",
            "Epoch: 451/500, Train loglik = -5.649, Test loglik = -6.034, Train RMSE = 96.823, Test RMSE = 106.558\n",
            "Epoch: 500/500, Train loglik = -5.749, Test loglik = -6.269, Train RMSE = 96.600, Test RMSE = 103.372\n",
            "FOLD 8:\n",
            "Epoch: 001/500, Train loglik = -8.204, Test loglik = -8.278, Train RMSE = 903.144, Test RMSE = 931.610\n",
            "Epoch: 051/500, Train loglik = -6.504, Test loglik = -8.440, Train RMSE = 134.500, Test RMSE = 426.864\n",
            "Epoch: 101/500, Train loglik = -6.164, Test loglik = -8.949, Train RMSE = 109.414, Test RMSE = 434.053\n",
            "Epoch: 151/500, Train loglik = -6.247, Test loglik = -8.227, Train RMSE = 109.821, Test RMSE = 441.570\n",
            "Epoch: 201/500, Train loglik = -5.996, Test loglik = -6.836, Train RMSE = 98.955, Test RMSE = 457.106\n",
            "Epoch: 251/500, Train loglik = -5.990, Test loglik = -7.114, Train RMSE = 101.154, Test RMSE = 422.259\n",
            "Epoch: 301/500, Train loglik = -6.028, Test loglik = -7.153, Train RMSE = 100.604, Test RMSE = 429.674\n",
            "Epoch: 351/500, Train loglik = -5.726, Test loglik = -7.451, Train RMSE = 96.288, Test RMSE = 383.655\n",
            "Epoch: 401/500, Train loglik = -5.759, Test loglik = -6.689, Train RMSE = 99.588, Test RMSE = 400.689\n",
            "Epoch: 451/500, Train loglik = -5.675, Test loglik = -6.642, Train RMSE = 94.977, Test RMSE = 342.326\n",
            "Epoch: 500/500, Train loglik = -5.628, Test loglik = -6.539, Train RMSE = 93.132, Test RMSE = 323.071\n",
            "FOLD 9:\n",
            "Epoch: 001/500, Train loglik = -8.227, Test loglik = -8.120, Train RMSE = 929.890, Test RMSE = 760.648\n",
            "Epoch: 051/500, Train loglik = -6.712, Test loglik = -6.755, Train RMSE = 173.398, Test RMSE = 165.947\n",
            "Epoch: 101/500, Train loglik = -6.430, Test loglik = -6.530, Train RMSE = 174.922, Test RMSE = 148.795\n",
            "Epoch: 151/500, Train loglik = -6.214, Test loglik = -6.282, Train RMSE = 167.062, Test RMSE = 134.678\n",
            "Epoch: 201/500, Train loglik = -6.047, Test loglik = -6.222, Train RMSE = 143.297, Test RMSE = 126.454\n",
            "Epoch: 251/500, Train loglik = -5.953, Test loglik = -6.097, Train RMSE = 121.964, Test RMSE = 124.160\n",
            "Epoch: 301/500, Train loglik = -5.918, Test loglik = -6.129, Train RMSE = 96.683, Test RMSE = 128.419\n",
            "Epoch: 351/500, Train loglik = -5.944, Test loglik = -6.092, Train RMSE = 97.927, Test RMSE = 119.274\n",
            "Epoch: 401/500, Train loglik = -5.673, Test loglik = -6.070, Train RMSE = 96.623, Test RMSE = 120.501\n",
            "Epoch: 451/500, Train loglik = -5.747, Test loglik = -6.090, Train RMSE = 95.693, Test RMSE = 121.562\n",
            "Epoch: 500/500, Train loglik = -5.694, Test loglik = -6.141, Train RMSE = 95.253, Test RMSE = 120.700\n",
            "Train log. lik. = -5.686 +/-  0.044\n",
            "Test  log. lik. = -5.954 +/-  0.233\n",
            "Train RMSE      = 96.725 +/-  2.820\n",
            "Test  RMSE      = 121.166 +/- 70.418\n"
          ]
        }
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "name": "bbp_hetero.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.8"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}