bbp_hetero_overfitting.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/EdWangLoDaSc/Dropout-as-a-Grid-Search_Representing-Model-Uncertainty-in-Deep-Learning/blob/main/bbp_hetero_overfitting.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pVmDXZ3MUJfB",
        "outputId": "722c483f-9b7b-4258-891a-c43d7f28e984"
      },
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xAYPKSFrG8AF",
        "outputId": "61a77c5a-30a5-415b-999c-5dafcbcf9891"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "\u001b[31mERROR: torch-0.4.1-cp36-cp36m-linux_x86_64.whl is not a supported wheel on this platform.\u001b[0m\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.7/dist-packages (0.13.0+cu113)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torchvision) (4.1.1)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from torchvision) (2.23.0)\n",
            "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/local/lib/python3.7/dist-packages (from torchvision) (7.1.2)\n",
            "Requirement already satisfied: torch==1.12.0 in /usr/local/lib/python3.7/dist-packages (from torchvision) (1.12.0+cu113)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from torchvision) (1.21.6)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->torchvision) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->torchvision) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->torchvision) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->torchvision) (2022.6.15)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting GPy\n",
            "  Downloading GPy-1.10.0.tar.gz (959 kB)\n",
            "\u001b[K     |████████████████████████████████| 959 kB 31.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.7 in /usr/local/lib/python3.7/dist-packages (from GPy) (1.21.6)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from GPy) (1.15.0)\n",
            "Collecting paramz>=0.9.0\n",
            "  Downloading paramz-0.9.5.tar.gz (71 kB)\n",
            "\u001b[K     |████████████████████████████████| 71 kB 10.5 MB/s \n",
            "\u001b[?25hRequirement already satisfied: cython>=0.29 in /usr/local/lib/python3.7/dist-packages (from GPy) (0.29.30)\n",
            "Requirement already satisfied: scipy>=1.3.0 in /usr/local/lib/python3.7/dist-packages (from GPy) (1.7.3)\n",
            "Requirement already satisfied: decorator>=4.0.10 in /usr/local/lib/python3.7/dist-packages (from paramz>=0.9.0->GPy) (4.4.2)\n",
            "Building wheels for collected packages: GPy, paramz\n",
            "  Building wheel for GPy (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for GPy: filename=GPy-1.10.0-cp37-cp37m-linux_x86_64.whl size=2565136 sha256=cd894c20a13df542d2efaa651da5f5977e7cc1aeb6644a9531f39a088041552f\n",
            "  Stored in directory: /root/.cache/pip/wheels/f7/18/28/dd1ce0192a81b71a3b086fd952511d088b21e8359ea496860a\n",
            "  Building wheel for paramz (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for paramz: filename=paramz-0.9.5-py3-none-any.whl size=102566 sha256=7f88fe8ca034c943c43c8e8627e25de1a5be8e54dd1be84868881d7feaf64706\n",
            "  Stored in directory: /root/.cache/pip/wheels/c8/95/f5/ce28482da28162e6028c4b3a32c41d147395825b3cd62bc810\n",
            "Successfully built GPy paramz\n",
            "Installing collected packages: paramz, GPy\n",
            "Successfully installed GPy-1.10.0 paramz-0.9.5\n"
          ]
        }
      ],
      "source": [
        "!pip3 install http://download.pytorch.org/whl/cu92/torch-0.4.1-cp36-cp36m-linux_x86_64.whl\n",
        "!pip3 install torchvision\n",
        "!pip install GPy\n",
        "import GPy\n",
        "import pandas as pd\n",
        "import zipfile\n",
        "import urllib.request\n",
        "import time\n",
        "import copy\n",
        "import math\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import seaborn as sns\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torch.autograd import Variable\n",
        "from torch.optim import Optimizer\n",
        "from sklearn.model_selection import KFold\n",
        "\n",
        "from torchvision import datasets, transforms\n",
        "from torchvision.utils import make_grid\n",
        "from tqdm import tqdm, trange\n",
        "from google.colab import files\n",
        "%config InlineBackend.figure_format = 'svg'"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "id": "1-7dNcVmHA3I",
        "outputId": "a3df7525-d432-4955-a898-f9229d205313"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Tesla T4'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 3
        }
      ],
      "source": [
        "torch.cuda.device(0)\n",
        "torch.cuda.get_device_name(torch.cuda.current_device())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "_BzTB5WsP9Kx"
      },
      "outputs": [],
      "source": [
        "def to_variable(var=(), cuda=True, volatile=False):\n",
        "    out = []\n",
        "    for v in var:\n",
        "        \n",
        "        if isinstance(v, np.ndarray):\n",
        "            v = torch.from_numpy(v).type(torch.FloatTensor)\n",
        "\n",
        "        if not v.is_cuda and cuda:\n",
        "            v = v.cuda()\n",
        "\n",
        "        if not isinstance(v, Variable):\n",
        "            v = Variable(v, volatile=volatile)\n",
        "\n",
        "        out.append(v)\n",
        "    return out"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "Va8V78eFFsc9"
      },
      "outputs": [],
      "source": [
        "def log_gaussian_loss(output, target, sigma, no_dim, sum_reduce=True):\n",
        "    exponent = -0.5*(target - output)**2/sigma**2\n",
        "    log_coeff = -no_dim*torch.log(sigma) - 0.5*no_dim*np.log(2*np.pi)\n",
        "    \n",
        "    if sum_reduce:\n",
        "        return -(log_coeff + exponent).sum()\n",
        "    else:\n",
        "        return -(log_coeff + exponent)\n",
        "\n",
        "\n",
        "def get_kl_divergence(weights, prior, varpost):\n",
        "    prior_loglik = prior.loglik(weights)\n",
        "    \n",
        "    varpost_loglik = varpost.loglik(weights)\n",
        "    varpost_lik = varpost_loglik.exp()\n",
        "    \n",
        "    return (varpost_lik*(varpost_loglik - prior_loglik)).sum()\n",
        "\n",
        "\n",
        "class gaussian:\n",
        "    def __init__(self, mu, sigma):\n",
        "        self.mu = mu\n",
        "        self.sigma = sigma\n",
        "        \n",
        "    def loglik(self, weights):\n",
        "        exponent = -0.5*(weights - self.mu)**2/self.sigma**2\n",
        "        log_coeff = -0.5*(np.log(2*np.pi) + 2*np.log(self.sigma))\n",
        "        \n",
        "        return (exponent + log_coeff).sum()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "ASGi2Ecx5G-F"
      },
      "outputs": [],
      "source": [
        "class BayesLinear_Normalq(nn.Module):\n",
        "    def __init__(self, input_dim, output_dim, prior):\n",
        "        super(BayesLinear_Normalq, self).__init__()\n",
        "        self.input_dim = input_dim\n",
        "        self.output_dim = output_dim\n",
        "        self.prior = prior\n",
        "        \n",
        "        scale = (2/self.input_dim)**0.5\n",
        "        rho_init = np.log(np.exp((2/self.input_dim)**0.5) - 1)\n",
        "        self.weight_mus = nn.Parameter(torch.Tensor(self.input_dim, self.output_dim).uniform_(-0.01, 0.01))\n",
        "        self.weight_rhos = nn.Parameter(torch.Tensor(self.input_dim, self.output_dim).uniform_(-3, -3))\n",
        "        \n",
        "        self.bias_mus = nn.Parameter(torch.Tensor(self.output_dim).uniform_(-0.01, 0.01))\n",
        "        self.bias_rhos = nn.Parameter(torch.Tensor(self.output_dim).uniform_(-4, -3))\n",
        "        \n",
        "    def forward(self, x, sample = True):\n",
        "        \n",
        "        if sample:\n",
        "            # sample gaussian noise for each weight and each bias\n",
        "            weight_epsilons = Variable(self.weight_mus.data.new(self.weight_mus.size()).normal_())\n",
        "            bias_epsilons =  Variable(self.bias_mus.data.new(self.bias_mus.size()).normal_())\n",
        "            \n",
        "            # calculate the weight and bias stds from the rho parameters\n",
        "            weight_stds = torch.log(1 + torch.exp(self.weight_rhos))\n",
        "            bias_stds = torch.log(1 + torch.exp(self.bias_rhos))\n",
        "            \n",
        "            # calculate samples from the posterior from the sampled noise and mus/stds\n",
        "            weight_sample = self.weight_mus + weight_epsilons*weight_stds\n",
        "            bias_sample = self.bias_mus + bias_epsilons*bias_stds\n",
        "            \n",
        "            output = torch.mm(x, weight_sample) + bias_sample\n",
        "            \n",
        "            # computing the KL loss term\n",
        "            prior_cov, varpost_cov = self.prior.sigma**2, weight_stds**2\n",
        "            KL_loss = 0.5*(torch.log(prior_cov/varpost_cov)).sum() - 0.5*weight_stds.numel()\n",
        "            KL_loss = KL_loss + 0.5*(varpost_cov/prior_cov).sum()\n",
        "            KL_loss = KL_loss + 0.5*((self.weight_mus - self.prior.mu)**2/prior_cov).sum()\n",
        "            \n",
        "            prior_cov, varpost_cov = self.prior.sigma**2, bias_stds**2\n",
        "            KL_loss = KL_loss + 0.5*(torch.log(prior_cov/varpost_cov)).sum() - 0.5*bias_stds.numel()\n",
        "            KL_loss = KL_loss + 0.5*(varpost_cov/prior_cov).sum()\n",
        "            KL_loss = KL_loss + 0.5*((self.bias_mus - self.prior.mu)**2/prior_cov).sum()\n",
        "            \n",
        "            return output, KL_loss\n",
        "        \n",
        "        else:\n",
        "            output = torch.mm(x, self.weight_mus) + self.bias_mus\n",
        "            return output, KL_loss\n",
        "        \n",
        "    def sample_layer(self, no_samples):\n",
        "        all_samples = []\n",
        "        for i in range(no_samples):\n",
        "            # sample gaussian noise for each weight and each bias\n",
        "            weight_epsilons = Variable(self.weight_mus.data.new(self.weight_mus.size()).normal_())\n",
        "            \n",
        "            # calculate the weight and bias stds from the rho parameters\n",
        "            weight_stds = torch.log(1 + torch.exp(self.weight_rhos))\n",
        "            \n",
        "            # calculate samples from the posterior from the sampled noise and mus/stds\n",
        "            weight_sample = self.weight_mus + weight_epsilons*weight_stds\n",
        "            \n",
        "            all_samples += weight_sample.view(-1).cpu().data.numpy().tolist()\n",
        "            \n",
        "        return all_samples"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "_8dV-QIq5G-I"
      },
      "outputs": [],
      "source": [
        "class BBP_Heteroscedastic_Model(nn.Module):\n",
        "    def __init__(self, input_dim, output_dim, num_units):\n",
        "        super(BBP_Heteroscedastic_Model, self).__init__()\n",
        "        \n",
        "        self.input_dim = input_dim\n",
        "        self.output_dim = output_dim\n",
        "        \n",
        "        # network with two hidden and one output layer\n",
        "        self.layer1 = BayesLinear_Normalq(input_dim, num_units, gaussian(0, 1))\n",
        "        self.layer2 = BayesLinear_Normalq(num_units, 2*output_dim, gaussian(0, 1))\n",
        "        \n",
        "        # activation to be used between hidden layers\n",
        "        self.activation = nn.ReLU(inplace = True)\n",
        "    \n",
        "    def forward(self, x):\n",
        "        \n",
        "        KL_loss_total = 0\n",
        "        x = x.view(-1, self.input_dim)\n",
        "        \n",
        "        x, KL_loss = self.layer1(x)\n",
        "        KL_loss_total = KL_loss_total + KL_loss\n",
        "        x = self.activation(x)\n",
        "        \n",
        "        x, KL_loss = self.layer2(x)\n",
        "        KL_loss_total = KL_loss_total + KL_loss\n",
        "        \n",
        "        return x, KL_loss_total"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "oAYelw3B5G-K"
      },
      "outputs": [],
      "source": [
        "class BBP_Heteroscedastic_Model_Wrapper:\n",
        "    def __init__(self, network, learn_rate, batch_size, no_batches):\n",
        "        \n",
        "        self.learn_rate = learn_rate\n",
        "        self.batch_size = batch_size\n",
        "        self.no_batches = no_batches\n",
        "        \n",
        "        self.network = network\n",
        "        self.network.cuda()\n",
        "        \n",
        "        self.optimizer = torch.optim.Adam(self.network.parameters(), lr = self.learn_rate)\n",
        "        self.loss_func = log_gaussian_loss\n",
        "    \n",
        "    def fit(self, x, y, no_samples):\n",
        "        x, y = to_variable(var=(x, y), cuda=True)\n",
        "        \n",
        "        # reset gradient and total loss\n",
        "        self.optimizer.zero_grad()\n",
        "        fit_loss_total = 0\n",
        "        \n",
        "        for i in range(no_samples):\n",
        "            output, KL_loss_total = self.network(x)\n",
        "\n",
        "            # calculate fit loss based on mean and standard deviation of output\n",
        "            fit_loss = self.loss_func(output[:, :1], y, output[:, 1:].exp(), 1)\n",
        "            fit_loss_total = fit_loss_total + fit_loss\n",
        "        \n",
        "        KL_loss_total = KL_loss_total/self.no_batches\n",
        "        total_loss = (fit_loss_total + KL_loss_total)/(no_samples*x.shape[0])\n",
        "        total_loss.backward()\n",
        "        self.optimizer.step()\n",
        "\n",
        "        return fit_loss_total/no_samples, KL_loss_total\n",
        "    \n",
        "    def get_loss_and_rmse(self, x, y, no_samples):\n",
        "        x, y = to_variable(var=(x, y), cuda=True)\n",
        "        \n",
        "        means, stds = [], []\n",
        "        for i in range(no_samples):\n",
        "            output, KL_loss_total = self.network(x)\n",
        "            means.append(output[:, :1, None])\n",
        "            stds.append(output[:, 1:, None].exp())\n",
        "            \n",
        "        means, stds = torch.cat(means, 2), torch.cat(stds, 2)\n",
        "        mean = means.mean(dim=2)\n",
        "        std = (means.var(dim=2) + stds.mean(dim=2)**2)**0.5\n",
        "            \n",
        "        # calculate fit loss based on mean and standard deviation of output\n",
        "        logliks = self.loss_func(output[:, :1], y, output[:, 1:].exp(), 1, sum_reduce=False)\n",
        "        rmse = float((((mean - y)**2).mean()**0.5).cpu().data)\n",
        "\n",
        "        return logliks, rmse"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "oN9rvBjyVKxt"
      },
      "outputs": [],
      "source": [
        "class BBP_Heteroscedastic_Model_UCI(nn.Module):\n",
        "    def __init__(self, input_dim, output_dim, num_units):\n",
        "        super(BBP_Heteroscedastic_Model_UCI, self).__init__()\n",
        "        \n",
        "        self.input_dim = input_dim\n",
        "        self.output_dim = output_dim\n",
        "        \n",
        "        # network with two hidden and one output layer\n",
        "        self.layer1 = BayesLinear_Normalq(input_dim, num_units, gaussian(0, 1))\n",
        "        self.layer2 = BayesLinear_Normalq(num_units, num_units, gaussian(0, 1))\n",
        "        self.layer3 = BayesLinear_Normalq(num_units, 2*output_dim, gaussian(0, 1))\n",
        "        \n",
        "        # activation to be used between hidden layers\n",
        "        self.activation = nn.ReLU(inplace = True)\n",
        "    \n",
        "    def forward(self, x):\n",
        "        \n",
        "        KL_loss_total = 0\n",
        "        x = x.view(-1, self.input_dim)\n",
        "        \n",
        "        x, KL_loss = self.layer1(x)\n",
        "        KL_loss_total = KL_loss_total + KL_loss\n",
        "        x = self.activation(x)\n",
        "        \n",
        "        x, KL_loss = self.layer2(x)\n",
        "        KL_loss_total = KL_loss_total + KL_loss\n",
        "        \n",
        "        return x, KL_loss_total\n",
        "\n",
        "def train_BBP(data, n_splits, num_epochs, num_units, learn_rate, log_every):\n",
        "    kf = KFold(n_splits=n_splits)\n",
        "    in_dim = data.shape[1] - 1\n",
        "    train_logliks, test_logliks = [], []\n",
        "    train_rmses, test_rmses = [], []\n",
        "\n",
        "    for i, idx in enumerate(kf.split(data)):\n",
        "        print('FOLD %d:' % i)\n",
        "\n",
        "        train_index, test_index = idx\n",
        "\n",
        "        x_train, y_train = data[train_index, :in_dim], data[train_index, in_dim:]\n",
        "        x_test, y_test = data[test_index, :in_dim], data[test_index, in_dim:]\n",
        "\n",
        "        x_means, x_stds = x_train.mean(axis = 0), x_train.var(axis = 0)**0.5\n",
        "        y_means, y_stds = y_train.mean(axis = 0), y_train.var(axis = 0)**0.5\n",
        "\n",
        "        #x_train = (x_train - x_means)/x_stds\n",
        "        #y_train = (y_train - y_means)/y_stds\n",
        "\n",
        "        #x_test = (x_test - x_means)/x_stds\n",
        "        #y_test = (y_test - y_means)/y_stds\n",
        "\n",
        "        batch_size, nb_train = len(x_train), len(x_train)\n",
        "\n",
        "        net = BBP_Heteroscedastic_Model_Wrapper(network=BBP_Heteroscedastic_Model_UCI(input_dim=x_test.shape[-1], output_dim=1, num_units=num_units),\n",
        "                                                learn_rate=1e-2, batch_size=batch_size, no_batches=1)\n",
        "\n",
        "        fit_loss_train = np.zeros(num_epochs)\n",
        "        KL_loss_train = np.zeros(num_epochs)\n",
        "        total_loss = np.zeros(num_epochs)\n",
        "\n",
        "        best_net, best_loss = None, float('inf')\n",
        "\n",
        "        for i in range(num_epochs):\n",
        "\n",
        "            fit_loss, KL_loss = net.fit(x_train, y_train, no_samples = 20)\n",
        "            fit_loss_train[i] += fit_loss.cpu().data.numpy()\n",
        "            KL_loss_train[i] += KL_loss.cpu().data.numpy()\n",
        "\n",
        "            total_loss[i] = fit_loss_train[i] + KL_loss_train[i]\n",
        "\n",
        "            if fit_loss < best_loss:\n",
        "                best_loss = fit_loss\n",
        "                best_net = copy.deepcopy(net.network)\n",
        "\n",
        "            if i % log_every == 0 or i == num_epochs - 1:\n",
        "\n",
        "                train_losses, train_rmse = net.get_loss_and_rmse(x_train, y_train, 20)\n",
        "                test_losses, test_rmse = net.get_loss_and_rmse(x_test, y_test, 20)\n",
        "\n",
        "                print('Epoch: %s/%d, Train loglik = %.3f, Test loglik = %.3f, Train RMSE = %.3f, Test RMSE = %.3f' %\\\n",
        "                      (str(i+1).zfill(3), num_epochs, -train_losses.mean() - np.log(y_stds)[0],\n",
        "                       -test_losses.mean() - np.log(y_stds)[0], y_stds*train_rmse, y_stds*test_rmse))\n",
        "\n",
        "\n",
        "        train_losses, train_rmse = net.get_loss_and_rmse(x_train, y_train, 20)\n",
        "        test_losses, test_rmse = net.get_loss_and_rmse(x_test, y_test, 20)\n",
        "\n",
        "        train_logliks.append((train_losses.cpu().data.numpy().mean() + np.log(y_stds)[0]))\n",
        "        test_logliks.append((test_losses.cpu().data.numpy().mean() + np.log(y_stds)[0]))\n",
        "\n",
        "        train_rmses.append(y_stds*train_rmse)\n",
        "        test_rmses.append(y_stds*test_rmse)\n",
        "\n",
        "    print('Train log. lik. = %6.3f +/- %6.3f' % (-np.array(train_logliks).mean(), np.array(train_logliks).var()**0.5))\n",
        "    print('Test  log. lik. = %6.3f +/- %6.3f' % (-np.array(test_logliks).mean(), np.array(test_logliks).var()**0.5))\n",
        "    print('Train RMSE      = %6.3f +/- %6.3f' % (np.array(train_rmses).mean(), np.array(train_rmses).var()**0.5))\n",
        "    print('Test  RMSE      = %6.3f +/- %6.3f' % (np.array(test_rmses).mean(), np.array(test_rmses).var()**0.5))\n",
        "\n",
        "    return best_net"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cOq4tM_PLRUV",
        "outputId": "485192ea-cd9a-4919-9ecc-7db38faf9b6c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "FOLD 0:\n",
            "Epoch: 001/500, Train loglik = -14.754, Test loglik = -10.313, Train RMSE = 6.213, Test RMSE = 5.206\n",
            "Epoch: 101/500, Train loglik = -2.067, Test loglik = -2.193, Train RMSE = 0.559, Test RMSE = 1.414\n",
            "Epoch: 201/500, Train loglik = -1.228, Test loglik = -3.930, Train RMSE = 0.398, Test RMSE = 1.302\n",
            "Epoch: 301/500, Train loglik = -0.191, Test loglik = -5.813, Train RMSE = 0.167, Test RMSE = 1.171\n",
            "Epoch: 401/500, Train loglik = -0.962, Test loglik = -2.428, Train RMSE = 0.468, Test RMSE = 1.412\n",
            "Epoch: 500/500, Train loglik = -0.010, Test loglik = -7.062, Train RMSE = 0.252, Test RMSE = 1.125\n",
            "FOLD 1:\n",
            "Epoch: 001/500, Train loglik = -13.962, Test loglik = -15.307, Train RMSE = 6.419, Test RMSE = 6.842\n",
            "Epoch: 101/500, Train loglik = -0.939, Test loglik = -2.593, Train RMSE = 0.493, Test RMSE = 1.279\n",
            "Epoch: 201/500, Train loglik = -0.568, Test loglik = -6.533, Train RMSE = 0.279, Test RMSE = 1.512\n",
            "Epoch: 301/500, Train loglik = -0.275, Test loglik = -7.037, Train RMSE = 0.199, Test RMSE = 1.427\n",
            "Epoch: 401/500, Train loglik = -0.404, Test loglik = -4.483, Train RMSE = 0.237, Test RMSE = 1.407\n",
            "Epoch: 500/500, Train loglik = 0.183, Test loglik = -30.506, Train RMSE = 0.249, Test RMSE = 1.535\n",
            "FOLD 2:\n",
            "Epoch: 001/500, Train loglik = -13.132, Test loglik = -13.664, Train RMSE = 6.440, Test RMSE = 6.727\n",
            "Epoch: 101/500, Train loglik = -0.927, Test loglik = -2.766, Train RMSE = 0.392, Test RMSE = 1.269\n",
            "Epoch: 201/500, Train loglik = -0.544, Test loglik = -4.341, Train RMSE = 0.236, Test RMSE = 1.262\n",
            "Epoch: 301/500, Train loglik = -0.014, Test loglik = -16.874, Train RMSE = 0.188, Test RMSE = 1.260\n",
            "Epoch: 401/500, Train loglik = -2.941, Test loglik = -3.194, Train RMSE = 4.097, Test RMSE = 4.579\n",
            "Epoch: 500/500, Train loglik = -0.122, Test loglik = -9.609, Train RMSE = 0.229, Test RMSE = 1.267\n",
            "FOLD 3:\n",
            "Epoch: 001/500, Train loglik = -13.194, Test loglik = -18.582, Train RMSE = 6.429, Test RMSE = 7.057\n",
            "Epoch: 101/500, Train loglik = -0.987, Test loglik = -2.211, Train RMSE = 0.469, Test RMSE = 1.143\n",
            "Epoch: 201/500, Train loglik = -0.637, Test loglik = -1.779, Train RMSE = 0.260, Test RMSE = 1.032\n",
            "Epoch: 301/500, Train loglik = -0.189, Test loglik = -4.579, Train RMSE = 0.242, Test RMSE = 1.186\n",
            "Epoch: 401/500, Train loglik = -0.126, Test loglik = -3.860, Train RMSE = 0.195, Test RMSE = 1.031\n",
            "Epoch: 500/500, Train loglik = 0.124, Test loglik = -7.646, Train RMSE = 0.189, Test RMSE = 1.109\n",
            "FOLD 4:\n",
            "Epoch: 001/500, Train loglik = -14.725, Test loglik = -12.990, Train RMSE = 6.372, Test RMSE = 6.146\n",
            "Epoch: 101/500, Train loglik = -0.918, Test loglik = -4.466, Train RMSE = 0.493, Test RMSE = 1.581\n",
            "Epoch: 201/500, Train loglik = -0.480, Test loglik = -48.477, Train RMSE = 0.249, Test RMSE = 1.660\n",
            "Epoch: 301/500, Train loglik = -0.772, Test loglik = -11.734, Train RMSE = 0.296, Test RMSE = 1.627\n",
            "Epoch: 401/500, Train loglik = 0.322, Test loglik = -70.433, Train RMSE = 0.153, Test RMSE = 1.683\n",
            "Epoch: 500/500, Train loglik = 0.113, Test loglik = -23.998, Train RMSE = 0.122, Test RMSE = 1.687\n",
            "Train log. lik. =  0.033 +/-  0.165\n",
            "Test  log. lik. = -14.862 +/-  8.637\n",
            "Train RMSE      =  0.197 +/-  0.049\n",
            "Test  RMSE      =  1.338 +/-  0.223\n"
          ]
        }
      ],
      "source": [
        "batch_size = 64\n",
        "data = pd.read_csv('/content/drive/MyDrive/bayesian-deep-learning-master/data/Datasets.csv').values\n",
        "model = train_BBP(data, n_splits=5, num_epochs=500, num_units=100, learn_rate=0.3, log_every=100)"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "name": "bbp_hetero.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.8"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}