SGLD_!!!_256_AIDD.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/EdWangLoDaSc/Dropout-as-a-Grid-Search_Representing-Model-Uncertainty-in-Deep-Learning/blob/main/SGLD_!!!_256_AIDD.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YneWjImThPub",
        "outputId": "84f2a13a-587f-49a0-a3ba-18ffff624bf5"
      },
      "id": "YneWjImThPub",
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "id": "94c44267",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "94c44267",
        "outputId": "dfe4af2c-d5fb-42e0-9695-5f56a750c0c9"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "\u001b[31mERROR: torch-0.4.1-cp36-cp36m-linux_x86_64.whl is not a supported wheel on this platform.\u001b[0m\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.7/dist-packages (0.13.0+cu113)\n",
            "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/local/lib/python3.7/dist-packages (from torchvision) (7.1.2)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from torchvision) (2.23.0)\n",
            "Requirement already satisfied: torch==1.12.0 in /usr/local/lib/python3.7/dist-packages (from torchvision) (1.12.0+cu113)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from torchvision) (1.21.6)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torchvision) (4.1.1)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->torchvision) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->torchvision) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->torchvision) (2022.6.15)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->torchvision) (1.24.3)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: GPy in /usr/local/lib/python3.7/dist-packages (1.10.0)\n",
            "Requirement already satisfied: paramz>=0.9.0 in /usr/local/lib/python3.7/dist-packages (from GPy) (0.9.5)\n",
            "Requirement already satisfied: scipy>=1.3.0 in /usr/local/lib/python3.7/dist-packages (from GPy) (1.7.3)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from GPy) (1.15.0)\n",
            "Requirement already satisfied: cython>=0.29 in /usr/local/lib/python3.7/dist-packages (from GPy) (0.29.30)\n",
            "Requirement already satisfied: numpy>=1.7 in /usr/local/lib/python3.7/dist-packages (from GPy) (1.21.6)\n",
            "Requirement already satisfied: decorator>=4.0.10 in /usr/local/lib/python3.7/dist-packages (from paramz>=0.9.0->GPy) (4.4.2)\n"
          ]
        }
      ],
      "source": [
        "!pip3 install http://download.pytorch.org/whl/cu92/torch-0.4.1-cp36-cp36m-linux_x86_64.whl\n",
        "!pip3 install torchvision\n",
        "!pip3 install GPy\n",
        "import pandas as pd\n",
        "import zipfile\n",
        "import urllib.request\n",
        "import GPy\n",
        "import time\n",
        "import copy\n",
        "import math\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torch.autograd import Variable\n",
        "from torch.optim import Optimizer\n",
        "from torch.optim.sgd import SGD\n",
        "from sklearn.model_selection import KFold\n",
        "\n",
        "from torchvision import datasets, transforms\n",
        "from torchvision.utils import make_grid\n",
        "from tqdm import tqdm, trange\n",
        "from google.colab import files\n",
        "%config InlineBackend.figure_format = 'svg'"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "XkP-XKI5mOhv"
      },
      "id": "XkP-XKI5mOhv",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "id": "7f11def2",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 353
        },
        "id": "7f11def2",
        "outputId": "e3edfd2c-36c7-4245-da1b-9a239da9da6c"
      },
      "outputs": [
        {
          "output_type": "error",
          "ename": "RuntimeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-5-e559b70a4868>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_device_name\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcurrent_device\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/cuda/__init__.py\u001b[0m in \u001b[0;36mcurrent_device\u001b[0;34m()\u001b[0m\n\u001b[1;32m    480\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mcurrent_device\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    481\u001b[0m     \u001b[0;34mr\"\"\"Returns the index of a currently selected device.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 482\u001b[0;31m     \u001b[0m_lazy_init\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    483\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_C\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_cuda_getDevice\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    484\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/cuda/__init__.py\u001b[0m in \u001b[0;36m_lazy_init\u001b[0;34m()\u001b[0m\n\u001b[1;32m    215\u001b[0m         \u001b[0;31m# This function throws if there's a driver initialization error, no GPUs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    216\u001b[0m         \u001b[0;31m# are found or any other error occurs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 217\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_C\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_cuda_init\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    218\u001b[0m         \u001b[0;31m# Some of the queued calls may reentrantly call _lazy_init();\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    219\u001b[0m         \u001b[0;31m# we need to just return without initializing in that case.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mRuntimeError\u001b[0m: No CUDA GPUs are available"
          ]
        }
      ],
      "source": [
        "\n",
        "torch.cuda.device(0)\n",
        "torch.cuda.get_device_name(torch.cuda.current_device())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "id": "613d2696",
      "metadata": {
        "id": "613d2696"
      },
      "outputs": [],
      "source": [
        "def to_variable(var=(), cuda=True, volatile=False):\n",
        "    out = []\n",
        "    for v in var:\n",
        "        \n",
        "        if isinstance(v, np.ndarray):\n",
        "            v = torch.from_numpy(v).type(torch.FloatTensor)\n",
        "\n",
        "        if not v.is_cuda and cuda:\n",
        "            v = v.cuda()\n",
        "\n",
        "        if not isinstance(v, Variable):\n",
        "            v = Variable(v, volatile=volatile)\n",
        "\n",
        "        out.append(v)\n",
        "    return out\n",
        "class Langevin_SGD(Optimizer):\n",
        "\n",
        "    def __init__(self, params, lr, weight_decay=0, nesterov=False):\n",
        "        if lr < 0.0:\n",
        "            raise ValueError(\"Invalid learning rate: {}\".format(lr))\n",
        "        if weight_decay < 0.0:\n",
        "            raise ValueError(\"Invalid weight_decay value: {}\".format(weight_decay))\n",
        "\n",
        "        defaults = dict(lr=lr, weight_decay=weight_decay)\n",
        "        \n",
        "        super(Langevin_SGD, self).__init__(params, defaults)\n",
        "\n",
        "    def __setstate__(self, state):\n",
        "        super(SGD, self).__setstate__(state)\n",
        "        for group in self.param_groups:\n",
        "            group.setdefault('nesterov', False)\n",
        "\n",
        "    def step(self, closure=None):\n",
        "        \n",
        "        loss = None\n",
        "        if closure is not None:\n",
        "            loss = closure()\n",
        "\n",
        "        for group in self.param_groups:\n",
        "            \n",
        "            weight_decay = group['weight_decay']\n",
        "            \n",
        "            for p in group['params']:\n",
        "                if p.grad is None:\n",
        "                    continue\n",
        "                d_p = p.grad.data\n",
        "                \n",
        "                if len(p.shape) == 1 and p.shape[0] == 1:\n",
        "                    p.data.add_(-group['lr'], d_p)\n",
        "                    \n",
        "                else:\n",
        "                    if weight_decay != 0:\n",
        "                        d_p.add_(weight_decay, p.data)\n",
        "\n",
        "                    unit_noise = Variable(p.data.new(p.size()).normal_())\n",
        "\n",
        "                    p.data.add_(-group['lr'], 0.5*d_p + unit_noise/group['lr']**0.5)\n",
        "\n",
        "        return loss\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "id": "b95dd8e4",
      "metadata": {
        "id": "b95dd8e4"
      },
      "outputs": [],
      "source": [
        "\n",
        "def log_gaussian_loss(output, target, sigma, no_dim):\n",
        "    exponent = -0.5*(target - output)**2/sigma**2\n",
        "    log_coeff = -no_dim*torch.log(sigma)\n",
        "    \n",
        "    return - (log_coeff + exponent).sum()\n",
        "\n",
        "\n",
        "def get_kl_divergence(weights, prior, varpost):\n",
        "    prior_loglik = prior.loglik(weights)\n",
        "    \n",
        "    varpost_loglik = varpost.loglik(weights)\n",
        "    varpost_lik = varpost_loglik.exp()\n",
        "    \n",
        "    return (varpost_lik*(varpost_loglik - prior_loglik)).sum()\n",
        "\n",
        "\n",
        "class gaussian:\n",
        "    def __init__(self, mu, sigma):\n",
        "        self.mu = mu\n",
        "        self.sigma = sigma\n",
        "        \n",
        "    def loglik(self, weights):\n",
        "        exponent = -0.5*(weights - self.mu)**2/self.sigma**2\n",
        "        log_coeff = -0.5*(np.log(2*np.pi) + 2*np.log(self.sigma))\n",
        "        \n",
        "        return (exponent + log_coeff).sum()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "id": "6a5ba769",
      "metadata": {
        "id": "6a5ba769"
      },
      "outputs": [],
      "source": [
        "class Langevin_Layer(nn.Module):\n",
        "    def __init__(self, input_dim, output_dim):\n",
        "        super(Langevin_Layer, self).__init__()\n",
        "        self.input_dim = input_dim\n",
        "        self.output_dim = output_dim\n",
        "        \n",
        "        self.weights = nn.Parameter(torch.Tensor(self.input_dim, self.output_dim).uniform_(-0.01, 0.01))\n",
        "        self.biases = nn.Parameter(torch.Tensor(self.output_dim).uniform_(-0.01, 0.01))\n",
        "        \n",
        "    def forward(self, x):\n",
        "        \n",
        "        return torch.mm(x, self.weights) + self.biases\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "id": "dc555aa8",
      "metadata": {
        "id": "dc555aa8"
      },
      "outputs": [],
      "source": [
        "class Langevin_Model(nn.Module):\n",
        "    def __init__(self, input_dim, output_dim, num_units):\n",
        "        super(Langevin_Model, self).__init__()\n",
        "        \n",
        "        self.input_dim = input_dim\n",
        "        self.output_dim = output_dim\n",
        "        \n",
        "        # network with two hidden and one output layer\n",
        "        self.layer1 = Langevin_Layer(input_dim, num_units)\n",
        "        self.layer2 = Langevin_Layer(num_units, 2*output_dim)\n",
        "        \n",
        "        # activation to be used between hidden layers\n",
        "        self.activation = nn.ReLU(inplace = True)\n",
        "    \n",
        "    def forward(self, x):\n",
        "        \n",
        "        x = x.view(-1, self.input_dim)\n",
        "        \n",
        "        x = self.layer1(x)\n",
        "        x = self.activation(x)\n",
        "        \n",
        "        x = self.layer2(x)\n",
        "        \n",
        "        return x\n",
        "\n",
        "\n",
        "class Langevin_Wrapper:\n",
        "    def __init__(self, network, learn_rate, batch_size, no_batches, weight_decay):\n",
        "        \n",
        "        self.learn_rate = learn_rate\n",
        "        self.batch_size = batch_size\n",
        "        self.no_batches = no_batches\n",
        "        \n",
        "        self.network = network\n",
        "        self.network.cuda()\n",
        "        \n",
        "        self.optimizer = Langevin_SGD(self.network.parameters(), lr=self.learn_rate, weight_decay=weight_decay)\n",
        "        self.loss_func = log_gaussian_loss\n",
        "    \n",
        "    def fit(self, x, y):\n",
        "        x, y = to_variable(var=(x, y), cuda=True)\n",
        "        \n",
        "        # reset gradient and total loss\n",
        "        self.optimizer.zero_grad()\n",
        "        \n",
        "        output = self.network(x)\n",
        "        loss = self.loss_func(output[:, :1], y, output[:, 1:].exp(), 1)\n",
        "        \n",
        "        loss.backward()\n",
        "        self.optimizer.step()\n",
        "\n",
        "        return loss\n",
        "    \n",
        "    \n",
        "    def test_loss(self, x, y):\n",
        "        x, y = to_variable(var=(x, y), cuda=True)\n",
        "        \n",
        "        output = self.network(x)\n",
        "        loss = self.loss_func(output[:, :1], y, output[:, 1:].exp(), 1)\n",
        "\n",
        "        return loss"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "id": "24ee5406",
      "metadata": {
        "id": "24ee5406"
      },
      "outputs": [],
      "source": [
        "def eval_ensemble(x, y, ensemble):\n",
        "    \n",
        "    x, y = to_variable(var=(x, y), cuda=True)\n",
        "        \n",
        "    means, stds = [], []\n",
        "    for net in ensemble:\n",
        "        output = net(x)\n",
        "        means.append(output[:, :1, None])\n",
        "        stds.append(output[:, 1:, None].exp())\n",
        "            \n",
        "    means, stds = torch.cat(means, 2), torch.cat(stds, 2)\n",
        "    mean = means.mean(dim=2)\n",
        "    std = (means.var(dim=2) + stds.mean(dim=2)**2)**0.5\n",
        "    loss = log_gaussian_loss(mean, y, std, 1)/len(x)\n",
        "    \n",
        "    rmse = ((mean - y)**2).mean()**0.5\n",
        "\n",
        "    return loss, rmse\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "id": "189bc6ae",
      "metadata": {
        "id": "189bc6ae"
      },
      "outputs": [],
      "source": [
        "class Langevin_Model_UCI(nn.Module):\n",
        "    def __init__(self, input_dim, output_dim, num_units):\n",
        "        super(Langevin_Model_UCI, self).__init__()\n",
        "        \n",
        "        self.input_dim = input_dim\n",
        "        self.output_dim = output_dim\n",
        "        \n",
        "        # network with two hidden and one output layer\n",
        "        self.layer1 = Langevin_Layer(input_dim, num_units)\n",
        "        self.layer2 = Langevin_Layer(num_units, num_units)\n",
        "        self.layer3 = Langevin_Layer(num_units, 2*output_dim)\n",
        "        \n",
        "        # activation to be used between hidden layers\n",
        "        self.activation = nn.ReLU(inplace = True)\n",
        "    \n",
        "    def forward(self, x):\n",
        "        \n",
        "        x = x.view(-1, self.input_dim)\n",
        "        \n",
        "        x = self.layer1(x)\n",
        "        x = self.activation(x)\n",
        "        \n",
        "        x = self.layer2(x)\n",
        "        x = self.activation(x)\n",
        "        \n",
        "        x = self.layer3(x)\n",
        "        \n",
        "        return x\n",
        "\n",
        "\n",
        "def train_mc_dropout(data, n_splits, burn_in, mix_time, num_nets, num_units, learn_rate, weight_decay, log_every):\n",
        "    \n",
        "    kf = KFold(n_splits=n_splits)\n",
        "    in_dim = data.shape[1] - 1\n",
        "    train_logliks, test_logliks = [], []\n",
        "    train_rmses, test_rmses = [], []\n",
        "\n",
        "    for j, idx in enumerate(kf.split(data)):\n",
        "        print('FOLD %d:' % j)\n",
        "        train_index, test_index = idx\n",
        "\n",
        "        x_train, y_train = data[train_index, :in_dim], data[train_index, in_dim:]\n",
        "        x_test, y_test = data[test_index, :in_dim], data[test_index, in_dim:]\n",
        "\n",
        "        x_means, x_stds = x_train.mean(axis = 0), x_train.var(axis = 0)**0.5\n",
        "        y_means, y_stds = y_train.mean(axis = 0), y_train.var(axis = 0)**0.5\n",
        "\n",
        "        #x_train = (x_train - x_means)/x_stds\n",
        "        #y_train = (y_train - y_means)/y_stds\n",
        "\n",
        "        #x_test = (x_test - x_means)/x_stds\n",
        "        #y_test = (y_test - y_means)/y_stds\n",
        "\n",
        "        net = Langevin_Wrapper(network=Langevin_Model_UCI(input_dim=in_dim, output_dim=1, num_units=num_units),\n",
        "                               learn_rate=learn_rate, batch_size=batch_size, no_batches=1, weight_decay=weight_decay)\n",
        "\n",
        "        nets, losses = [], []\n",
        "        num_epochs = burn_in + mix_time*num_nets + 1\n",
        "        fit_loss_train = np.zeros(num_epochs)\n",
        "\n",
        "        for i in range(num_epochs):\n",
        "            loss = net.fit(x_train, y_train)\n",
        "\n",
        "            if i % mix_time == 0 and i > burn_in:\n",
        "                nets.append(copy.deepcopy(net.network))\n",
        "                \n",
        "            if i % log_every == 0 or i == num_epochs - 1:\n",
        "                test_loss = net.test_loss(x_test, y_test).cpu().data.numpy()\n",
        "\n",
        "                if len(nets) > 0: ensemble_loss, rmse = eval_ensemble(x_test, y_test, nets)\n",
        "                else: ensemble_loss, rmse = float('nan'), float('nan')\n",
        "\n",
        "                print('Epoch: %4d, Train loss: %6.3f Test loss: %6.3f Ensemble loss: %6.3f RMSE: %.3f Num. networks: %2d' %\n",
        "                      (i, loss.cpu().data.numpy()/len(x_train), test_loss/len(x_test), ensemble_loss, rmse*y_stds[0], len(nets)))\n",
        "\n",
        "\n",
        "        train_loss, train_rmse = eval_ensemble(x_train, y_train, nets)\n",
        "        test_loss, test_rmse = eval_ensemble(x_test, y_test, nets)\n",
        "\n",
        "        train_logliks.append(-(train_loss.cpu().data.numpy() + np.log(y_stds)[0]))\n",
        "        test_logliks.append(-(test_loss.cpu().data.numpy() + np.log(y_stds)[0]))\n",
        "\n",
        "        train_rmses.append(y_stds[0]*train_rmse.cpu().data.numpy())\n",
        "        test_rmses.append(y_stds[0]*test_rmse.cpu().data.numpy())\n",
        "\n",
        "\n",
        "    print('Train log. lik. = %7.3f +/- %6.3f' % (np.array(train_logliks).mean(), np.array(train_logliks).var()**0.5))\n",
        "    print('Test  log. lik. = %7.3f +/- %6.3f' % (np.array(test_logliks).mean(), np.array(test_logliks).var()**0.5))\n",
        "    print('Train RMSE      = %7.3f +/- %6.3f' % (np.array(train_rmses).mean(), np.array(train_rmses).var()**0.5))\n",
        "    print('Test  RMSE      = %7.3f +/- %6.3f' % (np.array(test_rmses).mean(), np.array(test_rmses).var()**0.5))\n",
        "    \n",
        "    return nets\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "id": "3b9afbcd",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 371
        },
        "id": "3b9afbcd",
        "outputId": "f8e6432b-dd60-427b-c55f-704fb181dd28"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "FOLD 0:\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "RuntimeError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-11-df5c9fce4c74>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'/content/drive/MyDrive/bayesian-deep-learning-master/data/3_256_datsets.csv'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m ensemble = train_mc_dropout(data=data, n_splits=10, burn_in=350, mix_time=300, num_nets=30,\n\u001b[0;32m----> 4\u001b[0;31m                             num_units=110, learn_rate=1e-2/len(data), weight_decay=0.9, log_every=10)\n\u001b[0m",
            "\u001b[0;32m<ipython-input-10-bd5428551fad>\u001b[0m in \u001b[0;36mtrain_mc_dropout\u001b[0;34m(data, n_splits, burn_in, mix_time, num_nets, num_units, learn_rate, weight_decay, log_every)\u001b[0m\n\u001b[1;32m     53\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     54\u001b[0m         net = Langevin_Wrapper(network=Langevin_Model_UCI(input_dim=in_dim, output_dim=1, num_units=num_units),\n\u001b[0;32m---> 55\u001b[0;31m                                learn_rate=learn_rate, batch_size=batch_size, no_batches=1, weight_decay=weight_decay)\n\u001b[0m\u001b[1;32m     56\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m         \u001b[0mnets\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlosses\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<ipython-input-8-599ec6a40a35>\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, network, learn_rate, batch_size, no_batches, weight_decay)\u001b[0m\n\u001b[1;32m     33\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     34\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnetwork\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnetwork\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 35\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnetwork\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     36\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     37\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimizer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mLangevin_SGD\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnetwork\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparameters\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlr\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlearn_rate\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mweight_decay\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mweight_decay\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36mcuda\u001b[0;34m(self, device)\u001b[0m\n\u001b[1;32m    687\u001b[0m             \u001b[0mModule\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    688\u001b[0m         \"\"\"\n\u001b[0;32m--> 689\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    690\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    691\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mipu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mT\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mOptional\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mUnion\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mT\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_apply\u001b[0;34m(self, fn)\u001b[0m\n\u001b[1;32m    577\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    578\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchildren\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 579\u001b[0;31m             \u001b[0mmodule\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    580\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    581\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mcompute_should_use_set_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtensor_applied\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_apply\u001b[0;34m(self, fn)\u001b[0m\n\u001b[1;32m    600\u001b[0m             \u001b[0;31m# `with torch.no_grad():`\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    601\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mno_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 602\u001b[0;31m                 \u001b[0mparam_applied\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparam\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    603\u001b[0m             \u001b[0mshould_use_set_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompute_should_use_set_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparam\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparam_applied\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    604\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mshould_use_set_data\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(t)\u001b[0m\n\u001b[1;32m    687\u001b[0m             \u001b[0mModule\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    688\u001b[0m         \"\"\"\n\u001b[0;32m--> 689\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    690\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    691\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mipu\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mT\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mOptional\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mUnion\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m->\u001b[0m \u001b[0mT\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/torch/cuda/__init__.py\u001b[0m in \u001b[0;36m_lazy_init\u001b[0;34m()\u001b[0m\n\u001b[1;32m    215\u001b[0m         \u001b[0;31m# This function throws if there's a driver initialization error, no GPUs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    216\u001b[0m         \u001b[0;31m# are found or any other error occurs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 217\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_C\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_cuda_init\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    218\u001b[0m         \u001b[0;31m# Some of the queued calls may reentrantly call _lazy_init();\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    219\u001b[0m         \u001b[0;31m# we need to just return without initializing in that case.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mRuntimeError\u001b[0m: No CUDA GPUs are available"
          ]
        }
      ],
      "source": [
        "batch_size = 64\n",
        "data = pd.read_csv('/content/drive/MyDrive/bayesian-deep-learning-master/data/3_256_datsets.csv').values\n",
        "ensemble = train_mc_dropout(data=data, n_splits=10, burn_in=350, mix_time=300, num_nets=30,\n",
        "                            num_units=110, learn_rate=1e-2/len(data), weight_decay=0.9, log_every=10)"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.13"
    },
    "colab": {
      "name": "Untitled1.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "nbformat": 4,
  "nbformat_minor": 5
}