SGLD_Hybrid.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/EdWangLoDaSc/Dropout-as-a-Grid-Search_Representing-Model-Uncertainty-in-Deep-Learning/blob/main/SGLD_Hybrid.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YneWjImThPub",
        "outputId": "964eb184-50a9-4fa0-d26e-6c541e3e26db"
      },
      "id": "YneWjImThPub",
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "id": "94c44267",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "94c44267",
        "outputId": "cc7f44b5-8498-4089-92b8-b9c16844ac78"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "\u001b[31mERROR: torch-0.4.1-cp36-cp36m-linux_x86_64.whl is not a supported wheel on this platform.\u001b[0m\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.7/dist-packages (0.13.0+cu113)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from torchvision) (2.23.0)\n",
            "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/local/lib/python3.7/dist-packages (from torchvision) (7.1.2)\n",
            "Requirement already satisfied: torch==1.12.0 in /usr/local/lib/python3.7/dist-packages (from torchvision) (1.12.0+cu113)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from torchvision) (1.21.6)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torchvision) (4.1.1)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->torchvision) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->torchvision) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->torchvision) (2022.6.15)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->torchvision) (3.0.4)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting GPy\n",
            "  Downloading GPy-1.10.0.tar.gz (959 kB)\n",
            "\u001b[K     |████████████████████████████████| 959 kB 7.6 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.7 in /usr/local/lib/python3.7/dist-packages (from GPy) (1.21.6)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from GPy) (1.15.0)\n",
            "Collecting paramz>=0.9.0\n",
            "  Downloading paramz-0.9.5.tar.gz (71 kB)\n",
            "\u001b[K     |████████████████████████████████| 71 kB 10.7 MB/s \n",
            "\u001b[?25hRequirement already satisfied: cython>=0.29 in /usr/local/lib/python3.7/dist-packages (from GPy) (0.29.30)\n",
            "Requirement already satisfied: scipy>=1.3.0 in /usr/local/lib/python3.7/dist-packages (from GPy) (1.7.3)\n",
            "Requirement already satisfied: decorator>=4.0.10 in /usr/local/lib/python3.7/dist-packages (from paramz>=0.9.0->GPy) (4.4.2)\n",
            "Building wheels for collected packages: GPy, paramz\n",
            "  Building wheel for GPy (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for GPy: filename=GPy-1.10.0-cp37-cp37m-linux_x86_64.whl size=2565105 sha256=7ca15d376a6d05a92e78b520fc3f93e8eee104956159d9aad521d87dc90d8bfa\n",
            "  Stored in directory: /root/.cache/pip/wheels/f7/18/28/dd1ce0192a81b71a3b086fd952511d088b21e8359ea496860a\n",
            "  Building wheel for paramz (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for paramz: filename=paramz-0.9.5-py3-none-any.whl size=102566 sha256=ecbbf57810d2de8fd48af15d76895c70afab74eee05ca9f18365f7c7dad57ef2\n",
            "  Stored in directory: /root/.cache/pip/wheels/c8/95/f5/ce28482da28162e6028c4b3a32c41d147395825b3cd62bc810\n",
            "Successfully built GPy paramz\n",
            "Installing collected packages: paramz, GPy\n",
            "Successfully installed GPy-1.10.0 paramz-0.9.5\n"
          ]
        }
      ],
      "source": [
        "!pip3 install http://download.pytorch.org/whl/cu92/torch-0.4.1-cp36-cp36m-linux_x86_64.whl\n",
        "!pip3 install torchvision\n",
        "!pip3 install GPy\n",
        "import pandas as pd\n",
        "import zipfile\n",
        "import urllib.request\n",
        "import GPy\n",
        "import time\n",
        "import copy\n",
        "import math\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from torch.autograd import Variable\n",
        "from torch.optim import Optimizer\n",
        "from torch.optim.sgd import SGD\n",
        "from sklearn.model_selection import KFold\n",
        "\n",
        "from torchvision import datasets, transforms\n",
        "from torchvision.utils import make_grid\n",
        "from tqdm import tqdm, trange\n",
        "from google.colab import files\n",
        "%config InlineBackend.figure_format = 'svg'"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "XkP-XKI5mOhv"
      },
      "id": "XkP-XKI5mOhv",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "id": "7f11def2",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "7f11def2",
        "outputId": "23604b07-47f2-4cb2-fc59-02a3a3612a12"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Tesla T4'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 3
        }
      ],
      "source": [
        "\n",
        "torch.cuda.device(0)\n",
        "torch.cuda.get_device_name(torch.cuda.current_device())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "id": "613d2696",
      "metadata": {
        "id": "613d2696"
      },
      "outputs": [],
      "source": [
        "def to_variable(var=(), cuda=True, volatile=False):\n",
        "    out = []\n",
        "    for v in var:\n",
        "        \n",
        "        if isinstance(v, np.ndarray):\n",
        "            v = torch.from_numpy(v).type(torch.FloatTensor)\n",
        "\n",
        "        if not v.is_cuda and cuda:\n",
        "            v = v.cuda()\n",
        "\n",
        "        if not isinstance(v, Variable):\n",
        "            v = Variable(v, volatile=volatile)\n",
        "\n",
        "        out.append(v)\n",
        "    return out\n",
        "class Langevin_SGD(Optimizer):\n",
        "\n",
        "    def __init__(self, params, lr, weight_decay=0, nesterov=False):\n",
        "        if lr < 0.0:\n",
        "            raise ValueError(\"Invalid learning rate: {}\".format(lr))\n",
        "        if weight_decay < 0.0:\n",
        "            raise ValueError(\"Invalid weight_decay value: {}\".format(weight_decay))\n",
        "\n",
        "        defaults = dict(lr=lr, weight_decay=weight_decay)\n",
        "        \n",
        "        super(Langevin_SGD, self).__init__(params, defaults)\n",
        "\n",
        "    def __setstate__(self, state):\n",
        "        super(SGD, self).__setstate__(state)\n",
        "        for group in self.param_groups:\n",
        "            group.setdefault('nesterov', False)\n",
        "\n",
        "    def step(self, closure=None):\n",
        "        \n",
        "        loss = None\n",
        "        if closure is not None:\n",
        "            loss = closure()\n",
        "\n",
        "        for group in self.param_groups:\n",
        "            \n",
        "            weight_decay = group['weight_decay']\n",
        "            \n",
        "            for p in group['params']:\n",
        "                if p.grad is None:\n",
        "                    continue\n",
        "                d_p = p.grad.data\n",
        "                \n",
        "                if len(p.shape) == 1 and p.shape[0] == 1:\n",
        "                    p.data.add_(-group['lr'], d_p)\n",
        "                    \n",
        "                else:\n",
        "                    if weight_decay != 0:\n",
        "                        d_p.add_(weight_decay, p.data)\n",
        "\n",
        "                    unit_noise = Variable(p.data.new(p.size()).normal_())\n",
        "\n",
        "                    p.data.add_(-group['lr'], 0.5*d_p + unit_noise/group['lr']**0.5)\n",
        "\n",
        "        return loss\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "id": "b95dd8e4",
      "metadata": {
        "id": "b95dd8e4"
      },
      "outputs": [],
      "source": [
        "\n",
        "def log_gaussian_loss(output, target, sigma, no_dim):\n",
        "    exponent = -0.5*(target - output)**2/sigma**2\n",
        "    log_coeff = -no_dim*torch.log(sigma)\n",
        "    \n",
        "    return - (log_coeff + exponent).sum()\n",
        "\n",
        "\n",
        "def get_kl_divergence(weights, prior, varpost):\n",
        "    prior_loglik = prior.loglik(weights)\n",
        "    \n",
        "    varpost_loglik = varpost.loglik(weights)\n",
        "    varpost_lik = varpost_loglik.exp()\n",
        "    \n",
        "    return (varpost_lik*(varpost_loglik - prior_loglik)).sum()\n",
        "\n",
        "\n",
        "class gaussian:\n",
        "    def __init__(self, mu, sigma):\n",
        "        self.mu = mu\n",
        "        self.sigma = sigma\n",
        "        \n",
        "    def loglik(self, weights):\n",
        "        exponent = -0.5*(weights - self.mu)**2/self.sigma**2\n",
        "        log_coeff = -0.5*(np.log(2*np.pi) + 2*np.log(self.sigma))\n",
        "        \n",
        "        return (exponent + log_coeff).sum()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "id": "6a5ba769",
      "metadata": {
        "id": "6a5ba769"
      },
      "outputs": [],
      "source": [
        "class Langevin_Layer(nn.Module):\n",
        "    def __init__(self, input_dim, output_dim):\n",
        "        super(Langevin_Layer, self).__init__()\n",
        "        self.input_dim = input_dim\n",
        "        self.output_dim = output_dim\n",
        "        \n",
        "        self.weights = nn.Parameter(torch.Tensor(self.input_dim, self.output_dim).uniform_(-0.01, 0.01))\n",
        "        self.biases = nn.Parameter(torch.Tensor(self.output_dim).uniform_(-0.01, 0.01))\n",
        "        \n",
        "    def forward(self, x):\n",
        "        \n",
        "        return torch.mm(x, self.weights) + self.biases\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "id": "dc555aa8",
      "metadata": {
        "id": "dc555aa8"
      },
      "outputs": [],
      "source": [
        "class Langevin_Model(nn.Module):\n",
        "    def __init__(self, input_dim, output_dim, num_units):\n",
        "        super(Langevin_Model, self).__init__()\n",
        "        \n",
        "        self.input_dim = input_dim\n",
        "        self.output_dim = output_dim\n",
        "        \n",
        "        # network with two hidden and one output layer\n",
        "        self.layer1 = Langevin_Layer(input_dim, num_units)\n",
        "        self.layer2 = Langevin_Layer(num_units, 2*output_dim)\n",
        "        \n",
        "        # activation to be used between hidden layers\n",
        "        self.activation = nn.ReLU(inplace = True)\n",
        "    \n",
        "    def forward(self, x):\n",
        "        \n",
        "        x = x.view(-1, self.input_dim)\n",
        "        \n",
        "        x = self.layer1(x)\n",
        "        x = self.activation(x)\n",
        "        \n",
        "        x = self.layer2(x)\n",
        "        \n",
        "        return x\n",
        "\n",
        "\n",
        "class Langevin_Wrapper:\n",
        "    def __init__(self, network, learn_rate, batch_size, no_batches, weight_decay):\n",
        "        \n",
        "        self.learn_rate = learn_rate\n",
        "        self.batch_size = batch_size\n",
        "        self.no_batches = no_batches\n",
        "        \n",
        "        self.network = network\n",
        "        self.network.cuda()\n",
        "        \n",
        "        self.optimizer = Langevin_SGD(self.network.parameters(), lr=self.learn_rate, weight_decay=weight_decay)\n",
        "        self.loss_func = log_gaussian_loss\n",
        "    \n",
        "    def fit(self, x, y):\n",
        "        x, y = to_variable(var=(x, y), cuda=True)\n",
        "        \n",
        "        # reset gradient and total loss\n",
        "        self.optimizer.zero_grad()\n",
        "        \n",
        "        output = self.network(x)\n",
        "        loss = self.loss_func(output[:, :1], y, output[:, 1:].exp(), 1)\n",
        "        \n",
        "        loss.backward()\n",
        "        self.optimizer.step()\n",
        "\n",
        "        return loss\n",
        "    \n",
        "    \n",
        "    def test_loss(self, x, y):\n",
        "        x, y = to_variable(var=(x, y), cuda=True)\n",
        "        \n",
        "        output = self.network(x)\n",
        "        loss = self.loss_func(output[:, :1], y, output[:, 1:].exp(), 1)\n",
        "\n",
        "        return loss"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "id": "24ee5406",
      "metadata": {
        "id": "24ee5406"
      },
      "outputs": [],
      "source": [
        "def eval_ensemble(x, y, ensemble):\n",
        "    \n",
        "    x, y = to_variable(var=(x, y), cuda=True)\n",
        "        \n",
        "    means, stds = [], []\n",
        "    for net in ensemble:\n",
        "        output = net(x)\n",
        "        means.append(output[:, :1, None])\n",
        "        stds.append(output[:, 1:, None].exp())\n",
        "            \n",
        "    means, stds = torch.cat(means, 2), torch.cat(stds, 2)\n",
        "    mean = means.mean(dim=2)\n",
        "    std = (means.var(dim=2) + stds.mean(dim=2)**2)**0.5\n",
        "    loss = log_gaussian_loss(mean, y, std, 1)/len(x)\n",
        "    \n",
        "    rmse = ((mean - y)**2).mean()**0.5\n",
        "\n",
        "    return loss, rmse\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "id": "189bc6ae",
      "metadata": {
        "id": "189bc6ae"
      },
      "outputs": [],
      "source": [
        "class Langevin_Model_UCI(nn.Module):\n",
        "    def __init__(self, input_dim, output_dim, num_units):\n",
        "        super(Langevin_Model_UCI, self).__init__()\n",
        "        \n",
        "        self.input_dim = input_dim\n",
        "        self.output_dim = output_dim\n",
        "        \n",
        "        # network with two hidden and one output layer\n",
        "        self.layer1 = Langevin_Layer(input_dim, num_units)\n",
        "        self.layer2 = Langevin_Layer(num_units, num_units)\n",
        "        self.layer3 = Langevin_Layer(num_units, 2*output_dim)\n",
        "        \n",
        "        # activation to be used between hidden layers\n",
        "        self.activation = nn.ReLU(inplace = True)\n",
        "    \n",
        "    def forward(self, x):\n",
        "        \n",
        "        x = x.view(-1, self.input_dim)\n",
        "        \n",
        "        x = self.layer1(x)\n",
        "        x = self.activation(x)\n",
        "        \n",
        "        x = self.layer2(x)\n",
        "        x = self.activation(x)\n",
        "        \n",
        "        x = self.layer3(x)\n",
        "        \n",
        "        return x\n",
        "\n",
        "\n",
        "def train_mc_dropout(data, n_splits, burn_in, mix_time, num_nets, num_units, learn_rate, weight_decay, log_every):\n",
        "    \n",
        "    kf = KFold(n_splits=n_splits)\n",
        "    in_dim = data.shape[1] - 1\n",
        "    train_logliks, test_logliks = [], []\n",
        "    train_rmses, test_rmses = [], []\n",
        "\n",
        "    for j, idx in enumerate(kf.split(data)):\n",
        "        print('FOLD %d:' % j)\n",
        "        train_index, test_index = idx\n",
        "\n",
        "        x_train, y_train = data[train_index, :in_dim], data[train_index, in_dim:]\n",
        "        x_test, y_test = data[test_index, :in_dim], data[test_index, in_dim:]\n",
        "\n",
        "        x_means, x_stds = x_train.mean(axis = 0), x_train.var(axis = 0)**0.5\n",
        "        y_means, y_stds = y_train.mean(axis = 0), y_train.var(axis = 0)**0.5\n",
        "\n",
        "        x_train = (x_train - x_means)/x_stds\n",
        "        y_train = (y_train - y_means)/y_stds\n",
        "\n",
        "        x_test = (x_test - x_means)/x_stds\n",
        "        y_test = (y_test - y_means)/y_stds\n",
        "\n",
        "        net = Langevin_Wrapper(network=Langevin_Model_UCI(input_dim=in_dim, output_dim=1, num_units=num_units),\n",
        "                               learn_rate=learn_rate, batch_size=batch_size, no_batches=1, weight_decay=weight_decay)\n",
        "\n",
        "        nets, losses = [], []\n",
        "        num_epochs = burn_in + mix_time*num_nets + 1\n",
        "        fit_loss_train = np.zeros(num_epochs)\n",
        "\n",
        "        for i in range(num_epochs):\n",
        "            loss = net.fit(x_train, y_train)\n",
        "\n",
        "            if i % mix_time == 0 and i > burn_in:\n",
        "                nets.append(copy.deepcopy(net.network))\n",
        "                \n",
        "            if i % log_every == 0 or i == num_epochs - 1:\n",
        "                test_loss = net.test_loss(x_test, y_test).cpu().data.numpy()\n",
        "\n",
        "                if len(nets) > 0: ensemble_loss, rmse = eval_ensemble(x_test, y_test, nets)\n",
        "                else: ensemble_loss, rmse = float('nan'), float('nan')\n",
        "\n",
        "                print('Epoch: %4d, Train loss: %6.3f Test loss: %6.3f Ensemble loss: %6.3f RMSE: %.3f Num. networks: %2d' %\n",
        "                      (i, loss.cpu().data.numpy()/len(x_train), test_loss/len(x_test), ensemble_loss, rmse*y_stds[0], len(nets)))\n",
        "\n",
        "\n",
        "        train_loss, train_rmse = eval_ensemble(x_train, y_train, nets)\n",
        "        test_loss, test_rmse = eval_ensemble(x_test, y_test, nets)\n",
        "\n",
        "        train_logliks.append(-(train_loss.cpu().data.numpy() + np.log(y_stds)[0]))\n",
        "        test_logliks.append(-(test_loss.cpu().data.numpy() + np.log(y_stds)[0]))\n",
        "\n",
        "        train_rmses.append(y_stds[0]*train_rmse.cpu().data.numpy())\n",
        "        test_rmses.append(y_stds[0]*test_rmse.cpu().data.numpy())\n",
        "\n",
        "\n",
        "    print('Train log. lik. = %7.3f +/- %6.3f' % (np.array(train_logliks).mean(), np.array(train_logliks).var()**0.5))\n",
        "    print('Test  log. lik. = %7.3f +/- %6.3f' % (np.array(test_logliks).mean(), np.array(test_logliks).var()**0.5))\n",
        "    print('Train RMSE      = %7.3f +/- %6.3f' % (np.array(train_rmses).mean(), np.array(train_rmses).var()**0.5))\n",
        "    print('Test  RMSE      = %7.3f +/- %6.3f' % (np.array(test_rmses).mean(), np.array(test_rmses).var()**0.5))\n",
        "    \n",
        "    return nets\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#from DataProcessing import DataProcessing as dp\n",
        "import os\n",
        "os.chdir('/content/drive/MyDrive')\n"
      ],
      "metadata": {
        "id": "engVhJi45CpW"
      },
      "id": "engVhJi45CpW",
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "pip install miceforest"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NT9MGz466Nnr",
        "outputId": "b17901f5-b732-4a5e-c433-57deac0ed889"
      },
      "id": "NT9MGz466Nnr",
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting miceforest\n",
            "  Downloading miceforest-5.5.4-py3-none-any.whl (49 kB)\n",
            "\u001b[K     |████████████████████████████████| 49 kB 3.5 MB/s \n",
            "\u001b[?25hCollecting blosc\n",
            "  Downloading blosc-1.10.6-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (2.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 2.6 MB 56.8 MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from miceforest) (1.21.6)\n",
            "Requirement already satisfied: dill in /usr/local/lib/python3.7/dist-packages (from miceforest) (0.3.5.1)\n",
            "Collecting lightgbm>=3.3.1\n",
            "  Downloading lightgbm-3.3.2-py3-none-manylinux1_x86_64.whl (2.0 MB)\n",
            "\u001b[K     |████████████████████████████████| 2.0 MB 21.2 MB/s \n",
            "\u001b[?25hRequirement already satisfied: wheel in /usr/local/lib/python3.7/dist-packages (from lightgbm>=3.3.1->miceforest) (0.37.1)\n",
            "Requirement already satisfied: scikit-learn!=0.22.0 in /usr/local/lib/python3.7/dist-packages (from lightgbm>=3.3.1->miceforest) (1.0.2)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.7/dist-packages (from lightgbm>=3.3.1->miceforest) (1.7.3)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn!=0.22.0->lightgbm>=3.3.1->miceforest) (1.1.0)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn!=0.22.0->lightgbm>=3.3.1->miceforest) (3.1.0)\n",
            "Installing collected packages: lightgbm, blosc, miceforest\n",
            "  Attempting uninstall: lightgbm\n",
            "    Found existing installation: lightgbm 2.2.3\n",
            "    Uninstalling lightgbm-2.2.3:\n",
            "      Successfully uninstalled lightgbm-2.2.3\n",
            "Successfully installed blosc-1.10.6 lightgbm-3.3.2 miceforest-5.5.4\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "os.chdir('/content/drive/MyDrive/DropoutUncertaintyExps-master/Hydrid_modeling')"
      ],
      "metadata": {
        "id": "sKERd3es7XXI"
      },
      "id": "sKERd3es7XXI",
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "batch_size = 1024\n",
        "data = BNN_20\n",
        "ensemble = train_mc_dropout(data=data, n_splits=20, burn_in=400, mix_time=300, num_nets=40,\n",
        "                            num_units=100, learn_rate=1e-1/len(data), weight_decay=0.7, log_every=1000)"
      ],
      "metadata": {
        "id": "VhnrNNL3MApD",
        "outputId": "0db8a6cc-63d1-46ce-dc73-1f4c90497411",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "id": "VhnrNNL3MApD",
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "FOLD 0:\n",
            "Epoch:    0, Train loss:  0.500 Test loss:  0.299 Ensemble loss:    nan RMSE: nan Num. networks:  0\n",
            "Epoch: 1000, Train loss: -0.858 Test loss: -0.864 Ensemble loss: -0.636 RMSE: 224.615 Num. networks:  2\n",
            "Epoch: 2000, Train loss: -1.402 Test loss: -0.526 Ensemble loss: -0.982 RMSE: 94.593 Num. networks:  5\n",
            "Epoch: 3000, Train loss: -0.454 Test loss: -0.345 Ensemble loss: -0.965 RMSE: 76.019 Num. networks:  9\n",
            "Epoch: 4000, Train loss: -0.816 Test loss: -0.820 Ensemble loss: -0.563 RMSE: 116.201 Num. networks: 12\n",
            "Epoch: 5000, Train loss: -0.322 Test loss: -0.272 Ensemble loss: -0.611 RMSE: 93.674 Num. networks: 15\n",
            "Epoch: 6000, Train loss:  6.161 Test loss:  0.215 Ensemble loss: -0.560 RMSE: 106.680 Num. networks: 19\n",
            "Epoch: 7000, Train loss: -1.231 Test loss: -1.123 Ensemble loss: -0.587 RMSE: 122.210 Num. networks: 22\n",
            "Epoch: 8000, Train loss: -0.937 Test loss: -0.808 Ensemble loss: -0.622 RMSE: 97.980 Num. networks: 25\n",
            "Epoch: 9000, Train loss: -1.640 Test loss: -1.396 Ensemble loss: -0.681 RMSE: 94.916 Num. networks: 29\n",
            "Epoch: 10000, Train loss: -1.629 Test loss: -1.121 Ensemble loss: -0.548 RMSE: 83.218 Num. networks: 32\n",
            "Epoch: 11000, Train loss: -1.482 Test loss: -1.246 Ensemble loss: -0.566 RMSE: 88.016 Num. networks: 35\n",
            "Epoch: 12000, Train loss:  7.237 Test loss:  0.454 Ensemble loss: -0.573 RMSE: 88.483 Num. networks: 39\n",
            "Epoch: 12400, Train loss: -0.296 Test loss: -0.185 Ensemble loss: -0.583 RMSE: 89.312 Num. networks: 40\n",
            "FOLD 1:\n",
            "Epoch:    0, Train loss:  0.500 Test loss:  0.347 Ensemble loss:    nan RMSE: nan Num. networks:  0\n",
            "Epoch: 1000, Train loss: -1.359 Test loss: -1.309 Ensemble loss: -0.641 RMSE: 208.131 Num. networks:  2\n",
            "Epoch: 2000, Train loss: -1.168 Test loss: -0.145 Ensemble loss: -0.620 RMSE: 179.792 Num. networks:  5\n",
            "Epoch: 3000, Train loss: -1.348 Test loss: -1.286 Ensemble loss: -0.724 RMSE: 141.021 Num. networks:  9\n",
            "Epoch: 4000, Train loss: -0.664 Test loss: -0.676 Ensemble loss: -0.811 RMSE: 137.629 Num. networks: 12\n",
            "Epoch: 5000, Train loss: -1.261 Test loss: -1.209 Ensemble loss: -0.771 RMSE: 153.500 Num. networks: 15\n",
            "Epoch: 6000, Train loss: -1.533 Test loss: -1.429 Ensemble loss: -0.794 RMSE: 129.101 Num. networks: 19\n",
            "Epoch: 7000, Train loss: -1.066 Test loss: -1.124 Ensemble loss: -0.801 RMSE: 141.499 Num. networks: 22\n",
            "Epoch: 8000, Train loss: -1.697 Test loss: -1.342 Ensemble loss: -0.831 RMSE: 139.123 Num. networks: 25\n",
            "Epoch: 9000, Train loss: -1.123 Test loss:  1.290 Ensemble loss: -0.661 RMSE: 104.146 Num. networks: 29\n",
            "Epoch: 10000, Train loss: -1.047 Test loss: -1.111 Ensemble loss: -0.707 RMSE: 105.183 Num. networks: 32\n",
            "Epoch: 11000, Train loss: -0.720 Test loss: -0.808 Ensemble loss: -0.702 RMSE: 103.472 Num. networks: 35\n",
            "Epoch: 12000, Train loss:  5.012 Test loss: -0.381 Ensemble loss: -0.719 RMSE: 103.792 Num. networks: 39\n",
            "Epoch: 12400, Train loss: -1.661 Test loss: -1.396 Ensemble loss: -0.732 RMSE: 103.968 Num. networks: 40\n",
            "FOLD 2:\n",
            "Epoch:    0, Train loss:  0.500 Test loss:  0.219 Ensemble loss:    nan RMSE: nan Num. networks:  0\n",
            "Epoch: 1000, Train loss: -0.730 Test loss: -0.980 Ensemble loss: -0.678 RMSE: 156.489 Num. networks:  2\n",
            "Epoch: 2000, Train loss: -1.335 Test loss: -1.377 Ensemble loss: -0.913 RMSE: 126.976 Num. networks:  5\n",
            "Epoch: 3000, Train loss: -0.820 Test loss: -1.022 Ensemble loss: -0.834 RMSE: 120.334 Num. networks:  9\n",
            "Epoch: 4000, Train loss: -0.702 Test loss: -0.938 Ensemble loss: -0.735 RMSE: 101.012 Num. networks: 12\n",
            "Epoch: 5000, Train loss: -1.573 Test loss: -1.630 Ensemble loss: -0.733 RMSE: 105.966 Num. networks: 15\n",
            "Epoch: 6000, Train loss: -0.492 Test loss: -0.714 Ensemble loss: -0.792 RMSE: 109.084 Num. networks: 19\n",
            "Epoch: 7000, Train loss: -1.527 Test loss: -1.678 Ensemble loss: -0.669 RMSE: 96.579 Num. networks: 22\n",
            "Epoch: 8000, Train loss: -0.818 Test loss: -1.032 Ensemble loss: -0.713 RMSE: 97.207 Num. networks: 25\n",
            "Epoch: 9000, Train loss: -1.632 Test loss: -1.778 Ensemble loss: -0.746 RMSE: 99.264 Num. networks: 29\n",
            "Epoch: 10000, Train loss:  0.226 Test loss: -0.202 Ensemble loss: -0.778 RMSE: 100.047 Num. networks: 32\n",
            "Epoch: 11000, Train loss: -1.525 Test loss: -1.655 Ensemble loss: -0.812 RMSE: 99.798 Num. networks: 35\n",
            "Epoch: 12000, Train loss: -0.547 Test loss: -0.949 Ensemble loss: -0.847 RMSE: 99.048 Num. networks: 39\n",
            "Epoch: 12400, Train loss: -1.593 Test loss: -1.711 Ensemble loss: -0.859 RMSE: 98.899 Num. networks: 40\n",
            "FOLD 3:\n",
            "Epoch:    0, Train loss:  0.500 Test loss:  1.059 Ensemble loss:    nan RMSE: nan Num. networks:  0\n",
            "Epoch: 1000, Train loss: -1.289 Test loss: -1.033 Ensemble loss: -1.072 RMSE: 133.048 Num. networks:  2\n",
            "Epoch: 2000, Train loss: -0.758 Test loss: -0.727 Ensemble loss: -0.834 RMSE: 206.789 Num. networks:  5\n",
            "Epoch: 3000, Train loss: -0.399 Test loss: -0.284 Ensemble loss: -0.641 RMSE: 238.897 Num. networks:  9\n",
            "Epoch: 4000, Train loss: -1.088 Test loss: -0.975 Ensemble loss: -0.677 RMSE: 213.568 Num. networks: 12\n",
            "Epoch: 5000, Train loss: -0.695 Test loss: -0.616 Ensemble loss: -0.671 RMSE: 201.662 Num. networks: 15\n",
            "Epoch: 6000, Train loss: -0.741 Test loss: -0.572 Ensemble loss: -0.554 RMSE: 193.482 Num. networks: 19\n",
            "Epoch: 7000, Train loss: -1.596 Test loss: -1.497 Ensemble loss: -0.508 RMSE: 199.459 Num. networks: 22\n",
            "Epoch: 8000, Train loss: -1.487 Test loss: -1.465 Ensemble loss: -0.574 RMSE: 184.865 Num. networks: 25\n",
            "Epoch: 9000, Train loss: -1.255 Test loss: -1.143 Ensemble loss: -0.634 RMSE: 167.286 Num. networks: 29\n",
            "Epoch: 10000, Train loss: -0.774 Test loss: -0.672 Ensemble loss: -0.641 RMSE: 158.134 Num. networks: 32\n",
            "Epoch: 11000, Train loss: -1.726 Test loss: -1.624 Ensemble loss: -0.632 RMSE: 158.895 Num. networks: 35\n",
            "Epoch: 12000, Train loss: -0.643 Test loss: -0.484 Ensemble loss: -0.626 RMSE: 137.580 Num. networks: 39\n",
            "Epoch: 12400, Train loss: -1.599 Test loss: -0.739 Ensemble loss: -0.629 RMSE: 137.576 Num. networks: 40\n",
            "FOLD 4:\n",
            "Epoch:    0, Train loss:  0.500 Test loss:  0.516 Ensemble loss:    nan RMSE: nan Num. networks:  0\n",
            "Epoch: 1000, Train loss: -1.326 Test loss: -1.335 Ensemble loss: -0.603 RMSE: 210.653 Num. networks:  2\n",
            "Epoch: 2000, Train loss: -0.209 Test loss:  1.617 Ensemble loss:  0.022 RMSE: 241.008 Num. networks:  5\n",
            "Epoch: 3000, Train loss: -0.976 Test loss: -0.925 Ensemble loss: -0.255 RMSE: 145.789 Num. networks:  9\n",
            "Epoch: 4000, Train loss:  0.721 Test loss:  9.873 Ensemble loss: -0.176 RMSE: 199.844 Num. networks: 12\n",
            "Epoch: 5000, Train loss: -1.175 Test loss: -1.144 Ensemble loss: -0.288 RMSE: 168.739 Num. networks: 15\n",
            "Epoch: 6000, Train loss: -1.544 Test loss: -1.489 Ensemble loss: -0.300 RMSE: 113.362 Num. networks: 19\n",
            "Epoch: 7000, Train loss: -1.586 Test loss: -1.560 Ensemble loss: -0.350 RMSE: 121.438 Num. networks: 22\n",
            "Epoch: 8000, Train loss: -1.431 Test loss:  0.611 Ensemble loss: -0.400 RMSE: 109.801 Num. networks: 25\n",
            "Epoch: 9000, Train loss: -1.159 Test loss: -1.201 Ensemble loss: -0.432 RMSE: 93.360 Num. networks: 29\n",
            "Epoch: 10000, Train loss: -0.980 Test loss: -0.912 Ensemble loss: -0.434 RMSE: 85.759 Num. networks: 32\n",
            "Epoch: 11000, Train loss: -0.158 Test loss: -0.044 Ensemble loss: -0.455 RMSE: 87.595 Num. networks: 35\n",
            "Epoch: 12000, Train loss: -1.094 Test loss: -1.098 Ensemble loss: -0.471 RMSE: 89.583 Num. networks: 39\n",
            "Epoch: 12400, Train loss: -1.007 Test loss: -0.940 Ensemble loss: -0.483 RMSE: 89.147 Num. networks: 40\n",
            "FOLD 5:\n",
            "Epoch:    0, Train loss:  0.500 Test loss:  0.834 Ensemble loss:    nan RMSE: nan Num. networks:  0\n",
            "Epoch: 1000, Train loss: -1.090 Test loss: -0.955 Ensemble loss: -0.956 RMSE: 129.944 Num. networks:  2\n",
            "Epoch: 2000, Train loss: -1.207 Test loss: -1.065 Ensemble loss: -0.870 RMSE: 143.792 Num. networks:  5\n",
            "Epoch: 3000, Train loss: -1.446 Test loss: -1.301 Ensemble loss: -0.856 RMSE: 106.690 Num. networks:  9\n",
            "Epoch: 4000, Train loss: -0.878 Test loss: -0.810 Ensemble loss: -0.744 RMSE: 131.058 Num. networks: 12\n",
            "Epoch: 5000, Train loss: -1.372 Test loss: -1.341 Ensemble loss: -0.819 RMSE: 124.843 Num. networks: 15\n",
            "Epoch: 6000, Train loss: -1.332 Test loss:  1.424 Ensemble loss: -0.895 RMSE: 123.113 Num. networks: 19\n",
            "Epoch: 7000, Train loss: -1.451 Test loss: -1.498 Ensemble loss: -0.889 RMSE: 108.668 Num. networks: 22\n",
            "Epoch: 8000, Train loss: -1.069 Test loss: -0.996 Ensemble loss: -0.932 RMSE: 106.233 Num. networks: 25\n",
            "Epoch: 9000, Train loss: -1.639 Test loss: -1.638 Ensemble loss: -0.975 RMSE: 102.070 Num. networks: 29\n",
            "Epoch: 10000, Train loss:  5.776 Test loss:  0.998 Ensemble loss: -0.964 RMSE: 97.655 Num. networks: 32\n",
            "Epoch: 11000, Train loss: -0.470 Test loss: -0.493 Ensemble loss: -0.992 RMSE: 98.668 Num. networks: 35\n",
            "Epoch: 12000, Train loss:  0.544 Test loss:  0.573 Ensemble loss: -0.890 RMSE: 92.306 Num. networks: 39\n",
            "Epoch: 12400, Train loss: -1.722 Test loss: -1.520 Ensemble loss: -0.892 RMSE: 92.840 Num. networks: 40\n",
            "FOLD 6:\n",
            "Epoch:    0, Train loss:  0.500 Test loss:  0.413 Ensemble loss:    nan RMSE: nan Num. networks:  0\n",
            "Epoch: 1000, Train loss: -1.118 Test loss: -1.246 Ensemble loss: -1.308 RMSE: 126.806 Num. networks:  2\n",
            "Epoch: 2000, Train loss: -0.331 Test loss: -0.348 Ensemble loss: -1.243 RMSE: 117.603 Num. networks:  5\n",
            "Epoch: 3000, Train loss: -0.908 Test loss: -1.041 Ensemble loss: -1.129 RMSE: 128.969 Num. networks:  9\n",
            "Epoch: 4000, Train loss: -0.916 Test loss: -0.956 Ensemble loss: -1.176 RMSE: 124.172 Num. networks: 12\n",
            "Epoch: 5000, Train loss:  0.044 Test loss: -0.068 Ensemble loss: -0.855 RMSE: 152.489 Num. networks: 15\n",
            "Epoch: 6000, Train loss: -0.569 Test loss: -0.663 Ensemble loss: -0.849 RMSE: 117.242 Num. networks: 19\n",
            "Epoch: 7000, Train loss: -0.323 Test loss: -0.347 Ensemble loss: -0.749 RMSE: 96.964 Num. networks: 22\n",
            "Epoch: 8000, Train loss:  0.281 Test loss:  0.150 Ensemble loss: -0.785 RMSE: 97.690 Num. networks: 25\n",
            "Epoch: 9000, Train loss: -0.407 Test loss: -0.441 Ensemble loss: -0.801 RMSE: 95.232 Num. networks: 29\n",
            "Epoch: 10000, Train loss: -1.525 Test loss:  0.800 Ensemble loss: -0.823 RMSE: 94.307 Num. networks: 32\n",
            "Epoch: 11000, Train loss: -1.542 Test loss: -0.356 Ensemble loss: -0.855 RMSE: 94.486 Num. networks: 35\n",
            "Epoch: 12000, Train loss: -0.913 Test loss: -0.909 Ensemble loss: -0.803 RMSE: 90.965 Num. networks: 39\n",
            "Epoch: 12400, Train loss: -0.261 Test loss: -0.367 Ensemble loss: -0.798 RMSE: 90.575 Num. networks: 40\n",
            "FOLD 7:\n",
            "Epoch:    0, Train loss:  0.500 Test loss:  0.514 Ensemble loss:    nan RMSE: nan Num. networks:  0\n",
            "Epoch: 1000, Train loss: -1.107 Test loss: -1.217 Ensemble loss: -1.073 RMSE: 119.658 Num. networks:  2\n",
            "Epoch: 2000, Train loss:  0.025 Test loss:  0.042 Ensemble loss: -1.182 RMSE: 103.310 Num. networks:  5\n",
            "Epoch: 3000, Train loss: -0.629 Test loss: -0.717 Ensemble loss: -0.863 RMSE: 109.262 Num. networks:  9\n",
            "Epoch: 4000, Train loss: -0.127 Test loss:  0.004 Ensemble loss: -0.856 RMSE: 114.015 Num. networks: 12\n",
            "Epoch: 5000, Train loss: -1.428 Test loss: -1.428 Ensemble loss: -0.861 RMSE: 111.787 Num. networks: 15\n",
            "Epoch: 6000, Train loss: -1.229 Test loss: -1.230 Ensemble loss: -0.807 RMSE: 103.125 Num. networks: 19\n",
            "Epoch: 7000, Train loss:  7.597 Test loss:  2.723 Ensemble loss: -0.785 RMSE: 104.581 Num. networks: 22\n",
            "Epoch: 8000, Train loss: -1.523 Test loss: -1.530 Ensemble loss: -0.818 RMSE: 101.327 Num. networks: 25\n",
            "Epoch: 9000, Train loss: -1.414 Test loss: -1.398 Ensemble loss: -0.806 RMSE: 96.044 Num. networks: 29\n",
            "Epoch: 10000, Train loss: -0.570 Test loss: -0.576 Ensemble loss: -0.841 RMSE: 93.463 Num. networks: 32\n",
            "Epoch: 11000, Train loss: -0.580 Test loss: -0.550 Ensemble loss: -0.768 RMSE: 88.397 Num. networks: 35\n",
            "Epoch: 12000, Train loss:  3.014 Test loss:  9.537 Ensemble loss: -0.651 RMSE: 80.811 Num. networks: 39\n",
            "Epoch: 12400, Train loss: -0.287 Test loss: -0.323 Ensemble loss: -0.661 RMSE: 80.891 Num. networks: 40\n",
            "FOLD 8:\n",
            "Epoch:    0, Train loss:  0.500 Test loss:  0.839 Ensemble loss:    nan RMSE: nan Num. networks:  0\n",
            "Epoch: 1000, Train loss: -1.246 Test loss: -1.239 Ensemble loss: -1.191 RMSE: 108.160 Num. networks:  2\n",
            "Epoch: 2000, Train loss: -1.354 Test loss: -1.444 Ensemble loss: -0.559 RMSE: 291.523 Num. networks:  5\n",
            "Epoch: 3000, Train loss: -0.957 Test loss: -1.008 Ensemble loss: -0.511 RMSE: 277.764 Num. networks:  9\n",
            "Epoch: 4000, Train loss: -1.564 Test loss: -1.611 Ensemble loss: -0.582 RMSE: 252.924 Num. networks: 12\n",
            "Epoch: 5000, Train loss: -1.378 Test loss: -1.392 Ensemble loss: -0.660 RMSE: 215.289 Num. networks: 15\n",
            "Epoch: 6000, Train loss: -0.852 Test loss:  3.986 Ensemble loss: -0.728 RMSE: 165.765 Num. networks: 19\n",
            "Epoch: 7000, Train loss: -0.540 Test loss: -0.488 Ensemble loss: -0.727 RMSE: 151.412 Num. networks: 22\n",
            "Epoch: 8000, Train loss: -1.518 Test loss: -1.520 Ensemble loss: -0.727 RMSE: 138.357 Num. networks: 25\n",
            "Epoch: 9000, Train loss: -1.459 Test loss: -1.464 Ensemble loss: -0.797 RMSE: 128.707 Num. networks: 29\n",
            "Epoch: 10000, Train loss: -0.242 Test loss: -0.173 Ensemble loss: -0.832 RMSE: 122.088 Num. networks: 32\n",
            "Epoch: 11000, Train loss: -1.647 Test loss: -1.664 Ensemble loss: -0.856 RMSE: 117.883 Num. networks: 35\n",
            "Epoch: 12000, Train loss: -1.611 Test loss: -1.607 Ensemble loss: -0.713 RMSE: 98.723 Num. networks: 39\n",
            "Epoch: 12400, Train loss:  8.707 Test loss:  0.074 Ensemble loss: -0.709 RMSE: 96.431 Num. networks: 40\n",
            "FOLD 9:\n",
            "Epoch:    0, Train loss:  0.500 Test loss:  0.544 Ensemble loss:    nan RMSE: nan Num. networks:  0\n",
            "Epoch: 1000, Train loss: -1.176 Test loss:  0.779 Ensemble loss: -0.942 RMSE: 180.152 Num. networks:  2\n",
            "Epoch: 2000, Train loss: -0.990 Test loss: -1.007 Ensemble loss: -0.909 RMSE: 187.165 Num. networks:  5\n",
            "Epoch: 3000, Train loss: -0.110 Test loss: -0.239 Ensemble loss: -0.896 RMSE: 183.744 Num. networks:  9\n",
            "Epoch: 4000, Train loss: -0.986 Test loss: -0.805 Ensemble loss: -0.860 RMSE: 194.834 Num. networks: 12\n",
            "Epoch: 5000, Train loss: -0.753 Test loss: -0.623 Ensemble loss: -0.945 RMSE: 182.267 Num. networks: 15\n",
            "Epoch: 6000, Train loss: -0.269 Test loss: -0.186 Ensemble loss: -0.803 RMSE: 164.596 Num. networks: 19\n",
            "Epoch: 7000, Train loss: -1.016 Test loss: -0.874 Ensemble loss: -0.842 RMSE: 165.736 Num. networks: 22\n",
            "Epoch: 8000, Train loss: -1.078 Test loss: -0.910 Ensemble loss: -0.729 RMSE: 170.167 Num. networks: 25\n",
            "Epoch: 9000, Train loss:  0.003 Test loss:  0.163 Ensemble loss: -0.668 RMSE: 163.866 Num. networks: 29\n",
            "Epoch: 10000, Train loss:  7.716 Test loss:  1.481 Ensemble loss: -0.665 RMSE: 159.972 Num. networks: 32\n",
            "Epoch: 11000, Train loss: -1.651 Test loss: -1.445 Ensemble loss: -0.694 RMSE: 158.956 Num. networks: 35\n",
            "Epoch: 12000, Train loss: -0.849 Test loss: -0.859 Ensemble loss: -0.699 RMSE: 157.584 Num. networks: 39\n",
            "Epoch: 12400, Train loss: -0.858 Test loss: -0.844 Ensemble loss: -0.712 RMSE: 157.297 Num. networks: 40\n",
            "FOLD 10:\n",
            "Epoch:    0, Train loss:  0.500 Test loss:  0.811 Ensemble loss:    nan RMSE: nan Num. networks:  0\n",
            "Epoch: 1000, Train loss: -1.033 Test loss: -0.960 Ensemble loss: -0.460 RMSE: 300.815 Num. networks:  2\n",
            "Epoch: 2000, Train loss: -0.888 Test loss: -0.786 Ensemble loss: -0.538 RMSE: 291.155 Num. networks:  5\n",
            "Epoch: 3000, Train loss: -0.266 Test loss: -0.198 Ensemble loss: -0.651 RMSE: 232.630 Num. networks:  9\n",
            "Epoch: 4000, Train loss: -1.615 Test loss: -1.495 Ensemble loss: -0.709 RMSE: 206.056 Num. networks: 12\n",
            "Epoch: 5000, Train loss: -1.613 Test loss: -1.455 Ensemble loss: -0.331 RMSE: 206.397 Num. networks: 15\n",
            "Epoch: 6000, Train loss: -1.001 Test loss: -0.811 Ensemble loss: -0.394 RMSE: 188.891 Num. networks: 19\n",
            "Epoch: 7000, Train loss:  0.318 Test loss:  0.377 Ensemble loss: -0.461 RMSE: 175.897 Num. networks: 22\n",
            "Epoch: 8000, Train loss: -1.443 Test loss: -1.345 Ensemble loss: -0.507 RMSE: 168.691 Num. networks: 25\n",
            "Epoch: 9000, Train loss: -1.374 Test loss: -1.262 Ensemble loss: -0.513 RMSE: 172.531 Num. networks: 29\n",
            "Epoch: 10000, Train loss: -1.469 Test loss: -1.360 Ensemble loss: -0.541 RMSE: 173.091 Num. networks: 32\n",
            "Epoch: 11000, Train loss: -0.836 Test loss: -0.735 Ensemble loss: -0.574 RMSE: 158.671 Num. networks: 35\n",
            "Epoch: 12000, Train loss: -1.442 Test loss: -1.332 Ensemble loss: -0.578 RMSE: 141.221 Num. networks: 39\n",
            "Epoch: 12400, Train loss: -1.677 Test loss: -1.555 Ensemble loss: -0.579 RMSE: 138.271 Num. networks: 40\n",
            "FOLD 11:\n",
            "Epoch:    0, Train loss:  0.500 Test loss:  0.506 Ensemble loss:    nan RMSE: nan Num. networks:  0\n",
            "Epoch: 1000, Train loss: -1.118 Test loss: -1.229 Ensemble loss: -0.612 RMSE: 247.187 Num. networks:  2\n",
            "Epoch: 2000, Train loss: -0.925 Test loss:  2.440 Ensemble loss: -0.808 RMSE: 203.565 Num. networks:  5\n",
            "Epoch: 3000, Train loss: -0.544 Test loss: -0.617 Ensemble loss: -0.958 RMSE: 157.354 Num. networks:  9\n",
            "Epoch: 4000, Train loss: -1.298 Test loss: -1.228 Ensemble loss: -0.955 RMSE: 142.321 Num. networks: 12\n",
            "Epoch: 5000, Train loss: -1.010 Test loss: -0.798 Ensemble loss: -0.716 RMSE: 93.783 Num. networks: 15\n",
            "Epoch: 6000, Train loss: 17.328 Test loss:  1.470 Ensemble loss: -0.380 RMSE: 95.689 Num. networks: 19\n",
            "Epoch: 7000, Train loss: -1.386 Test loss: -1.527 Ensemble loss: -0.366 RMSE: 96.626 Num. networks: 22\n",
            "Epoch: 8000, Train loss: -1.442 Test loss: -1.440 Ensemble loss: -0.424 RMSE: 92.433 Num. networks: 25\n",
            "Epoch: 9000, Train loss: -0.168 Test loss: -0.278 Ensemble loss: -0.466 RMSE: 83.528 Num. networks: 29\n",
            "Epoch: 10000, Train loss: -1.307 Test loss: -1.356 Ensemble loss: -0.426 RMSE: 89.151 Num. networks: 32\n",
            "Epoch: 11000, Train loss: -0.540 Test loss: -0.602 Ensemble loss: -0.437 RMSE: 81.636 Num. networks: 35\n",
            "Epoch: 12000, Train loss: -1.273 Test loss: -1.281 Ensemble loss: -0.458 RMSE: 84.076 Num. networks: 39\n",
            "Epoch: 12400, Train loss: -0.437 Test loss: -0.574 Ensemble loss: -0.469 RMSE: 83.428 Num. networks: 40\n",
            "FOLD 12:\n",
            "Epoch:    0, Train loss:  0.500 Test loss:  0.591 Ensemble loss:    nan RMSE: nan Num. networks:  0\n",
            "Epoch: 1000, Train loss: -1.363 Test loss: -1.231 Ensemble loss: -0.721 RMSE: 280.436 Num. networks:  2\n",
            "Epoch: 2000, Train loss: -1.252 Test loss: -1.284 Ensemble loss: -0.118 RMSE: 276.450 Num. networks:  5\n",
            "Epoch: 3000, Train loss:  5.670 Test loss:  0.322 Ensemble loss: -0.213 RMSE: 168.714 Num. networks:  9\n",
            "Epoch: 4000, Train loss:  4.477 Test loss: 14.346 Ensemble loss: -0.210 RMSE: 168.124 Num. networks: 12\n",
            "Epoch: 5000, Train loss: -0.829 Test loss: -0.821 Ensemble loss: -0.323 RMSE: 151.848 Num. networks: 15\n",
            "Epoch: 6000, Train loss: -0.745 Test loss: -0.732 Ensemble loss: -0.390 RMSE: 145.951 Num. networks: 19\n",
            "Epoch: 7000, Train loss: -0.935 Test loss: -0.907 Ensemble loss: -0.417 RMSE: 143.257 Num. networks: 22\n",
            "Epoch: 8000, Train loss: -1.572 Test loss: -1.512 Ensemble loss: -0.453 RMSE: 145.953 Num. networks: 25\n",
            "Epoch: 9000, Train loss: -1.550 Test loss: -1.505 Ensemble loss: -0.518 RMSE: 137.114 Num. networks: 29\n",
            "Epoch: 10000, Train loss: -1.510 Test loss: -1.463 Ensemble loss: -0.562 RMSE: 132.718 Num. networks: 32\n",
            "Epoch: 11000, Train loss: -1.310 Test loss: -1.283 Ensemble loss: -0.556 RMSE: 120.024 Num. networks: 35\n",
            "Epoch: 12000, Train loss: -0.659 Test loss: -0.675 Ensemble loss: -0.582 RMSE: 114.191 Num. networks: 39\n",
            "Epoch: 12400, Train loss: -0.630 Test loss: -0.642 Ensemble loss: -0.590 RMSE: 113.566 Num. networks: 40\n",
            "FOLD 13:\n",
            "Epoch:    0, Train loss:  0.500 Test loss:  0.338 Ensemble loss:    nan RMSE: nan Num. networks:  0\n",
            "Epoch: 1000, Train loss: -1.206 Test loss: -1.289 Ensemble loss: -0.759 RMSE: 259.446 Num. networks:  2\n",
            "Epoch: 2000, Train loss: -1.176 Test loss: -1.201 Ensemble loss: -0.950 RMSE: 200.539 Num. networks:  5\n",
            "Epoch: 3000, Train loss: -0.210 Test loss: -0.341 Ensemble loss: -0.889 RMSE: 190.150 Num. networks:  9\n",
            "Epoch: 4000, Train loss: -1.453 Test loss: -1.196 Ensemble loss: -0.831 RMSE: 199.308 Num. networks: 12\n",
            "Epoch: 5000, Train loss: -0.620 Test loss: -0.750 Ensemble loss: -0.852 RMSE: 188.849 Num. networks: 15\n",
            "Epoch: 6000, Train loss: -1.031 Test loss: -0.820 Ensemble loss: -0.731 RMSE: 205.916 Num. networks: 19\n",
            "Epoch: 7000, Train loss: -1.284 Test loss: -1.088 Ensemble loss: -0.743 RMSE: 199.854 Num. networks: 22\n",
            "Epoch: 8000, Train loss: -1.693 Test loss: -1.034 Ensemble loss: -0.734 RMSE: 189.667 Num. networks: 25\n",
            "Epoch: 9000, Train loss: -1.636 Test loss: -1.261 Ensemble loss: -0.755 RMSE: 188.950 Num. networks: 29\n",
            "Epoch: 10000, Train loss: -0.531 Test loss: -0.495 Ensemble loss: -0.740 RMSE: 181.621 Num. networks: 32\n",
            "Epoch: 11000, Train loss: -0.687 Test loss: -0.697 Ensemble loss: -0.768 RMSE: 179.299 Num. networks: 35\n",
            "Epoch: 12000, Train loss:  0.236 Test loss:  0.064 Ensemble loss: -0.630 RMSE: 169.089 Num. networks: 39\n",
            "Epoch: 12400, Train loss: -0.149 Test loss: -0.282 Ensemble loss: -0.638 RMSE: 168.227 Num. networks: 40\n",
            "FOLD 14:\n",
            "Epoch:    0, Train loss:  0.500 Test loss:  0.341 Ensemble loss:    nan RMSE: nan Num. networks:  0\n",
            "Epoch: 1000, Train loss: -0.378 Test loss: -0.790 Ensemble loss: -0.938 RMSE: 180.510 Num. networks:  2\n",
            "Epoch: 2000, Train loss: -1.093 Test loss: -1.254 Ensemble loss: -0.985 RMSE: 139.255 Num. networks:  5\n",
            "Epoch: 3000, Train loss: -0.666 Test loss: -0.752 Ensemble loss: -0.905 RMSE: 118.638 Num. networks:  9\n",
            "Epoch: 4000, Train loss: -0.309 Test loss: -0.587 Ensemble loss: -0.878 RMSE: 111.965 Num. networks: 12\n",
            "Epoch: 5000, Train loss: -1.016 Test loss: -1.165 Ensemble loss: -0.889 RMSE: 110.410 Num. networks: 15\n",
            "Epoch: 6000, Train loss: -1.546 Test loss: -1.617 Ensemble loss: -0.893 RMSE: 106.400 Num. networks: 19\n",
            "Epoch: 7000, Train loss: -1.242 Test loss: -1.325 Ensemble loss: -0.937 RMSE: 104.548 Num. networks: 22\n",
            "Epoch: 8000, Train loss:  3.330 Test loss:  0.561 Ensemble loss: -0.946 RMSE: 104.140 Num. networks: 25\n",
            "Epoch: 9000, Train loss: -1.335 Test loss: -1.463 Ensemble loss: -0.897 RMSE: 108.506 Num. networks: 29\n",
            "Epoch: 10000, Train loss: -0.573 Test loss: -0.769 Ensemble loss: -0.898 RMSE: 108.265 Num. networks: 32\n",
            "Epoch: 11000, Train loss:  2.430 Test loss:  0.073 Ensemble loss: -0.922 RMSE: 106.932 Num. networks: 35\n",
            "Epoch: 12000, Train loss: -0.797 Test loss: -0.934 Ensemble loss: -0.906 RMSE: 102.853 Num. networks: 39\n",
            "Epoch: 12400, Train loss: -1.493 Test loss: -1.554 Ensemble loss: -0.903 RMSE: 103.326 Num. networks: 40\n",
            "FOLD 15:\n",
            "Epoch:    0, Train loss:  0.500 Test loss:  0.403 Ensemble loss:    nan RMSE: nan Num. networks:  0\n",
            "Epoch: 1000, Train loss: -0.939 Test loss: -1.032 Ensemble loss: -1.239 RMSE: 146.773 Num. networks:  2\n",
            "Epoch: 2000, Train loss: -1.380 Test loss: -1.305 Ensemble loss: -0.907 RMSE: 148.690 Num. networks:  5\n",
            "Epoch: 3000, Train loss: -1.245 Test loss:  0.461 Ensemble loss: -0.733 RMSE: 171.432 Num. networks:  9\n",
            "Epoch: 4000, Train loss: -0.988 Test loss: -1.014 Ensemble loss: -0.807 RMSE: 158.127 Num. networks: 12\n",
            "Epoch: 5000, Train loss: -1.387 Test loss: -1.373 Ensemble loss: -0.764 RMSE: 149.567 Num. networks: 15\n",
            "Epoch: 6000, Train loss: -0.080 Test loss: -0.345 Ensemble loss: -0.749 RMSE: 139.363 Num. networks: 19\n",
            "Epoch: 7000, Train loss: -1.227 Test loss: -1.266 Ensemble loss: -0.611 RMSE: 157.724 Num. networks: 22\n",
            "Epoch: 8000, Train loss: -0.955 Test loss: -1.036 Ensemble loss: -0.633 RMSE: 158.130 Num. networks: 25\n",
            "Epoch: 9000, Train loss: -1.495 Test loss: -1.486 Ensemble loss: -0.684 RMSE: 146.647 Num. networks: 29\n",
            "Epoch: 10000, Train loss: -0.986 Test loss: -1.036 Ensemble loss: -0.718 RMSE: 142.366 Num. networks: 32\n",
            "Epoch: 11000, Train loss: -0.385 Test loss: -0.478 Ensemble loss: -0.713 RMSE: 144.296 Num. networks: 35\n",
            "Epoch: 12000, Train loss: -0.970 Test loss: -1.012 Ensemble loss: -0.737 RMSE: 143.274 Num. networks: 39\n",
            "Epoch: 12400, Train loss: -1.409 Test loss: -1.391 Ensemble loss: -0.750 RMSE: 141.659 Num. networks: 40\n",
            "FOLD 16:\n",
            "Epoch:    0, Train loss:  0.500 Test loss:  0.338 Ensemble loss:    nan RMSE: nan Num. networks:  0\n",
            "Epoch: 1000, Train loss: -1.084 Test loss: -1.094 Ensemble loss: -0.996 RMSE: 153.516 Num. networks:  2\n",
            "Epoch: 2000, Train loss: -0.782 Test loss: -1.124 Ensemble loss: -0.879 RMSE: 166.328 Num. networks:  5\n",
            "Epoch: 3000, Train loss: -1.052 Test loss: -1.113 Ensemble loss: -0.760 RMSE: 178.211 Num. networks:  9\n",
            "Epoch: 4000, Train loss: -0.492 Test loss: -0.760 Ensemble loss: -0.717 RMSE: 176.614 Num. networks: 12\n",
            "Epoch: 5000, Train loss: -1.045 Test loss: -1.117 Ensemble loss: -0.608 RMSE: 173.751 Num. networks: 15\n",
            "Epoch: 6000, Train loss: -1.468 Test loss: -1.523 Ensemble loss: -0.694 RMSE: 164.588 Num. networks: 19\n",
            "Epoch: 7000, Train loss: -1.380 Test loss: -1.391 Ensemble loss: -0.687 RMSE: 166.868 Num. networks: 22\n",
            "Epoch: 8000, Train loss: -0.525 Test loss: -0.752 Ensemble loss: -0.631 RMSE: 150.722 Num. networks: 25\n",
            "Epoch: 9000, Train loss: -0.176 Test loss: -0.536 Ensemble loss: -0.666 RMSE: 154.756 Num. networks: 29\n",
            "Epoch: 10000, Train loss:  0.732 Test loss:  0.497 Ensemble loss: -0.627 RMSE: 136.488 Num. networks: 32\n",
            "Epoch: 11000, Train loss: -1.660 Test loss: -1.604 Ensemble loss: -0.655 RMSE: 134.904 Num. networks: 35\n",
            "Epoch: 12000, Train loss: -1.337 Test loss: -1.428 Ensemble loss: -0.698 RMSE: 133.549 Num. networks: 39\n",
            "Epoch: 12400, Train loss: -1.571 Test loss: -1.556 Ensemble loss: -0.668 RMSE: 138.586 Num. networks: 40\n",
            "FOLD 17:\n",
            "Epoch:    0, Train loss:  0.500 Test loss:  0.745 Ensemble loss:    nan RMSE: nan Num. networks:  0\n",
            "Epoch: 1000, Train loss: -1.310 Test loss:  0.949 Ensemble loss: -0.130 RMSE: 844.696 Num. networks:  2\n",
            "Epoch: 2000, Train loss: -1.105 Test loss: -0.427 Ensemble loss: -0.345 RMSE: 523.281 Num. networks:  5\n",
            "Epoch: 3000, Train loss:  0.118 Test loss:  0.161 Ensemble loss: -0.373 RMSE: 370.586 Num. networks:  9\n",
            "Epoch: 4000, Train loss: -1.514 Test loss: -0.773 Ensemble loss: -0.427 RMSE: 302.081 Num. networks: 12\n",
            "Epoch: 5000, Train loss: -0.022 Test loss:  0.082 Ensemble loss: -0.416 RMSE: 224.245 Num. networks: 15\n",
            "Epoch: 6000, Train loss: -0.889 Test loss: -0.788 Ensemble loss: -0.493 RMSE: 186.015 Num. networks: 19\n",
            "Epoch: 7000, Train loss: -0.556 Test loss: -0.549 Ensemble loss: -0.491 RMSE: 179.233 Num. networks: 22\n",
            "Epoch: 8000, Train loss:  0.886 Test loss:  0.598 Ensemble loss: -0.542 RMSE: 169.448 Num. networks: 25\n",
            "Epoch: 9000, Train loss: -1.307 Test loss: -1.244 Ensemble loss: -0.567 RMSE: 156.842 Num. networks: 29\n",
            "Epoch: 10000, Train loss: -0.786 Test loss: -0.797 Ensemble loss: -0.593 RMSE: 149.141 Num. networks: 32\n",
            "Epoch: 11000, Train loss: -1.534 Test loss: -1.457 Ensemble loss: -0.615 RMSE: 144.422 Num. networks: 35\n",
            "Epoch: 12000, Train loss: -1.149 Test loss: -1.067 Ensemble loss: -0.555 RMSE: 148.902 Num. networks: 39\n",
            "Epoch: 12400, Train loss:  0.980 Test loss:  0.049 Ensemble loss: -0.558 RMSE: 147.440 Num. networks: 40\n",
            "FOLD 18:\n",
            "Epoch:    0, Train loss:  0.500 Test loss:  0.162 Ensemble loss:    nan RMSE: nan Num. networks:  0\n",
            "Epoch: 1000, Train loss: -1.153 Test loss: -1.202 Ensemble loss: -1.195 RMSE: 162.991 Num. networks:  2\n",
            "Epoch: 2000, Train loss: -1.320 Test loss: -1.127 Ensemble loss: -0.778 RMSE: 179.199 Num. networks:  5\n",
            "Epoch: 3000, Train loss:  3.864 Test loss:  6.747 Ensemble loss: -0.483 RMSE: 219.122 Num. networks:  9\n",
            "Epoch: 4000, Train loss: -1.238 Test loss: -1.197 Ensemble loss: -0.533 RMSE: 214.675 Num. networks: 12\n",
            "Epoch: 5000, Train loss: -1.184 Test loss: -1.259 Ensemble loss: -0.593 RMSE: 216.651 Num. networks: 15\n",
            "Epoch: 6000, Train loss: -1.619 Test loss: -1.365 Ensemble loss: -0.668 RMSE: 207.031 Num. networks: 19\n",
            "Epoch: 7000, Train loss:  0.267 Test loss: -0.033 Ensemble loss: -0.694 RMSE: 199.020 Num. networks: 22\n",
            "Epoch: 8000, Train loss: -0.283 Test loss: -0.535 Ensemble loss: -0.744 RMSE: 190.253 Num. networks: 25\n",
            "Epoch: 9000, Train loss: -1.240 Test loss: -1.256 Ensemble loss: -0.802 RMSE: 181.589 Num. networks: 29\n",
            "Epoch: 10000, Train loss: -1.016 Test loss: -1.113 Ensemble loss: -0.826 RMSE: 177.288 Num. networks: 32\n",
            "Epoch: 11000, Train loss: -0.714 Test loss: -0.914 Ensemble loss: -0.851 RMSE: 173.517 Num. networks: 35\n",
            "Epoch: 12000, Train loss:  0.158 Test loss: -0.154 Ensemble loss: -0.832 RMSE: 165.382 Num. networks: 39\n",
            "Epoch: 12400, Train loss: -0.599 Test loss: -0.819 Ensemble loss: -0.839 RMSE: 164.800 Num. networks: 40\n",
            "FOLD 19:\n",
            "Epoch:    0, Train loss:  0.500 Test loss:  0.539 Ensemble loss:    nan RMSE: nan Num. networks:  0\n",
            "Epoch: 1000, Train loss: -1.031 Test loss:  0.913 Ensemble loss: -0.556 RMSE: 206.971 Num. networks:  2\n",
            "Epoch: 2000, Train loss: -0.510 Test loss: -0.570 Ensemble loss: -0.910 RMSE: 154.771 Num. networks:  5\n",
            "Epoch: 3000, Train loss:  0.459 Test loss:  0.267 Ensemble loss: -0.447 RMSE: 182.402 Num. networks:  9\n",
            "Epoch: 4000, Train loss: -1.063 Test loss:  2.635 Ensemble loss: -0.498 RMSE: 161.712 Num. networks: 12\n",
            "Epoch: 5000, Train loss: -1.115 Test loss: -1.124 Ensemble loss: -0.508 RMSE: 164.285 Num. networks: 15\n",
            "Epoch: 6000, Train loss: -1.531 Test loss: -1.522 Ensemble loss: -0.585 RMSE: 150.027 Num. networks: 19\n",
            "Epoch: 7000, Train loss: -1.202 Test loss: -1.201 Ensemble loss: -0.637 RMSE: 139.423 Num. networks: 22\n",
            "Epoch: 8000, Train loss:  0.440 Test loss:  0.450 Ensemble loss: -0.683 RMSE: 138.631 Num. networks: 25\n",
            "Epoch: 9000, Train loss: -1.286 Test loss: -1.267 Ensemble loss: -0.662 RMSE: 140.788 Num. networks: 29\n",
            "Epoch: 10000, Train loss: -0.496 Test loss: -0.552 Ensemble loss: -0.672 RMSE: 131.956 Num. networks: 32\n",
            "Epoch: 11000, Train loss:  0.418 Test loss:  0.365 Ensemble loss: -0.672 RMSE: 132.474 Num. networks: 35\n",
            "Epoch: 12000, Train loss: -0.705 Test loss: -0.709 Ensemble loss: -0.674 RMSE: 124.724 Num. networks: 39\n",
            "Epoch: 12400, Train loss:  3.156 Test loss:  5.729 Ensemble loss: -0.673 RMSE: 122.438 Num. networks: 40\n",
            "Train log. lik. =  -6.104 +/-  0.114\n",
            "Test  log. lik. =  -6.133 +/-  0.121\n",
            "Train RMSE      = 109.882 +/-  7.609\n",
            "Test  RMSE      = 117.934 +/- 28.104\n"
          ]
        }
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.13"
    },
    "colab": {
      "name": "Untitled1.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "nbformat": 4,
  "nbformat_minor": 5
}