Bootstrap MAP Ensemble.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "map_hetero.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/EdWangLoDaSc/Dropout-as-a-Grid-Search_Representing-Model-Uncertainty-in-Deep-Learning/blob/main/Bootstrap%20MAP%20Ensemble.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "id": "qt-k8IhFgUBr",
        "outputId": "f8f31a72-cffb-434f-ae39-fc8cb2902258",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xAYPKSFrG8AF",
        "outputId": "894abb4c-a543-4767-f62e-b3e2ddf10b67",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "!pip3 install http://download.pytorch.org/whl/cu92/torch-0.4.1-cp36-cp36m-linux_x86_64.whl\n",
        "!pip3 install torchvision\n",
        "!pip install GPy\n",
        "import pandas as pd\n",
        "import zipfile\n",
        "import urllib.request\n",
        "import os\n",
        "import GPy\n",
        "import time\n",
        "import copy\n",
        "import math\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.nn.functional as F\n",
        "from sklearn.model_selection import KFold\n",
        "from torch.autograd import Variable\n",
        "from torch.optim import Optimizer\n",
        "from torch.optim.sgd import SGD\n",
        "\n",
        "from torchvision import datasets, transforms\n",
        "from torchvision.utils import make_grid\n",
        "from tqdm import tqdm, trange\n",
        "from google.colab import files\n",
        "%config InlineBackend.figure_format = 'svg'"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "\u001b[31mERROR: torch-0.4.1-cp36-cp36m-linux_x86_64.whl is not a supported wheel on this platform.\u001b[0m\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.7/dist-packages (0.13.0+cu113)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torchvision) (4.1.1)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from torchvision) (2.23.0)\n",
            "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/local/lib/python3.7/dist-packages (from torchvision) (7.1.2)\n",
            "Requirement already satisfied: torch==1.12.0 in /usr/local/lib/python3.7/dist-packages (from torchvision) (1.12.0+cu113)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from torchvision) (1.21.6)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->torchvision) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->torchvision) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->torchvision) (2022.6.15)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->torchvision) (1.24.3)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: GPy in /usr/local/lib/python3.7/dist-packages (1.10.0)\n",
            "Requirement already satisfied: scipy>=1.3.0 in /usr/local/lib/python3.7/dist-packages (from GPy) (1.7.3)\n",
            "Requirement already satisfied: cython>=0.29 in /usr/local/lib/python3.7/dist-packages (from GPy) (0.29.30)\n",
            "Requirement already satisfied: paramz>=0.9.0 in /usr/local/lib/python3.7/dist-packages (from GPy) (0.9.5)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from GPy) (1.15.0)\n",
            "Requirement already satisfied: numpy>=1.7 in /usr/local/lib/python3.7/dist-packages (from GPy) (1.21.6)\n",
            "Requirement already satisfied: decorator>=4.0.10 in /usr/local/lib/python3.7/dist-packages (from paramz>=0.9.0->GPy) (4.4.2)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1-7dNcVmHA3I",
        "outputId": "df66e5d9-f927-4dcd-bd8c-bc38a83d9265",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "torch.cuda.device(0)\n",
        "torch.cuda.get_device_name(torch.cuda.current_device())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'Tesla T4'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_1S5kt0omQ-N"
      },
      "source": [
        "def to_variable(var=(), cuda=False, volatile=False):\n",
        "    out = []\n",
        "    for v in var:\n",
        "        \n",
        "        if isinstance(v, np.ndarray):\n",
        "            v = torch.from_numpy(v).type(torch.FloatTensor)\n",
        "\n",
        "        if not v.is_cuda and cuda:\n",
        "            v = v.cuda()\n",
        "\n",
        "        if not isinstance(v, Variable):\n",
        "            v = Variable(v, volatile=volatile)\n",
        "\n",
        "        out.append(v)\n",
        "    return out"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Va8V78eFFsc9"
      },
      "source": [
        "def log_gaussian_loss(output, target, sigma, no_dim):\n",
        "    exponent = -0.5*(target - output)**2/sigma**2\n",
        "    log_coeff = -no_dim*torch.log(sigma)\n",
        "    \n",
        "    return - (log_coeff + exponent).sum()\n",
        "\n",
        "\n",
        "def get_kl_divergence(weights, prior, varpost):\n",
        "    prior_loglik = prior.loglik(weights)\n",
        "    \n",
        "    varpost_loglik = varpost.loglik(weights)\n",
        "    varpost_lik = varpost_loglik.exp()\n",
        "    \n",
        "    return (varpost_lik*(varpost_loglik - prior_loglik)).sum()\n",
        "\n",
        "\n",
        "class gaussian:\n",
        "    def __init__(self, mu, sigma):\n",
        "        self.mu = mu\n",
        "        self.sigma = sigma\n",
        "        \n",
        "    def loglik(self, weights):\n",
        "        exponent = -0.5*(weights - self.mu)**2/self.sigma**2\n",
        "        log_coeff = -0.5*(np.log(2*np.pi) + 2*np.log(self.sigma))\n",
        "        \n",
        "        return (exponent + log_coeff).sum()"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rPqBGJbPKaxU"
      },
      "source": [
        "class Net(torch.nn.Module):\n",
        "    def __init__(self, n_feature, n_hidden, n_output, learn_rate, weight_decay):\n",
        "        super(Net, self).__init__()\n",
        "        self.layer1 = torch.nn.Linear(n_feature, n_hidden)\n",
        "        self.layer2 = torch.nn.Linear(n_hidden, 2*n_output)\n",
        "        \n",
        "        self.loss_func = log_gaussian_loss\n",
        "        self.optimizer = torch.optim.SGD(self.parameters(), lr=learn_rate, weight_decay=weight_decay)\n",
        "        \n",
        "    def forward(self, x):\n",
        "        x = F.relu(self.layer1(x))\n",
        "        x = self.layer2(x)\n",
        "        return x\n",
        "    \n",
        "    def fit(self, x, y):\n",
        "        x, y = to_variable(var=(x, y), cuda=False)\n",
        "        \n",
        "        # reset gradient and total loss\n",
        "        self.optimizer.zero_grad()\n",
        "        \n",
        "        output = self.forward(x)\n",
        "        loss = self.loss_func(output[:, :1], y, output[:, 1:].exp(), 1)/x.shape[0]\n",
        "        \n",
        "        loss.backward()\n",
        "        self.optimizer.step()\n",
        "\n",
        "        return loss"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_J_HMo9VJd5C"
      },
      "source": [
        "def eval_ensemble(x, y, ensemble):\n",
        "    \n",
        "    x, y = to_variable(var=(x, y), cuda=False)\n",
        "        \n",
        "    means, stds = [], []\n",
        "    for net in ensemble:\n",
        "        output = net(x)\n",
        "        means.append(output[:, :1, None])\n",
        "        stds.append(output[:, 1:, None].exp())\n",
        "            \n",
        "    means, stds = torch.cat(means, 2), torch.cat(stds, 2)\n",
        "    mean = means.mean(dim=2)\n",
        "    std = (means.var(dim=2) + stds.mean(dim=2)**2)**0.5\n",
        "    loss = log_gaussian_loss(mean, y, std, 1)/len(x)\n",
        "    \n",
        "    rmse = ((mean - y)**2).mean()**0.5\n",
        "\n",
        "    return loss, rmse"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nQFXw0se_4bE"
      },
      "source": [
        "class Net_UCI(torch.nn.Module):\n",
        "    def __init__(self, n_feature, n_hidden, n_output, learn_rate, weight_decay):\n",
        "        super(Net_UCI, self).__init__()\n",
        "        self.hidden1 = torch.nn.Linear(n_feature, n_hidden)\n",
        "        self.hidden2 = torch.nn.Linear(n_hidden, n_hidden)\n",
        "        self.predict = torch.nn.Linear(n_hidden, 2*n_output)\n",
        "        \n",
        "        self.loss_func = log_gaussian_loss\n",
        "        self.optimizer = torch.optim.SGD(self.parameters(), lr=learn_rate, weight_decay=weight_decay)\n",
        "        \n",
        "    def forward(self, x):\n",
        "        x = F.relu(self.hidden1(x))\n",
        "        x = F.relu(self.hidden2(x))\n",
        "        x = self.predict(x)\n",
        "        return x\n",
        "    \n",
        "    def fit(self, x, y):\n",
        "        x, y = to_variable(var=(x, y), cuda=False)\n",
        "        \n",
        "        # reset gradient and total loss\n",
        "        self.optimizer.zero_grad()\n",
        "        \n",
        "        output = self.forward(x)\n",
        "        loss = self.loss_func(output[:, :1], y, output[:, 1:].exp(), 1)/x.shape[0]\n",
        "        \n",
        "        loss.backward()\n",
        "        self.optimizer.step()\n",
        "\n",
        "        return loss"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j0gfEDv6KgSr"
      },
      "source": [
        "def train_ensemble(data, n_splits, num_epochs, num_nets, num_hidden, learn_rate, weight_decay, data_fraction, log_every):\n",
        "    \n",
        "    kf = KFold(n_splits=n_splits)\n",
        "    in_dim = data.shape[1] - 1\n",
        "    train_logliks, test_logliks = [], []\n",
        "    train_rmses, test_rmses = [], []\n",
        "\n",
        "    for j, idx in enumerate(kf.split(data)):\n",
        "        train_index, test_index = idx\n",
        "\n",
        "        x_train, y_train = data[train_index, :in_dim], data[train_index, in_dim:]\n",
        "        x_test, y_test = data[test_index, :in_dim], data[test_index, in_dim:]\n",
        "\n",
        "        #x_means, x_stds = x_train.mean(axis = 0), x_train.var(axis = 0)**0.5\n",
        "        y_means, y_stds = y_train.mean(axis = 0), y_train.var(axis = 0)**0.5\n",
        "\n",
        "        #x_train = (x_train - x_means)/x_stds\n",
        "        #y_train = (y_train - y_means)/y_stds\n",
        "\n",
        "        #x_test = (x_test - x_means)/x_stds\n",
        "        #y_test = (y_test - y_means)/y_stds\n",
        "\n",
        "        batch_size = len(x_train)\n",
        "\n",
        "        fit_loss_train = np.zeros(num_epochs)\n",
        "        best_net, best_loss = None, float('inf')\n",
        "        nets = []\n",
        "\n",
        "        for n in range(num_nets):\n",
        "            net = Net_UCI(n_feature=in_dim, n_hidden=num_hidden, n_output=1, learn_rate=learn_rate, weight_decay=weight_decay)\n",
        "\n",
        "            sub_idx = np.random.choice(np.arange(0, len(x_train)), size = (int(len(x_train)*data_fraction),), replace=True)\n",
        "            x_train_sub, y_train_sub = x_train[sub_idx], y_train[sub_idx]\n",
        "\n",
        "            for i in range(num_epochs):\n",
        "\n",
        "                loss = net.fit(x_train_sub, y_train_sub)\n",
        "\n",
        "                if log_every is not False and (i % log_every == 0 or i == num_epochs - 1) and len(nets) > 0:\n",
        "                    train_loss, train_rmse = eval_ensemble(x_train, y_train, nets)\n",
        "                    test_loss, test_rmse = eval_ensemble(x_test, y_test, nets)\n",
        "                    print('Epoch %3d, network %2d, Loss train/test %.3f/%.3f, RMSE train/test %.3f/%.3f' % \\\n",
        "                          (i+1, len(nets), train_loss.cpu().data.numpy(), test_loss.cpu().data.numpy(),\n",
        "                          train_rmse.cpu().data.numpy(), test_rmse.cpu().data.numpy()))\n",
        "\n",
        "            nets.append(copy.deepcopy(net))\n",
        "\n",
        "\n",
        "        train_loss, train_rmse = eval_ensemble(x_train, y_train, nets)\n",
        "        test_loss, test_rmse = eval_ensemble(x_test, y_test, nets)\n",
        "\n",
        "        train_logliks.append(-train_loss.cpu().data.numpy() - np.log(y_stds)[0])\n",
        "        test_logliks.append(-test_loss.cpu().data.numpy() - np.log(y_stds)[0])\n",
        "\n",
        "        train_rmses.append(y_stds[0]*train_rmse.cpu().data.numpy())\n",
        "        test_rmses.append(y_stds[0]*test_rmse.cpu().data.numpy())\n",
        "\n",
        "    print('Train log. lik. = %7.3f +/- %.3f' % (np.array(train_logliks).mean(), np.array(train_logliks).var()**0.5))\n",
        "    print('Test  log. lik. = %7.3f +/- %.3f' % (np.array(test_logliks).mean(), np.array(test_logliks).var()**0.5))\n",
        "    print('Train RMSE      = %7.3f +/- %.3f' % (np.array(train_rmses).mean(), np.array(train_rmses).var()**0.5))\n",
        "    print('Test  RMSE      = %7.3f +/- %.3f' % (np.array(test_rmses).mean(), np.array(test_rmses).var()**0.5))\n",
        "    \n",
        "    return nets"
      ],
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HXpA9VBGMsxc"
      },
      "source": [
        "# Housing dataset"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "LnzGF4t4gPFx"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "kmZj3mdJgPDa"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "GtL3YiZCgPBW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cOq4tM_PLRUV"
      },
      "source": [
        "batch_size = 64\n",
        "data = pd.read_csv('/content/drive/MyDrive/bayesian-deep-learning-master/data/Datasets.csv').values"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "outputId": "80d11b19-ade4-4888-b16b-f72d3a26cd74",
        "id": "KE6ieDPHloUg",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "source": [
        "ensemble = train_ensemble(data=data, n_splits=10, num_epochs=100, num_nets=10, num_hidden=100,\n",
        "                          learn_rate=1e-2, weight_decay=1e-2, data_fraction=0.8, log_every=False)"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train log. lik. =  -2.133 +/- 0.019\n",
            "Test  log. lik. =  -2.142 +/- 0.099\n",
            "Train RMSE      =   5.143 +/- 0.095\n",
            "Test  RMSE      =   5.146 +/- 0.524\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vnGD_RefRxkZ"
      },
      "source": [
        "# Concrete compressive dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "outputId": "8ff3f494-4060-4010-c878-c37e16195d7b",
        "id": "s5G3-z_jRxkZ",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        }
      },
      "source": [
        "np.random.seed(0)\n",
        "!wget \"https://archive.ics.uci.edu/ml/machine-learning-databases/concrete/compressive/Concrete_Data.xls\" --no-check-certificate\n",
        "data = pd.read_excel('Concrete_Data.xls', header=0, delimiter=\"\\s+\").values\n",
        "data = data[np.random.permutation(np.arange(len(data)))]\n",
        "data.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2019-05-17 22:40:30--  https://archive.ics.uci.edu/ml/machine-learning-databases/concrete/compressive/Concrete_Data.xls\n",
            "Resolving archive.ics.uci.edu (archive.ics.uci.edu)... 128.195.10.252\n",
            "Connecting to archive.ics.uci.edu (archive.ics.uci.edu)|128.195.10.252|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 124928 (122K) [application/x-httpd-php]\n",
            "Saving to: ‘Concrete_Data.xls.6’\n",
            "\n",
            "Concrete_Data.xls.6 100%[===================>] 122.00K   290KB/s    in 0.4s    \n",
            "\n",
            "2019-05-17 22:40:31 (290 KB/s) - ‘Concrete_Data.xls.6’ saved [124928/124928]\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1030, 9)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "outputId": "b8c8dd99-6461-48b1-db91-880546b88f5a",
        "id": "Sij7vIlqRxke",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "ensemble = train_ensemble(data=data, n_splits=10, num_epochs=100, num_nets=20, num_hidden=100,\n",
        "                          learn_rate=1e-1, weight_decay=0*1e-2, data_fraction=0.8, log_every=False)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train log. lik. =  -2.849 +/- 0.085\n",
            "Test  log. lik. =  -2.864 +/- 0.087\n",
            "Train RMSE      =   9.078 +/- 0.481\n",
            "Test  RMSE      =   9.407 +/- 0.969\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uxatwTIqVrZ1"
      },
      "source": [
        "# Energy efficiency dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pZDQX5l3ZfLV",
        "outputId": "9e2d1f8e-7436-46a3-fa8d-413dcc08bb55",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        }
      },
      "source": [
        "np.random.seed(0)\n",
        "!wget \"http://archive.ics.uci.edu/ml/machine-learning-databases/00242/ENB2012_data.xlsx\" --no-check-certificate\n",
        "data = pd.read_excel('ENB2012_data.xlsx', header=0, delimiter=\"\\s+\").values\n",
        "data = data[np.random.permutation(np.arange(len(data)))]\n",
        "data.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2019-05-17 22:14:45--  http://archive.ics.uci.edu/ml/machine-learning-databases/00242/ENB2012_data.xlsx\n",
            "Resolving archive.ics.uci.edu (archive.ics.uci.edu)... 128.195.10.252\n",
            "Connecting to archive.ics.uci.edu (archive.ics.uci.edu)|128.195.10.252|:80... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 76189 (74K) [application/x-httpd-php]\n",
            "Saving to: ‘ENB2012_data.xlsx.1’\n",
            "\n",
            "\rENB2012_data.xlsx.1   0%[                    ]       0  --.-KB/s               \rENB2012_data.xlsx.1  64%[===========>        ]  47.70K   150KB/s               \rENB2012_data.xlsx.1 100%[===================>]  74.40K   234KB/s    in 0.3s    \n",
            "\n",
            "2019-05-17 22:14:46 (234 KB/s) - ‘ENB2012_data.xlsx.1’ saved [76189/76189]\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(768, 10)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ajazc6zeZfLY",
        "outputId": "5c14187a-7a26-4109-a796-8ac6c16a873a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "ensemble = train_ensemble(data=data, n_splits=10, num_epochs=100, num_nets=20, num_hidden=100,\n",
        "                          learn_rate=1e-2, weight_decay=1e-2, data_fraction=0.8, log_every=False)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train log. lik. =  -1.490 +/- 0.113\n",
            "Test  log. lik. =  -1.501 +/- 0.107\n",
            "Train RMSE      =   2.633 +/- 0.121\n",
            "Test  RMSE      =   2.638 +/- 0.269\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vFsWDzx8NSry"
      },
      "source": [
        "# Power dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hzdZLChPNQ9X",
        "outputId": "1e30e1df-8735-4a41-d6e0-effb1bbaa033",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        }
      },
      "source": [
        "np.random.seed(0)\n",
        "!wget \"https://archive.ics.uci.edu/ml/machine-learning-databases/00294/CCPP.zip\" --no-check-certificate \n",
        "zipped = zipfile.ZipFile(\"CCPP.zip\")\n",
        "data = pd.read_excel(zipped.open('CCPP/Folds5x2_pp.xlsx'), header=0, delimiter=\"\\t\").values\n",
        "np.random.shuffle(data)\n",
        "data.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2019-05-17 22:15:13--  https://archive.ics.uci.edu/ml/machine-learning-databases/00294/CCPP.zip\n",
            "Resolving archive.ics.uci.edu (archive.ics.uci.edu)... 128.195.10.252\n",
            "Connecting to archive.ics.uci.edu (archive.ics.uci.edu)|128.195.10.252|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 3674852 (3.5M) [application/x-httpd-php]\n",
            "Saving to: ‘CCPP.zip.1’\n",
            "\n",
            "CCPP.zip.1          100%[===================>]   3.50M  3.13MB/s    in 1.1s    \n",
            "\n",
            "2019-05-17 22:15:14 (3.13 MB/s) - ‘CCPP.zip.1’ saved [3674852/3674852]\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(9568, 5)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ld95mVAKyvo9",
        "outputId": "9dd205d5-b252-4e35-cf7b-659717aaeb3e",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "ensemble = train_ensemble(data=data, n_splits=10, num_epochs=100, num_nets=20, num_hidden=100,\n",
        "                          learn_rate=1e-2, weight_decay=1e-2, data_fraction=0.8, log_every=False)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train log. lik. =  -2.034 +/- 0.012\n",
            "Test  log. lik. =  -2.038 +/- 0.020\n",
            "Train RMSE      =   4.498 +/- 0.053\n",
            "Test  RMSE      =   4.500 +/- 0.147\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HRPICBiXCegI"
      },
      "source": [
        "# Red wine dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "outputId": "6f3c7d92-a720-4d9f-899d-dc4663364982",
        "id": "KOqgIBXcCegJ",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        }
      },
      "source": [
        "np.random.seed(0)\n",
        "!wget \"https://archive.ics.uci.edu/ml/machine-learning-databases/wine-quality/winequality-red.csv\" --no-check-certificate \n",
        "data = pd.read_csv('winequality-red.csv', header=1, delimiter=';').values\n",
        "data = data[np.random.permutation(np.arange(len(data)))]\n",
        "data.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2019-05-17 22:15:47--  https://archive.ics.uci.edu/ml/machine-learning-databases/wine-quality/winequality-red.csv\n",
            "Resolving archive.ics.uci.edu (archive.ics.uci.edu)... 128.195.10.252\n",
            "Connecting to archive.ics.uci.edu (archive.ics.uci.edu)|128.195.10.252|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 84199 (82K) [application/x-httpd-php]\n",
            "Saving to: ‘winequality-red.csv.2’\n",
            "\n",
            "winequality-red.csv 100%[===================>]  82.23K   291KB/s    in 0.3s    \n",
            "\n",
            "2019-05-17 22:15:47 (291 KB/s) - ‘winequality-red.csv.2’ saved [84199/84199]\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1598, 12)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "outputId": "f61b48c7-5fc0-4a5c-930a-7b081cbf7283",
        "id": "h8gWjBbKCegM",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "ensemble = train_ensemble(data=data, n_splits=10, num_epochs=100, num_nets=20, num_hidden=100,\n",
        "                          learn_rate=1e-2, weight_decay=1e-2, data_fraction=0.8, log_every=False)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train log. lik. =  -0.141 +/- 0.011\n",
            "Test  log. lik. =  -0.152 +/- 0.086\n",
            "Train RMSE      =   0.719 +/- 0.007\n",
            "Test  RMSE      =   0.719 +/- 0.069\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PVZw0uJzDgdy"
      },
      "source": [
        "# Yacht dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "outputId": "2f1406b1-341c-4490-996f-27317e9905c5",
        "id": "ja_kIet3Dgdz",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 221
        }
      },
      "source": [
        "np.random.seed(0)\n",
        "!wget \"http://archive.ics.uci.edu/ml/machine-learning-databases/00243/yacht_hydrodynamics.data\" --no-check-certificate \n",
        "data = pd.read_csv('yacht_hydrodynamics.data', header=1, delimiter='\\s+').values\n",
        "data = data[np.random.permutation(np.arange(len(data)))]\n",
        "data.shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2019-05-17 22:16:19--  http://archive.ics.uci.edu/ml/machine-learning-databases/00243/yacht_hydrodynamics.data\n",
            "Resolving archive.ics.uci.edu (archive.ics.uci.edu)... 128.195.10.252\n",
            "Connecting to archive.ics.uci.edu (archive.ics.uci.edu)|128.195.10.252|:80... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 11487 (11K) [application/x-httpd-php]\n",
            "Saving to: ‘yacht_hydrodynamics.data.2’\n",
            "\n",
            "\r          yacht_hyd   0%[                    ]       0  --.-KB/s               \ryacht_hydrodynamics 100%[===================>]  11.22K  --.-KB/s    in 0s      \n",
            "\n",
            "2019-05-17 22:16:19 (311 MB/s) - ‘yacht_hydrodynamics.data.2’ saved [11487/11487]\n",
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(306, 7)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "outputId": "71dce1a6-f1e7-45fd-9fba-440605233b07",
        "id": "clxNDH6dDgd3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "ensemble = train_ensemble(data=data, n_splits=10, num_epochs=100, num_nets=20, num_hidden=100,\n",
        "                          learn_rate=1e-2, weight_decay=1e-2, data_fraction=0.8, log_every=False)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train log. lik. =  -2.503 +/- 0.112\n",
            "Test  log. lik. =  -2.540 +/- 0.114\n",
            "Train RMSE      =  11.073 +/- 0.250\n",
            "Test  RMSE      =  10.882 +/- 2.943\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1NwNZ9WgBkfp"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}