Deep Ensemble.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "69f0c434",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow.compat.v1 as tf\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "from sklearn.utils import shuffle"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e65ccef7",
   "metadata": {},
   "source": [
    "# Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b4ca46a7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Parameters of training\n",
    "Learning_rate = 0.0001\n",
    "epsilon = 1e-8\n",
    "\n",
    "num_iter = 10000\n",
    "batch_size = 128\n",
    "\n",
    "test_ratio = 0.20\n",
    "gpu_fraction = 0.5\n",
    "\n",
    "# Ensemble networks (5 networks)\n",
    "networks = ['network1', 'network2', 'network3', 'network4', 'network5']\n",
    "\n",
    "# Import CSV File\n",
    "data = pd.read_csv('C:/Users/ssyhw7/Desktop/UncertaintyEstimation-master/data/AIDD/Datasets.csv')\n",
    "column_names = data.columns\n",
    "\n",
    "num_rows = len(data)\n",
    "num_columns = len(column_names) \n",
    "num_data = num_columns - 1\n",
    "\n",
    "# Dense [input size, output size]\n",
    "dense1 = [num_data, 256]\n",
    "dense2 = [256, 512]\n",
    "dense3 = [512, 1024]\n",
    "dense_mu  = [1024, 1]\n",
    "dense_sig = [1024, 1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "55b14337",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train data shape: (744, 512)\n",
      "Test data shape: (187, 512)\n"
     ]
    }
   ],
   "source": [
    "data_x = np.zeros([num_rows, num_columns - 1])\n",
    "data_y = np.zeros([num_rows, 1])\n",
    "\n",
    "for i in range(num_rows):\n",
    "    for j in range(num_columns - 1):\n",
    "        data_x[i, j] = data[column_names[j]][i]\n",
    "    data_y[i,0] = data[column_names[-1]][i]\n",
    "\n",
    "num_train_data = int(num_rows * (1 - test_ratio))\n",
    "num_test_data  = num_rows - num_train_data\n",
    "\n",
    "train_x = data_x[:num_train_data, :]\n",
    "train_y = data_y[:num_train_data, :]\n",
    "test_x  = data_x[num_train_data:, :]\n",
    "test_y  = data_y[num_train_data:, :]\n",
    "\n",
    "print(\"Train data shape: \" + str(train_x.shape))\n",
    "print(\"Test data shape: \" + str(test_x.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ec596b84",
   "metadata": {},
   "source": [
    "# Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "a6d5e260",
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "tf.compat.v1.disable_eager_execution()\n",
    "\n",
    "def weight_variable(name, shape):\n",
    "    return tf.get_variable(name, shape = shape, initializer = tf.keras.initializers.glorot_uniform(), dtype = tf.float64)\n",
    "\n",
    "def bias_variable(name, shape):\n",
    "    return tf.get_variable(name, shape = shape, initializer = tf.keras.initializers.glorot_uniform(), dtype = tf.float64)\n",
    "\n",
    "# Get networks\n",
    "def get_network(network_name):\n",
    "    input_x = tf.placeholder(tf.float64, shape = [None, num_data])\n",
    "    \n",
    "    with tf.variable_scope(network_name):\n",
    "        # Densely connect layer variables\n",
    "        w_fc1 = weight_variable(network_name + '_w_fc1', dense1)\n",
    "        b_fc1 = bias_variable(network_name + '_b_fc1', [dense1[1]])\n",
    "        \n",
    "        w_fc2 = weight_variable(network_name + '_w_fc2', dense2)\n",
    "        b_fc2 = bias_variable(network_name + '_b_fc2', [dense2[1]])\n",
    "\n",
    "        w_fc3 = weight_variable(network_name + '_w_fc3', dense3)\n",
    "        b_fc3 = bias_variable(network_name + '_b_fc3', [dense3[1]])\n",
    "        \n",
    "        w_fc_mu = weight_variable(network_name + '_w_fc_mu', dense_mu)\n",
    "        b_fc_mu = bias_variable(network_name + '_b_fc_mu', [dense_mu[1]])\n",
    "\n",
    "        w_fc_sig = weight_variable(network_name + '_w_fc_sig', dense_sig)\n",
    "        b_fc_sig = bias_variable(network_name + '_b_fc_sig', [dense_sig[1]])\n",
    "\n",
    "    # Network\n",
    "    fc1 = tf.nn.relu(tf.matmul(input_x, w_fc1) + b_fc1)\n",
    "    fc2 = tf.nn.relu(tf.matmul(fc1, w_fc2) + b_fc2)\n",
    "    fc3 = tf.nn.relu(tf.matmul(fc2, w_fc3) + b_fc3)\n",
    "    output_mu  = tf.matmul(fc3, w_fc_mu) + b_fc_mu\n",
    "    output_sig = tf.matmul(fc3, w_fc_sig) + b_fc_sig\n",
    "    output_sig_pos = tf.log(1 + tf.exp(output_sig)) + 1e-06\n",
    "    \n",
    "    y = tf.placeholder(tf.float64, shape = [None, 1])\n",
    "    \n",
    "    # Negative Log Likelihood(NLL) \n",
    "    loss = tf.reduce_mean(0.5*tf.log(output_sig_pos) + 0.5*tf.div(tf.square(y - output_mu),output_sig_pos)) + 5\n",
    "\n",
    "    # Get trainable variables\n",
    "    train_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES, network_name) \n",
    "    \n",
    "    # Gradient clipping for preventing nan\n",
    "    optimizer = tf.train.AdamOptimizer(learning_rate = Learning_rate)\n",
    "    gvs = optimizer.compute_gradients(loss, var_list = train_vars)\n",
    "    capped_gvs = [(tf.clip_by_value(grad, -1., 1.), var) for grad, var in gvs]\n",
    "    train_opt = optimizer.apply_gradients(capped_gvs)\n",
    "\n",
    "    return input_x, y, output_mu, output_sig_pos, loss, train_opt, train_vars\n",
    "\n",
    "\n",
    "# Make batch data \n",
    "def making_batch(data_size, sample_size, data_x, data_y):\n",
    "    \n",
    "    # Making batches(testing)\n",
    "    batch_idx = np.random.choice(data_size, sample_size)\n",
    "    \n",
    "    batch_x = np.zeros([sample_size, num_data])\n",
    "    batch_y = np.zeros([sample_size, 1])\n",
    "        \n",
    "    for i in range(batch_idx.shape[0]):\n",
    "        batch_x[i,:] = data_x[batch_idx[i], :]\n",
    "        batch_y[i,:] = data_y[batch_idx[i], :] \n",
    "        \n",
    "    return batch_x, batch_y   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "4ce08f77",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\ssyhw7\\AppData\\Local\\Temp\\ipykernel_5888\\3876758076.py:42: div (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Deprecated in favor of operator or tf.math.divide.\n"
     ]
    }
   ],
   "source": [
    "x_list = []\n",
    "y_list = []\n",
    "output_mu_list = []\n",
    "output_sig_list = []\n",
    "loss_list = []\n",
    "train_list = []\n",
    "train_var_list = []\n",
    "output_test_list = []\n",
    "\n",
    "# Train each ensemble network\n",
    "for i in range(len(networks)):\n",
    "    x_input, y, output_mu, output_sig, loss, train_opt, train_vars = get_network(networks[i])\n",
    "\n",
    "    x_list.append(x_input)\n",
    "    y_list.append(y)\n",
    "    output_mu_list.append(output_mu)\n",
    "    output_sig_list.append(output_sig)\n",
    "    loss_list.append(loss)\n",
    "    train_list.append(train_opt)\n",
    "    train_var_list.append(train_vars)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "5f216cd9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create Session\n",
    "config = tf.ConfigProto()\n",
    "config.gpu_options.per_process_gpu_memory_fraction = gpu_fraction\n",
    "\n",
    "sess = tf.InteractiveSession(config=config)\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f63fff2f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------- Iteration: 500 -------------------------\n",
      "Average Loss(NLL): [4.72929699 5.45740944 6.83309566 4.41405178 5.16356714]\n",
      "mu: [5.48541932 5.61554797 5.98603204 6.05703234 5.49469738]\n",
      "std: [0.16311852 0.10249108 0.1676848  0.0723784  0.09015389]\n",
      "Final mu: 5.727745809398786\n",
      "Final std: 0.2754487485534476\n",
      "Real Value: [5.21]\n",
      "\n",
      "\n",
      "------------------------- Iteration: 1000 -------------------------\n",
      "Average Loss(NLL): [2.65537419 2.82220005 2.99498943 2.5162789  2.76406511]\n",
      "mu: [6.33566649 6.74367169 6.56682724 6.6069956  6.35532836]\n",
      "std: [0.02975884 0.0378075  0.04743208 0.03536943 0.06170712]\n",
      "Final mu: 6.521697877309919\n",
      "Final std: 0.16155172162739934\n",
      "Real Value: [6.69]\n",
      "\n",
      "\n",
      "------------------------- Iteration: 1500 -------------------------\n",
      "Average Loss(NLL): [2.32598075 2.37163463 2.43107575 2.28526488 2.4074036 ]\n",
      "mu: [5.13274892 5.20568521 4.41318278 5.23692211 5.12673119]\n",
      "std: [0.0462765  0.03963865 0.06394147 0.02798294 0.05117983]\n",
      "Final mu: 5.023054040941782\n",
      "Final std: 0.311455369991624\n",
      "Real Value: [4.66]\n",
      "\n",
      "\n",
      "------------------------- Iteration: 2000 -------------------------\n",
      "Average Loss(NLL): [2.1373632  2.21144998 2.27296866 2.04501204 2.279732  ]\n",
      "mu: [5.92883392 6.32414379 6.43768257 6.12558722 5.96728846]\n",
      "std: [0.02047227 0.02070021 0.02868423 0.01740978 0.04962405]\n",
      "Final mu: 6.156707192821676\n",
      "Final std: 0.2000961574445898\n",
      "Real Value: [7.05]\n",
      "\n",
      "\n",
      "------------------------- Iteration: 2500 -------------------------\n",
      "Average Loss(NLL): [1.9743294  2.07428115 2.14697724 1.97512052 2.16463884]\n",
      "mu: [6.36812585 5.28362308 6.07559274 6.25005949 6.14548103]\n",
      "std: [0.03201266 0.03349518 0.03060724 0.03314908 0.02978772]\n",
      "Final mu: 6.024576440037096\n",
      "Final std: 0.38475212388147273\n",
      "Real Value: [4.64]\n",
      "\n",
      "\n",
      "------------------------- Iteration: 3000 -------------------------\n",
      "Average Loss(NLL): [1.89332328 2.07134936 2.05398743 1.84349242 2.02437869]\n",
      "mu: [4.95741262 4.70510043 4.37556873 5.25691208 4.66622957]\n",
      "std: [0.05583305 0.05982864 0.04422809 0.032112   0.07166573]\n",
      "Final mu: 4.792244685480648\n",
      "Final std: 0.3016982245712652\n",
      "Real Value: [6.07]\n",
      "\n",
      "\n",
      "------------------------- Iteration: 3500 -------------------------\n",
      "Average Loss(NLL): [1.78706387 1.92952563 1.97921628 1.80605476 2.06397865]\n",
      "mu: [6.41329869 5.78580702 6.68873897 6.76859393 6.56841147]\n",
      "std: [0.01662307 0.03089052 0.02916025 0.0267656  0.02716507]\n",
      "Final mu: 6.444970016407266\n",
      "Final std: 0.3516895409285475\n",
      "Real Value: [4.67]\n",
      "\n",
      "\n",
      "------------------------- Iteration: 4000 -------------------------\n",
      "Average Loss(NLL): [1.75127655 1.86597755 1.91560234 1.77787324 1.96220491]\n",
      "mu: [5.18297852 5.01912533 5.50131689 5.46979725 5.23271772]\n",
      "std: [0.03119857 0.02616959 0.02418659 0.02125564 0.03813275]\n",
      "Final mu: 5.281187141124392\n",
      "Final std: 0.1837664761108718\n",
      "Real Value: [6.52]\n",
      "\n",
      "\n",
      "------------------------- Iteration: 4500 -------------------------\n",
      "Average Loss(NLL): [1.72154051 1.77175122 1.81804327 1.69578482 2.0370371 ]\n",
      "mu: [5.28414139 5.46286961 4.54787736 5.30887262 5.27520026]\n",
      "std: [0.03607935 0.02330074 0.03243306 0.02039024 0.0346365 ]\n",
      "Final mu: 5.175792246870882\n",
      "Final std: 0.3226559755897847\n",
      "Real Value: [4.92]\n",
      "\n",
      "\n",
      "------------------------- Iteration: 5000 -------------------------\n",
      "Average Loss(NLL): [1.63948537 1.81120989 1.82908982 1.67426962 1.87857624]\n",
      "mu: [6.19375112 6.19858904 6.1854827  7.38125436 6.0623148 ]\n",
      "std: [0.01146757 0.02425215 0.01256496 0.00804215 0.01384912]\n",
      "Final mu: 6.404278404324465\n",
      "Final std: 0.49133625397318376\n",
      "Real Value: [4.23]\n",
      "\n",
      "\n",
      "------------------------- Iteration: 5500 -------------------------\n",
      "Average Loss(NLL): [1.66980922 1.76027049 1.82726463 1.62158033 1.79391963]\n",
      "mu: [5.60146455 4.72020667 5.78075009 5.94362591 5.49282328]\n",
      "std: [0.02017045 0.03075822 0.05052818 0.02203384 0.05352289]\n",
      "Final mu: 5.507774100662729\n",
      "Final std: 0.42450036057202095\n",
      "Real Value: [4.85]\n",
      "\n",
      "\n",
      "------------------------- Iteration: 6000 -------------------------\n",
      "Average Loss(NLL): [1.59017556 1.7967752  1.77476772 1.56664428 1.81974766]\n",
      "mu: [5.0727799  5.65062686 5.86474605 5.27813258 5.88888244]\n",
      "std: [0.01997696 0.02386178 0.02052033 0.01701054 0.015401  ]\n",
      "Final mu: 5.551033565591477\n",
      "Final std: 0.32483651921382883\n",
      "Real Value: [7.84]\n",
      "\n",
      "\n",
      "------------------------- Iteration: 6500 -------------------------\n",
      "Average Loss(NLL): [1.61713193 1.63964551 1.76689731 1.69413277 1.7500896 ]\n",
      "mu: [5.94577958 6.2434996  6.1981903  5.41263083 5.80391247]\n",
      "std: [0.01680112 0.00995637 0.0134903  0.02054661 0.01499112]\n",
      "Final mu: 5.920802556649905\n",
      "Final std: 0.30160016137039847\n",
      "Real Value: [7.16]\n",
      "\n",
      "\n",
      "------------------------- Iteration: 7000 -------------------------\n",
      "Average Loss(NLL): [1.53601896 1.65832168 1.72061931 1.60165965 1.79437013]\n",
      "mu: [4.68090323 4.32124796 4.49399922 4.77085102 5.31238807]\n",
      "std: [0.01617329 0.03063246 0.0212424  0.0287639  0.02345794]\n",
      "Final mu: 4.715877900758317\n",
      "Final std: 0.33706420617693816\n",
      "Real Value: [4.47]\n",
      "\n",
      "\n",
      "------------------------- Iteration: 7500 -------------------------\n",
      "Average Loss(NLL): [1.60955161 1.68569661 1.67377994 1.55738833 1.86589715]\n",
      "mu: [5.96948876 5.16383602 5.54632566 6.38022609 5.70551759]\n",
      "std: [0.01843157 0.03082798 0.04136146 0.02749047 0.03785992]\n",
      "Final mu: 5.753078822308848\n",
      "Final std: 0.40928057227657844\n",
      "Real Value: [4.59]\n",
      "\n",
      "\n",
      "------------------------- Iteration: 8000 -------------------------\n",
      "Average Loss(NLL): [1.5991434  1.71255874 1.63490148 1.55644721 1.77487665]\n",
      "mu: [6.56090484 6.4001953  6.6141113  5.9965864  6.75044541]\n",
      "std: [0.02250688 0.03990961 0.01444369 0.02354163 0.02738653]\n",
      "Final mu: 6.464448648072656\n",
      "Final std: 0.2608189474850764\n",
      "Real Value: [6.96]\n",
      "\n",
      "\n",
      "------------------------- Iteration: 8500 -------------------------\n",
      "Average Loss(NLL): [1.47782072 1.75264834 1.69007369 1.48645947 1.75359926]\n",
      "mu: [6.75770502 6.54094292 6.49291137 6.48718374 6.94210376]\n",
      "std: [0.01228699 0.01687094 0.02130905 0.01430042 0.01591271]\n",
      "Final mu: 6.64416936383499\n",
      "Final std: 0.17954487244906364\n",
      "Real Value: [6.96]\n",
      "\n",
      "\n",
      "------------------------- Iteration: 9000 -------------------------\n",
      "Average Loss(NLL): [1.52978813 1.63525616 1.71347945 1.59718593 1.7846406 ]\n",
      "mu: [5.70364818 5.88933888 6.22792513 5.62388536 5.70504768]\n",
      "std: [0.02393532 0.02403285 0.01253855 0.05234253 0.01085366]\n",
      "Final mu: 5.829969047418981\n",
      "Final std: 0.21913094432674834\n",
      "Real Value: [8.12]\n",
      "\n",
      "\n",
      "------------------------- Iteration: 9500 -------------------------\n",
      "Average Loss(NLL): [1.51000994 1.69870992 1.6456714  1.56268183 1.80846002]\n",
      "mu: [5.1205643  5.60265166 5.40500841 5.06845955 5.04962957]\n",
      "std: [0.00987886 0.02272836 0.01431587 0.01313064 0.03159411]\n",
      "Final mu: 5.249262698127872\n",
      "Final std: 0.21919700759173102\n",
      "Real Value: [5.23]\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Set parameters for printing and testing\n",
    "num_print = 500\n",
    "test_size = 10\n",
    "\n",
    "train_data_num = train_x.shape[0]\n",
    "test_data_num  = test_x.shape[0]\n",
    "\n",
    "loss_train = np.zeros([len(networks)])\n",
    "out_mu     = np.zeros([test_size, len(networks)])\n",
    "out_sig    = np.zeros([test_size, len(networks)])\n",
    "\n",
    "for iter in range(num_iter):\n",
    "    # Making batches(testing)\n",
    "    batch_x_test, batch_y_test = making_batch(test_data_num, test_size, test_x, test_y)\n",
    "        \n",
    "    for i in range(len(networks)):\n",
    "        # Making batches(training)\n",
    "        batch_x, batch_y = making_batch(train_data_num, batch_size, train_x, train_y)\n",
    "       \n",
    "        # Training\n",
    "        _, loss, mu, sig = sess.run([train_list[i], loss_list[i], output_mu_list[i], output_sig_list[i]], \n",
    "                                               feed_dict = {x_list[i]: batch_x, y_list[i]: batch_y})\n",
    "  \n",
    "      \n",
    "        # Testing\n",
    "        loss_test, mu_test, sig_test = sess.run([loss_list[i], output_mu_list[i], output_sig_list[i]], \n",
    "                                    feed_dict = {x_list[i]: batch_x_test, y_list[i]: batch_y_test})\n",
    "\n",
    "        if np.any(np.isnan(loss)):\n",
    "            raise ValueError('There is Nan in loss')\n",
    "        \n",
    "        loss_train[i] += loss\n",
    "        out_mu[:, i] = np.reshape(mu_test, (test_size))\n",
    "        out_sig[:, i] = np.reshape(sig_test, (test_size))\n",
    "        \n",
    "    # Get final test result\n",
    "    out_mu_final = np.mean(out_mu, axis = 1)\n",
    "    out_sig_final = np.sqrt(np.mean(out_sig, axis=1) + np.mean(np.square(out_mu), axis = 1) - np.square(out_mu_final))\n",
    "    \n",
    "    if iter % num_print == 0 and iter != 0:\n",
    "        print(('-------------------------') + ' Iteration: ' + str(iter) + ' -------------------------')\n",
    "        print('Average Loss(NLL): ' + str(loss_train / num_print))\n",
    "        print('mu: ' + str(out_mu[0, :]))\n",
    "        print('std: ' + str(np.sqrt(out_sig[0, :])))\n",
    "        print('Final mu: ' + str(out_mu_final[0]))\n",
    "        print('Final std: ' + str(out_sig_final[0]))\n",
    "        print('Real Value: ' + str(batch_y_test[0]))\n",
    "        print('\\n')\n",
    "        \n",
    "        loss_train = np.zeros(len(networks))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fbf0e75e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}