notebooks/4.1 Evaluation metrics of CVAE.ipynb

{"cells":[{"cell_type":"code","execution_count":1,"metadata":{"executionInfo":{"elapsed":4,"status":"ok","timestamp":1711170805638,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"},"user_tz":-480},"id":"BfcF5LB6snwi"},"outputs":[],"source":["model_name = 'CVAE_mse'"]},{"cell_type":"code","execution_count":2,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":7830,"status":"ok","timestamp":1711170813465,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"},"user_tz":-480},"id":"a4xPxcPtGpza","outputId":"f5e6d0de-96b4-4b35-cad6-82b1d8f53f0a"},"outputs":[{"output_type":"stream","name":"stdout","text":["Requirement already satisfied: scipy==1.11.1 in /usr/local/lib/python3.10/dist-packages (1.11.1)\n","Requirement already satisfied: numpy<1.28.0,>=1.21.6 in /usr/local/lib/python3.10/dist-packages (from scipy==1.11.1) (1.25.2)\n"]}],"source":["!pip install scipy==1.11.1"]},{"cell_type":"code","execution_count":3,"metadata":{"executionInfo":{"elapsed":8357,"status":"ok","timestamp":1711170821819,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"},"user_tz":-480},"id":"SxVKvknVGyOi"},"outputs":[],"source":["from keras.layers import Input, Dense, Conv2D, Flatten, Reshape, Conv2DTranspose\n","from keras.models import Model\n","from keras.datasets import mnist\n","from keras.utils import to_categorical\n","from keras import backend as K\n","from keras.callbacks import Callback\n","import tensorflow as tf\n","\n","from scipy.stats import entropy\n","from scipy.linalg import sqrtm\n","from scipy.stats import pearsonr\n","\n","from keras.applications.inception_v3 import InceptionV3\n","\n","import numpy as np\n","import pandas as pd\n","\n","import matplotlib.pyplot as plt\n","\n","from sklearn.preprocessing import LabelEncoder\n","\n","from tensorflow.keras.models import load_model\n","\n","import cv2\n","\n","from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score\n","from scipy.stats import entropy\n","from scipy.spatial.distance import cosine\n","\n","import tensorflow as tf\n","from tensorflow.keras.applications.inception_v3 import InceptionV3\n","from scipy.linalg import sqrtm"]},{"cell_type":"code","execution_count":4,"metadata":{"executionInfo":{"elapsed":21,"status":"ok","timestamp":1711170821820,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"},"user_tz":-480},"id":"NoC5qEIyGzOC"},"outputs":[],"source":["import random\n","\n","seed = 42\n","\n","# Python's built-in random library\n","random.seed(seed)\n","\n","# Scikit-learn (only affects some parts of scikit-learn)\n","from sklearn.utils import check_random_state\n","check_random_state(seed)\n","\n","np.random.seed(seed)\n","tf.random.set_seed(seed)\n","tf.keras.utils.set_random_seed(seed)"]},{"cell_type":"code","execution_count":5,"metadata":{"executionInfo":{"elapsed":19,"status":"ok","timestamp":1711170821820,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"},"user_tz":-480},"id":"XibW79oBGzpU"},"outputs":[],"source":["img_rows = 168\n","img_cols = 168\n","channels = 1\n","\n","# Input image dimensions\n","img_shape = (img_rows, img_cols, channels)"]},{"cell_type":"code","execution_count":6,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":2159,"status":"ok","timestamp":1711170823960,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"},"user_tz":-480},"id":"ba19WKsvGzrw","outputId":"4dd06965-30c9-4a80-c0fd-730d4403935c"},"outputs":[{"output_type":"stream","name":"stdout","text":["Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount(\"/content/gdrive\", force_remount=True).\n"]}],"source":["# Set path for data source\n","import os\n","from google.colab import drive\n","drive.mount('/content/gdrive')\n","os.chdir(\"/content/gdrive/My Drive/Load Diffusion/dataset/\")"]},{"cell_type":"code","execution_count":7,"metadata":{"executionInfo":{"elapsed":5442,"status":"ok","timestamp":1711170829399,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"},"user_tz":-480},"id":"T3Q_l6x5Gzue"},"outputs":[],"source":["# Set the path to the folder containing saved files\n","load_path = \"./preprocessed_data_168x168_2years\"\n","\n","# Load saved NumPy arrays\n","X_train = np.load(os.path.join(load_path, 'X_train.npy'))\n","X_test = np.load(os.path.join(load_path, 'X_test.npy'))\n","y_train = np.load(os.path.join(load_path, 'y_train.npy'))\n","y_test = np.load(os.path.join(load_path, 'y_test.npy'))\n","\n","# Load saved Pandas DataFrames\n","train_data = pd.read_pickle(os.path.join(load_path, 'train_data.pkl'))\n","test_data = pd.read_pickle(os.path.join(load_path, 'test_data.pkl'))\n","metadata = pd.read_pickle(os.path.join(load_path, 'metadata.pkl'))\n","metadata_original = pd.read_pickle(os.path.join(load_path, 'metadata_original.pkl'))"]},{"cell_type":"code","execution_count":8,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":495},"executionInfo":{"elapsed":15,"status":"ok","timestamp":1711170829400,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"},"user_tz":-480},"id":"DoYSmVXKGzxJ","outputId":"ddeae785-83f0-4a96-fdb5-3f7fe08f8877"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["                                              lat         lng  year  \\\n","electricity_Lamb_office_Caitlin_2016    51.497838   -3.186246  2016   \n","electricity_Rat_public_Kelle_2016       38.903504  -77.005349  2016   \n","electricity_Rat_assembly_Ezequiel_2016  38.903504  -77.005349  2016   \n","electricity_Bear_assembly_Roxy_2016     37.871903 -122.260729  2016   \n","gas_Shrew_office_Rose_2017              51.499840   -0.124663  2017   \n","...                                           ...         ...   ...   \n","steam_Moose_education_Omar_2016         45.421500  -75.697200  2016   \n","electricity_Rat_public_Chrissy_2017     38.903504  -77.005349  2017   \n","electricity_Hog_office_Merilyn_2016     44.978782  -93.255398  2016   \n","steam_Bull_assembly_Amalia_2016         30.267200  -97.743100  2016   \n","electricity_Rat_office_Jeannie_2017     38.903504  -77.005349  2017   \n","\n","                                         meter_type  \\\n","electricity_Lamb_office_Caitlin_2016    electricity   \n","electricity_Rat_public_Kelle_2016       electricity   \n","electricity_Rat_assembly_Ezequiel_2016  electricity   \n","electricity_Bear_assembly_Roxy_2016     electricity   \n","gas_Shrew_office_Rose_2017                      gas   \n","...                                             ...   \n","steam_Moose_education_Omar_2016               steam   \n","electricity_Rat_public_Chrissy_2017     electricity   \n","electricity_Hog_office_Merilyn_2016     electricity   \n","steam_Bull_assembly_Amalia_2016               steam   \n","electricity_Rat_office_Jeannie_2017     electricity   \n","\n","                                                        building_type  \n","electricity_Lamb_office_Caitlin_2016                           Office  \n","electricity_Rat_public_Kelle_2016                     Public services  \n","electricity_Rat_assembly_Ezequiel_2016  Entertainment/public assembly  \n","electricity_Bear_assembly_Roxy_2016     Entertainment/public assembly  \n","gas_Shrew_office_Rose_2017                                     Office  \n","...                                                               ...  \n","steam_Moose_education_Omar_2016                             Education  \n","electricity_Rat_public_Chrissy_2017                   Public services  \n","electricity_Hog_office_Merilyn_2016                            Office  \n","steam_Bull_assembly_Amalia_2016         Entertainment/public assembly  \n","electricity_Rat_office_Jeannie_2017                            Office  \n","\n","[901 rows x 5 columns]"],"text/html":["\n","  <div id=\"df-a31b42e2-f1f9-4faa-89dd-791d18104a47\" class=\"colab-df-container\">\n","    <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>lat</th>\n","      <th>lng</th>\n","      <th>year</th>\n","      <th>meter_type</th>\n","      <th>building_type</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>electricity_Lamb_office_Caitlin_2016</th>\n","      <td>51.497838</td>\n","      <td>-3.186246</td>\n","      <td>2016</td>\n","      <td>electricity</td>\n","      <td>Office</td>\n","    </tr>\n","    <tr>\n","      <th>electricity_Rat_public_Kelle_2016</th>\n","      <td>38.903504</td>\n","      <td>-77.005349</td>\n","      <td>2016</td>\n","      <td>electricity</td>\n","      <td>Public services</td>\n","    </tr>\n","    <tr>\n","      <th>electricity_Rat_assembly_Ezequiel_2016</th>\n","      <td>38.903504</td>\n","      <td>-77.005349</td>\n","      <td>2016</td>\n","      <td>electricity</td>\n","      <td>Entertainment/public assembly</td>\n","    </tr>\n","    <tr>\n","      <th>electricity_Bear_assembly_Roxy_2016</th>\n","      <td>37.871903</td>\n","      <td>-122.260729</td>\n","      <td>2016</td>\n","      <td>electricity</td>\n","      <td>Entertainment/public assembly</td>\n","    </tr>\n","    <tr>\n","      <th>gas_Shrew_office_Rose_2017</th>\n","      <td>51.499840</td>\n","      <td>-0.124663</td>\n","      <td>2017</td>\n","      <td>gas</td>\n","      <td>Office</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>steam_Moose_education_Omar_2016</th>\n","      <td>45.421500</td>\n","      <td>-75.697200</td>\n","      <td>2016</td>\n","      <td>steam</td>\n","      <td>Education</td>\n","    </tr>\n","    <tr>\n","      <th>electricity_Rat_public_Chrissy_2017</th>\n","      <td>38.903504</td>\n","      <td>-77.005349</td>\n","      <td>2017</td>\n","      <td>electricity</td>\n","      <td>Public services</td>\n","    </tr>\n","    <tr>\n","      <th>electricity_Hog_office_Merilyn_2016</th>\n","      <td>44.978782</td>\n","      <td>-93.255398</td>\n","      <td>2016</td>\n","      <td>electricity</td>\n","      <td>Office</td>\n","    </tr>\n","    <tr>\n","      <th>steam_Bull_assembly_Amalia_2016</th>\n","      <td>30.267200</td>\n","      <td>-97.743100</td>\n","      <td>2016</td>\n","      <td>steam</td>\n","      <td>Entertainment/public assembly</td>\n","    </tr>\n","    <tr>\n","      <th>electricity_Rat_office_Jeannie_2017</th>\n","      <td>38.903504</td>\n","      <td>-77.005349</td>\n","      <td>2017</td>\n","      <td>electricity</td>\n","      <td>Office</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>901 rows × 5 columns</p>\n","</div>\n","    <div class=\"colab-df-buttons\">\n","\n","  <div class=\"colab-df-container\">\n","    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a31b42e2-f1f9-4faa-89dd-791d18104a47')\"\n","            title=\"Convert this dataframe to an interactive table.\"\n","            style=\"display:none;\">\n","\n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n","    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n","  </svg>\n","    </button>\n","\n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    .colab-df-buttons div {\n","      margin-bottom: 4px;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","    <script>\n","      const buttonEl =\n","        document.querySelector('#df-a31b42e2-f1f9-4faa-89dd-791d18104a47 button.colab-df-convert');\n","      buttonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","      async function convertToInteractive(key) {\n","        const element = document.querySelector('#df-a31b42e2-f1f9-4faa-89dd-791d18104a47');\n","        const dataTable =\n","          await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                    [key], {});\n","        if (!dataTable) return;\n","\n","        const docLinkHtml = 'Like what you see? Visit the ' +\n","          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","          + ' to learn more about interactive tables.';\n","        element.innerHTML = '';\n","        dataTable['output_type'] = 'display_data';\n","        await google.colab.output.renderOutput(dataTable, element);\n","        const docLink = document.createElement('div');\n","        docLink.innerHTML = docLinkHtml;\n","        element.appendChild(docLink);\n","      }\n","    </script>\n","  </div>\n","\n","\n","<div id=\"df-4bee9784-0de1-45ff-879a-0b835defe80e\">\n","  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-4bee9784-0de1-45ff-879a-0b835defe80e')\"\n","            title=\"Suggest charts\"\n","            style=\"display:none;\">\n","\n","<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","     width=\"24px\">\n","    <g>\n","        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n","    </g>\n","</svg>\n","  </button>\n","\n","<style>\n","  .colab-df-quickchart {\n","      --bg-color: #E8F0FE;\n","      --fill-color: #1967D2;\n","      --hover-bg-color: #E2EBFA;\n","      --hover-fill-color: #174EA6;\n","      --disabled-fill-color: #AAA;\n","      --disabled-bg-color: #DDD;\n","  }\n","\n","  [theme=dark] .colab-df-quickchart {\n","      --bg-color: #3B4455;\n","      --fill-color: #D2E3FC;\n","      --hover-bg-color: #434B5C;\n","      --hover-fill-color: #FFFFFF;\n","      --disabled-bg-color: #3B4455;\n","      --disabled-fill-color: #666;\n","  }\n","\n","  .colab-df-quickchart {\n","    background-color: var(--bg-color);\n","    border: none;\n","    border-radius: 50%;\n","    cursor: pointer;\n","    display: none;\n","    fill: var(--fill-color);\n","    height: 32px;\n","    padding: 0;\n","    width: 32px;\n","  }\n","\n","  .colab-df-quickchart:hover {\n","    background-color: var(--hover-bg-color);\n","    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n","    fill: var(--button-hover-fill-color);\n","  }\n","\n","  .colab-df-quickchart-complete:disabled,\n","  .colab-df-quickchart-complete:disabled:hover {\n","    background-color: var(--disabled-bg-color);\n","    fill: var(--disabled-fill-color);\n","    box-shadow: none;\n","  }\n","\n","  .colab-df-spinner {\n","    border: 2px solid var(--fill-color);\n","    border-color: transparent;\n","    border-bottom-color: var(--fill-color);\n","    animation:\n","      spin 1s steps(1) infinite;\n","  }\n","\n","  @keyframes spin {\n","    0% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","      border-left-color: var(--fill-color);\n","    }\n","    20% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    30% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","      border-right-color: var(--fill-color);\n","    }\n","    40% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    60% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","    }\n","    80% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-bottom-color: var(--fill-color);\n","    }\n","    90% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","    }\n","  }\n","</style>\n","\n","  <script>\n","    async function quickchart(key) {\n","      const quickchartButtonEl =\n","        document.querySelector('#' + key + ' button');\n","      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n","      quickchartButtonEl.classList.add('colab-df-spinner');\n","      try {\n","        const charts = await google.colab.kernel.invokeFunction(\n","            'suggestCharts', [key], {});\n","      } catch (error) {\n","        console.error('Error during call to suggestCharts:', error);\n","      }\n","      quickchartButtonEl.classList.remove('colab-df-spinner');\n","      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n","    }\n","    (() => {\n","      let quickchartButtonEl =\n","        document.querySelector('#df-4bee9784-0de1-45ff-879a-0b835defe80e button');\n","      quickchartButtonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","    })();\n","  </script>\n","</div>\n","\n","  <div id=\"id_78ce9ca2-1c9e-4934-83e8-43c27a3765b8\">\n","    <style>\n","      .colab-df-generate {\n","        background-color: #E8F0FE;\n","        border: none;\n","        border-radius: 50%;\n","        cursor: pointer;\n","        display: none;\n","        fill: #1967D2;\n","        height: 32px;\n","        padding: 0 0 0 0;\n","        width: 32px;\n","      }\n","\n","      .colab-df-generate:hover {\n","        background-color: #E2EBFA;\n","        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","        fill: #174EA6;\n","      }\n","\n","      [theme=dark] .colab-df-generate {\n","        background-color: #3B4455;\n","        fill: #D2E3FC;\n","      }\n","\n","      [theme=dark] .colab-df-generate:hover {\n","        background-color: #434B5C;\n","        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","        fill: #FFFFFF;\n","      }\n","    </style>\n","    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('test_metadata')\"\n","            title=\"Generate code using this dataframe.\"\n","            style=\"display:none;\">\n","\n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n","  </svg>\n","    </button>\n","    <script>\n","      (() => {\n","      const buttonEl =\n","        document.querySelector('#id_78ce9ca2-1c9e-4934-83e8-43c27a3765b8 button.colab-df-generate');\n","      buttonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","      buttonEl.onclick = () => {\n","        google.colab.notebook.generateWithVariable('test_metadata');\n","      }\n","      })();\n","    </script>\n","  </div>\n","\n","    </div>\n","  </div>\n"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"dataframe","variable_name":"test_metadata","summary":"{\n  \"name\": \"test_metadata\",\n  \"rows\": 901,\n  \"fields\": [\n    {\n      \"column\": \"lat\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 7.208395818560617,\n        \"min\": 28.5176885788321,\n        \"max\": 53.34979999999997,\n        \"num_unique_values\": 13,\n        \"samples\": [\n          40.349999999999966,\n          53.34979999999997,\n          51.49783804108846\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"lng\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 36.55924667468882,\n        \"min\": -122.26072860000008,\n        \"max\": 6.2603,\n        \"num_unique_values\": 14,\n        \"samples\": [\n          -111.92814039999979,\n          -0.1345562307692307,\n          -3.186245571244898\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"year\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 0,\n        \"min\": 2016,\n        \"max\": 2017,\n        \"num_unique_values\": 2,\n        \"samples\": [\n          2017,\n          2016\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"meter_type\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 5,\n        \"samples\": [\n          \"gas\",\n          \"steam\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"building_type\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 5,\n        \"samples\": [\n          \"Public services\",\n          \"Lodging/residential\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"}},"metadata":{},"execution_count":8}],"source":["def merge_dummified_columns(df, dummified_columns, new_column_name):\n","    \"\"\"\n","    Merges dummified columns into a single column in a DataFrame.\n","\n","    Parameters:\n","        df (pd.DataFrame): The DataFrame containing the dummified columns.\n","        dummified_columns (list): A list of the names of the dummified columns.\n","        new_column_name (str): The name of the new column to create.\n","\n","    Returns:\n","        pd.DataFrame: The DataFrame with the dummified columns merged into a single new column.\n","    \"\"\"\n","    # Create a new column containing the column name where the value is 1 for each row\n","    df[new_column_name] = df[dummified_columns].idxmax(axis=1)\n","\n","    # Drop the dummified columns\n","    df.drop(dummified_columns, axis=1, inplace=True)\n","\n","    return df\n","\n","test_metadata = metadata_original.loc[test_data.columns]\n","\n","test_metadata = merge_dummified_columns(test_metadata,\n","                     dummified_columns=['chilledwater', 'electricity', 'gas', 'hotwater', 'steam'],\n","                     new_column_name='meter_type')\n","test_metadata = merge_dummified_columns(test_metadata,\n","                     dummified_columns=['Education', 'Entertainment/public assembly', 'Lodging/residential', 'Office', 'Public services'],\n","                     new_column_name='building_type')\n","\n","test_metadata\n"]},{"cell_type":"code","execution_count":9,"metadata":{"executionInfo":{"elapsed":14,"status":"ok","timestamp":1711170829400,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"},"user_tz":-480},"id":"E1i1UbW-G8-A"},"outputs":[],"source":["def calculate_rmse(true_data, predicted_data):\n","    return np.sqrt(mean_squared_error(true_data, predicted_data))\n","\n","def calculate_mae(true_data, predicted_data):\n","    return mean_absolute_error(true_data, predicted_data)\n","\n","def calculate_r2(true_data, predicted_data):\n","    return r2_score(true_data, predicted_data)\n","\n","def calculate_kl_divergence(true_data, predicted_data):\n","    return entropy(true_data+0.00000001, predicted_data+0.00000001)\n","\n","def calculate_cosine_similarity(true_data, predicted_data):\n","    return 1 - cosine(true_data, predicted_data)"]},{"cell_type":"code","execution_count":10,"metadata":{"executionInfo":{"elapsed":13,"status":"ok","timestamp":1711170829400,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"},"user_tz":-480},"id":"aslAFReUG9A1"},"outputs":[],"source":["def evaluate_model(test_data, generated_series):\n","    metrics_result = {\n","        'rmse': [],\n","        'mae': [],\n","        'r2': [],\n","        'kl_divergence': [],\n","        'cosine_similarity': [],\n","        'pearson_correlation': []\n","    }\n","\n","    for i in range(test_data.shape[1]):\n","        try:\n","            true_data = test_data[:, i]\n","            predicted_data = generated_series[:, i]\n","        except:\n","            true_data = test_data.values[:, i]\n","            predicted_data = generated_series.values[:, i]\n","\n","        # Calculate Pearson correlation\n","        pearson_corr, _ = pearsonr(true_data, predicted_data)\n","\n","        metrics_result['rmse'].append(calculate_rmse(true_data, predicted_data))\n","        metrics_result['mae'].append(calculate_mae(true_data, predicted_data))\n","        metrics_result['r2'].append(calculate_r2(true_data, predicted_data))\n","        metrics_result['kl_divergence'].append(calculate_kl_divergence(true_data, predicted_data))\n","        metrics_result['cosine_similarity'].append(calculate_cosine_similarity(true_data, predicted_data))\n","        metrics_result['pearson_correlation'].append(pearson_corr)\n","\n","    return metrics_result\n","\n","# Function to calculate FID between two sets of images\n","def calculate_fid(model, images1, images2):\n","    # Calculate activations\n","    act1 = model.predict(images1)\n","    act2 = model.predict(images2)\n","\n","    # Calculate mean and covariance statistics\n","    mu1, sigma1 = act1.mean(axis=0), np.cov(act1, rowvar=False)\n","    mu2, sigma2 = act2.mean(axis=0), np.cov(act2, rowvar=False)\n","\n","    # Calculate sum squared difference between means\n","    ssdiff = np.sum((mu1 - mu2)**2.0)\n","\n","    # Calculate sqrt of product between covariances\n","    covmean = sqrtm(sigma1.dot(sigma2))\n","\n","    # Check and correct imaginary numbers from sqrt\n","    if np.iscomplexobj(covmean):\n","        covmean = covmean.real\n","\n","    # Calculate the FID score\n","    fid = ssdiff + np.trace(sigma1 + sigma2 - 2.0 * covmean)\n","    return fid"]},{"cell_type":"code","execution_count":11,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"bmyXFGbTG9GI","executionInfo":{"status":"ok","timestamp":1711172801772,"user_tz":-480,"elapsed":1972385,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"}},"outputId":"454a4c08-f184-407e-bf42-7992fffb7e50"},"outputs":[{"output_type":"stream","name":"stdout","text":["generated_images_seed_1.npy\n","29/29 [==============================] - 25s 797ms/step\n","29/29 [==============================] - 12s 430ms/step\n","FID: 943.8997217467477\n","generated_images_seed_2.npy\n","29/29 [==============================] - 16s 503ms/step\n","29/29 [==============================] - 12s 395ms/step\n","FID: 950.0023913314024\n","generated_images_seed_3.npy\n","29/29 [==============================] - 13s 425ms/step\n","29/29 [==============================] - 12s 431ms/step\n","FID: 954.7249892711368\n","generated_images_seed_4.npy\n","29/29 [==============================] - 14s 432ms/step\n","29/29 [==============================] - 14s 491ms/step\n","FID: 922.9765406772386\n","generated_images_seed_5.npy\n","29/29 [==============================] - 13s 423ms/step\n","29/29 [==============================] - 12s 424ms/step\n","FID: 946.2291509290717\n","generated_images_seed_6.npy\n","29/29 [==============================] - 13s 413ms/step\n","29/29 [==============================] - 11s 370ms/step\n","FID: 954.7388571774534\n","generated_images_seed_7.npy\n","29/29 [==============================] - 13s 390ms/step\n","29/29 [==============================] - 12s 408ms/step\n","FID: 950.9189169755365\n","generated_images_seed_8.npy\n","29/29 [==============================] - 13s 420ms/step\n","29/29 [==============================] - 11s 378ms/step\n","FID: 945.2703776499475\n","generated_images_seed_9.npy\n","29/29 [==============================] - 13s 417ms/step\n","29/29 [==============================] - 14s 480ms/step\n","FID: 933.0082830456026\n","generated_images_seed_10.npy\n","29/29 [==============================] - 13s 382ms/step\n","29/29 [==============================] - 12s 413ms/step\n","FID: 953.5179391045322\n","generated_images_seed_11.npy\n","29/29 [==============================] - 13s 403ms/step\n","29/29 [==============================] - 12s 408ms/step\n","FID: 938.8493044822949\n","generated_images_seed_12.npy\n","29/29 [==============================] - 13s 409ms/step\n","29/29 [==============================] - 12s 418ms/step\n","FID: 942.6499447530638\n","generated_images_seed_13.npy\n","29/29 [==============================] - 13s 400ms/step\n","29/29 [==============================] - 12s 421ms/step\n","FID: 956.804778857194\n","generated_images_seed_14.npy\n","29/29 [==============================] - 14s 399ms/step\n","29/29 [==============================] - 11s 380ms/step\n","FID: 944.6020794201584\n","generated_images_seed_15.npy\n","29/29 [==============================] - 13s 419ms/step\n","29/29 [==============================] - 12s 422ms/step\n","FID: 944.4963603968007\n","generated_images_seed_16.npy\n","29/29 [==============================] - 13s 382ms/step\n","29/29 [==============================] - 12s 423ms/step\n","FID: 941.7445229389566\n","generated_images_seed_17.npy\n","29/29 [==============================] - 13s 420ms/step\n","29/29 [==============================] - 12s 423ms/step\n","FID: 938.8715912171656\n","generated_images_seed_18.npy\n","29/29 [==============================] - 14s 428ms/step\n","29/29 [==============================] - 12s 431ms/step\n","FID: 937.0572584092405\n","generated_images_seed_19.npy\n","29/29 [==============================] - 13s 414ms/step\n","29/29 [==============================] - 20s 700ms/step\n","FID: 971.1960588751617\n","generated_images_seed_20.npy\n","29/29 [==============================] - 14s 429ms/step\n","29/29 [==============================] - 13s 459ms/step\n","FID: 945.1524748938738\n","generated_images_seed_21.npy\n","29/29 [==============================] - 13s 417ms/step\n","29/29 [==============================] - 12s 418ms/step\n","FID: 959.0957520004312\n","generated_images_seed_22.npy\n","29/29 [==============================] - 14s 441ms/step\n","29/29 [==============================] - 12s 422ms/step\n","FID: 945.8265452167911\n","generated_images_seed_23.npy\n","29/29 [==============================] - 13s 421ms/step\n","29/29 [==============================] - 12s 424ms/step\n","FID: 951.2301967861173\n","generated_images_seed_24.npy\n","29/29 [==============================] - 14s 402ms/step\n","29/29 [==============================] - 12s 410ms/step\n","FID: 961.7676828513117\n","generated_images_seed_25.npy\n","29/29 [==============================] - 13s 378ms/step\n","29/29 [==============================] - 12s 408ms/step\n","FID: 925.0406949517171\n","generated_images_seed_26.npy\n","29/29 [==============================] - 13s 374ms/step\n","29/29 [==============================] - 12s 408ms/step\n","FID: 946.055513062226\n","generated_images_seed_27.npy\n","29/29 [==============================] - 13s 408ms/step\n","29/29 [==============================] - 12s 397ms/step\n","FID: 951.1305298887328\n","generated_images_seed_28.npy\n","29/29 [==============================] - 17s 555ms/step\n","29/29 [==============================] - 13s 430ms/step\n","FID: 954.7976549208882\n","generated_images_seed_29.npy\n","29/29 [==============================] - 13s 417ms/step\n","29/29 [==============================] - 12s 415ms/step\n","FID: 939.0366286296603\n","generated_images_seed_30.npy\n","29/29 [==============================] - 15s 478ms/step\n","29/29 [==============================] - 12s 427ms/step\n","FID: 952.1945275509344\n"]}],"source":["# Specify the path where the datasets are saved\n","generation_path = \"./generation/\" + model_name + '_168x168'\n","\n","# Initialize an empty DataFrame to store metrics\n","df_metrics = []\n","\n","for file_name in [item for item in os.listdir(generation_path) if item.startswith('generated_images_seed')]:\n","  # Load the dataset\n","  print(file_name)\n","  generated_images = np.load(os.path.join(generation_path, file_name))\n","  images_seed = int(file_name.split('.')[0].split('_')[-1])\n","\n","  # Set the target shape\n","  target_shape = (generated_images.shape[0], 52, 168, 1)\n","\n","  # Initialize an array to hold the resized images\n","  resized_images = np.zeros(target_shape)\n","\n","  # Loop through each image to resize it\n","  for i in range(generated_images.shape[0]):\n","      resized_image = cv2.resize(generated_images[i], (168, 52))  # Note the dimensions are (width, height)\n","      resized_images[i] = np.expand_dims(resized_image, axis=-1)  # Add back the last dimension\n","\n","  # Assume resized_images has a shape of (905, 52, 168, 1)\n","  flattened_images = resized_images.reshape(generated_images.shape[0], -1).T  # Transpose to get shape (8736, generated_images.shape[0])\n","\n","  generated_series = pd.DataFrame(flattened_images)\n","  generated_series.columns = test_data.columns\n","\n","  generated_series = (generated_series-generated_series.min())/(generated_series.max()-generated_series.min())\n","  test_data = (test_data-test_data.min())/(test_data.max()-test_data.min())\n","\n","  test_data_resampled = test_data.rolling(24).mean()[23::24]\n","  generated_series_resampled = generated_series.rolling(24).mean()[23::24]\n","\n","  evaluation_results = evaluate_model(test_data, generated_series)\n","  evaluation_results_resampled = evaluate_model(test_data_resampled, generated_series_resampled)\n","\n","  df_evaluation_results = pd.DataFrame(evaluation_results)\n","  df_evaluation_results_resampled = pd.DataFrame(evaluation_results_resampled)\n","  df_evaluation_results_resampled.columns = df_evaluation_results_resampled.columns+'_daily_avg'\n","\n","  # Prepare the Inception v3 model\n","  model = InceptionV3(include_top=False, pooling='avg', weights=\"imagenet\", input_shape=(99,99,3))\n","\n","  # Convert to RGB and upscale to the necessary input size for the Inception model\n","  X_test_rgb = tf.image.grayscale_to_rgb(tf.convert_to_tensor(X_test))\n","  generated_images_rgb = tf.image.grayscale_to_rgb(tf.convert_to_tensor(generated_images))\n","\n","  X_test_upscaled = tf.image.resize(X_test_rgb, [99, 99])\n","  generated_images_upscaled = tf.image.resize(generated_images_rgb, [99, 99])\n","\n","  # Calculate FID\n","  fid_score = calculate_fid(model, X_test_upscaled, generated_images_upscaled)\n","  print('FID: '+str(fid_score))\n","\n","  metrics = pd.concat([df_evaluation_results.mean(), df_evaluation_results_resampled.mean()])\n","  metrics['fid_score'] = fid_score\n","  metrics['images_seed'] = images_seed\n","  metrics['model'] = model_name\n","  metrics = metrics.to_frame().T\n","\n","  # Append the metrics to df_metrics\n","  df_metrics.append(metrics)"]},{"cell_type":"code","execution_count":12,"metadata":{"id":"08yTJJqVGz8a","colab":{"base_uri":"https://localhost:8080/","height":1000},"executionInfo":{"status":"ok","timestamp":1711172801773,"user_tz":-480,"elapsed":15,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"}},"outputId":"a4ac9bb4-c88a-40a5-c83d-2f314a997e6d"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["        rmse       mae        r2 kl_divergence cosine_similarity  \\\n","0   0.258466   0.21079 -3.969388      0.455561          0.772334   \n","1    0.26031  0.213112 -3.169984      0.459104          0.769955   \n","2   0.259448  0.211934 -2.652959      0.455319          0.772245   \n","3    0.25636  0.209263 -2.616288      0.452215          0.773616   \n","4   0.257445  0.210197 -2.655894      0.449045          0.775444   \n","5   0.257744  0.210114 -2.771796      0.453282          0.773415   \n","6   0.258673  0.211351 -2.917075      0.456499          0.772843   \n","7    0.25938  0.211985 -3.201531      0.455643          0.772015   \n","8   0.258008  0.211309 -2.725111      0.453468          0.773472   \n","9   0.258851  0.211121 -3.351192      0.453145          0.772833   \n","10  0.259562  0.212889 -3.454872      0.454413          0.772682   \n","11  0.261159  0.213976 -3.783544      0.453785          0.771405   \n","12  0.261413  0.213784  -3.00243      0.457289          0.768739   \n","13  0.258146  0.210963 -2.484273         0.452          0.773514   \n","14  0.259145  0.211684 -3.109877      0.460888          0.769893   \n","15  0.255237  0.208375 -2.922585      0.447489          0.776296   \n","16  0.257491  0.209961 -3.524372      0.452944          0.772418   \n","17  0.256513  0.209548 -3.062147      0.451038          0.773625   \n","18  0.256583  0.209012 -3.808926      0.450834          0.775997   \n","19  0.259383  0.212309  -3.25853      0.457462          0.771876   \n","20  0.260697  0.213744 -3.015391      0.453703          0.772587   \n","21  0.260367  0.213282 -3.138918      0.455478          0.772829   \n","22  0.255266  0.208833 -3.281251      0.443798           0.77846   \n","23  0.260194  0.212041 -3.163655      0.460831          0.768149   \n","24  0.258743  0.212027 -3.774706       0.44892          0.775371   \n","25   0.25676  0.209428 -2.401888      0.452932          0.773939   \n","26   0.25632  0.209452 -3.447599      0.456669          0.772239   \n","27  0.257663  0.210442  -2.79033      0.453584          0.773463   \n","28  0.257916  0.211071 -3.976184      0.448238          0.775062   \n","29  0.257153  0.210358 -2.905799      0.453371          0.773993   \n","\n","   pearson_correlation rmse_daily_avg mae_daily_avg r2_daily_avg  \\\n","0             0.351359       0.207696      0.175321   -69.883774   \n","1             0.344622       0.208965      0.177203   -47.161156   \n","2              0.35019       0.208889      0.176443   -38.684801   \n","3             0.352346       0.205994      0.173661   -35.084782   \n","4             0.362717       0.207305      0.175191   -37.910288   \n","5             0.352731       0.207042      0.174645   -39.689465   \n","6              0.35693       0.208143      0.176076   -41.189101   \n","7             0.349538       0.209877       0.17743   -49.392402   \n","8             0.348279       0.208841      0.176579   -40.014183   \n","9             0.354481        0.20809      0.175601   -52.829711   \n","10            0.350188       0.209752      0.177566   -55.559161   \n","11            0.343954        0.21104      0.178777   -62.238977   \n","12            0.341413       0.210658      0.178486   -42.638697   \n","13            0.357743       0.206964      0.174939   -33.601993   \n","14            0.345238        0.20717      0.175232   -48.112517   \n","15            0.362625       0.205114      0.172998    -45.77485   \n","16            0.345368       0.207548      0.174814   -54.484138   \n","17            0.353942       0.207348      0.175147   -48.570369   \n","18            0.370658       0.206075      0.173863   -54.375017   \n","19            0.346583       0.209362      0.177045   -54.367282   \n","20            0.346738       0.209341      0.177693   -44.232364   \n","21            0.355998       0.208861      0.176987   -44.251667   \n","22            0.366267       0.205808      0.174103   -53.682502   \n","23            0.340827       0.210104      0.177054   -45.360348   \n","24            0.354972       0.208401      0.176634   -63.410052   \n","25            0.355073        0.20617      0.173563   -34.630062   \n","26             0.35557       0.206211       0.17384   -54.221495   \n","27            0.357141       0.206079      0.174103   -38.081208   \n","28            0.353792       0.207203      0.175645   -68.502499   \n","29            0.350675       0.207748      0.175934   -45.183902   \n","\n","   kl_divergence_daily_avg cosine_similarity_daily_avg  \\\n","0                  0.29105                    0.840831   \n","1                 0.292518                     0.83893   \n","2                 0.292948                    0.838754   \n","3                  0.28985                    0.840365   \n","4                 0.284989                    0.843379   \n","5                  0.28853                    0.841722   \n","6                 0.292665                    0.840258   \n","7                 0.292947                    0.838624   \n","8                 0.291423                    0.839946   \n","9                 0.287568                    0.841677   \n","10                0.290311                    0.839768   \n","11                0.289278                    0.839507   \n","12                0.293464                    0.837449   \n","13                0.287219                    0.841902   \n","14                0.291758                     0.83968   \n","15                0.285794                    0.843104   \n","16                0.290647                    0.839423   \n","17                0.289616                    0.839971   \n","18                0.287525                    0.842902   \n","19                0.292952                    0.838912   \n","20                0.288098                     0.84077   \n","21                0.287967                    0.842544   \n","22                 0.28443                    0.844119   \n","23                0.295605                    0.835703   \n","24                0.286222                    0.843342   \n","25                 0.28992                    0.841011   \n","26                0.293232                    0.839855   \n","27                0.286909                    0.843087   \n","28                0.286667                     0.84228   \n","29                0.290852                    0.840483   \n","\n","   pearson_correlation_daily_avg   fid_score images_seed     model  \n","0                       0.331587  943.899722         1.0  CVAE_mse  \n","1                       0.323282  950.002391         2.0  CVAE_mse  \n","2                       0.322743  954.724989         3.0  CVAE_mse  \n","3                       0.328916  922.976541         4.0  CVAE_mse  \n","4                       0.340809  946.229151         5.0  CVAE_mse  \n","5                       0.332532  954.738857         6.0  CVAE_mse  \n","6                       0.338983  950.918917         7.0  CVAE_mse  \n","7                       0.319999  945.270378         8.0  CVAE_mse  \n","8                       0.320422  933.008283         9.0  CVAE_mse  \n","9                       0.330217  953.517939        10.0  CVAE_mse  \n","10                      0.319277  938.849304        11.0  CVAE_mse  \n","11                      0.321467  942.649945        12.0  CVAE_mse  \n","12                      0.314136  956.804779        13.0  CVAE_mse  \n","13                      0.342211  944.602079        14.0  CVAE_mse  \n","14                      0.327678   944.49636        15.0  CVAE_mse  \n","15                      0.342759  941.744523        16.0  CVAE_mse  \n","16                      0.315266  938.871591        17.0  CVAE_mse  \n","17                      0.327504  937.057258        18.0  CVAE_mse  \n","18                      0.346447  971.196059        19.0  CVAE_mse  \n","19                      0.319173  945.152475        20.0  CVAE_mse  \n","20                      0.319931  959.095752        21.0  CVAE_mse  \n","21                      0.338453  945.826545        22.0  CVAE_mse  \n","22                      0.345811  951.230197        23.0  CVAE_mse  \n","23                       0.30605  961.767683        24.0  CVAE_mse  \n","24                      0.339773  925.040695        25.0  CVAE_mse  \n","25                      0.329555  946.055513        26.0  CVAE_mse  \n","26                      0.335316   951.13053        27.0  CVAE_mse  \n","27                      0.340607  954.797655        28.0  CVAE_mse  \n","28                      0.329855  939.036629        29.0  CVAE_mse  \n","29                      0.316575  952.194528        30.0  CVAE_mse  "],"text/html":["\n","  <div id=\"df-11de29c4-bdff-4518-9a50-5283e537953d\" class=\"colab-df-container\">\n","    <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>rmse</th>\n","      <th>mae</th>\n","      <th>r2</th>\n","      <th>kl_divergence</th>\n","      <th>cosine_similarity</th>\n","      <th>pearson_correlation</th>\n","      <th>rmse_daily_avg</th>\n","      <th>mae_daily_avg</th>\n","      <th>r2_daily_avg</th>\n","      <th>kl_divergence_daily_avg</th>\n","      <th>cosine_similarity_daily_avg</th>\n","      <th>pearson_correlation_daily_avg</th>\n","      <th>fid_score</th>\n","      <th>images_seed</th>\n","      <th>model</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>0.258466</td>\n","      <td>0.21079</td>\n","      <td>-3.969388</td>\n","      <td>0.455561</td>\n","      <td>0.772334</td>\n","      <td>0.351359</td>\n","      <td>0.207696</td>\n","      <td>0.175321</td>\n","      <td>-69.883774</td>\n","      <td>0.29105</td>\n","      <td>0.840831</td>\n","      <td>0.331587</td>\n","      <td>943.899722</td>\n","      <td>1.0</td>\n","      <td>CVAE_mse</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>0.26031</td>\n","      <td>0.213112</td>\n","      <td>-3.169984</td>\n","      <td>0.459104</td>\n","      <td>0.769955</td>\n","      <td>0.344622</td>\n","      <td>0.208965</td>\n","      <td>0.177203</td>\n","      <td>-47.161156</td>\n","      <td>0.292518</td>\n","      <td>0.83893</td>\n","      <td>0.323282</td>\n","      <td>950.002391</td>\n","      <td>2.0</td>\n","      <td>CVAE_mse</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>0.259448</td>\n","      <td>0.211934</td>\n","      <td>-2.652959</td>\n","      <td>0.455319</td>\n","      <td>0.772245</td>\n","      <td>0.35019</td>\n","      <td>0.208889</td>\n","      <td>0.176443</td>\n","      <td>-38.684801</td>\n","      <td>0.292948</td>\n","      <td>0.838754</td>\n","      <td>0.322743</td>\n","      <td>954.724989</td>\n","      <td>3.0</td>\n","      <td>CVAE_mse</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>0.25636</td>\n","      <td>0.209263</td>\n","      <td>-2.616288</td>\n","      <td>0.452215</td>\n","      <td>0.773616</td>\n","      <td>0.352346</td>\n","      <td>0.205994</td>\n","      <td>0.173661</td>\n","      <td>-35.084782</td>\n","      <td>0.28985</td>\n","      <td>0.840365</td>\n","      <td>0.328916</td>\n","      <td>922.976541</td>\n","      <td>4.0</td>\n","      <td>CVAE_mse</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>0.257445</td>\n","      <td>0.210197</td>\n","      <td>-2.655894</td>\n","      <td>0.449045</td>\n","      <td>0.775444</td>\n","      <td>0.362717</td>\n","      <td>0.207305</td>\n","      <td>0.175191</td>\n","      <td>-37.910288</td>\n","      <td>0.284989</td>\n","      <td>0.843379</td>\n","      <td>0.340809</td>\n","      <td>946.229151</td>\n","      <td>5.0</td>\n","      <td>CVAE_mse</td>\n","    </tr>\n","    <tr>\n","      <th>5</th>\n","      <td>0.257744</td>\n","      <td>0.210114</td>\n","      <td>-2.771796</td>\n","      <td>0.453282</td>\n","      <td>0.773415</td>\n","      <td>0.352731</td>\n","      <td>0.207042</td>\n","      <td>0.174645</td>\n","      <td>-39.689465</td>\n","      <td>0.28853</td>\n","      <td>0.841722</td>\n","      <td>0.332532</td>\n","      <td>954.738857</td>\n","      <td>6.0</td>\n","      <td>CVAE_mse</td>\n","    </tr>\n","    <tr>\n","      <th>6</th>\n","      <td>0.258673</td>\n","      <td>0.211351</td>\n","      <td>-2.917075</td>\n","      <td>0.456499</td>\n","      <td>0.772843</td>\n","      <td>0.35693</td>\n","      <td>0.208143</td>\n","      <td>0.176076</td>\n","      <td>-41.189101</td>\n","      <td>0.292665</td>\n","      <td>0.840258</td>\n","      <td>0.338983</td>\n","      <td>950.918917</td>\n","      <td>7.0</td>\n","      <td>CVAE_mse</td>\n","    </tr>\n","    <tr>\n","      <th>7</th>\n","      <td>0.25938</td>\n","      <td>0.211985</td>\n","      <td>-3.201531</td>\n","      <td>0.455643</td>\n","      <td>0.772015</td>\n","      <td>0.349538</td>\n","      <td>0.209877</td>\n","      <td>0.17743</td>\n","      <td>-49.392402</td>\n","      <td>0.292947</td>\n","      <td>0.838624</td>\n","      <td>0.319999</td>\n","      <td>945.270378</td>\n","      <td>8.0</td>\n","      <td>CVAE_mse</td>\n","    </tr>\n","    <tr>\n","      <th>8</th>\n","      <td>0.258008</td>\n","      <td>0.211309</td>\n","      <td>-2.725111</td>\n","      <td>0.453468</td>\n","      <td>0.773472</td>\n","      <td>0.348279</td>\n","      <td>0.208841</td>\n","      <td>0.176579</td>\n","      <td>-40.014183</td>\n","      <td>0.291423</td>\n","      <td>0.839946</td>\n","      <td>0.320422</td>\n","      <td>933.008283</td>\n","      <td>9.0</td>\n","      <td>CVAE_mse</td>\n","    </tr>\n","    <tr>\n","      <th>9</th>\n","      <td>0.258851</td>\n","      <td>0.211121</td>\n","      <td>-3.351192</td>\n","      <td>0.453145</td>\n","      <td>0.772833</td>\n","      <td>0.354481</td>\n","      <td>0.20809</td>\n","      <td>0.175601</td>\n","      <td>-52.829711</td>\n","      <td>0.287568</td>\n","      <td>0.841677</td>\n","      <td>0.330217</td>\n","      <td>953.517939</td>\n","      <td>10.0</td>\n","      <td>CVAE_mse</td>\n","    </tr>\n","    <tr>\n","      <th>10</th>\n","      <td>0.259562</td>\n","      <td>0.212889</td>\n","      <td>-3.454872</td>\n","      <td>0.454413</td>\n","      <td>0.772682</td>\n","      <td>0.350188</td>\n","      <td>0.209752</td>\n","      <td>0.177566</td>\n","      <td>-55.559161</td>\n","      <td>0.290311</td>\n","      <td>0.839768</td>\n","      <td>0.319277</td>\n","      <td>938.849304</td>\n","      <td>11.0</td>\n","      <td>CVAE_mse</td>\n","    </tr>\n","    <tr>\n","      <th>11</th>\n","      <td>0.261159</td>\n","      <td>0.213976</td>\n","      <td>-3.783544</td>\n","      <td>0.453785</td>\n","      <td>0.771405</td>\n","      <td>0.343954</td>\n","      <td>0.21104</td>\n","      <td>0.178777</td>\n","      <td>-62.238977</td>\n","      <td>0.289278</td>\n","      <td>0.839507</td>\n","      <td>0.321467</td>\n","      <td>942.649945</td>\n","      <td>12.0</td>\n","      <td>CVAE_mse</td>\n","    </tr>\n","    <tr>\n","      <th>12</th>\n","      <td>0.261413</td>\n","      <td>0.213784</td>\n","      <td>-3.00243</td>\n","      <td>0.457289</td>\n","      <td>0.768739</td>\n","      <td>0.341413</td>\n","      <td>0.210658</td>\n","      <td>0.178486</td>\n","      <td>-42.638697</td>\n","      <td>0.293464</td>\n","      <td>0.837449</td>\n","      <td>0.314136</td>\n","      <td>956.804779</td>\n","      <td>13.0</td>\n","      <td>CVAE_mse</td>\n","    </tr>\n","    <tr>\n","      <th>13</th>\n","      <td>0.258146</td>\n","      <td>0.210963</td>\n","      <td>-2.484273</td>\n","      <td>0.452</td>\n","      <td>0.773514</td>\n","      <td>0.357743</td>\n","      <td>0.206964</td>\n","      <td>0.174939</td>\n","      <td>-33.601993</td>\n","      <td>0.287219</td>\n","      <td>0.841902</td>\n","      <td>0.342211</td>\n","      <td>944.602079</td>\n","      <td>14.0</td>\n","      <td>CVAE_mse</td>\n","    </tr>\n","    <tr>\n","      <th>14</th>\n","      <td>0.259145</td>\n","      <td>0.211684</td>\n","      <td>-3.109877</td>\n","      <td>0.460888</td>\n","      <td>0.769893</td>\n","      <td>0.345238</td>\n","      <td>0.20717</td>\n","      <td>0.175232</td>\n","      <td>-48.112517</td>\n","      <td>0.291758</td>\n","      <td>0.83968</td>\n","      <td>0.327678</td>\n","      <td>944.49636</td>\n","      <td>15.0</td>\n","      <td>CVAE_mse</td>\n","    </tr>\n","    <tr>\n","      <th>15</th>\n","      <td>0.255237</td>\n","      <td>0.208375</td>\n","      <td>-2.922585</td>\n","      <td>0.447489</td>\n","      <td>0.776296</td>\n","      <td>0.362625</td>\n","      <td>0.205114</td>\n","      <td>0.172998</td>\n","      <td>-45.77485</td>\n","      <td>0.285794</td>\n","      <td>0.843104</td>\n","      <td>0.342759</td>\n","      <td>941.744523</td>\n","      <td>16.0</td>\n","      <td>CVAE_mse</td>\n","    </tr>\n","    <tr>\n","      <th>16</th>\n","      <td>0.257491</td>\n","      <td>0.209961</td>\n","      <td>-3.524372</td>\n","      <td>0.452944</td>\n","      <td>0.772418</td>\n","      <td>0.345368</td>\n","      <td>0.207548</td>\n","      <td>0.174814</td>\n","      <td>-54.484138</td>\n","      <td>0.290647</td>\n","      <td>0.839423</td>\n","      <td>0.315266</td>\n","      <td>938.871591</td>\n","      <td>17.0</td>\n","      <td>CVAE_mse</td>\n","    </tr>\n","    <tr>\n","      <th>17</th>\n","      <td>0.256513</td>\n","      <td>0.209548</td>\n","      <td>-3.062147</td>\n","      <td>0.451038</td>\n","      <td>0.773625</td>\n","      <td>0.353942</td>\n","      <td>0.207348</td>\n","      <td>0.175147</td>\n","      <td>-48.570369</td>\n","      <td>0.289616</td>\n","      <td>0.839971</td>\n","      <td>0.327504</td>\n","      <td>937.057258</td>\n","      <td>18.0</td>\n","      <td>CVAE_mse</td>\n","    </tr>\n","    <tr>\n","      <th>18</th>\n","      <td>0.256583</td>\n","      <td>0.209012</td>\n","      <td>-3.808926</td>\n","      <td>0.450834</td>\n","      <td>0.775997</td>\n","      <td>0.370658</td>\n","      <td>0.206075</td>\n","      <td>0.173863</td>\n","      <td>-54.375017</td>\n","      <td>0.287525</td>\n","      <td>0.842902</td>\n","      <td>0.346447</td>\n","      <td>971.196059</td>\n","      <td>19.0</td>\n","      <td>CVAE_mse</td>\n","    </tr>\n","    <tr>\n","      <th>19</th>\n","      <td>0.259383</td>\n","      <td>0.212309</td>\n","      <td>-3.25853</td>\n","      <td>0.457462</td>\n","      <td>0.771876</td>\n","      <td>0.346583</td>\n","      <td>0.209362</td>\n","      <td>0.177045</td>\n","      <td>-54.367282</td>\n","      <td>0.292952</td>\n","      <td>0.838912</td>\n","      <td>0.319173</td>\n","      <td>945.152475</td>\n","      <td>20.0</td>\n","      <td>CVAE_mse</td>\n","    </tr>\n","    <tr>\n","      <th>20</th>\n","      <td>0.260697</td>\n","      <td>0.213744</td>\n","      <td>-3.015391</td>\n","      <td>0.453703</td>\n","      <td>0.772587</td>\n","      <td>0.346738</td>\n","      <td>0.209341</td>\n","      <td>0.177693</td>\n","      <td>-44.232364</td>\n","      <td>0.288098</td>\n","      <td>0.84077</td>\n","      <td>0.319931</td>\n","      <td>959.095752</td>\n","      <td>21.0</td>\n","      <td>CVAE_mse</td>\n","    </tr>\n","    <tr>\n","      <th>21</th>\n","      <td>0.260367</td>\n","      <td>0.213282</td>\n","      <td>-3.138918</td>\n","      <td>0.455478</td>\n","      <td>0.772829</td>\n","      <td>0.355998</td>\n","      <td>0.208861</td>\n","      <td>0.176987</td>\n","      <td>-44.251667</td>\n","      <td>0.287967</td>\n","      <td>0.842544</td>\n","      <td>0.338453</td>\n","      <td>945.826545</td>\n","      <td>22.0</td>\n","      <td>CVAE_mse</td>\n","    </tr>\n","    <tr>\n","      <th>22</th>\n","      <td>0.255266</td>\n","      <td>0.208833</td>\n","      <td>-3.281251</td>\n","      <td>0.443798</td>\n","      <td>0.77846</td>\n","      <td>0.366267</td>\n","      <td>0.205808</td>\n","      <td>0.174103</td>\n","      <td>-53.682502</td>\n","      <td>0.28443</td>\n","      <td>0.844119</td>\n","      <td>0.345811</td>\n","      <td>951.230197</td>\n","      <td>23.0</td>\n","      <td>CVAE_mse</td>\n","    </tr>\n","    <tr>\n","      <th>23</th>\n","      <td>0.260194</td>\n","      <td>0.212041</td>\n","      <td>-3.163655</td>\n","      <td>0.460831</td>\n","      <td>0.768149</td>\n","      <td>0.340827</td>\n","      <td>0.210104</td>\n","      <td>0.177054</td>\n","      <td>-45.360348</td>\n","      <td>0.295605</td>\n","      <td>0.835703</td>\n","      <td>0.30605</td>\n","      <td>961.767683</td>\n","      <td>24.0</td>\n","      <td>CVAE_mse</td>\n","    </tr>\n","    <tr>\n","      <th>24</th>\n","      <td>0.258743</td>\n","      <td>0.212027</td>\n","      <td>-3.774706</td>\n","      <td>0.44892</td>\n","      <td>0.775371</td>\n","      <td>0.354972</td>\n","      <td>0.208401</td>\n","      <td>0.176634</td>\n","      <td>-63.410052</td>\n","      <td>0.286222</td>\n","      <td>0.843342</td>\n","      <td>0.339773</td>\n","      <td>925.040695</td>\n","      <td>25.0</td>\n","      <td>CVAE_mse</td>\n","    </tr>\n","    <tr>\n","      <th>25</th>\n","      <td>0.25676</td>\n","      <td>0.209428</td>\n","      <td>-2.401888</td>\n","      <td>0.452932</td>\n","      <td>0.773939</td>\n","      <td>0.355073</td>\n","      <td>0.20617</td>\n","      <td>0.173563</td>\n","      <td>-34.630062</td>\n","      <td>0.28992</td>\n","      <td>0.841011</td>\n","      <td>0.329555</td>\n","      <td>946.055513</td>\n","      <td>26.0</td>\n","      <td>CVAE_mse</td>\n","    </tr>\n","    <tr>\n","      <th>26</th>\n","      <td>0.25632</td>\n","      <td>0.209452</td>\n","      <td>-3.447599</td>\n","      <td>0.456669</td>\n","      <td>0.772239</td>\n","      <td>0.35557</td>\n","      <td>0.206211</td>\n","      <td>0.17384</td>\n","      <td>-54.221495</td>\n","      <td>0.293232</td>\n","      <td>0.839855</td>\n","      <td>0.335316</td>\n","      <td>951.13053</td>\n","      <td>27.0</td>\n","      <td>CVAE_mse</td>\n","    </tr>\n","    <tr>\n","      <th>27</th>\n","      <td>0.257663</td>\n","      <td>0.210442</td>\n","      <td>-2.79033</td>\n","      <td>0.453584</td>\n","      <td>0.773463</td>\n","      <td>0.357141</td>\n","      <td>0.206079</td>\n","      <td>0.174103</td>\n","      <td>-38.081208</td>\n","      <td>0.286909</td>\n","      <td>0.843087</td>\n","      <td>0.340607</td>\n","      <td>954.797655</td>\n","      <td>28.0</td>\n","      <td>CVAE_mse</td>\n","    </tr>\n","    <tr>\n","      <th>28</th>\n","      <td>0.257916</td>\n","      <td>0.211071</td>\n","      <td>-3.976184</td>\n","      <td>0.448238</td>\n","      <td>0.775062</td>\n","      <td>0.353792</td>\n","      <td>0.207203</td>\n","      <td>0.175645</td>\n","      <td>-68.502499</td>\n","      <td>0.286667</td>\n","      <td>0.84228</td>\n","      <td>0.329855</td>\n","      <td>939.036629</td>\n","      <td>29.0</td>\n","      <td>CVAE_mse</td>\n","    </tr>\n","    <tr>\n","      <th>29</th>\n","      <td>0.257153</td>\n","      <td>0.210358</td>\n","      <td>-2.905799</td>\n","      <td>0.453371</td>\n","      <td>0.773993</td>\n","      <td>0.350675</td>\n","      <td>0.207748</td>\n","      <td>0.175934</td>\n","      <td>-45.183902</td>\n","      <td>0.290852</td>\n","      <td>0.840483</td>\n","      <td>0.316575</td>\n","      <td>952.194528</td>\n","      <td>30.0</td>\n","      <td>CVAE_mse</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","    <div class=\"colab-df-buttons\">\n","\n","  <div class=\"colab-df-container\">\n","    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-11de29c4-bdff-4518-9a50-5283e537953d')\"\n","            title=\"Convert this dataframe to an interactive table.\"\n","            style=\"display:none;\">\n","\n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n","    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n","  </svg>\n","    </button>\n","\n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    .colab-df-buttons div {\n","      margin-bottom: 4px;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","    <script>\n","      const buttonEl =\n","        document.querySelector('#df-11de29c4-bdff-4518-9a50-5283e537953d button.colab-df-convert');\n","      buttonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","      async function convertToInteractive(key) {\n","        const element = document.querySelector('#df-11de29c4-bdff-4518-9a50-5283e537953d');\n","        const dataTable =\n","          await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                    [key], {});\n","        if (!dataTable) return;\n","\n","        const docLinkHtml = 'Like what you see? Visit the ' +\n","          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","          + ' to learn more about interactive tables.';\n","        element.innerHTML = '';\n","        dataTable['output_type'] = 'display_data';\n","        await google.colab.output.renderOutput(dataTable, element);\n","        const docLink = document.createElement('div');\n","        docLink.innerHTML = docLinkHtml;\n","        element.appendChild(docLink);\n","      }\n","    </script>\n","  </div>\n","\n","\n","<div id=\"df-0fb6258e-7c55-47a4-adc6-ddcf658ae28c\">\n","  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-0fb6258e-7c55-47a4-adc6-ddcf658ae28c')\"\n","            title=\"Suggest charts\"\n","            style=\"display:none;\">\n","\n","<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","     width=\"24px\">\n","    <g>\n","        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n","    </g>\n","</svg>\n","  </button>\n","\n","<style>\n","  .colab-df-quickchart {\n","      --bg-color: #E8F0FE;\n","      --fill-color: #1967D2;\n","      --hover-bg-color: #E2EBFA;\n","      --hover-fill-color: #174EA6;\n","      --disabled-fill-color: #AAA;\n","      --disabled-bg-color: #DDD;\n","  }\n","\n","  [theme=dark] .colab-df-quickchart {\n","      --bg-color: #3B4455;\n","      --fill-color: #D2E3FC;\n","      --hover-bg-color: #434B5C;\n","      --hover-fill-color: #FFFFFF;\n","      --disabled-bg-color: #3B4455;\n","      --disabled-fill-color: #666;\n","  }\n","\n","  .colab-df-quickchart {\n","    background-color: var(--bg-color);\n","    border: none;\n","    border-radius: 50%;\n","    cursor: pointer;\n","    display: none;\n","    fill: var(--fill-color);\n","    height: 32px;\n","    padding: 0;\n","    width: 32px;\n","  }\n","\n","  .colab-df-quickchart:hover {\n","    background-color: var(--hover-bg-color);\n","    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n","    fill: var(--button-hover-fill-color);\n","  }\n","\n","  .colab-df-quickchart-complete:disabled,\n","  .colab-df-quickchart-complete:disabled:hover {\n","    background-color: var(--disabled-bg-color);\n","    fill: var(--disabled-fill-color);\n","    box-shadow: none;\n","  }\n","\n","  .colab-df-spinner {\n","    border: 2px solid var(--fill-color);\n","    border-color: transparent;\n","    border-bottom-color: var(--fill-color);\n","    animation:\n","      spin 1s steps(1) infinite;\n","  }\n","\n","  @keyframes spin {\n","    0% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","      border-left-color: var(--fill-color);\n","    }\n","    20% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    30% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","      border-right-color: var(--fill-color);\n","    }\n","    40% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    60% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","    }\n","    80% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-bottom-color: var(--fill-color);\n","    }\n","    90% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","    }\n","  }\n","</style>\n","\n","  <script>\n","    async function quickchart(key) {\n","      const quickchartButtonEl =\n","        document.querySelector('#' + key + ' button');\n","      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n","      quickchartButtonEl.classList.add('colab-df-spinner');\n","      try {\n","        const charts = await google.colab.kernel.invokeFunction(\n","            'suggestCharts', [key], {});\n","      } catch (error) {\n","        console.error('Error during call to suggestCharts:', error);\n","      }\n","      quickchartButtonEl.classList.remove('colab-df-spinner');\n","      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n","    }\n","    (() => {\n","      let quickchartButtonEl =\n","        document.querySelector('#df-0fb6258e-7c55-47a4-adc6-ddcf658ae28c button');\n","      quickchartButtonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","    })();\n","  </script>\n","</div>\n","\n","  <div id=\"id_47b4642a-c1da-4f5c-ae6b-0bd60b175971\">\n","    <style>\n","      .colab-df-generate {\n","        background-color: #E8F0FE;\n","        border: none;\n","        border-radius: 50%;\n","        cursor: pointer;\n","        display: none;\n","        fill: #1967D2;\n","        height: 32px;\n","        padding: 0 0 0 0;\n","        width: 32px;\n","      }\n","\n","      .colab-df-generate:hover {\n","        background-color: #E2EBFA;\n","        box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","        fill: #174EA6;\n","      }\n","\n","      [theme=dark] .colab-df-generate {\n","        background-color: #3B4455;\n","        fill: #D2E3FC;\n","      }\n","\n","      [theme=dark] .colab-df-generate:hover {\n","        background-color: #434B5C;\n","        box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","        filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","        fill: #FFFFFF;\n","      }\n","    </style>\n","    <button class=\"colab-df-generate\" onclick=\"generateWithVariable('df_metrics')\"\n","            title=\"Generate code using this dataframe.\"\n","            style=\"display:none;\">\n","\n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M7,19H8.4L18.45,9,17,7.55,7,17.6ZM5,21V16.75L18.45,3.32a2,2,0,0,1,2.83,0l1.4,1.43a1.91,1.91,0,0,1,.58,1.4,1.91,1.91,0,0,1-.58,1.4L9.25,21ZM18.45,9,17,7.55Zm-12,3A5.31,5.31,0,0,0,4.9,8.1,5.31,5.31,0,0,0,1,6.5,5.31,5.31,0,0,0,4.9,4.9,5.31,5.31,0,0,0,6.5,1,5.31,5.31,0,0,0,8.1,4.9,5.31,5.31,0,0,0,12,6.5,5.46,5.46,0,0,0,6.5,12Z\"/>\n","  </svg>\n","    </button>\n","    <script>\n","      (() => {\n","      const buttonEl =\n","        document.querySelector('#id_47b4642a-c1da-4f5c-ae6b-0bd60b175971 button.colab-df-generate');\n","      buttonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","      buttonEl.onclick = () => {\n","        google.colab.notebook.generateWithVariable('df_metrics');\n","      }\n","      })();\n","    </script>\n","  </div>\n","\n","    </div>\n","  </div>\n"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"dataframe","variable_name":"df_metrics","summary":"{\n  \"name\": \"df_metrics\",\n  \"rows\": 30,\n  \"fields\": [\n    {\n      \"column\": \"rmse\",\n      \"properties\": {\n        \"dtype\": \"date\",\n        \"min\": 0.2552372521694933,\n        \"max\": 0.26141288991157685,\n        \"num_unique_values\": 30,\n        \"samples\": [\n          0.25766290367206607,\n          0.2552372521694933,\n          0.2601939457679112\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"mae\",\n      \"properties\": {\n        \"dtype\": \"date\",\n        \"min\": 0.20837500098092598,\n        \"max\": 0.2139757032615761,\n        \"num_unique_values\": 30,\n        \"samples\": [\n          0.21044201847285507,\n          0.20837500098092598,\n          0.21204109394545118\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"r2\",\n      \"properties\": {\n        \"dtype\": \"date\",\n        \"min\": -3.9761835697418517,\n        \"max\": -2.4018876602164694,\n        \"num_unique_values\": 30,\n        \"samples\": [\n          -2.7903300488982303,\n          -2.922585202326125,\n          -3.163655434229526\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"kl_divergence\",\n      \"properties\": {\n        \"dtype\": \"date\",\n        \"min\": 0.4437981922714198,\n        \"max\": 0.4608878693195827,\n        \"num_unique_values\": 30,\n        \"samples\": [\n          0.4535841183928365,\n          0.44748925346660573,\n          0.46083115601432084\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"cosine_similarity\",\n      \"properties\": {\n        \"dtype\": \"date\",\n        \"min\": 0.7681489333363887,\n        \"max\": 0.7784598954525852,\n        \"num_unique_values\": 30,\n        \"samples\": [\n          0.7734631485632224,\n          0.7762960373120332,\n          0.7681489333363887\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"pearson_correlation\",\n      \"properties\": {\n        \"dtype\": \"date\",\n        \"min\": 0.3408271526128195,\n        \"max\": 0.37065792520839036,\n        \"num_unique_values\": 30,\n        \"samples\": [\n          0.3571410202815119,\n          0.36262532978331574,\n          0.3408271526128195\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"rmse_daily_avg\",\n      \"properties\": {\n        \"dtype\": \"date\",\n        \"min\": 0.20511377890976448,\n        \"max\": 0.21103985937372538,\n        \"num_unique_values\": 30,\n        \"samples\": [\n          0.20607946756758808,\n          0.20511377890976448,\n          0.21010405282874878\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"mae_daily_avg\",\n      \"properties\": {\n        \"dtype\": \"date\",\n        \"min\": 0.17299815793703005,\n        \"max\": 0.17877663528433216,\n        \"num_unique_values\": 30,\n        \"samples\": [\n          0.17410345962785176,\n          0.17299815793703005,\n          0.17705369115611733\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"r2_daily_avg\",\n      \"properties\": {\n        \"dtype\": \"date\",\n        \"min\": -69.88377449794518,\n        \"max\": -33.60199280379133,\n        \"num_unique_values\": 30,\n        \"samples\": [\n          -38.08120832163936,\n          -45.774849923847015,\n          -45.36034811806801\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"kl_divergence_daily_avg\",\n      \"properties\": {\n        \"dtype\": \"date\",\n        \"min\": 0.28442957894471,\n        \"max\": 0.2956053920693863,\n        \"num_unique_values\": 30,\n        \"samples\": [\n          0.2869087648162251,\n          0.28579440866589606,\n          0.2956053920693863\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"cosine_similarity_daily_avg\",\n      \"properties\": {\n        \"dtype\": \"date\",\n        \"min\": 0.8357032016991339,\n        \"max\": 0.8441186692398862,\n        \"num_unique_values\": 30,\n        \"samples\": [\n          0.8430866542659158,\n          0.8431038342228296,\n          0.8357032016991339\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"pearson_correlation_daily_avg\",\n      \"properties\": {\n        \"dtype\": \"date\",\n        \"min\": 0.30605015847211586,\n        \"max\": 0.3464472991081006,\n        \"num_unique_values\": 30,\n        \"samples\": [\n          0.34060708123783384,\n          0.34275945386349244,\n          0.30605015847211586\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"fid_score\",\n      \"properties\": {\n        \"dtype\": \"date\",\n        \"min\": 922.9765406772386,\n        \"max\": 971.1960588751617,\n        \"num_unique_values\": 30,\n        \"samples\": [\n          954.7976549208882,\n          941.7445229389566,\n          961.7676828513117\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"images_seed\",\n      \"properties\": {\n        \"dtype\": \"date\",\n        \"min\": 1.0,\n        \"max\": 30.0,\n        \"num_unique_values\": 30,\n        \"samples\": [\n          28.0,\n          16.0,\n          24.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"model\",\n      \"properties\": {\n        \"dtype\": \"category\",\n        \"num_unique_values\": 1,\n        \"samples\": [\n          \"CVAE_mse\"\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"}},"metadata":{},"execution_count":12}],"source":["df_metrics = pd.concat(df_metrics, ignore_index=True)\n","df_metrics"]},{"cell_type":"code","execution_count":13,"metadata":{"id":"RtNXcng6Gz_H","executionInfo":{"status":"ok","timestamp":1711172801773,"user_tz":-480,"elapsed":4,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"}}},"outputs":[],"source":["generation_path = \"./evaluations\"\n","df_metrics.to_csv(os.path.join(generation_path, model_name+'_metrics.csv'),index=False)"]},{"cell_type":"code","execution_count":14,"metadata":{"id":"ODEBAmjCG0CD","colab":{"base_uri":"https://localhost:8080/","height":35},"executionInfo":{"status":"ok","timestamp":1711172801773,"user_tz":-480,"elapsed":4,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"}},"outputId":"62be6592-cda8-4317-8f3c-df07fb5c7ec0"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["'./evaluations/CVAE_mse_metrics.csv'"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"}},"metadata":{},"execution_count":14}],"source":["os.path.join(generation_path, model_name+'_metrics.csv')"]},{"cell_type":"code","execution_count":23,"metadata":{"id":"0BCfni6qG0KW","executionInfo":{"status":"ok","timestamp":1711173376396,"user_tz":-480,"elapsed":345,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"}},"colab":{"base_uri":"https://localhost:8080/","height":300},"outputId":"8635d363-660e-4616-c1c5-f0604c847d41"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["        fid_score  kl_divergence  pearson_correlation       rmse\n","count   30.000000      30.000000            30.000000  30.000000\n","mean   946.762909       0.453632             0.352599   0.258347\n","std     10.138314       0.003815             0.007058   0.001666\n","min    922.976541       0.443798             0.340827   0.255237\n","25%    941.970878       0.452053             0.347123   0.257226\n","50%    945.941029       0.453526             0.352538   0.258306\n","75%    953.187086       0.455622             0.355891   0.259432\n","max    971.196059       0.460888             0.370658   0.261413"],"text/html":["\n","  <div id=\"df-5326d05c-3608-41e2-b440-30e666fcb733\" class=\"colab-df-container\">\n","    <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>fid_score</th>\n","      <th>kl_divergence</th>\n","      <th>pearson_correlation</th>\n","      <th>rmse</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>count</th>\n","      <td>30.000000</td>\n","      <td>30.000000</td>\n","      <td>30.000000</td>\n","      <td>30.000000</td>\n","    </tr>\n","    <tr>\n","      <th>mean</th>\n","      <td>946.762909</td>\n","      <td>0.453632</td>\n","      <td>0.352599</td>\n","      <td>0.258347</td>\n","    </tr>\n","    <tr>\n","      <th>std</th>\n","      <td>10.138314</td>\n","      <td>0.003815</td>\n","      <td>0.007058</td>\n","      <td>0.001666</td>\n","    </tr>\n","    <tr>\n","      <th>min</th>\n","      <td>922.976541</td>\n","      <td>0.443798</td>\n","      <td>0.340827</td>\n","      <td>0.255237</td>\n","    </tr>\n","    <tr>\n","      <th>25%</th>\n","      <td>941.970878</td>\n","      <td>0.452053</td>\n","      <td>0.347123</td>\n","      <td>0.257226</td>\n","    </tr>\n","    <tr>\n","      <th>50%</th>\n","      <td>945.941029</td>\n","      <td>0.453526</td>\n","      <td>0.352538</td>\n","      <td>0.258306</td>\n","    </tr>\n","    <tr>\n","      <th>75%</th>\n","      <td>953.187086</td>\n","      <td>0.455622</td>\n","      <td>0.355891</td>\n","      <td>0.259432</td>\n","    </tr>\n","    <tr>\n","      <th>max</th>\n","      <td>971.196059</td>\n","      <td>0.460888</td>\n","      <td>0.370658</td>\n","      <td>0.261413</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","    <div class=\"colab-df-buttons\">\n","\n","  <div class=\"colab-df-container\">\n","    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-5326d05c-3608-41e2-b440-30e666fcb733')\"\n","            title=\"Convert this dataframe to an interactive table.\"\n","            style=\"display:none;\">\n","\n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n","    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n","  </svg>\n","    </button>\n","\n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    .colab-df-buttons div {\n","      margin-bottom: 4px;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","    <script>\n","      const buttonEl =\n","        document.querySelector('#df-5326d05c-3608-41e2-b440-30e666fcb733 button.colab-df-convert');\n","      buttonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","      async function convertToInteractive(key) {\n","        const element = document.querySelector('#df-5326d05c-3608-41e2-b440-30e666fcb733');\n","        const dataTable =\n","          await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                    [key], {});\n","        if (!dataTable) return;\n","\n","        const docLinkHtml = 'Like what you see? Visit the ' +\n","          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","          + ' to learn more about interactive tables.';\n","        element.innerHTML = '';\n","        dataTable['output_type'] = 'display_data';\n","        await google.colab.output.renderOutput(dataTable, element);\n","        const docLink = document.createElement('div');\n","        docLink.innerHTML = docLinkHtml;\n","        element.appendChild(docLink);\n","      }\n","    </script>\n","  </div>\n","\n","\n","<div id=\"df-17736f9e-6425-4a16-8173-911e3fbf8159\">\n","  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-17736f9e-6425-4a16-8173-911e3fbf8159')\"\n","            title=\"Suggest charts\"\n","            style=\"display:none;\">\n","\n","<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","     width=\"24px\">\n","    <g>\n","        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n","    </g>\n","</svg>\n","  </button>\n","\n","<style>\n","  .colab-df-quickchart {\n","      --bg-color: #E8F0FE;\n","      --fill-color: #1967D2;\n","      --hover-bg-color: #E2EBFA;\n","      --hover-fill-color: #174EA6;\n","      --disabled-fill-color: #AAA;\n","      --disabled-bg-color: #DDD;\n","  }\n","\n","  [theme=dark] .colab-df-quickchart {\n","      --bg-color: #3B4455;\n","      --fill-color: #D2E3FC;\n","      --hover-bg-color: #434B5C;\n","      --hover-fill-color: #FFFFFF;\n","      --disabled-bg-color: #3B4455;\n","      --disabled-fill-color: #666;\n","  }\n","\n","  .colab-df-quickchart {\n","    background-color: var(--bg-color);\n","    border: none;\n","    border-radius: 50%;\n","    cursor: pointer;\n","    display: none;\n","    fill: var(--fill-color);\n","    height: 32px;\n","    padding: 0;\n","    width: 32px;\n","  }\n","\n","  .colab-df-quickchart:hover {\n","    background-color: var(--hover-bg-color);\n","    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n","    fill: var(--button-hover-fill-color);\n","  }\n","\n","  .colab-df-quickchart-complete:disabled,\n","  .colab-df-quickchart-complete:disabled:hover {\n","    background-color: var(--disabled-bg-color);\n","    fill: var(--disabled-fill-color);\n","    box-shadow: none;\n","  }\n","\n","  .colab-df-spinner {\n","    border: 2px solid var(--fill-color);\n","    border-color: transparent;\n","    border-bottom-color: var(--fill-color);\n","    animation:\n","      spin 1s steps(1) infinite;\n","  }\n","\n","  @keyframes spin {\n","    0% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","      border-left-color: var(--fill-color);\n","    }\n","    20% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    30% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","      border-right-color: var(--fill-color);\n","    }\n","    40% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    60% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","    }\n","    80% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-bottom-color: var(--fill-color);\n","    }\n","    90% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","    }\n","  }\n","</style>\n","\n","  <script>\n","    async function quickchart(key) {\n","      const quickchartButtonEl =\n","        document.querySelector('#' + key + ' button');\n","      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n","      quickchartButtonEl.classList.add('colab-df-spinner');\n","      try {\n","        const charts = await google.colab.kernel.invokeFunction(\n","            'suggestCharts', [key], {});\n","      } catch (error) {\n","        console.error('Error during call to suggestCharts:', error);\n","      }\n","      quickchartButtonEl.classList.remove('colab-df-spinner');\n","      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n","    }\n","    (() => {\n","      let quickchartButtonEl =\n","        document.querySelector('#df-17736f9e-6425-4a16-8173-911e3fbf8159 button');\n","      quickchartButtonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","    })();\n","  </script>\n","</div>\n","\n","    </div>\n","  </div>\n"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"dataframe","summary":"{\n  \"name\": \"df_metrics\",\n  \"rows\": 8,\n  \"fields\": [\n    {\n      \"column\": \"fid_score\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 429.3253412550755,\n        \"min\": 10.138314265205176,\n        \"max\": 971.1960588751617,\n        \"num_unique_values\": 8,\n        \"samples\": [\n          946.7629089337129,\n          945.9410291395086,\n          30.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"kl_divergence\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 10.470234859521566,\n        \"min\": 0.0038146246285615513,\n        \"max\": 30.0,\n        \"num_unique_values\": 8,\n        \"samples\": [\n          0.45363151007832475,\n          0.4535261535363422,\n          30.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"pearson_correlation\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 10.499889783333337,\n        \"min\": 0.007058048044739086,\n        \"max\": 30.0,\n        \"num_unique_values\": 8,\n        \"samples\": [\n          0.35259862515941226,\n          0.3525384437675513,\n          30.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    },\n    {\n      \"column\": \"rmse\",\n      \"properties\": {\n        \"dtype\": \"number\",\n        \"std\": 10.528615931490796,\n        \"min\": 0.0016664405504814158,\n        \"max\": 30.0,\n        \"num_unique_values\": 8,\n        \"samples\": [\n          0.25834653748467523,\n          0.25830634903837,\n          30.0\n        ],\n        \"semantic_type\": \"\",\n        \"description\": \"\"\n      }\n    }\n  ]\n}"}},"metadata":{},"execution_count":23}],"source":["df_metrics.drop('model',axis=1).astype('float').describe()[['fid_score','kl_divergence','pearson_correlation','rmse']]"]},{"cell_type":"code","source":[],"metadata":{"id":"iiF8FIz_4XjI"},"execution_count":null,"outputs":[]}],"metadata":{"colab":{"provenance":[],"authorship_tag":"ABX9TyNuo0f20yOTPhJejcQ/LQfK"},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"name":"python"}},"nbformat":4,"nbformat_minor":0}