notebooks/4.2 Evaluation metrics of CGAN.ipynb

{"cells":[{"cell_type":"code","execution_count":1,"metadata":{"executionInfo":{"elapsed":2,"status":"ok","timestamp":1697896928326,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"},"user_tz":-480},"id":"BfcF5LB6snwi"},"outputs":[],"source":["model_name = 'CGAN'"]},{"cell_type":"code","execution_count":2,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":7034,"status":"ok","timestamp":1697896935358,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"},"user_tz":-480},"id":"a4xPxcPtGpza","outputId":"40646cc2-0a5c-4b52-8828-efee52f2751f"},"outputs":[{"output_type":"stream","name":"stdout","text":["Requirement already satisfied: scipy==1.11.1 in /usr/local/lib/python3.10/dist-packages (1.11.1)\n","Requirement already satisfied: numpy<1.28.0,>=1.21.6 in /usr/local/lib/python3.10/dist-packages (from scipy==1.11.1) (1.23.5)\n"]}],"source":["!pip install scipy==1.11.1"]},{"cell_type":"code","execution_count":3,"metadata":{"executionInfo":{"elapsed":5028,"status":"ok","timestamp":1697896940383,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"},"user_tz":-480},"id":"SxVKvknVGyOi"},"outputs":[],"source":["from keras.layers import Input, Dense, Conv2D, Flatten, Reshape, Conv2DTranspose\n","from keras.models import Model\n","from keras.datasets import mnist\n","from keras.utils import to_categorical\n","from keras import backend as K\n","from keras.callbacks import Callback\n","import tensorflow as tf\n","\n","from scipy.stats import entropy\n","from scipy.linalg import sqrtm\n","from scipy.stats import pearsonr\n","\n","from keras.applications.inception_v3 import InceptionV3\n","\n","import numpy as np\n","import pandas as pd\n","\n","import matplotlib.pyplot as plt\n","\n","from sklearn.preprocessing import LabelEncoder\n","\n","from tensorflow.keras.models import load_model\n","\n","import cv2\n","\n","from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score\n","from scipy.stats import entropy\n","from scipy.spatial.distance import cosine\n","\n","import tensorflow as tf\n","from tensorflow.keras.applications.inception_v3 import InceptionV3\n","from scipy.linalg import sqrtm"]},{"cell_type":"code","execution_count":4,"metadata":{"executionInfo":{"elapsed":4,"status":"ok","timestamp":1697896940383,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"},"user_tz":-480},"id":"NoC5qEIyGzOC"},"outputs":[],"source":["import random\n","\n","seed = 42\n","\n","# Python's built-in random library\n","random.seed(seed)\n","\n","# Scikit-learn (only affects some parts of scikit-learn)\n","from sklearn.utils import check_random_state\n","check_random_state(seed)\n","\n","np.random.seed(seed)\n","tf.random.set_seed(seed)\n","tf.keras.utils.set_random_seed(seed)"]},{"cell_type":"code","execution_count":5,"metadata":{"executionInfo":{"elapsed":4,"status":"ok","timestamp":1697896940383,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"},"user_tz":-480},"id":"XibW79oBGzpU"},"outputs":[],"source":["img_rows = 168\n","img_cols = 168\n","channels = 1\n","\n","# Input image dimensions\n","img_shape = (img_rows, img_cols, channels)"]},{"cell_type":"code","execution_count":6,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":2157,"status":"ok","timestamp":1697896942537,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"},"user_tz":-480},"id":"ba19WKsvGzrw","outputId":"313ab6fc-4f4d-4440-f908-a2cd1deb82f3"},"outputs":[{"output_type":"stream","name":"stdout","text":["Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount(\"/content/gdrive\", force_remount=True).\n"]}],"source":["# Set path for data source\n","import os\n","from google.colab import drive\n","drive.mount('/content/gdrive')\n","os.chdir(\"/content/gdrive/My Drive/Load Diffusion/dataset/\")"]},{"cell_type":"code","execution_count":7,"metadata":{"executionInfo":{"elapsed":1501,"status":"ok","timestamp":1697896944036,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"},"user_tz":-480},"id":"T3Q_l6x5Gzue"},"outputs":[],"source":["# Set the path to the folder containing saved files\n","load_path = \"./preprocessed_data_168x168_2years\"\n","\n","# Load saved NumPy arrays\n","X_train = np.load(os.path.join(load_path, 'X_train.npy'))\n","X_test = np.load(os.path.join(load_path, 'X_test.npy'))\n","y_train = np.load(os.path.join(load_path, 'y_train.npy'))\n","y_test = np.load(os.path.join(load_path, 'y_test.npy'))\n","\n","# Load saved Pandas DataFrames\n","train_data = pd.read_pickle(os.path.join(load_path, 'train_data.pkl'))\n","test_data = pd.read_pickle(os.path.join(load_path, 'test_data.pkl'))\n","metadata = pd.read_pickle(os.path.join(load_path, 'metadata.pkl'))\n","metadata_original = pd.read_pickle(os.path.join(load_path, 'metadata_original.pkl'))"]},{"cell_type":"code","execution_count":8,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":423},"executionInfo":{"elapsed":11,"status":"ok","timestamp":1697896944037,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"},"user_tz":-480},"id":"DoYSmVXKGzxJ","outputId":"1f2f3a41-c8de-4d09-d3e0-9c9f1c930edc"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["                                              lat         lng  year  \\\n","electricity_Lamb_office_Caitlin_2016    51.497838   -3.186246  2016   \n","electricity_Rat_public_Kelle_2016       38.903504  -77.005349  2016   \n","electricity_Rat_assembly_Ezequiel_2016  38.903504  -77.005349  2016   \n","electricity_Bear_assembly_Roxy_2016     37.871903 -122.260729  2016   \n","gas_Shrew_office_Rose_2017              51.499840   -0.124663  2017   \n","...                                           ...         ...   ...   \n","steam_Moose_education_Omar_2016         45.421500  -75.697200  2016   \n","electricity_Rat_public_Chrissy_2017     38.903504  -77.005349  2017   \n","electricity_Hog_office_Merilyn_2016     44.978782  -93.255398  2016   \n","steam_Bull_assembly_Amalia_2016         30.267200  -97.743100  2016   \n","electricity_Rat_office_Jeannie_2017     38.903504  -77.005349  2017   \n","\n","                                         meter_type  \\\n","electricity_Lamb_office_Caitlin_2016    electricity   \n","electricity_Rat_public_Kelle_2016       electricity   \n","electricity_Rat_assembly_Ezequiel_2016  electricity   \n","electricity_Bear_assembly_Roxy_2016     electricity   \n","gas_Shrew_office_Rose_2017                      gas   \n","...                                             ...   \n","steam_Moose_education_Omar_2016               steam   \n","electricity_Rat_public_Chrissy_2017     electricity   \n","electricity_Hog_office_Merilyn_2016     electricity   \n","steam_Bull_assembly_Amalia_2016               steam   \n","electricity_Rat_office_Jeannie_2017     electricity   \n","\n","                                                        building_type  \n","electricity_Lamb_office_Caitlin_2016                           Office  \n","electricity_Rat_public_Kelle_2016                     Public services  \n","electricity_Rat_assembly_Ezequiel_2016  Entertainment/public assembly  \n","electricity_Bear_assembly_Roxy_2016     Entertainment/public assembly  \n","gas_Shrew_office_Rose_2017                                     Office  \n","...                                                               ...  \n","steam_Moose_education_Omar_2016                             Education  \n","electricity_Rat_public_Chrissy_2017                   Public services  \n","electricity_Hog_office_Merilyn_2016                            Office  \n","steam_Bull_assembly_Amalia_2016         Entertainment/public assembly  \n","electricity_Rat_office_Jeannie_2017                            Office  \n","\n","[901 rows x 5 columns]"],"text/html":["\n","  <div id=\"df-50854eef-bb72-4e63-88b2-9db7178e0089\" class=\"colab-df-container\">\n","    <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>lat</th>\n","      <th>lng</th>\n","      <th>year</th>\n","      <th>meter_type</th>\n","      <th>building_type</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>electricity_Lamb_office_Caitlin_2016</th>\n","      <td>51.497838</td>\n","      <td>-3.186246</td>\n","      <td>2016</td>\n","      <td>electricity</td>\n","      <td>Office</td>\n","    </tr>\n","    <tr>\n","      <th>electricity_Rat_public_Kelle_2016</th>\n","      <td>38.903504</td>\n","      <td>-77.005349</td>\n","      <td>2016</td>\n","      <td>electricity</td>\n","      <td>Public services</td>\n","    </tr>\n","    <tr>\n","      <th>electricity_Rat_assembly_Ezequiel_2016</th>\n","      <td>38.903504</td>\n","      <td>-77.005349</td>\n","      <td>2016</td>\n","      <td>electricity</td>\n","      <td>Entertainment/public assembly</td>\n","    </tr>\n","    <tr>\n","      <th>electricity_Bear_assembly_Roxy_2016</th>\n","      <td>37.871903</td>\n","      <td>-122.260729</td>\n","      <td>2016</td>\n","      <td>electricity</td>\n","      <td>Entertainment/public assembly</td>\n","    </tr>\n","    <tr>\n","      <th>gas_Shrew_office_Rose_2017</th>\n","      <td>51.499840</td>\n","      <td>-0.124663</td>\n","      <td>2017</td>\n","      <td>gas</td>\n","      <td>Office</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>steam_Moose_education_Omar_2016</th>\n","      <td>45.421500</td>\n","      <td>-75.697200</td>\n","      <td>2016</td>\n","      <td>steam</td>\n","      <td>Education</td>\n","    </tr>\n","    <tr>\n","      <th>electricity_Rat_public_Chrissy_2017</th>\n","      <td>38.903504</td>\n","      <td>-77.005349</td>\n","      <td>2017</td>\n","      <td>electricity</td>\n","      <td>Public services</td>\n","    </tr>\n","    <tr>\n","      <th>electricity_Hog_office_Merilyn_2016</th>\n","      <td>44.978782</td>\n","      <td>-93.255398</td>\n","      <td>2016</td>\n","      <td>electricity</td>\n","      <td>Office</td>\n","    </tr>\n","    <tr>\n","      <th>steam_Bull_assembly_Amalia_2016</th>\n","      <td>30.267200</td>\n","      <td>-97.743100</td>\n","      <td>2016</td>\n","      <td>steam</td>\n","      <td>Entertainment/public assembly</td>\n","    </tr>\n","    <tr>\n","      <th>electricity_Rat_office_Jeannie_2017</th>\n","      <td>38.903504</td>\n","      <td>-77.005349</td>\n","      <td>2017</td>\n","      <td>electricity</td>\n","      <td>Office</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>901 rows × 5 columns</p>\n","</div>\n","    <div class=\"colab-df-buttons\">\n","\n","  <div class=\"colab-df-container\">\n","    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-50854eef-bb72-4e63-88b2-9db7178e0089')\"\n","            title=\"Convert this dataframe to an interactive table.\"\n","            style=\"display:none;\">\n","\n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n","    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n","  </svg>\n","    </button>\n","\n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    .colab-df-buttons div {\n","      margin-bottom: 4px;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","    <script>\n","      const buttonEl =\n","        document.querySelector('#df-50854eef-bb72-4e63-88b2-9db7178e0089 button.colab-df-convert');\n","      buttonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","      async function convertToInteractive(key) {\n","        const element = document.querySelector('#df-50854eef-bb72-4e63-88b2-9db7178e0089');\n","        const dataTable =\n","          await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                    [key], {});\n","        if (!dataTable) return;\n","\n","        const docLinkHtml = 'Like what you see? Visit the ' +\n","          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","          + ' to learn more about interactive tables.';\n","        element.innerHTML = '';\n","        dataTable['output_type'] = 'display_data';\n","        await google.colab.output.renderOutput(dataTable, element);\n","        const docLink = document.createElement('div');\n","        docLink.innerHTML = docLinkHtml;\n","        element.appendChild(docLink);\n","      }\n","    </script>\n","  </div>\n","\n","\n","<div id=\"df-58cc44de-7055-4c4f-a26f-19f6477af460\">\n","  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-58cc44de-7055-4c4f-a26f-19f6477af460')\"\n","            title=\"Suggest charts.\"\n","            style=\"display:none;\">\n","\n","<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","     width=\"24px\">\n","    <g>\n","        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n","    </g>\n","</svg>\n","  </button>\n","\n","<style>\n","  .colab-df-quickchart {\n","      --bg-color: #E8F0FE;\n","      --fill-color: #1967D2;\n","      --hover-bg-color: #E2EBFA;\n","      --hover-fill-color: #174EA6;\n","      --disabled-fill-color: #AAA;\n","      --disabled-bg-color: #DDD;\n","  }\n","\n","  [theme=dark] .colab-df-quickchart {\n","      --bg-color: #3B4455;\n","      --fill-color: #D2E3FC;\n","      --hover-bg-color: #434B5C;\n","      --hover-fill-color: #FFFFFF;\n","      --disabled-bg-color: #3B4455;\n","      --disabled-fill-color: #666;\n","  }\n","\n","  .colab-df-quickchart {\n","    background-color: var(--bg-color);\n","    border: none;\n","    border-radius: 50%;\n","    cursor: pointer;\n","    display: none;\n","    fill: var(--fill-color);\n","    height: 32px;\n","    padding: 0;\n","    width: 32px;\n","  }\n","\n","  .colab-df-quickchart:hover {\n","    background-color: var(--hover-bg-color);\n","    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n","    fill: var(--button-hover-fill-color);\n","  }\n","\n","  .colab-df-quickchart-complete:disabled,\n","  .colab-df-quickchart-complete:disabled:hover {\n","    background-color: var(--disabled-bg-color);\n","    fill: var(--disabled-fill-color);\n","    box-shadow: none;\n","  }\n","\n","  .colab-df-spinner {\n","    border: 2px solid var(--fill-color);\n","    border-color: transparent;\n","    border-bottom-color: var(--fill-color);\n","    animation:\n","      spin 1s steps(1) infinite;\n","  }\n","\n","  @keyframes spin {\n","    0% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","      border-left-color: var(--fill-color);\n","    }\n","    20% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    30% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","      border-right-color: var(--fill-color);\n","    }\n","    40% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    60% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","    }\n","    80% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-bottom-color: var(--fill-color);\n","    }\n","    90% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","    }\n","  }\n","</style>\n","\n","  <script>\n","    async function quickchart(key) {\n","      const quickchartButtonEl =\n","        document.querySelector('#' + key + ' button');\n","      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n","      quickchartButtonEl.classList.add('colab-df-spinner');\n","      try {\n","        const charts = await google.colab.kernel.invokeFunction(\n","            'suggestCharts', [key], {});\n","      } catch (error) {\n","        console.error('Error during call to suggestCharts:', error);\n","      }\n","      quickchartButtonEl.classList.remove('colab-df-spinner');\n","      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n","    }\n","    (() => {\n","      let quickchartButtonEl =\n","        document.querySelector('#df-58cc44de-7055-4c4f-a26f-19f6477af460 button');\n","      quickchartButtonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","    })();\n","  </script>\n","</div>\n","    </div>\n","  </div>\n"]},"metadata":{},"execution_count":8}],"source":["def merge_dummified_columns(df, dummified_columns, new_column_name):\n","    \"\"\"\n","    Merges dummified columns into a single column in a DataFrame.\n","\n","    Parameters:\n","        df (pd.DataFrame): The DataFrame containing the dummified columns.\n","        dummified_columns (list): A list of the names of the dummified columns.\n","        new_column_name (str): The name of the new column to create.\n","\n","    Returns:\n","        pd.DataFrame: The DataFrame with the dummified columns merged into a single new column.\n","    \"\"\"\n","    # Create a new column containing the column name where the value is 1 for each row\n","    df[new_column_name] = df[dummified_columns].idxmax(axis=1)\n","\n","    # Drop the dummified columns\n","    df.drop(dummified_columns, axis=1, inplace=True)\n","\n","    return df\n","\n","test_metadata = metadata_original.loc[test_data.columns]\n","\n","test_metadata = merge_dummified_columns(test_metadata,\n","                     dummified_columns=['chilledwater', 'electricity', 'gas', 'hotwater', 'steam'],\n","                     new_column_name='meter_type')\n","test_metadata = merge_dummified_columns(test_metadata,\n","                     dummified_columns=['Education', 'Entertainment/public assembly', 'Lodging/residential', 'Office', 'Public services'],\n","                     new_column_name='building_type')\n","\n","test_metadata\n"]},{"cell_type":"code","execution_count":9,"metadata":{"executionInfo":{"elapsed":8,"status":"ok","timestamp":1697896944037,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"},"user_tz":-480},"id":"E1i1UbW-G8-A"},"outputs":[],"source":["def calculate_rmse(true_data, predicted_data):\n","    return np.sqrt(mean_squared_error(true_data, predicted_data))\n","\n","def calculate_mae(true_data, predicted_data):\n","    return mean_absolute_error(true_data, predicted_data)\n","\n","def calculate_r2(true_data, predicted_data):\n","    return r2_score(true_data, predicted_data)\n","\n","def calculate_kl_divergence(true_data, predicted_data):\n","    return entropy(true_data+0.00000001, predicted_data+0.00000001)\n","\n","def calculate_cosine_similarity(true_data, predicted_data):\n","    return 1 - cosine(true_data, predicted_data)"]},{"cell_type":"code","execution_count":10,"metadata":{"executionInfo":{"elapsed":8,"status":"ok","timestamp":1697896944037,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"},"user_tz":-480},"id":"aslAFReUG9A1"},"outputs":[],"source":["def evaluate_model(test_data, generated_series):\n","    metrics_result = {\n","        'rmse': [],\n","        'mae': [],\n","        'r2': [],\n","        'kl_divergence': [],\n","        'cosine_similarity': [],\n","        'pearson_correlation': []\n","    }\n","\n","    for i in range(test_data.shape[1]):\n","        try:\n","            true_data = test_data[:, i]\n","            predicted_data = generated_series[:, i]\n","        except:\n","            true_data = test_data.values[:, i]\n","            predicted_data = generated_series.values[:, i]\n","\n","        # Calculate Pearson correlation\n","        pearson_corr, _ = pearsonr(true_data, predicted_data)\n","\n","        metrics_result['rmse'].append(calculate_rmse(true_data, predicted_data))\n","        metrics_result['mae'].append(calculate_mae(true_data, predicted_data))\n","        metrics_result['r2'].append(calculate_r2(true_data, predicted_data))\n","        metrics_result['kl_divergence'].append(calculate_kl_divergence(true_data, predicted_data))\n","        metrics_result['cosine_similarity'].append(calculate_cosine_similarity(true_data, predicted_data))\n","        metrics_result['pearson_correlation'].append(pearson_corr)\n","\n","    return metrics_result\n","\n","# Function to calculate FID between two sets of images\n","def calculate_fid(model, images1, images2):\n","    # Calculate activations\n","    act1 = model.predict(images1)\n","    act2 = model.predict(images2)\n","\n","    # Calculate mean and covariance statistics\n","    mu1, sigma1 = act1.mean(axis=0), np.cov(act1, rowvar=False)\n","    mu2, sigma2 = act2.mean(axis=0), np.cov(act2, rowvar=False)\n","\n","    # Calculate sum squared difference between means\n","    ssdiff = np.sum((mu1 - mu2)**2.0)\n","\n","    # Calculate sqrt of product between covariances\n","    covmean = sqrtm(sigma1.dot(sigma2))\n","\n","    # Check and correct imaginary numbers from sqrt\n","    if np.iscomplexobj(covmean):\n","        covmean = covmean.real\n","\n","    # Calculate the FID score\n","    fid = ssdiff + np.trace(sigma1 + sigma2 - 2.0 * covmean)\n","    return fid"]},{"cell_type":"code","execution_count":11,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"bmyXFGbTG9GI","executionInfo":{"status":"ok","timestamp":1697899132718,"user_tz":-480,"elapsed":2188689,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"}},"outputId":"43621d29-d35a-428b-8f95-0cf4ad2eb5d9"},"outputs":[{"output_type":"stream","name":"stdout","text":["29/29 [==============================] - 22s 634ms/step\n","29/29 [==============================] - 15s 488ms/step\n","FID: 816.5609881443559\n","29/29 [==============================] - 17s 525ms/step\n","29/29 [==============================] - 16s 549ms/step\n","FID: 828.7441077955718\n","29/29 [==============================] - 17s 486ms/step\n","29/29 [==============================] - 15s 499ms/step\n","FID: 804.033063834877\n","29/29 [==============================] - 17s 526ms/step\n","29/29 [==============================] - 18s 594ms/step\n","FID: 794.3444852236125\n","29/29 [==============================] - 17s 527ms/step\n","29/29 [==============================] - 15s 528ms/step\n","FID: 795.024499957359\n","29/29 [==============================] - 17s 509ms/step\n","29/29 [==============================] - 15s 524ms/step\n","FID: 810.3023341954656\n","29/29 [==============================] - 17s 528ms/step\n","29/29 [==============================] - 15s 527ms/step\n","FID: 780.0850411266354\n","29/29 [==============================] - 17s 527ms/step\n","29/29 [==============================] - 15s 527ms/step\n","FID: 815.7035633924581\n","29/29 [==============================] - 17s 525ms/step\n","29/29 [==============================] - 15s 488ms/step\n","FID: 823.4186599522883\n","29/29 [==============================] - 17s 525ms/step\n","29/29 [==============================] - 14s 489ms/step\n","FID: 810.3648228120348\n","29/29 [==============================] - 17s 530ms/step\n","29/29 [==============================] - 15s 496ms/step\n","FID: 811.4135611267211\n","29/29 [==============================] - 17s 525ms/step\n","29/29 [==============================] - 15s 527ms/step\n","FID: 805.8549966168322\n","29/29 [==============================] - 17s 522ms/step\n","29/29 [==============================] - 15s 523ms/step\n","FID: 805.0226114992504\n","29/29 [==============================] - 17s 526ms/step\n","29/29 [==============================] - 15s 526ms/step\n","FID: 805.8179148142485\n","29/29 [==============================] - 17s 524ms/step\n","29/29 [==============================] - 15s 534ms/step\n","FID: 793.5509156242863\n","29/29 [==============================] - 17s 520ms/step\n","29/29 [==============================] - 15s 523ms/step\n","FID: 798.2811971996067\n","29/29 [==============================] - 17s 485ms/step\n","29/29 [==============================] - 15s 524ms/step\n","FID: 798.9583629268566\n","29/29 [==============================] - 17s 525ms/step\n","29/29 [==============================] - 15s 523ms/step\n","FID: 802.6497862742659\n","29/29 [==============================] - 17s 493ms/step\n","29/29 [==============================] - 15s 532ms/step\n","FID: 800.7973314389081\n","29/29 [==============================] - 17s 505ms/step\n","29/29 [==============================] - 15s 523ms/step\n","FID: 790.4527080749575\n","29/29 [==============================] - 19s 628ms/step\n","29/29 [==============================] - 15s 525ms/step\n","FID: 816.7018654445397\n","29/29 [==============================] - 17s 523ms/step\n","29/29 [==============================] - 15s 508ms/step\n","FID: 829.3509716914677\n","29/29 [==============================] - 17s 489ms/step\n","29/29 [==============================] - 15s 525ms/step\n","FID: 822.9641108001339\n","29/29 [==============================] - 17s 524ms/step\n","29/29 [==============================] - 15s 525ms/step\n","FID: 820.3282303291685\n","29/29 [==============================] - 16s 481ms/step\n","29/29 [==============================] - 15s 523ms/step\n","FID: 837.214287461948\n","29/29 [==============================] - 18s 516ms/step\n","29/29 [==============================] - 15s 525ms/step\n","FID: 798.2415512500236\n","29/29 [==============================] - 17s 526ms/step\n","29/29 [==============================] - 15s 521ms/step\n","FID: 822.6979179024485\n","29/29 [==============================] - 17s 524ms/step\n","29/29 [==============================] - 15s 525ms/step\n","FID: 796.9718060469538\n","29/29 [==============================] - 17s 524ms/step\n","29/29 [==============================] - 15s 527ms/step\n","FID: 829.7821928233834\n","29/29 [==============================] - 17s 519ms/step\n","29/29 [==============================] - 15s 526ms/step\n","FID: 813.8682357148607\n"]}],"source":["# Specify the path where the datasets are saved\n","generation_path = \"./generation/\" + model_name + '_168x168'\n","\n","# Initialize an empty DataFrame to store metrics\n","df_metrics = []\n","\n","for file_name in os.listdir(generation_path):\n","  # Load the dataset\n","  generated_images = np.load(os.path.join(generation_path, file_name))\n","  images_seed = int(file_name.split('.')[0].split('_')[-1])\n","\n","  # Set the target shape\n","  target_shape = (generated_images.shape[0], 52, 168, 1)\n","\n","  # Initialize an array to hold the resized images\n","  resized_images = np.zeros(target_shape)\n","\n","  # Loop through each image to resize it\n","  for i in range(generated_images.shape[0]):\n","      resized_image = cv2.resize(generated_images[i], (168, 52))  # Note the dimensions are (width, height)\n","      resized_images[i] = np.expand_dims(resized_image, axis=-1)  # Add back the last dimension\n","\n","  # Assume resized_images has a shape of (905, 52, 168, 1)\n","  flattened_images = resized_images.reshape(generated_images.shape[0], -1).T  # Transpose to get shape (8736, generated_images.shape[0])\n","\n","  generated_series = pd.DataFrame(flattened_images)\n","  generated_series.columns = test_data.columns\n","\n","  generated_series = (generated_series-generated_series.min())/(generated_series.max()-generated_series.min())\n","  test_data = (test_data-test_data.min())/(test_data.max()-test_data.min())\n","\n","  test_data_resampled = test_data.rolling(24).mean()[23::24]\n","  generated_series_resampled = generated_series.rolling(24).mean()[23::24]\n","\n","  evaluation_results = evaluate_model(test_data, generated_series)\n","  evaluation_results_resampled = evaluate_model(test_data_resampled, generated_series_resampled)\n","\n","  df_evaluation_results = pd.DataFrame(evaluation_results)\n","  df_evaluation_results_resampled = pd.DataFrame(evaluation_results_resampled)\n","  df_evaluation_results_resampled.columns = df_evaluation_results_resampled.columns+'_daily_avg'\n","\n","  # Prepare the Inception v3 model\n","  model = InceptionV3(include_top=False, pooling='avg', weights=\"imagenet\", input_shape=(99,99,3))\n","\n","  # Convert to RGB and upscale to the necessary input size for the Inception model\n","  X_test_rgb = tf.image.grayscale_to_rgb(tf.convert_to_tensor(X_test))\n","  generated_images_rgb = tf.image.grayscale_to_rgb(tf.convert_to_tensor(generated_images))\n","\n","  X_test_upscaled = tf.image.resize(X_test_rgb, [99, 99])\n","  generated_images_upscaled = tf.image.resize(generated_images_rgb, [99, 99])\n","\n","  # Calculate FID\n","  fid_score = calculate_fid(model, X_test_upscaled, generated_images_upscaled)\n","  print('FID: '+str(fid_score))\n","\n","  metrics = pd.concat([df_evaluation_results.mean(), df_evaluation_results_resampled.mean()])\n","  metrics['fid_score'] = fid_score\n","  metrics['images_seed'] = images_seed\n","  metrics['model'] = model_name\n","  metrics = metrics.to_frame().T\n","\n","  # Append the metrics to df_metrics\n","  df_metrics.append(metrics)"]},{"cell_type":"code","execution_count":12,"metadata":{"id":"08yTJJqVGz8a","colab":{"base_uri":"https://localhost:8080/","height":0},"executionInfo":{"status":"ok","timestamp":1697899132718,"user_tz":-480,"elapsed":14,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"}},"outputId":"5ffb09af-7aa4-4020-8f83-b11d448ea0df"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["        rmse       mae        r2 kl_divergence cosine_similarity  \\\n","0   0.266936  0.208033 -2.523189      0.850772           0.74209   \n","1   0.270168  0.211327 -2.847209      0.838104          0.741545   \n","2   0.270502  0.211735 -3.419544      0.856902          0.741182   \n","3   0.268234  0.208878 -2.655214      0.868167          0.739723   \n","4   0.268515  0.208963 -3.958449      0.911734          0.738204   \n","5    0.27003  0.210401 -3.784898      0.829131          0.742261   \n","6   0.267931  0.208775 -2.846658      0.879963          0.741315   \n","7   0.269019  0.210196 -2.829386      0.861085          0.739189   \n","8   0.268254  0.208678   -3.0233       0.88826          0.740813   \n","9    0.26851  0.209622 -2.611975       0.84178          0.744327   \n","10  0.269823  0.210008 -3.956749      0.899395          0.738671   \n","11  0.265792  0.207144 -3.802975      0.876353          0.744335   \n","12  0.270707  0.210652 -2.489832      0.915677          0.737351   \n","13  0.268169  0.208671 -3.107453      0.864439          0.742355   \n","14  0.266897  0.206903 -3.318603      0.899924          0.739344   \n","15  0.268833  0.209573 -2.532921      0.923681           0.73837   \n","16  0.269653  0.210733 -3.941026      0.876163          0.738176   \n","17  0.268504  0.208852 -2.875692      0.894204          0.738822   \n","18  0.268082  0.208923 -3.336794       0.88316          0.740693   \n","19  0.267693  0.208217 -3.497649      0.851894          0.744682   \n","20  0.272993  0.213913 -3.539575      0.904677          0.733921   \n","21  0.269802   0.21077 -3.065866      0.892874          0.738592   \n","22  0.267653  0.207815 -3.088294      0.850029           0.74203   \n","23  0.268168  0.209237 -3.690577      0.859341          0.744065   \n","24  0.268658  0.209328 -3.661011      0.899833          0.740397   \n","25  0.266345  0.207422 -3.468162      0.849513          0.743235   \n","26  0.268026  0.209156 -2.890155      0.840219          0.744034   \n","27  0.267609  0.208262 -3.539071      0.902399          0.742533   \n","28  0.268606  0.209024 -2.952013      0.866785           0.73923   \n","29  0.269891  0.210227 -2.967591      0.884932          0.739611   \n","\n","   pearson_correlation rmse_daily_avg mae_daily_avg r2_daily_avg  \\\n","0             0.330211       0.216724      0.174694    -33.39359   \n","1             0.326234       0.219242      0.177088   -37.329998   \n","2             0.325497       0.221313      0.179113   -50.812885   \n","3             0.322036       0.217699      0.175464   -34.127746   \n","4              0.32768       0.218281      0.176051   -63.880142   \n","5             0.322693       0.219046      0.176848   -60.331425   \n","6             0.332882       0.217981      0.175808   -37.382554   \n","7             0.323222       0.219239      0.177289   -36.466857   \n","8             0.326878       0.217642      0.175212   -39.949966   \n","9             0.332006        0.21794      0.176004   -34.534976   \n","10            0.324567       0.219123      0.176697   -61.925336   \n","11              0.3369       0.214909      0.173148   -61.337831   \n","12             0.32654        0.22069      0.177572   -30.797188   \n","13            0.330016       0.217875       0.17495    -45.27849   \n","14            0.333721       0.215804      0.173007   -48.995064   \n","15            0.337602       0.218124      0.176112   -32.003306   \n","16             0.31935       0.219966      0.177889   -63.243954   \n","17            0.328861       0.217749      0.175652   -39.702225   \n","18            0.331284       0.217733      0.175674   -50.289411   \n","19            0.332095       0.216649      0.174263   -51.621888   \n","20            0.313815        0.22331      0.180819   -53.685614   \n","21            0.316559       0.220222      0.177546   -41.403274   \n","22            0.331925       0.217436        0.1748   -43.034843   \n","23            0.330384       0.217873      0.175583   -59.274763   \n","24            0.329138       0.217789       0.17591   -56.600256   \n","25            0.336025       0.216528      0.174026   -54.521451   \n","26             0.33668       0.217968      0.175693   -39.044474   \n","27            0.337895       0.216948      0.174527   -53.586153   \n","28            0.320746       0.218729      0.176244   -38.828809   \n","29            0.326067       0.219739      0.176893   -38.777038   \n","\n","   kl_divergence_daily_avg cosine_similarity_daily_avg  \\\n","0                   0.5162                    0.814148   \n","1                 0.509275                    0.815061   \n","2                 0.522303                    0.812586   \n","3                 0.527621                    0.812875   \n","4                 0.554089                    0.811172   \n","5                 0.495656                    0.815656   \n","6                 0.534758                    0.814475   \n","7                 0.524616                    0.811824   \n","8                 0.536607                    0.814722   \n","9                 0.514387                    0.816939   \n","10                0.547826                    0.812075   \n","11                0.535703                    0.816496   \n","12                0.558231                    0.810656   \n","13                  0.5229                    0.817474   \n","14                0.541067                     0.81279   \n","15                0.561357                    0.813314   \n","16                0.536425                    0.809919   \n","17                0.541333                     0.81222   \n","18                0.534255                    0.813682   \n","19                0.518247                      0.8182   \n","20                0.551176                    0.807025   \n","21                0.553401                    0.811237   \n","22                 0.51723                    0.814008   \n","23                0.519787                    0.816908   \n","24                0.544152                    0.814761   \n","25                0.511408                    0.816251   \n","26                0.510794                    0.817231   \n","27                0.546511                     0.81601   \n","28                0.528856                    0.811056   \n","29                0.539018                    0.812626   \n","\n","   pearson_correlation_daily_avg   fid_score images_seed model  \n","0                       0.323147  816.560988         1.0  CGAN  \n","1                       0.327309  828.744108         3.0  CGAN  \n","2                       0.314495  804.033064         2.0  CGAN  \n","3                       0.312971  794.344485         4.0  CGAN  \n","4                       0.324242    795.0245         6.0  CGAN  \n","5                       0.316689  810.302334         5.0  CGAN  \n","6                        0.32841  780.085041         7.0  CGAN  \n","7                       0.315429  815.703563         8.0  CGAN  \n","8                       0.322402   823.41866         9.0  CGAN  \n","9                       0.326439  810.364823        10.0  CGAN  \n","10                      0.320449  811.413561        11.0  CGAN  \n","11                      0.332463  805.854997        12.0  CGAN  \n","12                      0.322718  805.022611        13.0  CGAN  \n","13                      0.331629  805.817915        14.0  CGAN  \n","14                      0.328582  793.550916        15.0  CGAN  \n","15                      0.333134  798.281197        17.0  CGAN  \n","16                      0.310077  798.958363        16.0  CGAN  \n","17                      0.321315  802.649786        18.0  CGAN  \n","18                      0.320436  800.797331        19.0  CGAN  \n","19                      0.327729  790.452708        20.0  CGAN  \n","20                      0.300739  816.701865        21.0  CGAN  \n","21                      0.305406  829.350972        22.0  CGAN  \n","22                      0.323678  822.964111        23.0  CGAN  \n","23                      0.323935   820.32823        24.0  CGAN  \n","24                      0.327638  837.214287        25.0  CGAN  \n","25                      0.334694  798.241551        27.0  CGAN  \n","26                      0.334397  822.697918        26.0  CGAN  \n","27                        0.3394  796.971806        28.0  CGAN  \n","28                      0.308413  829.782193        29.0  CGAN  \n","29                      0.320345  813.868236        30.0  CGAN  "],"text/html":["\n","  <div id=\"df-d16e7f20-b60c-4f52-8abb-52eb88f2173e\" class=\"colab-df-container\">\n","    <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>rmse</th>\n","      <th>mae</th>\n","      <th>r2</th>\n","      <th>kl_divergence</th>\n","      <th>cosine_similarity</th>\n","      <th>pearson_correlation</th>\n","      <th>rmse_daily_avg</th>\n","      <th>mae_daily_avg</th>\n","      <th>r2_daily_avg</th>\n","      <th>kl_divergence_daily_avg</th>\n","      <th>cosine_similarity_daily_avg</th>\n","      <th>pearson_correlation_daily_avg</th>\n","      <th>fid_score</th>\n","      <th>images_seed</th>\n","      <th>model</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>0.266936</td>\n","      <td>0.208033</td>\n","      <td>-2.523189</td>\n","      <td>0.850772</td>\n","      <td>0.74209</td>\n","      <td>0.330211</td>\n","      <td>0.216724</td>\n","      <td>0.174694</td>\n","      <td>-33.39359</td>\n","      <td>0.5162</td>\n","      <td>0.814148</td>\n","      <td>0.323147</td>\n","      <td>816.560988</td>\n","      <td>1.0</td>\n","      <td>CGAN</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>0.270168</td>\n","      <td>0.211327</td>\n","      <td>-2.847209</td>\n","      <td>0.838104</td>\n","      <td>0.741545</td>\n","      <td>0.326234</td>\n","      <td>0.219242</td>\n","      <td>0.177088</td>\n","      <td>-37.329998</td>\n","      <td>0.509275</td>\n","      <td>0.815061</td>\n","      <td>0.327309</td>\n","      <td>828.744108</td>\n","      <td>3.0</td>\n","      <td>CGAN</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>0.270502</td>\n","      <td>0.211735</td>\n","      <td>-3.419544</td>\n","      <td>0.856902</td>\n","      <td>0.741182</td>\n","      <td>0.325497</td>\n","      <td>0.221313</td>\n","      <td>0.179113</td>\n","      <td>-50.812885</td>\n","      <td>0.522303</td>\n","      <td>0.812586</td>\n","      <td>0.314495</td>\n","      <td>804.033064</td>\n","      <td>2.0</td>\n","      <td>CGAN</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>0.268234</td>\n","      <td>0.208878</td>\n","      <td>-2.655214</td>\n","      <td>0.868167</td>\n","      <td>0.739723</td>\n","      <td>0.322036</td>\n","      <td>0.217699</td>\n","      <td>0.175464</td>\n","      <td>-34.127746</td>\n","      <td>0.527621</td>\n","      <td>0.812875</td>\n","      <td>0.312971</td>\n","      <td>794.344485</td>\n","      <td>4.0</td>\n","      <td>CGAN</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>0.268515</td>\n","      <td>0.208963</td>\n","      <td>-3.958449</td>\n","      <td>0.911734</td>\n","      <td>0.738204</td>\n","      <td>0.32768</td>\n","      <td>0.218281</td>\n","      <td>0.176051</td>\n","      <td>-63.880142</td>\n","      <td>0.554089</td>\n","      <td>0.811172</td>\n","      <td>0.324242</td>\n","      <td>795.0245</td>\n","      <td>6.0</td>\n","      <td>CGAN</td>\n","    </tr>\n","    <tr>\n","      <th>5</th>\n","      <td>0.27003</td>\n","      <td>0.210401</td>\n","      <td>-3.784898</td>\n","      <td>0.829131</td>\n","      <td>0.742261</td>\n","      <td>0.322693</td>\n","      <td>0.219046</td>\n","      <td>0.176848</td>\n","      <td>-60.331425</td>\n","      <td>0.495656</td>\n","      <td>0.815656</td>\n","      <td>0.316689</td>\n","      <td>810.302334</td>\n","      <td>5.0</td>\n","      <td>CGAN</td>\n","    </tr>\n","    <tr>\n","      <th>6</th>\n","      <td>0.267931</td>\n","      <td>0.208775</td>\n","      <td>-2.846658</td>\n","      <td>0.879963</td>\n","      <td>0.741315</td>\n","      <td>0.332882</td>\n","      <td>0.217981</td>\n","      <td>0.175808</td>\n","      <td>-37.382554</td>\n","      <td>0.534758</td>\n","      <td>0.814475</td>\n","      <td>0.32841</td>\n","      <td>780.085041</td>\n","      <td>7.0</td>\n","      <td>CGAN</td>\n","    </tr>\n","    <tr>\n","      <th>7</th>\n","      <td>0.269019</td>\n","      <td>0.210196</td>\n","      <td>-2.829386</td>\n","      <td>0.861085</td>\n","      <td>0.739189</td>\n","      <td>0.323222</td>\n","      <td>0.219239</td>\n","      <td>0.177289</td>\n","      <td>-36.466857</td>\n","      <td>0.524616</td>\n","      <td>0.811824</td>\n","      <td>0.315429</td>\n","      <td>815.703563</td>\n","      <td>8.0</td>\n","      <td>CGAN</td>\n","    </tr>\n","    <tr>\n","      <th>8</th>\n","      <td>0.268254</td>\n","      <td>0.208678</td>\n","      <td>-3.0233</td>\n","      <td>0.88826</td>\n","      <td>0.740813</td>\n","      <td>0.326878</td>\n","      <td>0.217642</td>\n","      <td>0.175212</td>\n","      <td>-39.949966</td>\n","      <td>0.536607</td>\n","      <td>0.814722</td>\n","      <td>0.322402</td>\n","      <td>823.41866</td>\n","      <td>9.0</td>\n","      <td>CGAN</td>\n","    </tr>\n","    <tr>\n","      <th>9</th>\n","      <td>0.26851</td>\n","      <td>0.209622</td>\n","      <td>-2.611975</td>\n","      <td>0.84178</td>\n","      <td>0.744327</td>\n","      <td>0.332006</td>\n","      <td>0.21794</td>\n","      <td>0.176004</td>\n","      <td>-34.534976</td>\n","      <td>0.514387</td>\n","      <td>0.816939</td>\n","      <td>0.326439</td>\n","      <td>810.364823</td>\n","      <td>10.0</td>\n","      <td>CGAN</td>\n","    </tr>\n","    <tr>\n","      <th>10</th>\n","      <td>0.269823</td>\n","      <td>0.210008</td>\n","      <td>-3.956749</td>\n","      <td>0.899395</td>\n","      <td>0.738671</td>\n","      <td>0.324567</td>\n","      <td>0.219123</td>\n","      <td>0.176697</td>\n","      <td>-61.925336</td>\n","      <td>0.547826</td>\n","      <td>0.812075</td>\n","      <td>0.320449</td>\n","      <td>811.413561</td>\n","      <td>11.0</td>\n","      <td>CGAN</td>\n","    </tr>\n","    <tr>\n","      <th>11</th>\n","      <td>0.265792</td>\n","      <td>0.207144</td>\n","      <td>-3.802975</td>\n","      <td>0.876353</td>\n","      <td>0.744335</td>\n","      <td>0.3369</td>\n","      <td>0.214909</td>\n","      <td>0.173148</td>\n","      <td>-61.337831</td>\n","      <td>0.535703</td>\n","      <td>0.816496</td>\n","      <td>0.332463</td>\n","      <td>805.854997</td>\n","      <td>12.0</td>\n","      <td>CGAN</td>\n","    </tr>\n","    <tr>\n","      <th>12</th>\n","      <td>0.270707</td>\n","      <td>0.210652</td>\n","      <td>-2.489832</td>\n","      <td>0.915677</td>\n","      <td>0.737351</td>\n","      <td>0.32654</td>\n","      <td>0.22069</td>\n","      <td>0.177572</td>\n","      <td>-30.797188</td>\n","      <td>0.558231</td>\n","      <td>0.810656</td>\n","      <td>0.322718</td>\n","      <td>805.022611</td>\n","      <td>13.0</td>\n","      <td>CGAN</td>\n","    </tr>\n","    <tr>\n","      <th>13</th>\n","      <td>0.268169</td>\n","      <td>0.208671</td>\n","      <td>-3.107453</td>\n","      <td>0.864439</td>\n","      <td>0.742355</td>\n","      <td>0.330016</td>\n","      <td>0.217875</td>\n","      <td>0.17495</td>\n","      <td>-45.27849</td>\n","      <td>0.5229</td>\n","      <td>0.817474</td>\n","      <td>0.331629</td>\n","      <td>805.817915</td>\n","      <td>14.0</td>\n","      <td>CGAN</td>\n","    </tr>\n","    <tr>\n","      <th>14</th>\n","      <td>0.266897</td>\n","      <td>0.206903</td>\n","      <td>-3.318603</td>\n","      <td>0.899924</td>\n","      <td>0.739344</td>\n","      <td>0.333721</td>\n","      <td>0.215804</td>\n","      <td>0.173007</td>\n","      <td>-48.995064</td>\n","      <td>0.541067</td>\n","      <td>0.81279</td>\n","      <td>0.328582</td>\n","      <td>793.550916</td>\n","      <td>15.0</td>\n","      <td>CGAN</td>\n","    </tr>\n","    <tr>\n","      <th>15</th>\n","      <td>0.268833</td>\n","      <td>0.209573</td>\n","      <td>-2.532921</td>\n","      <td>0.923681</td>\n","      <td>0.73837</td>\n","      <td>0.337602</td>\n","      <td>0.218124</td>\n","      <td>0.176112</td>\n","      <td>-32.003306</td>\n","      <td>0.561357</td>\n","      <td>0.813314</td>\n","      <td>0.333134</td>\n","      <td>798.281197</td>\n","      <td>17.0</td>\n","      <td>CGAN</td>\n","    </tr>\n","    <tr>\n","      <th>16</th>\n","      <td>0.269653</td>\n","      <td>0.210733</td>\n","      <td>-3.941026</td>\n","      <td>0.876163</td>\n","      <td>0.738176</td>\n","      <td>0.31935</td>\n","      <td>0.219966</td>\n","      <td>0.177889</td>\n","      <td>-63.243954</td>\n","      <td>0.536425</td>\n","      <td>0.809919</td>\n","      <td>0.310077</td>\n","      <td>798.958363</td>\n","      <td>16.0</td>\n","      <td>CGAN</td>\n","    </tr>\n","    <tr>\n","      <th>17</th>\n","      <td>0.268504</td>\n","      <td>0.208852</td>\n","      <td>-2.875692</td>\n","      <td>0.894204</td>\n","      <td>0.738822</td>\n","      <td>0.328861</td>\n","      <td>0.217749</td>\n","      <td>0.175652</td>\n","      <td>-39.702225</td>\n","      <td>0.541333</td>\n","      <td>0.81222</td>\n","      <td>0.321315</td>\n","      <td>802.649786</td>\n","      <td>18.0</td>\n","      <td>CGAN</td>\n","    </tr>\n","    <tr>\n","      <th>18</th>\n","      <td>0.268082</td>\n","      <td>0.208923</td>\n","      <td>-3.336794</td>\n","      <td>0.88316</td>\n","      <td>0.740693</td>\n","      <td>0.331284</td>\n","      <td>0.217733</td>\n","      <td>0.175674</td>\n","      <td>-50.289411</td>\n","      <td>0.534255</td>\n","      <td>0.813682</td>\n","      <td>0.320436</td>\n","      <td>800.797331</td>\n","      <td>19.0</td>\n","      <td>CGAN</td>\n","    </tr>\n","    <tr>\n","      <th>19</th>\n","      <td>0.267693</td>\n","      <td>0.208217</td>\n","      <td>-3.497649</td>\n","      <td>0.851894</td>\n","      <td>0.744682</td>\n","      <td>0.332095</td>\n","      <td>0.216649</td>\n","      <td>0.174263</td>\n","      <td>-51.621888</td>\n","      <td>0.518247</td>\n","      <td>0.8182</td>\n","      <td>0.327729</td>\n","      <td>790.452708</td>\n","      <td>20.0</td>\n","      <td>CGAN</td>\n","    </tr>\n","    <tr>\n","      <th>20</th>\n","      <td>0.272993</td>\n","      <td>0.213913</td>\n","      <td>-3.539575</td>\n","      <td>0.904677</td>\n","      <td>0.733921</td>\n","      <td>0.313815</td>\n","      <td>0.22331</td>\n","      <td>0.180819</td>\n","      <td>-53.685614</td>\n","      <td>0.551176</td>\n","      <td>0.807025</td>\n","      <td>0.300739</td>\n","      <td>816.701865</td>\n","      <td>21.0</td>\n","      <td>CGAN</td>\n","    </tr>\n","    <tr>\n","      <th>21</th>\n","      <td>0.269802</td>\n","      <td>0.21077</td>\n","      <td>-3.065866</td>\n","      <td>0.892874</td>\n","      <td>0.738592</td>\n","      <td>0.316559</td>\n","      <td>0.220222</td>\n","      <td>0.177546</td>\n","      <td>-41.403274</td>\n","      <td>0.553401</td>\n","      <td>0.811237</td>\n","      <td>0.305406</td>\n","      <td>829.350972</td>\n","      <td>22.0</td>\n","      <td>CGAN</td>\n","    </tr>\n","    <tr>\n","      <th>22</th>\n","      <td>0.267653</td>\n","      <td>0.207815</td>\n","      <td>-3.088294</td>\n","      <td>0.850029</td>\n","      <td>0.74203</td>\n","      <td>0.331925</td>\n","      <td>0.217436</td>\n","      <td>0.1748</td>\n","      <td>-43.034843</td>\n","      <td>0.51723</td>\n","      <td>0.814008</td>\n","      <td>0.323678</td>\n","      <td>822.964111</td>\n","      <td>23.0</td>\n","      <td>CGAN</td>\n","    </tr>\n","    <tr>\n","      <th>23</th>\n","      <td>0.268168</td>\n","      <td>0.209237</td>\n","      <td>-3.690577</td>\n","      <td>0.859341</td>\n","      <td>0.744065</td>\n","      <td>0.330384</td>\n","      <td>0.217873</td>\n","      <td>0.175583</td>\n","      <td>-59.274763</td>\n","      <td>0.519787</td>\n","      <td>0.816908</td>\n","      <td>0.323935</td>\n","      <td>820.32823</td>\n","      <td>24.0</td>\n","      <td>CGAN</td>\n","    </tr>\n","    <tr>\n","      <th>24</th>\n","      <td>0.268658</td>\n","      <td>0.209328</td>\n","      <td>-3.661011</td>\n","      <td>0.899833</td>\n","      <td>0.740397</td>\n","      <td>0.329138</td>\n","      <td>0.217789</td>\n","      <td>0.17591</td>\n","      <td>-56.600256</td>\n","      <td>0.544152</td>\n","      <td>0.814761</td>\n","      <td>0.327638</td>\n","      <td>837.214287</td>\n","      <td>25.0</td>\n","      <td>CGAN</td>\n","    </tr>\n","    <tr>\n","      <th>25</th>\n","      <td>0.266345</td>\n","      <td>0.207422</td>\n","      <td>-3.468162</td>\n","      <td>0.849513</td>\n","      <td>0.743235</td>\n","      <td>0.336025</td>\n","      <td>0.216528</td>\n","      <td>0.174026</td>\n","      <td>-54.521451</td>\n","      <td>0.511408</td>\n","      <td>0.816251</td>\n","      <td>0.334694</td>\n","      <td>798.241551</td>\n","      <td>27.0</td>\n","      <td>CGAN</td>\n","    </tr>\n","    <tr>\n","      <th>26</th>\n","      <td>0.268026</td>\n","      <td>0.209156</td>\n","      <td>-2.890155</td>\n","      <td>0.840219</td>\n","      <td>0.744034</td>\n","      <td>0.33668</td>\n","      <td>0.217968</td>\n","      <td>0.175693</td>\n","      <td>-39.044474</td>\n","      <td>0.510794</td>\n","      <td>0.817231</td>\n","      <td>0.334397</td>\n","      <td>822.697918</td>\n","      <td>26.0</td>\n","      <td>CGAN</td>\n","    </tr>\n","    <tr>\n","      <th>27</th>\n","      <td>0.267609</td>\n","      <td>0.208262</td>\n","      <td>-3.539071</td>\n","      <td>0.902399</td>\n","      <td>0.742533</td>\n","      <td>0.337895</td>\n","      <td>0.216948</td>\n","      <td>0.174527</td>\n","      <td>-53.586153</td>\n","      <td>0.546511</td>\n","      <td>0.81601</td>\n","      <td>0.3394</td>\n","      <td>796.971806</td>\n","      <td>28.0</td>\n","      <td>CGAN</td>\n","    </tr>\n","    <tr>\n","      <th>28</th>\n","      <td>0.268606</td>\n","      <td>0.209024</td>\n","      <td>-2.952013</td>\n","      <td>0.866785</td>\n","      <td>0.73923</td>\n","      <td>0.320746</td>\n","      <td>0.218729</td>\n","      <td>0.176244</td>\n","      <td>-38.828809</td>\n","      <td>0.528856</td>\n","      <td>0.811056</td>\n","      <td>0.308413</td>\n","      <td>829.782193</td>\n","      <td>29.0</td>\n","      <td>CGAN</td>\n","    </tr>\n","    <tr>\n","      <th>29</th>\n","      <td>0.269891</td>\n","      <td>0.210227</td>\n","      <td>-2.967591</td>\n","      <td>0.884932</td>\n","      <td>0.739611</td>\n","      <td>0.326067</td>\n","      <td>0.219739</td>\n","      <td>0.176893</td>\n","      <td>-38.777038</td>\n","      <td>0.539018</td>\n","      <td>0.812626</td>\n","      <td>0.320345</td>\n","      <td>813.868236</td>\n","      <td>30.0</td>\n","      <td>CGAN</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","    <div class=\"colab-df-buttons\">\n","\n","  <div class=\"colab-df-container\">\n","    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d16e7f20-b60c-4f52-8abb-52eb88f2173e')\"\n","            title=\"Convert this dataframe to an interactive table.\"\n","            style=\"display:none;\">\n","\n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n","    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n","  </svg>\n","    </button>\n","\n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    .colab-df-buttons div {\n","      margin-bottom: 4px;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","    <script>\n","      const buttonEl =\n","        document.querySelector('#df-d16e7f20-b60c-4f52-8abb-52eb88f2173e button.colab-df-convert');\n","      buttonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","      async function convertToInteractive(key) {\n","        const element = document.querySelector('#df-d16e7f20-b60c-4f52-8abb-52eb88f2173e');\n","        const dataTable =\n","          await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                    [key], {});\n","        if (!dataTable) return;\n","\n","        const docLinkHtml = 'Like what you see? Visit the ' +\n","          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","          + ' to learn more about interactive tables.';\n","        element.innerHTML = '';\n","        dataTable['output_type'] = 'display_data';\n","        await google.colab.output.renderOutput(dataTable, element);\n","        const docLink = document.createElement('div');\n","        docLink.innerHTML = docLinkHtml;\n","        element.appendChild(docLink);\n","      }\n","    </script>\n","  </div>\n","\n","\n","<div id=\"df-bed92b5b-4858-45c9-a579-a36746107dc0\">\n","  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-bed92b5b-4858-45c9-a579-a36746107dc0')\"\n","            title=\"Suggest charts.\"\n","            style=\"display:none;\">\n","\n","<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","     width=\"24px\">\n","    <g>\n","        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n","    </g>\n","</svg>\n","  </button>\n","\n","<style>\n","  .colab-df-quickchart {\n","      --bg-color: #E8F0FE;\n","      --fill-color: #1967D2;\n","      --hover-bg-color: #E2EBFA;\n","      --hover-fill-color: #174EA6;\n","      --disabled-fill-color: #AAA;\n","      --disabled-bg-color: #DDD;\n","  }\n","\n","  [theme=dark] .colab-df-quickchart {\n","      --bg-color: #3B4455;\n","      --fill-color: #D2E3FC;\n","      --hover-bg-color: #434B5C;\n","      --hover-fill-color: #FFFFFF;\n","      --disabled-bg-color: #3B4455;\n","      --disabled-fill-color: #666;\n","  }\n","\n","  .colab-df-quickchart {\n","    background-color: var(--bg-color);\n","    border: none;\n","    border-radius: 50%;\n","    cursor: pointer;\n","    display: none;\n","    fill: var(--fill-color);\n","    height: 32px;\n","    padding: 0;\n","    width: 32px;\n","  }\n","\n","  .colab-df-quickchart:hover {\n","    background-color: var(--hover-bg-color);\n","    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n","    fill: var(--button-hover-fill-color);\n","  }\n","\n","  .colab-df-quickchart-complete:disabled,\n","  .colab-df-quickchart-complete:disabled:hover {\n","    background-color: var(--disabled-bg-color);\n","    fill: var(--disabled-fill-color);\n","    box-shadow: none;\n","  }\n","\n","  .colab-df-spinner {\n","    border: 2px solid var(--fill-color);\n","    border-color: transparent;\n","    border-bottom-color: var(--fill-color);\n","    animation:\n","      spin 1s steps(1) infinite;\n","  }\n","\n","  @keyframes spin {\n","    0% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","      border-left-color: var(--fill-color);\n","    }\n","    20% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    30% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","      border-right-color: var(--fill-color);\n","    }\n","    40% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    60% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","    }\n","    80% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-bottom-color: var(--fill-color);\n","    }\n","    90% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","    }\n","  }\n","</style>\n","\n","  <script>\n","    async function quickchart(key) {\n","      const quickchartButtonEl =\n","        document.querySelector('#' + key + ' button');\n","      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n","      quickchartButtonEl.classList.add('colab-df-spinner');\n","      try {\n","        const charts = await google.colab.kernel.invokeFunction(\n","            'suggestCharts', [key], {});\n","      } catch (error) {\n","        console.error('Error during call to suggestCharts:', error);\n","      }\n","      quickchartButtonEl.classList.remove('colab-df-spinner');\n","      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n","    }\n","    (() => {\n","      let quickchartButtonEl =\n","        document.querySelector('#df-bed92b5b-4858-45c9-a579-a36746107dc0 button');\n","      quickchartButtonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","    })();\n","  </script>\n","</div>\n","    </div>\n","  </div>\n"]},"metadata":{},"execution_count":12}],"source":["df_metrics = pd.concat(df_metrics, ignore_index=True)\n","df_metrics"]},{"cell_type":"code","execution_count":13,"metadata":{"id":"RtNXcng6Gz_H","executionInfo":{"status":"ok","timestamp":1697899132718,"user_tz":-480,"elapsed":5,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"}}},"outputs":[],"source":["generation_path = \"./evaluations\"\n","df_metrics.to_csv(os.path.join(generation_path, model_name+'_metrics.csv'),index=False)"]},{"cell_type":"code","execution_count":14,"metadata":{"id":"ODEBAmjCG0CD","colab":{"base_uri":"https://localhost:8080/","height":0},"executionInfo":{"status":"ok","timestamp":1697899132718,"user_tz":-480,"elapsed":5,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"}},"outputId":"7277c7c0-5b4a-43ba-a36e-5ae014d9c056"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["'./evaluations/CGAN_metrics.csv'"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"}},"metadata":{},"execution_count":14}],"source":["os.path.join(generation_path, model_name+'_metrics.csv')"]},{"cell_type":"code","execution_count":14,"metadata":{"id":"0BCfni6qG0KW","executionInfo":{"status":"ok","timestamp":1697899132718,"user_tz":-480,"elapsed":5,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"}}},"outputs":[],"source":[]}],"metadata":{"colab":{"provenance":[],"authorship_tag":"ABX9TyNv0JcXcTiHhESTcIO1yLxu"},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"name":"python"}},"nbformat":4,"nbformat_minor":0}