notebooks/4.3 Evaluation metrics of Diffusion.ipynb

{"cells":[{"cell_type":"code","execution_count":1,"metadata":{"executionInfo":{"elapsed":3,"status":"ok","timestamp":1697943551387,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"},"user_tz":-480},"id":"BfcF5LB6snwi"},"outputs":[],"source":["model_name = 'diffusion_model'"]},{"cell_type":"code","execution_count":2,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":31978,"status":"ok","timestamp":1697943583363,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"},"user_tz":-480},"id":"a4xPxcPtGpza","outputId":"7f19998f-1fa2-4a57-8f10-b5c2f74f03d0"},"outputs":[{"output_type":"stream","name":"stdout","text":["Collecting scipy==1.11.1\n","  Downloading scipy-1.11.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (36.3 MB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m36.3/36.3 MB\u001b[0m \u001b[31m33.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: numpy<1.28.0,>=1.21.6 in /usr/local/lib/python3.10/dist-packages (from scipy==1.11.1) (1.23.5)\n","Installing collected packages: scipy\n","  Attempting uninstall: scipy\n","    Found existing installation: scipy 1.11.3\n","    Uninstalling scipy-1.11.3:\n","      Successfully uninstalled scipy-1.11.3\n","\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n","lida 0.0.10 requires fastapi, which is not installed.\n","lida 0.0.10 requires kaleido, which is not installed.\n","lida 0.0.10 requires python-multipart, which is not installed.\n","lida 0.0.10 requires uvicorn, which is not installed.\u001b[0m\u001b[31m\n","\u001b[0mSuccessfully installed scipy-1.11.1\n"]}],"source":["!pip install scipy==1.11.1"]},{"cell_type":"code","execution_count":3,"metadata":{"executionInfo":{"elapsed":9206,"status":"ok","timestamp":1697943592566,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"},"user_tz":-480},"id":"SxVKvknVGyOi"},"outputs":[],"source":["from keras.layers import Input, Dense, Conv2D, Flatten, Reshape, Conv2DTranspose\n","from keras.models import Model\n","from keras.datasets import mnist\n","from keras.utils import to_categorical\n","from keras import backend as K\n","from keras.callbacks import Callback\n","import tensorflow as tf\n","\n","from scipy.stats import entropy\n","from scipy.linalg import sqrtm\n","from scipy.stats import pearsonr\n","\n","from keras.applications.inception_v3 import InceptionV3\n","\n","import numpy as np\n","import pandas as pd\n","\n","import matplotlib.pyplot as plt\n","\n","from sklearn.preprocessing import LabelEncoder\n","\n","from tensorflow.keras.models import load_model\n","\n","import cv2\n","\n","from sklearn.metrics import mean_squared_error, mean_absolute_error, r2_score\n","from scipy.stats import entropy\n","from scipy.spatial.distance import cosine\n","\n","import tensorflow as tf\n","from tensorflow.keras.applications.inception_v3 import InceptionV3\n","from scipy.linalg import sqrtm"]},{"cell_type":"code","execution_count":4,"metadata":{"executionInfo":{"elapsed":4,"status":"ok","timestamp":1697943592566,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"},"user_tz":-480},"id":"NoC5qEIyGzOC"},"outputs":[],"source":["import random\n","\n","seed = 42\n","\n","# Python's built-in random library\n","random.seed(seed)\n","\n","# Scikit-learn (only affects some parts of scikit-learn)\n","from sklearn.utils import check_random_state\n","check_random_state(seed)\n","\n","np.random.seed(seed)\n","tf.random.set_seed(seed)\n","tf.keras.utils.set_random_seed(seed)"]},{"cell_type":"code","execution_count":5,"metadata":{"executionInfo":{"elapsed":3,"status":"ok","timestamp":1697943592566,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"},"user_tz":-480},"id":"XibW79oBGzpU"},"outputs":[],"source":["img_rows = 168\n","img_cols = 168\n","channels = 1\n","\n","# Input image dimensions\n","img_shape = (img_rows, img_cols, channels)"]},{"cell_type":"code","execution_count":6,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":21331,"status":"ok","timestamp":1697943613894,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"},"user_tz":-480},"id":"ba19WKsvGzrw","outputId":"4745c9fe-5636-4bba-ccdb-7be374ea7959"},"outputs":[{"output_type":"stream","name":"stdout","text":["Mounted at /content/gdrive\n"]}],"source":["# Set path for data source\n","import os\n","from google.colab import drive\n","drive.mount('/content/gdrive')\n","os.chdir(\"/content/gdrive/My Drive/Load Diffusion/dataset/\")"]},{"cell_type":"code","execution_count":7,"metadata":{"executionInfo":{"elapsed":17347,"status":"ok","timestamp":1697943631239,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"},"user_tz":-480},"id":"T3Q_l6x5Gzue"},"outputs":[],"source":["# Set the path to the folder containing saved files\n","load_path = \"./preprocessed_data_168x168_2years\"\n","\n","# Load saved NumPy arrays\n","X_train = np.load(os.path.join(load_path, 'X_train.npy'))\n","X_test = np.load(os.path.join(load_path, 'X_test.npy'))\n","y_train = np.load(os.path.join(load_path, 'y_train.npy'))\n","y_test = np.load(os.path.join(load_path, 'y_test.npy'))\n","\n","# Load saved Pandas DataFrames\n","train_data = pd.read_pickle(os.path.join(load_path, 'train_data.pkl'))\n","test_data = pd.read_pickle(os.path.join(load_path, 'test_data.pkl'))\n","metadata = pd.read_pickle(os.path.join(load_path, 'metadata.pkl'))\n","metadata_original = pd.read_pickle(os.path.join(load_path, 'metadata_original.pkl'))"]},{"cell_type":"code","execution_count":8,"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":424},"executionInfo":{"elapsed":5,"status":"ok","timestamp":1697943631239,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"},"user_tz":-480},"id":"DoYSmVXKGzxJ","outputId":"61080945-ffcc-40d9-f12f-22cb9367c6c6"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["                                              lat         lng  year  \\\n","electricity_Lamb_office_Caitlin_2016    51.497838   -3.186246  2016   \n","electricity_Rat_public_Kelle_2016       38.903504  -77.005349  2016   \n","electricity_Rat_assembly_Ezequiel_2016  38.903504  -77.005349  2016   \n","electricity_Bear_assembly_Roxy_2016     37.871903 -122.260729  2016   \n","gas_Shrew_office_Rose_2017              51.499840   -0.124663  2017   \n","...                                           ...         ...   ...   \n","steam_Moose_education_Omar_2016         45.421500  -75.697200  2016   \n","electricity_Rat_public_Chrissy_2017     38.903504  -77.005349  2017   \n","electricity_Hog_office_Merilyn_2016     44.978782  -93.255398  2016   \n","steam_Bull_assembly_Amalia_2016         30.267200  -97.743100  2016   \n","electricity_Rat_office_Jeannie_2017     38.903504  -77.005349  2017   \n","\n","                                         meter_type  \\\n","electricity_Lamb_office_Caitlin_2016    electricity   \n","electricity_Rat_public_Kelle_2016       electricity   \n","electricity_Rat_assembly_Ezequiel_2016  electricity   \n","electricity_Bear_assembly_Roxy_2016     electricity   \n","gas_Shrew_office_Rose_2017                      gas   \n","...                                             ...   \n","steam_Moose_education_Omar_2016               steam   \n","electricity_Rat_public_Chrissy_2017     electricity   \n","electricity_Hog_office_Merilyn_2016     electricity   \n","steam_Bull_assembly_Amalia_2016               steam   \n","electricity_Rat_office_Jeannie_2017     electricity   \n","\n","                                                        building_type  \n","electricity_Lamb_office_Caitlin_2016                           Office  \n","electricity_Rat_public_Kelle_2016                     Public services  \n","electricity_Rat_assembly_Ezequiel_2016  Entertainment/public assembly  \n","electricity_Bear_assembly_Roxy_2016     Entertainment/public assembly  \n","gas_Shrew_office_Rose_2017                                     Office  \n","...                                                               ...  \n","steam_Moose_education_Omar_2016                             Education  \n","electricity_Rat_public_Chrissy_2017                   Public services  \n","electricity_Hog_office_Merilyn_2016                            Office  \n","steam_Bull_assembly_Amalia_2016         Entertainment/public assembly  \n","electricity_Rat_office_Jeannie_2017                            Office  \n","\n","[901 rows x 5 columns]"],"text/html":["\n","  <div id=\"df-a91db81a-02bb-41f7-91f7-8497aeb8de3a\" class=\"colab-df-container\">\n","    <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>lat</th>\n","      <th>lng</th>\n","      <th>year</th>\n","      <th>meter_type</th>\n","      <th>building_type</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>electricity_Lamb_office_Caitlin_2016</th>\n","      <td>51.497838</td>\n","      <td>-3.186246</td>\n","      <td>2016</td>\n","      <td>electricity</td>\n","      <td>Office</td>\n","    </tr>\n","    <tr>\n","      <th>electricity_Rat_public_Kelle_2016</th>\n","      <td>38.903504</td>\n","      <td>-77.005349</td>\n","      <td>2016</td>\n","      <td>electricity</td>\n","      <td>Public services</td>\n","    </tr>\n","    <tr>\n","      <th>electricity_Rat_assembly_Ezequiel_2016</th>\n","      <td>38.903504</td>\n","      <td>-77.005349</td>\n","      <td>2016</td>\n","      <td>electricity</td>\n","      <td>Entertainment/public assembly</td>\n","    </tr>\n","    <tr>\n","      <th>electricity_Bear_assembly_Roxy_2016</th>\n","      <td>37.871903</td>\n","      <td>-122.260729</td>\n","      <td>2016</td>\n","      <td>electricity</td>\n","      <td>Entertainment/public assembly</td>\n","    </tr>\n","    <tr>\n","      <th>gas_Shrew_office_Rose_2017</th>\n","      <td>51.499840</td>\n","      <td>-0.124663</td>\n","      <td>2017</td>\n","      <td>gas</td>\n","      <td>Office</td>\n","    </tr>\n","    <tr>\n","      <th>...</th>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","      <td>...</td>\n","    </tr>\n","    <tr>\n","      <th>steam_Moose_education_Omar_2016</th>\n","      <td>45.421500</td>\n","      <td>-75.697200</td>\n","      <td>2016</td>\n","      <td>steam</td>\n","      <td>Education</td>\n","    </tr>\n","    <tr>\n","      <th>electricity_Rat_public_Chrissy_2017</th>\n","      <td>38.903504</td>\n","      <td>-77.005349</td>\n","      <td>2017</td>\n","      <td>electricity</td>\n","      <td>Public services</td>\n","    </tr>\n","    <tr>\n","      <th>electricity_Hog_office_Merilyn_2016</th>\n","      <td>44.978782</td>\n","      <td>-93.255398</td>\n","      <td>2016</td>\n","      <td>electricity</td>\n","      <td>Office</td>\n","    </tr>\n","    <tr>\n","      <th>steam_Bull_assembly_Amalia_2016</th>\n","      <td>30.267200</td>\n","      <td>-97.743100</td>\n","      <td>2016</td>\n","      <td>steam</td>\n","      <td>Entertainment/public assembly</td>\n","    </tr>\n","    <tr>\n","      <th>electricity_Rat_office_Jeannie_2017</th>\n","      <td>38.903504</td>\n","      <td>-77.005349</td>\n","      <td>2017</td>\n","      <td>electricity</td>\n","      <td>Office</td>\n","    </tr>\n","  </tbody>\n","</table>\n","<p>901 rows × 5 columns</p>\n","</div>\n","    <div class=\"colab-df-buttons\">\n","\n","  <div class=\"colab-df-container\">\n","    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a91db81a-02bb-41f7-91f7-8497aeb8de3a')\"\n","            title=\"Convert this dataframe to an interactive table.\"\n","            style=\"display:none;\">\n","\n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n","    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n","  </svg>\n","    </button>\n","\n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    .colab-df-buttons div {\n","      margin-bottom: 4px;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","    <script>\n","      const buttonEl =\n","        document.querySelector('#df-a91db81a-02bb-41f7-91f7-8497aeb8de3a button.colab-df-convert');\n","      buttonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","      async function convertToInteractive(key) {\n","        const element = document.querySelector('#df-a91db81a-02bb-41f7-91f7-8497aeb8de3a');\n","        const dataTable =\n","          await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                    [key], {});\n","        if (!dataTable) return;\n","\n","        const docLinkHtml = 'Like what you see? Visit the ' +\n","          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","          + ' to learn more about interactive tables.';\n","        element.innerHTML = '';\n","        dataTable['output_type'] = 'display_data';\n","        await google.colab.output.renderOutput(dataTable, element);\n","        const docLink = document.createElement('div');\n","        docLink.innerHTML = docLinkHtml;\n","        element.appendChild(docLink);\n","      }\n","    </script>\n","  </div>\n","\n","\n","<div id=\"df-310f31ae-48d0-487b-92bb-4df4decedddd\">\n","  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-310f31ae-48d0-487b-92bb-4df4decedddd')\"\n","            title=\"Suggest charts.\"\n","            style=\"display:none;\">\n","\n","<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","     width=\"24px\">\n","    <g>\n","        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n","    </g>\n","</svg>\n","  </button>\n","\n","<style>\n","  .colab-df-quickchart {\n","      --bg-color: #E8F0FE;\n","      --fill-color: #1967D2;\n","      --hover-bg-color: #E2EBFA;\n","      --hover-fill-color: #174EA6;\n","      --disabled-fill-color: #AAA;\n","      --disabled-bg-color: #DDD;\n","  }\n","\n","  [theme=dark] .colab-df-quickchart {\n","      --bg-color: #3B4455;\n","      --fill-color: #D2E3FC;\n","      --hover-bg-color: #434B5C;\n","      --hover-fill-color: #FFFFFF;\n","      --disabled-bg-color: #3B4455;\n","      --disabled-fill-color: #666;\n","  }\n","\n","  .colab-df-quickchart {\n","    background-color: var(--bg-color);\n","    border: none;\n","    border-radius: 50%;\n","    cursor: pointer;\n","    display: none;\n","    fill: var(--fill-color);\n","    height: 32px;\n","    padding: 0;\n","    width: 32px;\n","  }\n","\n","  .colab-df-quickchart:hover {\n","    background-color: var(--hover-bg-color);\n","    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n","    fill: var(--button-hover-fill-color);\n","  }\n","\n","  .colab-df-quickchart-complete:disabled,\n","  .colab-df-quickchart-complete:disabled:hover {\n","    background-color: var(--disabled-bg-color);\n","    fill: var(--disabled-fill-color);\n","    box-shadow: none;\n","  }\n","\n","  .colab-df-spinner {\n","    border: 2px solid var(--fill-color);\n","    border-color: transparent;\n","    border-bottom-color: var(--fill-color);\n","    animation:\n","      spin 1s steps(1) infinite;\n","  }\n","\n","  @keyframes spin {\n","    0% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","      border-left-color: var(--fill-color);\n","    }\n","    20% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    30% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","      border-right-color: var(--fill-color);\n","    }\n","    40% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    60% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","    }\n","    80% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-bottom-color: var(--fill-color);\n","    }\n","    90% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","    }\n","  }\n","</style>\n","\n","  <script>\n","    async function quickchart(key) {\n","      const quickchartButtonEl =\n","        document.querySelector('#' + key + ' button');\n","      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n","      quickchartButtonEl.classList.add('colab-df-spinner');\n","      try {\n","        const charts = await google.colab.kernel.invokeFunction(\n","            'suggestCharts', [key], {});\n","      } catch (error) {\n","        console.error('Error during call to suggestCharts:', error);\n","      }\n","      quickchartButtonEl.classList.remove('colab-df-spinner');\n","      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n","    }\n","    (() => {\n","      let quickchartButtonEl =\n","        document.querySelector('#df-310f31ae-48d0-487b-92bb-4df4decedddd button');\n","      quickchartButtonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","    })();\n","  </script>\n","</div>\n","    </div>\n","  </div>\n"]},"metadata":{},"execution_count":8}],"source":["def merge_dummified_columns(df, dummified_columns, new_column_name):\n","    \"\"\"\n","    Merges dummified columns into a single column in a DataFrame.\n","\n","    Parameters:\n","        df (pd.DataFrame): The DataFrame containing the dummified columns.\n","        dummified_columns (list): A list of the names of the dummified columns.\n","        new_column_name (str): The name of the new column to create.\n","\n","    Returns:\n","        pd.DataFrame: The DataFrame with the dummified columns merged into a single new column.\n","    \"\"\"\n","    # Create a new column containing the column name where the value is 1 for each row\n","    df[new_column_name] = df[dummified_columns].idxmax(axis=1)\n","\n","    # Drop the dummified columns\n","    df.drop(dummified_columns, axis=1, inplace=True)\n","\n","    return df\n","\n","test_metadata = metadata_original.loc[test_data.columns]\n","\n","test_metadata = merge_dummified_columns(test_metadata,\n","                     dummified_columns=['chilledwater', 'electricity', 'gas', 'hotwater', 'steam'],\n","                     new_column_name='meter_type')\n","test_metadata = merge_dummified_columns(test_metadata,\n","                     dummified_columns=['Education', 'Entertainment/public assembly', 'Lodging/residential', 'Office', 'Public services'],\n","                     new_column_name='building_type')\n","\n","test_metadata\n"]},{"cell_type":"code","execution_count":9,"metadata":{"executionInfo":{"elapsed":3,"status":"ok","timestamp":1697943631239,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"},"user_tz":-480},"id":"E1i1UbW-G8-A"},"outputs":[],"source":["def calculate_rmse(true_data, predicted_data):\n","    return np.sqrt(mean_squared_error(true_data, predicted_data))\n","\n","def calculate_mae(true_data, predicted_data):\n","    return mean_absolute_error(true_data, predicted_data)\n","\n","def calculate_r2(true_data, predicted_data):\n","    return r2_score(true_data, predicted_data)\n","\n","def calculate_kl_divergence(true_data, predicted_data):\n","    return entropy(true_data+0.00000001, predicted_data+0.00000001)\n","\n","def calculate_cosine_similarity(true_data, predicted_data):\n","    return 1 - cosine(true_data, predicted_data)"]},{"cell_type":"code","execution_count":10,"metadata":{"executionInfo":{"elapsed":517,"status":"ok","timestamp":1697943631754,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"},"user_tz":-480},"id":"aslAFReUG9A1"},"outputs":[],"source":["def evaluate_model(test_data, generated_series):\n","    metrics_result = {\n","        'rmse': [],\n","        'mae': [],\n","        'r2': [],\n","        'kl_divergence': [],\n","        'cosine_similarity': [],\n","        'pearson_correlation': []\n","    }\n","\n","    for i in range(test_data.shape[1]):\n","        try:\n","            true_data = test_data[:, i]\n","            predicted_data = generated_series[:, i]\n","        except:\n","            true_data = test_data.values[:, i]\n","            predicted_data = generated_series.values[:, i]\n","\n","        # Calculate Pearson correlation\n","        pearson_corr, _ = pearsonr(true_data, predicted_data)\n","\n","        metrics_result['rmse'].append(calculate_rmse(true_data, predicted_data))\n","        metrics_result['mae'].append(calculate_mae(true_data, predicted_data))\n","        metrics_result['r2'].append(calculate_r2(true_data, predicted_data))\n","        metrics_result['kl_divergence'].append(calculate_kl_divergence(true_data, predicted_data))\n","        metrics_result['cosine_similarity'].append(calculate_cosine_similarity(true_data, predicted_data))\n","        metrics_result['pearson_correlation'].append(pearson_corr)\n","\n","    return metrics_result\n","\n","# Function to calculate FID between two sets of images\n","def calculate_fid(model, images1, images2):\n","    # Calculate activations\n","    act1 = model.predict(images1)\n","    act2 = model.predict(images2)\n","\n","    # Calculate mean and covariance statistics\n","    mu1, sigma1 = act1.mean(axis=0), np.cov(act1, rowvar=False)\n","    mu2, sigma2 = act2.mean(axis=0), np.cov(act2, rowvar=False)\n","\n","    # Calculate sum squared difference between means\n","    ssdiff = np.sum((mu1 - mu2)**2.0)\n","\n","    # Calculate sqrt of product between covariances\n","    covmean = sqrtm(sigma1.dot(sigma2))\n","\n","    # Check and correct imaginary numbers from sqrt\n","    if np.iscomplexobj(covmean):\n","        covmean = covmean.real\n","\n","    # Calculate the FID score\n","    fid = ssdiff + np.trace(sigma1 + sigma2 - 2.0 * covmean)\n","    return fid"]},{"cell_type":"code","execution_count":11,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"bmyXFGbTG9GI","executionInfo":{"status":"ok","timestamp":1697946054442,"user_tz":-480,"elapsed":2422690,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"}},"outputId":"f460dab5-20ac-415b-ee9e-8b1eda0a05c5"},"outputs":[{"output_type":"stream","name":"stdout","text":["Downloading data from https://storage.googleapis.com/tensorflow/keras-applications/inception_v3/inception_v3_weights_tf_dim_ordering_tf_kernels_notop.h5\n","87910968/87910968 [==============================] - 1s 0us/step\n","29/29 [==============================] - 18s 557ms/step\n","29/29 [==============================] - 16s 555ms/step\n","FID: 517.7867001449742\n","29/29 [==============================] - 19s 562ms/step\n","29/29 [==============================] - 17s 576ms/step\n","FID: 512.1445150587934\n","29/29 [==============================] - 18s 548ms/step\n","29/29 [==============================] - 17s 573ms/step\n","FID: 516.6871320211224\n","29/29 [==============================] - 17s 547ms/step\n","29/29 [==============================] - 16s 542ms/step\n","FID: 513.6530691368752\n","29/29 [==============================] - 17s 545ms/step\n","29/29 [==============================] - 15s 534ms/step\n","FID: 512.779852066052\n","29/29 [==============================] - 18s 524ms/step\n","29/29 [==============================] - 15s 517ms/step\n","FID: 514.199088209566\n","29/29 [==============================] - 17s 550ms/step\n","29/29 [==============================] - 16s 548ms/step\n","FID: 521.3273984629789\n","29/29 [==============================] - 18s 519ms/step\n","29/29 [==============================] - 16s 521ms/step\n","FID: 512.2663605654045\n","29/29 [==============================] - 18s 523ms/step\n","29/29 [==============================] - 16s 545ms/step\n","FID: 515.3954691276849\n","29/29 [==============================] - 18s 519ms/step\n","29/29 [==============================] - 16s 546ms/step\n","FID: 528.0308168698559\n","29/29 [==============================] - 18s 549ms/step\n","29/29 [==============================] - 16s 554ms/step\n","FID: 522.3458433276011\n","29/29 [==============================] - 18s 537ms/step\n","29/29 [==============================] - 16s 546ms/step\n","FID: 524.4910823117362\n","29/29 [==============================] - 18s 517ms/step\n","29/29 [==============================] - 17s 597ms/step\n","FID: 515.2691007517745\n","29/29 [==============================] - 18s 547ms/step\n","29/29 [==============================] - 16s 547ms/step\n","FID: 513.724849623421\n","29/29 [==============================] - 17s 546ms/step\n","29/29 [==============================] - 16s 526ms/step\n","FID: 512.8448112373051\n","29/29 [==============================] - 17s 545ms/step\n","29/29 [==============================] - 16s 544ms/step\n","FID: 514.4795202128702\n","29/29 [==============================] - 17s 517ms/step\n","29/29 [==============================] - 16s 546ms/step\n","FID: 512.7547983340579\n","29/29 [==============================] - 17s 544ms/step\n","29/29 [==============================] - 16s 550ms/step\n","FID: 511.4417850623332\n","29/29 [==============================] - 17s 546ms/step\n","29/29 [==============================] - 16s 544ms/step\n","FID: 514.1464978103455\n","29/29 [==============================] - 20s 532ms/step\n","29/29 [==============================] - 16s 546ms/step\n","FID: 520.9541282705319\n","29/29 [==============================] - 18s 539ms/step\n","29/29 [==============================] - 16s 551ms/step\n","FID: 530.514411022435\n","29/29 [==============================] - 18s 554ms/step\n","29/29 [==============================] - 15s 519ms/step\n","FID: 511.9864315224865\n","29/29 [==============================] - 18s 547ms/step\n","29/29 [==============================] - 16s 548ms/step\n","FID: 524.7380753328418\n","29/29 [==============================] - 18s 548ms/step\n","29/29 [==============================] - 16s 549ms/step\n","FID: 513.6604835337291\n","29/29 [==============================] - 17s 546ms/step\n","29/29 [==============================] - 16s 544ms/step\n","FID: 523.3750387908341\n","29/29 [==============================] - 18s 543ms/step\n","29/29 [==============================] - 16s 543ms/step\n","FID: 524.1414459998848\n","29/29 [==============================] - 18s 545ms/step\n","29/29 [==============================] - 16s 547ms/step\n","FID: 523.488949329248\n","29/29 [==============================] - 17s 543ms/step\n","29/29 [==============================] - 16s 544ms/step\n","FID: 510.7262082755045\n","29/29 [==============================] - 18s 511ms/step\n","29/29 [==============================] - 15s 511ms/step\n","FID: 519.0475150848454\n","29/29 [==============================] - 18s 547ms/step\n","29/29 [==============================] - 16s 546ms/step\n","FID: 510.11447739711156\n"]}],"source":["# Specify the path where the datasets are saved\n","generation_path = \"./generation/\" + model_name + '_168x168'\n","\n","# Initialize an empty DataFrame to store metrics\n","df_metrics = []\n","\n","for file_name in os.listdir(generation_path):\n","  # Load the dataset\n","  generated_images = np.load(os.path.join(generation_path, file_name))\n","  images_seed = int(file_name.split('.')[0].split('_')[-1])\n","\n","  # Set the target shape\n","  target_shape = (generated_images.shape[0], 52, 168, 1)\n","\n","  # Initialize an array to hold the resized images\n","  resized_images = np.zeros(target_shape)\n","\n","  # Loop through each image to resize it\n","  for i in range(generated_images.shape[0]):\n","      resized_image = cv2.resize(generated_images[i], (168, 52))  # Note the dimensions are (width, height)\n","      resized_images[i] = np.expand_dims(resized_image, axis=-1)  # Add back the last dimension\n","\n","  # Assume resized_images has a shape of (905, 52, 168, 1)\n","  flattened_images = resized_images.reshape(generated_images.shape[0], -1).T  # Transpose to get shape (8736, generated_images.shape[0])\n","\n","  generated_series = pd.DataFrame(flattened_images)\n","  generated_series.columns = test_data.columns\n","\n","  generated_series = (generated_series-generated_series.min())/(generated_series.max()-generated_series.min())\n","  test_data = (test_data-test_data.min())/(test_data.max()-test_data.min())\n","\n","  test_data_resampled = test_data.rolling(24).mean()[23::24]\n","  generated_series_resampled = generated_series.rolling(24).mean()[23::24]\n","\n","  evaluation_results = evaluate_model(test_data, generated_series)\n","  evaluation_results_resampled = evaluate_model(test_data_resampled, generated_series_resampled)\n","\n","  df_evaluation_results = pd.DataFrame(evaluation_results)\n","  df_evaluation_results_resampled = pd.DataFrame(evaluation_results_resampled)\n","  df_evaluation_results_resampled.columns = df_evaluation_results_resampled.columns+'_daily_avg'\n","\n","  # Prepare the Inception v3 model\n","  model = InceptionV3(include_top=False, pooling='avg', weights=\"imagenet\", input_shape=(99,99,3))\n","\n","  # Convert to RGB and upscale to the necessary input size for the Inception model\n","  X_test_rgb = tf.image.grayscale_to_rgb(tf.convert_to_tensor(X_test))\n","  generated_images_rgb = tf.image.grayscale_to_rgb(tf.convert_to_tensor(generated_images))\n","\n","  X_test_upscaled = tf.image.resize(X_test_rgb, [99, 99])\n","  generated_images_upscaled = tf.image.resize(generated_images_rgb, [99, 99])\n","\n","  # Calculate FID\n","  fid_score = calculate_fid(model, X_test_upscaled, generated_images_upscaled)\n","  print('FID: '+str(fid_score))\n","\n","  metrics = pd.concat([df_evaluation_results.mean(), df_evaluation_results_resampled.mean()])\n","  metrics['fid_score'] = fid_score\n","  metrics['images_seed'] = images_seed\n","  metrics['model'] = model_name\n","  metrics = metrics.to_frame().T\n","\n","  # Append the metrics to df_metrics\n","  df_metrics.append(metrics)"]},{"cell_type":"code","execution_count":12,"metadata":{"id":"08yTJJqVGz8a","colab":{"base_uri":"https://localhost:8080/","height":1000},"executionInfo":{"status":"ok","timestamp":1697946054443,"user_tz":-480,"elapsed":13,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"}},"outputId":"e7fd1027-4dbe-4550-fcee-eab43ea8a78a"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["        rmse       mae        r2 kl_divergence cosine_similarity  \\\n","0   0.244876  0.200829 -3.438515      0.394846          0.806622   \n","1   0.245811  0.201123  -3.96638      0.400144          0.805375   \n","2   0.246997  0.202186 -3.231512      0.404612          0.803461   \n","3   0.246654  0.201918 -3.380347      0.404015          0.803037   \n","4   0.244766  0.200014 -3.196604      0.403005          0.804793   \n","5   0.244916  0.200764 -2.982125      0.399948          0.805831   \n","6   0.247307  0.202898 -3.142238      0.409961          0.801153   \n","7   0.246576   0.20224 -3.568715      0.404886          0.802713   \n","8   0.244471  0.199887 -3.110968      0.397083          0.805793   \n","9   0.244184   0.19936 -3.315773      0.401162          0.805245   \n","10  0.244177  0.199676 -3.024157      0.399259          0.806127   \n","11  0.245217   0.20042 -3.372237      0.398506          0.806286   \n","12  0.244733  0.200496 -3.578113      0.396743          0.808027   \n","13  0.244137  0.199964 -3.245593      0.399345           0.80611   \n","14  0.244143  0.199839  -3.23188      0.400473          0.806432   \n","15  0.244473  0.200616 -3.242585      0.396766           0.80642   \n","16   0.24507  0.200815 -3.410064      0.402601          0.803131   \n","17  0.245966  0.201677 -3.453844      0.404201          0.804262   \n","18  0.244626  0.200576 -2.923437      0.398196          0.806992   \n","19  0.244139  0.199711 -3.147514       0.40077           0.80535   \n","20  0.249893  0.205706 -3.079636      0.403738          0.803107   \n","21  0.245406  0.200846 -2.719608      0.398802          0.805715   \n","22   0.24693  0.202294 -3.301648      0.400613           0.80436   \n","23   0.24468  0.200422 -3.198893      0.399562          0.805834   \n","24  0.245428  0.201178 -3.120564      0.403024          0.804267   \n","25  0.246752   0.20219 -3.792168      0.403128          0.803305   \n","26  0.244206  0.200007 -2.949504      0.394871          0.808302   \n","27  0.242458  0.198017 -2.603606      0.395574          0.807679   \n","28  0.245789   0.20193 -2.994601      0.401432          0.805223   \n","29  0.245781  0.201424 -3.489873      0.399742          0.805516   \n","\n","   pearson_correlation rmse_daily_avg mae_daily_avg r2_daily_avg  \\\n","0             0.429623       0.200504      0.169569   -54.486312   \n","1              0.42258       0.201191      0.169687   -72.161198   \n","2             0.418322       0.203059      0.171561   -55.462353   \n","3             0.421451       0.202299      0.170656   -58.520661   \n","4             0.425432       0.200813       0.16941   -55.773813   \n","5             0.433424       0.200371      0.169467   -47.093638   \n","6             0.416386       0.202877       0.17166   -52.374453   \n","7             0.411928       0.202871      0.171678   -59.444721   \n","8             0.425788        0.20004      0.168665    -51.50081   \n","9             0.427535       0.200422      0.168582    -54.96043   \n","10            0.429203       0.199702      0.168319   -46.390869   \n","11            0.432197       0.201196      0.169709   -58.129681   \n","12            0.432537       0.200271      0.169166   -64.471481   \n","13            0.429292       0.199789       0.16881   -56.184076   \n","14            0.423963       0.200042      0.168883    -56.11982   \n","15            0.426764       0.199897      0.169167   -55.273213   \n","16            0.420236       0.201022      0.169809   -59.296351   \n","17            0.422317       0.201618      0.170283   -61.039848   \n","18             0.43425        0.20154      0.170527   -48.447568   \n","19            0.424226       0.199207      0.168242    -53.37313   \n","20            0.416578       0.206053      0.175139    -51.14419   \n","21            0.425523       0.201117      0.169542   -41.894575   \n","22            0.426758       0.203649      0.172542   -57.054606   \n","23            0.423342       0.200421      0.169134   -54.196927   \n","24            0.423859       0.200545      0.169301   -51.422472   \n","25             0.42013       0.203157      0.171768   -67.855957   \n","26            0.430474       0.200392      0.169257     -49.6078   \n","27            0.433339       0.197741      0.166903   -37.539905   \n","28            0.424648       0.201802      0.170911   -49.272898   \n","29            0.425333       0.201128      0.170185   -60.222573   \n","\n","   kl_divergence_daily_avg cosine_similarity_daily_avg  \\\n","0                  0.24564                    0.869799   \n","1                 0.249009                    0.868746   \n","2                 0.252931                    0.866385   \n","3                 0.254024                    0.865242   \n","4                 0.254909                    0.866135   \n","5                 0.249365                    0.868678   \n","6                 0.257968                    0.863645   \n","7                  0.25584                    0.863837   \n","8                 0.246662                    0.868643   \n","9                 0.252259                    0.866971   \n","10                 0.24946                    0.868819   \n","11                0.250779                    0.867287   \n","12                0.244964                    0.871182   \n","13                0.248571                    0.869005   \n","14                0.249464                    0.868689   \n","15                 0.24763                    0.869021   \n","16                0.251655                    0.865749   \n","17                0.252848                    0.866787   \n","18                0.249336                    0.868735   \n","19                0.250624                    0.867621   \n","20                0.254401                    0.865555   \n","21                0.248442                    0.868298   \n","22                0.251758                    0.866031   \n","23                0.249211                    0.868297   \n","24                0.251187                     0.86723   \n","25                0.252861                    0.865448   \n","26                0.245069                    0.870545   \n","27                0.246473                    0.869662   \n","28                0.249359                    0.868468   \n","29                0.249987                    0.867478   \n","\n","   pearson_correlation_daily_avg   fid_score images_seed            model  \n","0                       0.458994    517.7867         1.0  diffusion_model  \n","1                       0.452031  512.144515         2.0  diffusion_model  \n","2                       0.440564  516.687132         3.0  diffusion_model  \n","3                       0.440642  513.653069         4.0  diffusion_model  \n","4                       0.450104  512.779852         5.0  diffusion_model  \n","5                       0.459755  514.199088         6.0  diffusion_model  \n","6                       0.437406  521.327398         7.0  diffusion_model  \n","7                       0.427003  512.266361         8.0  diffusion_model  \n","8                        0.44877  515.395469         9.0  diffusion_model  \n","9                       0.452416  528.030817        10.0  diffusion_model  \n","10                      0.455039  522.345843        11.0  diffusion_model  \n","11                      0.453304  524.491082        12.0  diffusion_model  \n","12                      0.458298  515.269101        13.0  diffusion_model  \n","13                      0.457502   513.72485        14.0  diffusion_model  \n","14                      0.443886  512.844811        15.0  diffusion_model  \n","15                      0.450814   514.47952        16.0  diffusion_model  \n","16                      0.438239  512.754798        17.0  diffusion_model  \n","17                      0.445947  511.441785        18.0  diffusion_model  \n","18                      0.455676  514.146498        19.0  diffusion_model  \n","19                      0.449198  520.954128        20.0  diffusion_model  \n","20                      0.440538  530.514411        21.0  diffusion_model  \n","21                      0.445009  511.986432        23.0  diffusion_model  \n","22                      0.449146  524.738075        24.0  diffusion_model  \n","23                      0.444062  513.660484        25.0  diffusion_model  \n","24                      0.448604  523.375039        26.0  diffusion_model  \n","25                      0.437262  524.141446        27.0  diffusion_model  \n","26                      0.454003  523.488949        28.0  diffusion_model  \n","27                      0.457445  510.726208        29.0  diffusion_model  \n","28                      0.450887  519.047515        30.0  diffusion_model  \n","29                      0.449862  510.114477        22.0  diffusion_model  "],"text/html":["\n","  <div id=\"df-27f882fa-48b9-421e-ac10-2b44cc815bd3\" class=\"colab-df-container\">\n","    <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>rmse</th>\n","      <th>mae</th>\n","      <th>r2</th>\n","      <th>kl_divergence</th>\n","      <th>cosine_similarity</th>\n","      <th>pearson_correlation</th>\n","      <th>rmse_daily_avg</th>\n","      <th>mae_daily_avg</th>\n","      <th>r2_daily_avg</th>\n","      <th>kl_divergence_daily_avg</th>\n","      <th>cosine_similarity_daily_avg</th>\n","      <th>pearson_correlation_daily_avg</th>\n","      <th>fid_score</th>\n","      <th>images_seed</th>\n","      <th>model</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>0.244876</td>\n","      <td>0.200829</td>\n","      <td>-3.438515</td>\n","      <td>0.394846</td>\n","      <td>0.806622</td>\n","      <td>0.429623</td>\n","      <td>0.200504</td>\n","      <td>0.169569</td>\n","      <td>-54.486312</td>\n","      <td>0.24564</td>\n","      <td>0.869799</td>\n","      <td>0.458994</td>\n","      <td>517.7867</td>\n","      <td>1.0</td>\n","      <td>diffusion_model</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>0.245811</td>\n","      <td>0.201123</td>\n","      <td>-3.96638</td>\n","      <td>0.400144</td>\n","      <td>0.805375</td>\n","      <td>0.42258</td>\n","      <td>0.201191</td>\n","      <td>0.169687</td>\n","      <td>-72.161198</td>\n","      <td>0.249009</td>\n","      <td>0.868746</td>\n","      <td>0.452031</td>\n","      <td>512.144515</td>\n","      <td>2.0</td>\n","      <td>diffusion_model</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>0.246997</td>\n","      <td>0.202186</td>\n","      <td>-3.231512</td>\n","      <td>0.404612</td>\n","      <td>0.803461</td>\n","      <td>0.418322</td>\n","      <td>0.203059</td>\n","      <td>0.171561</td>\n","      <td>-55.462353</td>\n","      <td>0.252931</td>\n","      <td>0.866385</td>\n","      <td>0.440564</td>\n","      <td>516.687132</td>\n","      <td>3.0</td>\n","      <td>diffusion_model</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>0.246654</td>\n","      <td>0.201918</td>\n","      <td>-3.380347</td>\n","      <td>0.404015</td>\n","      <td>0.803037</td>\n","      <td>0.421451</td>\n","      <td>0.202299</td>\n","      <td>0.170656</td>\n","      <td>-58.520661</td>\n","      <td>0.254024</td>\n","      <td>0.865242</td>\n","      <td>0.440642</td>\n","      <td>513.653069</td>\n","      <td>4.0</td>\n","      <td>diffusion_model</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>0.244766</td>\n","      <td>0.200014</td>\n","      <td>-3.196604</td>\n","      <td>0.403005</td>\n","      <td>0.804793</td>\n","      <td>0.425432</td>\n","      <td>0.200813</td>\n","      <td>0.16941</td>\n","      <td>-55.773813</td>\n","      <td>0.254909</td>\n","      <td>0.866135</td>\n","      <td>0.450104</td>\n","      <td>512.779852</td>\n","      <td>5.0</td>\n","      <td>diffusion_model</td>\n","    </tr>\n","    <tr>\n","      <th>5</th>\n","      <td>0.244916</td>\n","      <td>0.200764</td>\n","      <td>-2.982125</td>\n","      <td>0.399948</td>\n","      <td>0.805831</td>\n","      <td>0.433424</td>\n","      <td>0.200371</td>\n","      <td>0.169467</td>\n","      <td>-47.093638</td>\n","      <td>0.249365</td>\n","      <td>0.868678</td>\n","      <td>0.459755</td>\n","      <td>514.199088</td>\n","      <td>6.0</td>\n","      <td>diffusion_model</td>\n","    </tr>\n","    <tr>\n","      <th>6</th>\n","      <td>0.247307</td>\n","      <td>0.202898</td>\n","      <td>-3.142238</td>\n","      <td>0.409961</td>\n","      <td>0.801153</td>\n","      <td>0.416386</td>\n","      <td>0.202877</td>\n","      <td>0.17166</td>\n","      <td>-52.374453</td>\n","      <td>0.257968</td>\n","      <td>0.863645</td>\n","      <td>0.437406</td>\n","      <td>521.327398</td>\n","      <td>7.0</td>\n","      <td>diffusion_model</td>\n","    </tr>\n","    <tr>\n","      <th>7</th>\n","      <td>0.246576</td>\n","      <td>0.20224</td>\n","      <td>-3.568715</td>\n","      <td>0.404886</td>\n","      <td>0.802713</td>\n","      <td>0.411928</td>\n","      <td>0.202871</td>\n","      <td>0.171678</td>\n","      <td>-59.444721</td>\n","      <td>0.25584</td>\n","      <td>0.863837</td>\n","      <td>0.427003</td>\n","      <td>512.266361</td>\n","      <td>8.0</td>\n","      <td>diffusion_model</td>\n","    </tr>\n","    <tr>\n","      <th>8</th>\n","      <td>0.244471</td>\n","      <td>0.199887</td>\n","      <td>-3.110968</td>\n","      <td>0.397083</td>\n","      <td>0.805793</td>\n","      <td>0.425788</td>\n","      <td>0.20004</td>\n","      <td>0.168665</td>\n","      <td>-51.50081</td>\n","      <td>0.246662</td>\n","      <td>0.868643</td>\n","      <td>0.44877</td>\n","      <td>515.395469</td>\n","      <td>9.0</td>\n","      <td>diffusion_model</td>\n","    </tr>\n","    <tr>\n","      <th>9</th>\n","      <td>0.244184</td>\n","      <td>0.19936</td>\n","      <td>-3.315773</td>\n","      <td>0.401162</td>\n","      <td>0.805245</td>\n","      <td>0.427535</td>\n","      <td>0.200422</td>\n","      <td>0.168582</td>\n","      <td>-54.96043</td>\n","      <td>0.252259</td>\n","      <td>0.866971</td>\n","      <td>0.452416</td>\n","      <td>528.030817</td>\n","      <td>10.0</td>\n","      <td>diffusion_model</td>\n","    </tr>\n","    <tr>\n","      <th>10</th>\n","      <td>0.244177</td>\n","      <td>0.199676</td>\n","      <td>-3.024157</td>\n","      <td>0.399259</td>\n","      <td>0.806127</td>\n","      <td>0.429203</td>\n","      <td>0.199702</td>\n","      <td>0.168319</td>\n","      <td>-46.390869</td>\n","      <td>0.24946</td>\n","      <td>0.868819</td>\n","      <td>0.455039</td>\n","      <td>522.345843</td>\n","      <td>11.0</td>\n","      <td>diffusion_model</td>\n","    </tr>\n","    <tr>\n","      <th>11</th>\n","      <td>0.245217</td>\n","      <td>0.20042</td>\n","      <td>-3.372237</td>\n","      <td>0.398506</td>\n","      <td>0.806286</td>\n","      <td>0.432197</td>\n","      <td>0.201196</td>\n","      <td>0.169709</td>\n","      <td>-58.129681</td>\n","      <td>0.250779</td>\n","      <td>0.867287</td>\n","      <td>0.453304</td>\n","      <td>524.491082</td>\n","      <td>12.0</td>\n","      <td>diffusion_model</td>\n","    </tr>\n","    <tr>\n","      <th>12</th>\n","      <td>0.244733</td>\n","      <td>0.200496</td>\n","      <td>-3.578113</td>\n","      <td>0.396743</td>\n","      <td>0.808027</td>\n","      <td>0.432537</td>\n","      <td>0.200271</td>\n","      <td>0.169166</td>\n","      <td>-64.471481</td>\n","      <td>0.244964</td>\n","      <td>0.871182</td>\n","      <td>0.458298</td>\n","      <td>515.269101</td>\n","      <td>13.0</td>\n","      <td>diffusion_model</td>\n","    </tr>\n","    <tr>\n","      <th>13</th>\n","      <td>0.244137</td>\n","      <td>0.199964</td>\n","      <td>-3.245593</td>\n","      <td>0.399345</td>\n","      <td>0.80611</td>\n","      <td>0.429292</td>\n","      <td>0.199789</td>\n","      <td>0.16881</td>\n","      <td>-56.184076</td>\n","      <td>0.248571</td>\n","      <td>0.869005</td>\n","      <td>0.457502</td>\n","      <td>513.72485</td>\n","      <td>14.0</td>\n","      <td>diffusion_model</td>\n","    </tr>\n","    <tr>\n","      <th>14</th>\n","      <td>0.244143</td>\n","      <td>0.199839</td>\n","      <td>-3.23188</td>\n","      <td>0.400473</td>\n","      <td>0.806432</td>\n","      <td>0.423963</td>\n","      <td>0.200042</td>\n","      <td>0.168883</td>\n","      <td>-56.11982</td>\n","      <td>0.249464</td>\n","      <td>0.868689</td>\n","      <td>0.443886</td>\n","      <td>512.844811</td>\n","      <td>15.0</td>\n","      <td>diffusion_model</td>\n","    </tr>\n","    <tr>\n","      <th>15</th>\n","      <td>0.244473</td>\n","      <td>0.200616</td>\n","      <td>-3.242585</td>\n","      <td>0.396766</td>\n","      <td>0.80642</td>\n","      <td>0.426764</td>\n","      <td>0.199897</td>\n","      <td>0.169167</td>\n","      <td>-55.273213</td>\n","      <td>0.24763</td>\n","      <td>0.869021</td>\n","      <td>0.450814</td>\n","      <td>514.47952</td>\n","      <td>16.0</td>\n","      <td>diffusion_model</td>\n","    </tr>\n","    <tr>\n","      <th>16</th>\n","      <td>0.24507</td>\n","      <td>0.200815</td>\n","      <td>-3.410064</td>\n","      <td>0.402601</td>\n","      <td>0.803131</td>\n","      <td>0.420236</td>\n","      <td>0.201022</td>\n","      <td>0.169809</td>\n","      <td>-59.296351</td>\n","      <td>0.251655</td>\n","      <td>0.865749</td>\n","      <td>0.438239</td>\n","      <td>512.754798</td>\n","      <td>17.0</td>\n","      <td>diffusion_model</td>\n","    </tr>\n","    <tr>\n","      <th>17</th>\n","      <td>0.245966</td>\n","      <td>0.201677</td>\n","      <td>-3.453844</td>\n","      <td>0.404201</td>\n","      <td>0.804262</td>\n","      <td>0.422317</td>\n","      <td>0.201618</td>\n","      <td>0.170283</td>\n","      <td>-61.039848</td>\n","      <td>0.252848</td>\n","      <td>0.866787</td>\n","      <td>0.445947</td>\n","      <td>511.441785</td>\n","      <td>18.0</td>\n","      <td>diffusion_model</td>\n","    </tr>\n","    <tr>\n","      <th>18</th>\n","      <td>0.244626</td>\n","      <td>0.200576</td>\n","      <td>-2.923437</td>\n","      <td>0.398196</td>\n","      <td>0.806992</td>\n","      <td>0.43425</td>\n","      <td>0.20154</td>\n","      <td>0.170527</td>\n","      <td>-48.447568</td>\n","      <td>0.249336</td>\n","      <td>0.868735</td>\n","      <td>0.455676</td>\n","      <td>514.146498</td>\n","      <td>19.0</td>\n","      <td>diffusion_model</td>\n","    </tr>\n","    <tr>\n","      <th>19</th>\n","      <td>0.244139</td>\n","      <td>0.199711</td>\n","      <td>-3.147514</td>\n","      <td>0.40077</td>\n","      <td>0.80535</td>\n","      <td>0.424226</td>\n","      <td>0.199207</td>\n","      <td>0.168242</td>\n","      <td>-53.37313</td>\n","      <td>0.250624</td>\n","      <td>0.867621</td>\n","      <td>0.449198</td>\n","      <td>520.954128</td>\n","      <td>20.0</td>\n","      <td>diffusion_model</td>\n","    </tr>\n","    <tr>\n","      <th>20</th>\n","      <td>0.249893</td>\n","      <td>0.205706</td>\n","      <td>-3.079636</td>\n","      <td>0.403738</td>\n","      <td>0.803107</td>\n","      <td>0.416578</td>\n","      <td>0.206053</td>\n","      <td>0.175139</td>\n","      <td>-51.14419</td>\n","      <td>0.254401</td>\n","      <td>0.865555</td>\n","      <td>0.440538</td>\n","      <td>530.514411</td>\n","      <td>21.0</td>\n","      <td>diffusion_model</td>\n","    </tr>\n","    <tr>\n","      <th>21</th>\n","      <td>0.245406</td>\n","      <td>0.200846</td>\n","      <td>-2.719608</td>\n","      <td>0.398802</td>\n","      <td>0.805715</td>\n","      <td>0.425523</td>\n","      <td>0.201117</td>\n","      <td>0.169542</td>\n","      <td>-41.894575</td>\n","      <td>0.248442</td>\n","      <td>0.868298</td>\n","      <td>0.445009</td>\n","      <td>511.986432</td>\n","      <td>23.0</td>\n","      <td>diffusion_model</td>\n","    </tr>\n","    <tr>\n","      <th>22</th>\n","      <td>0.24693</td>\n","      <td>0.202294</td>\n","      <td>-3.301648</td>\n","      <td>0.400613</td>\n","      <td>0.80436</td>\n","      <td>0.426758</td>\n","      <td>0.203649</td>\n","      <td>0.172542</td>\n","      <td>-57.054606</td>\n","      <td>0.251758</td>\n","      <td>0.866031</td>\n","      <td>0.449146</td>\n","      <td>524.738075</td>\n","      <td>24.0</td>\n","      <td>diffusion_model</td>\n","    </tr>\n","    <tr>\n","      <th>23</th>\n","      <td>0.24468</td>\n","      <td>0.200422</td>\n","      <td>-3.198893</td>\n","      <td>0.399562</td>\n","      <td>0.805834</td>\n","      <td>0.423342</td>\n","      <td>0.200421</td>\n","      <td>0.169134</td>\n","      <td>-54.196927</td>\n","      <td>0.249211</td>\n","      <td>0.868297</td>\n","      <td>0.444062</td>\n","      <td>513.660484</td>\n","      <td>25.0</td>\n","      <td>diffusion_model</td>\n","    </tr>\n","    <tr>\n","      <th>24</th>\n","      <td>0.245428</td>\n","      <td>0.201178</td>\n","      <td>-3.120564</td>\n","      <td>0.403024</td>\n","      <td>0.804267</td>\n","      <td>0.423859</td>\n","      <td>0.200545</td>\n","      <td>0.169301</td>\n","      <td>-51.422472</td>\n","      <td>0.251187</td>\n","      <td>0.86723</td>\n","      <td>0.448604</td>\n","      <td>523.375039</td>\n","      <td>26.0</td>\n","      <td>diffusion_model</td>\n","    </tr>\n","    <tr>\n","      <th>25</th>\n","      <td>0.246752</td>\n","      <td>0.20219</td>\n","      <td>-3.792168</td>\n","      <td>0.403128</td>\n","      <td>0.803305</td>\n","      <td>0.42013</td>\n","      <td>0.203157</td>\n","      <td>0.171768</td>\n","      <td>-67.855957</td>\n","      <td>0.252861</td>\n","      <td>0.865448</td>\n","      <td>0.437262</td>\n","      <td>524.141446</td>\n","      <td>27.0</td>\n","      <td>diffusion_model</td>\n","    </tr>\n","    <tr>\n","      <th>26</th>\n","      <td>0.244206</td>\n","      <td>0.200007</td>\n","      <td>-2.949504</td>\n","      <td>0.394871</td>\n","      <td>0.808302</td>\n","      <td>0.430474</td>\n","      <td>0.200392</td>\n","      <td>0.169257</td>\n","      <td>-49.6078</td>\n","      <td>0.245069</td>\n","      <td>0.870545</td>\n","      <td>0.454003</td>\n","      <td>523.488949</td>\n","      <td>28.0</td>\n","      <td>diffusion_model</td>\n","    </tr>\n","    <tr>\n","      <th>27</th>\n","      <td>0.242458</td>\n","      <td>0.198017</td>\n","      <td>-2.603606</td>\n","      <td>0.395574</td>\n","      <td>0.807679</td>\n","      <td>0.433339</td>\n","      <td>0.197741</td>\n","      <td>0.166903</td>\n","      <td>-37.539905</td>\n","      <td>0.246473</td>\n","      <td>0.869662</td>\n","      <td>0.457445</td>\n","      <td>510.726208</td>\n","      <td>29.0</td>\n","      <td>diffusion_model</td>\n","    </tr>\n","    <tr>\n","      <th>28</th>\n","      <td>0.245789</td>\n","      <td>0.20193</td>\n","      <td>-2.994601</td>\n","      <td>0.401432</td>\n","      <td>0.805223</td>\n","      <td>0.424648</td>\n","      <td>0.201802</td>\n","      <td>0.170911</td>\n","      <td>-49.272898</td>\n","      <td>0.249359</td>\n","      <td>0.868468</td>\n","      <td>0.450887</td>\n","      <td>519.047515</td>\n","      <td>30.0</td>\n","      <td>diffusion_model</td>\n","    </tr>\n","    <tr>\n","      <th>29</th>\n","      <td>0.245781</td>\n","      <td>0.201424</td>\n","      <td>-3.489873</td>\n","      <td>0.399742</td>\n","      <td>0.805516</td>\n","      <td>0.425333</td>\n","      <td>0.201128</td>\n","      <td>0.170185</td>\n","      <td>-60.222573</td>\n","      <td>0.249987</td>\n","      <td>0.867478</td>\n","      <td>0.449862</td>\n","      <td>510.114477</td>\n","      <td>22.0</td>\n","      <td>diffusion_model</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","    <div class=\"colab-df-buttons\">\n","\n","  <div class=\"colab-df-container\">\n","    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-27f882fa-48b9-421e-ac10-2b44cc815bd3')\"\n","            title=\"Convert this dataframe to an interactive table.\"\n","            style=\"display:none;\">\n","\n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n","    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n","  </svg>\n","    </button>\n","\n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    .colab-df-buttons div {\n","      margin-bottom: 4px;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","    <script>\n","      const buttonEl =\n","        document.querySelector('#df-27f882fa-48b9-421e-ac10-2b44cc815bd3 button.colab-df-convert');\n","      buttonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","      async function convertToInteractive(key) {\n","        const element = document.querySelector('#df-27f882fa-48b9-421e-ac10-2b44cc815bd3');\n","        const dataTable =\n","          await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                    [key], {});\n","        if (!dataTable) return;\n","\n","        const docLinkHtml = 'Like what you see? Visit the ' +\n","          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","          + ' to learn more about interactive tables.';\n","        element.innerHTML = '';\n","        dataTable['output_type'] = 'display_data';\n","        await google.colab.output.renderOutput(dataTable, element);\n","        const docLink = document.createElement('div');\n","        docLink.innerHTML = docLinkHtml;\n","        element.appendChild(docLink);\n","      }\n","    </script>\n","  </div>\n","\n","\n","<div id=\"df-399d7217-622a-4298-8e45-0e2e52bd1487\">\n","  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-399d7217-622a-4298-8e45-0e2e52bd1487')\"\n","            title=\"Suggest charts.\"\n","            style=\"display:none;\">\n","\n","<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","     width=\"24px\">\n","    <g>\n","        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n","    </g>\n","</svg>\n","  </button>\n","\n","<style>\n","  .colab-df-quickchart {\n","      --bg-color: #E8F0FE;\n","      --fill-color: #1967D2;\n","      --hover-bg-color: #E2EBFA;\n","      --hover-fill-color: #174EA6;\n","      --disabled-fill-color: #AAA;\n","      --disabled-bg-color: #DDD;\n","  }\n","\n","  [theme=dark] .colab-df-quickchart {\n","      --bg-color: #3B4455;\n","      --fill-color: #D2E3FC;\n","      --hover-bg-color: #434B5C;\n","      --hover-fill-color: #FFFFFF;\n","      --disabled-bg-color: #3B4455;\n","      --disabled-fill-color: #666;\n","  }\n","\n","  .colab-df-quickchart {\n","    background-color: var(--bg-color);\n","    border: none;\n","    border-radius: 50%;\n","    cursor: pointer;\n","    display: none;\n","    fill: var(--fill-color);\n","    height: 32px;\n","    padding: 0;\n","    width: 32px;\n","  }\n","\n","  .colab-df-quickchart:hover {\n","    background-color: var(--hover-bg-color);\n","    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n","    fill: var(--button-hover-fill-color);\n","  }\n","\n","  .colab-df-quickchart-complete:disabled,\n","  .colab-df-quickchart-complete:disabled:hover {\n","    background-color: var(--disabled-bg-color);\n","    fill: var(--disabled-fill-color);\n","    box-shadow: none;\n","  }\n","\n","  .colab-df-spinner {\n","    border: 2px solid var(--fill-color);\n","    border-color: transparent;\n","    border-bottom-color: var(--fill-color);\n","    animation:\n","      spin 1s steps(1) infinite;\n","  }\n","\n","  @keyframes spin {\n","    0% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","      border-left-color: var(--fill-color);\n","    }\n","    20% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    30% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","      border-right-color: var(--fill-color);\n","    }\n","    40% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    60% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","    }\n","    80% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-bottom-color: var(--fill-color);\n","    }\n","    90% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","    }\n","  }\n","</style>\n","\n","  <script>\n","    async function quickchart(key) {\n","      const quickchartButtonEl =\n","        document.querySelector('#' + key + ' button');\n","      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n","      quickchartButtonEl.classList.add('colab-df-spinner');\n","      try {\n","        const charts = await google.colab.kernel.invokeFunction(\n","            'suggestCharts', [key], {});\n","      } catch (error) {\n","        console.error('Error during call to suggestCharts:', error);\n","      }\n","      quickchartButtonEl.classList.remove('colab-df-spinner');\n","      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n","    }\n","    (() => {\n","      let quickchartButtonEl =\n","        document.querySelector('#df-399d7217-622a-4298-8e45-0e2e52bd1487 button');\n","      quickchartButtonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","    })();\n","  </script>\n","</div>\n","    </div>\n","  </div>\n"]},"metadata":{},"execution_count":12}],"source":["df_metrics = pd.concat(df_metrics, ignore_index=True)\n","df_metrics"]},{"cell_type":"code","execution_count":13,"metadata":{"id":"RtNXcng6Gz_H","executionInfo":{"status":"ok","timestamp":1697946054974,"user_tz":-480,"elapsed":535,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"}}},"outputs":[],"source":["generation_path = \"./evaluations\"\n","df_metrics.to_csv(os.path.join(generation_path, model_name+'_metrics.csv'),index=False)"]},{"cell_type":"code","execution_count":14,"metadata":{"id":"ODEBAmjCG0CD","colab":{"base_uri":"https://localhost:8080/","height":38},"executionInfo":{"status":"ok","timestamp":1697946054974,"user_tz":-480,"elapsed":7,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"}},"outputId":"79f5bf7d-5577-43f3-a9d9-0aec779b4a4c"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["'./evaluations/diffusion_model_metrics.csv'"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"}},"metadata":{},"execution_count":14}],"source":["os.path.join(generation_path, model_name+'_metrics.csv')"]},{"cell_type":"code","execution_count":14,"metadata":{"id":"0BCfni6qG0KW","executionInfo":{"status":"ok","timestamp":1697946054974,"user_tz":-480,"elapsed":6,"user":{"displayName":"Patrick Fu","userId":"17527192923561102849"}}},"outputs":[],"source":[]}],"metadata":{"colab":{"provenance":[],"authorship_tag":"ABX9TyPZPFEq4KK0r8DHBPMk3SXQ"},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"name":"python"}},"nbformat":4,"nbformat_minor":0}