tm_survival.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "2ff1c255-e0db-4a42-8daf-5fdeca8aa1d7",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import h5py as h5\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "import pickle\n",
    "import sys\n",
    "import optuna\n",
    "\n",
    "sys.path.append('/projects/EKOLEMEN/survival_tm/auton-survival/')\n",
    "\n",
    "from sklearn.model_selection import ParameterGrid\n",
    "from auton_survival.models.dsm import DeepSurvivalMachines\n",
    "from auton_survival.estimators import SurvivalModel\n",
    "from sksurv.metrics import concordance_index_ipcw, brier_score, cumulative_dynamic_auc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "92eb83c7-4afb-47dd-8030-7f4dd2e7fc4b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import h5py as h5\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "a27a94c0-d2c7-4c6c-8c06-e4887ae3ee5c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "data = h5.File('/projects/EKOLEMEN/profile_predictor/joe_hiro_models/diiid_data.h5', 'r')\n",
    "tm_data = h5.File('/projects/EKOLEMEN/survival_tm/tm_labels.h5', 'r')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "c93a5560",
   "metadata": {},
   "outputs": [],
   "source": [
    "shots = list(tm_data.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "df9a7761-b95b-4425-9b79-5155427b41c5",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([   0,   20,   40,   60,   80,  100,  120,  140,  160,  180,  200,\n",
       "        220,  240,  260,  280,  300,  320,  340,  360,  380,  400,  420,\n",
       "        440,  460,  480,  500,  520,  540,  560,  580,  600,  620,  640,\n",
       "        660,  680,  700,  720,  740,  760,  780,  800,  820,  840,  860,\n",
       "        880,  900,  920,  940,  960,  980, 1000, 1020, 1040, 1060, 1080,\n",
       "       1100, 1120, 1140, 1160, 1180, 1200, 1220, 1240, 1260, 1280, 1300,\n",
       "       1320, 1340, 1360, 1380, 1400, 1420, 1440, 1460, 1480, 1500, 1520,\n",
       "       1540, 1560, 1580, 1600, 1620, 1640, 1660, 1680, 1700, 1720, 1740,\n",
       "       1760, 1780, 1800, 1820, 1840, 1860, 1880, 1900, 1920, 1940, 1960,\n",
       "       1980, 2000, 2020, 2040, 2060, 2080, 2100, 2120, 2140, 2160, 2180,\n",
       "       2200, 2220, 2240, 2260, 2280, 2300, 2320, 2340, 2360, 2380, 2400,\n",
       "       2420, 2440, 2460, 2480, 2500, 2520, 2540, 2560, 2580, 2600, 2620,\n",
       "       2640, 2660, 2680, 2700, 2720, 2740, 2760, 2780, 2800, 2820, 2840,\n",
       "       2860, 2880, 2900, 2920, 2940, 2960, 2980, 3000, 3020, 3040, 3060,\n",
       "       3080, 3100, 3120, 3140, 3160, 3180, 3200, 3220, 3240, 3260, 3280,\n",
       "       3300, 3320, 3340, 3360, 3380, 3400, 3420, 3440, 3460, 3480, 3500,\n",
       "       3520, 3540, 3560, 3580, 3600, 3620, 3640, 3660, 3680, 3700, 3720,\n",
       "       3740, 3760, 3780, 3800, 3820, 3840, 3860, 3880, 3900, 3920, 3940,\n",
       "       3960, 3980, 4000, 4020, 4040, 4060, 4080, 4100, 4120, 4140, 4160,\n",
       "       4180, 4200, 4220, 4240, 4260, 4280, 4300, 4320, 4340, 4360, 4380,\n",
       "       4400, 4420, 4440, 4460, 4480, 4500, 4520, 4540, 4560, 4580, 4600,\n",
       "       4620, 4640, 4660, 4680, 4700, 4720, 4740, 4760, 4780, 4800, 4820,\n",
       "       4840, 4860, 4880, 4900, 4920, 4940, 4960, 4980, 5000, 5020, 5040,\n",
       "       5060, 5080, 5100, 5120, 5140, 5160, 5180, 5200, 5220, 5240, 5260,\n",
       "       5280, 5300, 5320, 5340, 5360, 5380, 5400, 5420, 5440, 5460, 5480,\n",
       "       5500, 5520, 5540, 5560, 5580, 5600, 5620, 5640, 5660, 5680, 5700,\n",
       "       5720, 5740, 5760, 5780, 5800, 5820, 5840, 5860, 5880, 5900, 5920,\n",
       "       5940, 5960, 5980])"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(tm_data[shots[0]]['time'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "6ff00bcf-3290-418a-8bc0-74c10e6cafb8",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
       "       0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tm_data[shots[0]]['label'][:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "6cd84f32-0834-4c24-8e3a-59aa2abb74f7",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Automatic pdb calling has been turned ON\n"
     ]
    }
   ],
   "source": [
    "%pdb"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5f675eda-5c94-40b7-b92d-20f6a85bc8f7",
   "metadata": {
    "tags": []
   },
   "source": [
    "# One Time Pre-processing"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6fa7851c-cc2d-4ed4-83c9-3f354120116c",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## Save TM Labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "cb94ae31-7e8e-45ef-babd-b6a8db7976db",
   "metadata": {},
   "outputs": [],
   "source": [
    "files = os.listdir(tm_label_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "d176df97-6c78-4c49-8ebf-a59b37fa0d29",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\nwith h5.File('/projects/EKOLEMEN/survival_tm/tm_labels.h5', 'w') as tm_labels:\\n    for f in files:\\n        temp_label = np.load(tm_label_dir+f)\\n\\n        grp = tm_labels.create_group(f[:6])\\n        grp['time'] = temp_label[0,:]\\n        grp['label'] = temp_label[1,:]\\n        grp['has_tm'] = np.sum(temp_label[1,:]) > 0\\n\""
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\"\"\"\n",
    "with h5.File('/projects/EKOLEMEN/survival_tm/tm_labels.h5', 'w') as tm_labels:\n",
    "    for f in files:\n",
    "        temp_label = np.load(tm_label_dir+f)\n",
    "\n",
    "        grp = tm_labels.create_group(f[:6])\n",
    "        grp['time'] = temp_label[0,:]\n",
    "        grp['label'] = temp_label[1,:]\n",
    "        grp['has_tm'] = np.sum(temp_label[1,:]) > 0\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "00c20838-4526-4bcf-92aa-30abae2aafd9",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "##  Get Shot Numbers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "91a2b8a1-3dad-40c5-bd5e-fb80ea381bb7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "full_shots = range(140000, 195502)\n",
    "shots = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "1eb586e3-11bf-4c5a-8562-f3bdf48b7e10",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 55502/55502 [00:00<00:00, 86600.69it/s]\n"
     ]
    }
   ],
   "source": [
    "for shot in tqdm(full_shots):\n",
    "    if str(shot) in tm_data.keys() and str(shot) in data.keys():\n",
    "        shots.append(shot)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "1ae31a3b-6614-4163-b471-2f20dd20ee39",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "sorted_shots = []\n",
    "signals = ['zipfit_etempfit_rho', 'zipfit_itempfit_rho', 'zipfit_edensfit_rho', 'zipfit_trotfit_rho', 'qpsi_EFIT01',\n",
    "          'pinj', 'tinj', 'betan_EFIT01', 'qmin_EFIT01', 'ech_pwr_total', 'ip', 'bt', 'li_EFIT01', 'aminor_EFIT01', \n",
    "          'rmaxis_EFIT01', 'tribot_EFIT01', 'tritop_EFIT01', 'kappa_EFIT01', 'volume_EFIT01']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "131ee767-2a3e-4954-a190-f9ce7095aac3",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 27753/27753 [00:04<00:00, 5954.57it/s]\n"
     ]
    }
   ],
   "source": [
    "for shot in tqdm(shots):\n",
    "    keys = data[str(shot)].keys()\n",
    "    \n",
    "    all_present = True\n",
    "    \n",
    "    for signal in signals:\n",
    "        if signal not in keys:\n",
    "            all_present = False\n",
    "            break\n",
    "    \n",
    "    if all_present:\n",
    "        sorted_shots.append(shot)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "76c419d5-d84b-44bd-aa7b-bb65b1d93353",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "np.save('/projects/EKOLEMEN/survival_tm/tm_shots.npy', sorted_shots)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "f6c42cfa-33ab-4c2b-8b30-a52b2c524823",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "18064"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(sorted_shots)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f2dd0fb4-5df4-4cdf-a3cf-7d49d81ea42b",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## Reformat TM Labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "55082a56-8aba-4cf7-b029-e352e22dceb4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "shots = np.load('/projects/EKOLEMEN/survival_tm/tm_shots.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "f5922916-2c02-40e0-8844-e84991707209",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "new_t = np.arange(0, 6000, 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "d3180b18-0f9a-43ba-a79f-5e65d8b455da",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 18064/18064 [02:05<00:00, 143.79it/s]\n"
     ]
    }
   ],
   "source": [
    "with h5.File('/projects/EKOLEMEN/survival_tm/tm_labels_2.h5', 'w') as slim_tm_data:\n",
    "    for shot in tqdm(shots):\n",
    "        new_labels = np.zeros(new_t.shape)\n",
    "        t = np.round(tm_data[str(shot)]['time']).astype(int)\n",
    "        label = tm_data[str(shot)]['label']\n",
    "\n",
    "        for i in range(1,300):\n",
    "            new_labels[i] = np.sum(label[20*(i-1):20*i]) > 0\n",
    "\n",
    "        grp = slim_tm_data.create_group(str(shot))\n",
    "        grp['time'] = new_t\n",
    "        grp['label'] = new_labels\n",
    "        grp['has_tm'] = np.sum(new_labels) > 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "60368490-1451-4dde-8b23-f9b14b0eef7d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "2ec2826e-f8aa-4bf0-9e2f-fa6b808a91a0",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Format Dataset for Model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5b93f482-49e3-49bb-94a2-f4b1fd9639d5",
   "metadata": {},
   "source": [
    "Note on dataset:\n",
    "3844 shots have TMs out of 18064 total so ~21% of dataset has TMs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e965087b-f991-4362-b5a4-9301ebfc99a3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "shots = np.load('/projects/EKOLEMEN/survival_tm/tm_shots.npy').astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "97141a87-422b-4d6d-8c6f-695f941847c3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "prof_signals = ['zipfit_etempfit_rho', 'zipfit_itempfit_rho', 'zipfit_edensfit_rho', 'zipfit_trotfit_rho', 'qpsi_EFIT01']\n",
    "\n",
    "signals = ['pinj', 'tinj', 'betan_EFIT01', 'qmin_EFIT01', 'ech_pwr_total', 'ip', 'bt', 'li_EFIT01', 'aminor_EFIT01', \n",
    "          'rmaxis_EFIT01', 'tribot_EFIT01', 'tritop_EFIT01', 'kappa_EFIT01', 'volume_EFIT01']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1720dcd1-a213-4776-82a2-abc97607e0bb",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def check_all_signals(shot_data, prof_signals, signals, ind):\n",
    "    # Check scalars for nan values\n",
    "    for sig in signals:\n",
    "        if np.isnan(shot_data[sig][ind]):\n",
    "            return False\n",
    "    \n",
    "    # Check profiles for nan values\n",
    "    for sig in prof_signals:\n",
    "        if np.sum(np.isnan(shot_data[sig][ind,:])) > 0:\n",
    "            return False\n",
    "       \n",
    "    return True\n",
    "\n",
    "def gather_feature(shot_data, prof_signals, signals, ind):\n",
    "    feature = np.zeros(len(prof_signals)*33+len(signals))\n",
    "    \n",
    "    # Load scalar signals\n",
    "    for i, sig in enumerate(signals):\n",
    "        feature[i] = shot_data[sig][ind]\n",
    "    \n",
    "    # Load profile signals\n",
    "    for i, sig in enumerate(prof_signals):\n",
    "        feature[33*i+len(signals):33*(i+1)+len(signals)] = shot_data[sig][ind,:]\n",
    "    \n",
    "    return feature"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "82f9ef37-711e-49fa-bc3c-fce7b3e166e8",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "x = [] # Feature vector\n",
    "t = [] # Time to measurment\n",
    "e = [] # Event of measurement (0 if no TM/end of shot, 1 if TM occurred)\n",
    "\n",
    "for shot in tqdm(shots[:10]):\n",
    "    # First figure out time of TM, or time of end of shot\n",
    "    label = tm_data[shot]['label'][:]\n",
    "    time = tm_data[shot]['time'][:]\n",
    "    tm_ind = np.argmax(label>0)\n",
    "    tm_time = time[tm_ind]\n",
    "    \n",
    "    # Find start and stop of flattop\n",
    "    start = np.array(data[shot]['t_ip_flat_sql'])\n",
    "    duration = np.array(data[shot]['ip_flat_duration_sql'])\n",
    "    stop = start+duration\n",
    "    # Get start and stop inds\n",
    "    start_ind = np.argmin(np.abs(time - start))\n",
    "    end_ind = np.argmin(np.abs(time - stop))\n",
    "    \n",
    "    # End shot when TM occurs\n",
    "    if end_ind > tm_ind:\n",
    "        end_ind = tm_ind\n",
    "    \n",
    "    # Go through valid indices and gather dataset\n",
    "    for ind in range(start_ind, end_ind):\n",
    "        # Check that we have all signals at this time\n",
    "        if check_all_signals(data[shot], prof_signals, signals, ind):\n",
    "            feature = gather_feature(data[shot], prof_signals, signals, ind)\n",
    "            x.append(feature)\n",
    "            \n",
    "            if tm_time > 0:\n",
    "                # Time to TM\n",
    "                t.append(tm_time - time[ind])\n",
    "                e.append(1)\n",
    "            else:\n",
    "                # Time to end of shot\n",
    "                t.append(time[end_ind] - time[ind])\n",
    "                e.append(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1eea3800-f8b7-49e6-93a4-b86dca3997fc",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "x = np.array(x)\n",
    "t = np.array(t)\n",
    "e = np.array(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "eb152628-238c-4cbe-8811-db01ba6afcf0",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18064\n"
     ]
    }
   ],
   "source": [
    "print(len(shots))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "379616fe-de41-440d-913c-fcb830f00615",
   "metadata": {},
   "source": [
    "# Train Model?"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "369e15a2-081d-44d0-b3b8-5c0b9995aa3f",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "### Get shots from new labels "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "id": "bf498feb-1d24-4efd-a2fd-966f2bb8914d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "files = os.listdir('/projects/EKOLEMEN/survival_tm/survival_labels/')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "f7cfa361-3d63-4497-a5ec-73bf69abe25a",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "shots = []\n",
    "\n",
    "for file in files:\n",
    "    shots.append(file[:6])\n",
    "\n",
    "shots = np.sort(np.array(shots))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "466d09ff-9134-45b6-9ff8-e1936b35dd1c",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "17336"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(shots)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "626d7399-5b34-43f0-97a5-e6b3e18b540f",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 17336/17336 [00:18<00:00, 957.70it/s] \n"
     ]
    }
   ],
   "source": [
    "tm_shots = []\n",
    "st_shots = []\n",
    "full_shots = []\n",
    "\n",
    "for shot in tqdm(shots):\n",
    "    with open(f'/projects/EKOLEMEN/survival_tm/survival_labels/{shot}.pkl', 'rb') as f:\n",
    "        pkl_data = pickle.load(f)\n",
    "        \n",
    "        if len(pkl_data['e']) == 0:\n",
    "            continue\n",
    "        \n",
    "        full_shots.append(shot)\n",
    "        if pkl_data['e'][0] == 1:\n",
    "            tm_shots.append(shot)\n",
    "        else:\n",
    "            st_shots.append(shot)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "ebfdd3a7-a7fa-4556-a14e-b6d966db420b",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "16951\n",
      "3598\n",
      "13353\n"
     ]
    }
   ],
   "source": [
    "print(len(full_shots))\n",
    "print(len(tm_shots))\n",
    "print(len(st_shots))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "8d18c21b-44e9-4f16-9066-f206d3d8a24c",
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save('/projects/EKOLEMEN/survival_tm/shots.npy', full_shots)\n",
    "np.save('/projects/EKOLEMEN/survival_tm/tm_shots.npy', tm_shots)\n",
    "np.save('/projects/EKOLEMEN/survival_tm/st_shots.npy', st_shots)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5e0622ee-b498-41c3-ae56-6fda47b8f757",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "### Saving Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2bd229ed-483c-4504-b9f5-add99f58bab3",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_survival_data(shots):\n",
    "    x = np.array([])\n",
    "    t = np.array([])\n",
    "    e = np.array([])\n",
    "    \n",
    "    for shot in tqdm(shots):\n",
    "        with open(f'/projects/EKOLEMEN/survival_tm/survival_labels/{shot}.pkl', 'rb') as f:\n",
    "            pkl_data = pickle.load(f)\n",
    "\n",
    "        if x.shape == (0,):\n",
    "            x = np.array(pkl_data['x'])\n",
    "            t = np.array(pkl_data['t'])\n",
    "            e = np.array(pkl_data['e'])\n",
    "        else:\n",
    "            x = np.append(x, pkl_data['x'], axis=0)\n",
    "            t = np.append(t, pkl_data['t'], axis=0)\n",
    "            e = np.append(e, pkl_data['e'], axis=0)\n",
    "    \n",
    "    return np.array(x), np.array(t), np.array(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "41256e89-5ccc-4383-9640-6845b1826ec2",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dict = dict()\n",
    "test_dict = dict()\n",
    "valid_dict = dict()\n",
    "\n",
    "train_dict['x'] = x_train\n",
    "train_dict['t'] = t_train\n",
    "train_dict['e'] = e_train\n",
    "\n",
    "test_dict['x'] = x_test\n",
    "test_dict['t'] = t_test\n",
    "test_dict['e'] = e_test\n",
    "\n",
    "valid_dict['x'] = x_valid\n",
    "valid_dict['t'] = t_valid\n",
    "valid_dict['e'] = e_valid\n",
    "\n",
    "with open('/projects/EKOLEMEN/survival_tm/formatted_labels/train.pkl', 'wb') as f:\n",
    "    pickle.dump(train_dict, f)\n",
    "    \n",
    "with open('/projects/EKOLEMEN/survival_tm/formatted_labels/test.pkl', 'wb') as f:\n",
    "    pickle.dump(test_dict, f)\n",
    "    \n",
    "with open('/projects/EKOLEMEN/survival_tm/formatted_labels/valid.pkl', 'wb') as f:\n",
    "    pickle.dump(valid_dict, f)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0b73e26f-9feb-48d6-9d34-e3e2f000c1c2",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "## Load Data from pkl files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "d98f5ed0-3815-4de3-a4b6-c5644dc401e4",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "shots = np.load('/projects/EKOLEMEN/survival_tm/shots.npy')\n",
    "tm_shots = np.load('/projects/EKOLEMEN/survival_tm/tm_shots.npy')\n",
    "st_shots = np.load('/projects/EKOLEMEN/survival_tm/st_shots.npy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "cd8f7e65-1693-4bb2-a3f0-001b13094c91",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def load_data(data_type):\n",
    "    with open(f'/projects/EKOLEMEN/survival_tm/formatted_labels/{data_type}.pkl', 'rb') as f:\n",
    "        data = pickle.load(f)\n",
    "    \n",
    "    return data['x'], data['t'], data['e']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "cdbd8629-01d1-4581-b2e4-659d4b9d0520",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Don't want whole dataset right now so 0.5 factor to reduce\n",
    "n = len(shots)\n",
    "\n",
    "tr_size = int(n*0.80)\n",
    "vl_size = int(n*0.10)\n",
    "te_size = int(n*0.10)\n",
    "\n",
    "train_shots = shots[:tr_size]\n",
    "test_shots = shots[-te_size:]\n",
    "valid_shots = shots[tr_size:tr_size+vl_size]\n",
    "\n",
    "x_train, t_train, e_train = load_data('train')\n",
    "x_test,  t_test,  e_test  = load_data('test')\n",
    "x_valid, t_valid, e_valid = load_data('valid')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "346d8329-7d2d-49ee-a9e0-2216802cb12c",
   "metadata": {
    "jp-MarkdownHeadingCollapsed": true,
    "tags": []
   },
   "source": [
    "### Trim dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "496bddb1-4016-4263-b7a4-ac2e01098fbc",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1427692\n"
     ]
    }
   ],
   "source": [
    "print(len(t_train))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "bae3dfd2-1722-405f-8740-2cdcace053ba",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Get inds for time <600ms\n",
    "inds = np.where(t_train < 600)[0]\n",
    "\n",
    "x_train = x_train[inds]\n",
    "t_train = t_train[inds]\n",
    "e_train = e_train[inds]\n",
    "\n",
    "tm_inds = np.where(e_train == 1)[0]\n",
    "st_inds = np.where(e_train == 0)[0]\n",
    "new_st_inds = np.random.choice(st_inds, size=len(tm_inds), replace=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "e217d814-17e9-4c87-a622-9625a5836967",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "246148\n",
      "58475\n",
      "187673\n",
      "58475\n"
     ]
    }
   ],
   "source": [
    "print(len(t_train))\n",
    "print(len(tm_inds))\n",
    "print(len(st_inds))\n",
    "print(len(new_st_inds))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "08cc20f7-f5db-45ae-9570-26b820efebce",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "x_train = np.concatenate((x_train[tm_inds], x_train[new_st_inds]), axis=0)\n",
    "t_train = np.concatenate((t_train[tm_inds], t_train[new_st_inds]), axis=0)\n",
    "e_train = np.concatenate((e_train[tm_inds], e_train[new_st_inds]), axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "2bffa0dd-1681-4d35-b1ea-48efdda536b1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Shuffle arrays because currently all 1s followed by all 0s\n",
    "p = np.random.permutation(len(t_train))\n",
    "x_train = x_train[p,:]\n",
    "t_train = t_train[p]\n",
    "e_train = e_train[p]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0eddf1a4-a2fb-4b6c-8c61-9c1d7f4cd23e",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Train Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "80c96655-99d4-4631-be4a-fe092e35c8dd",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'study' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m/home/hf8585/TMPredictor/survival_tm/train_models/tm_survival.ipynb Cell 54\u001b[0m line \u001b[0;36m1\n\u001b[0;32m----> <a href='vscode-notebook-cell://ssh-remote%2Bstellar.princeton.edu/home/hf8585/TMPredictor/survival_tm/train_models/tm_survival.ipynb#Y103sdnNjb2RlLXJlbW90ZQ%3D%3D?line=0'>1</a>\u001b[0m study\u001b[39m.\u001b[39mbest_trials[\u001b[39m0\u001b[39m]\u001b[39m.\u001b[39mparams\n",
      "\u001b[0;31mNameError\u001b[0m: name 'study' is not defined"
     ]
    }
   ],
   "source": [
    "study.best_trials[0].params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "c2625a1c-9d1c-4bb3-a853-33dd16ae9513",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "param_grid = {'k' : [2, 4, 6, 8],\n",
    "              'distribution' : ['LogNormal', 'Weibull'],\n",
    "              'learning_rate' : [ 1e-5, 1e-4, 1e-3 ],\n",
    "              'activation': ['ReLU6', 'ReLU', 'SeLU', 'Tanh'],\n",
    "              'lin1' : [50, 100, 150],\n",
    "              'lin2' : [30, 40, 50, 60],\n",
    "              'lin3' : [50, 100, 150],\n",
    "              'lin4' : [50, 100, 150],\n",
    "              'lin5' : [50, 100, 150],\n",
    "             }\n",
    "\n",
    "param_grid = {'k' : [3],\n",
    "              'distribution' : ['LogNormal'],\n",
    "              'learning_rate' : [ 0.00392 ],\n",
    "              'activation': ['SeLU'],\n",
    "              'lin1' : [100],\n",
    "              'lin2' : [60],\n",
    "              'lin3' : [175],\n",
    "              'lin4' : [225],\n",
    "              'lin5' : [120],\n",
    "             }\n",
    "params = ParameterGrid(param_grid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "c870220c-931f-4f9b-b230-5ee9a897e9a0",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "-1",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[1;32m/home/hf8585/TMPredictor/survival_tm/train_models/tm_survival.ipynb Cell 56\u001b[0m line \u001b[0;36m1\n\u001b[1;32m      <a href='vscode-notebook-cell://ssh-remote%2Bstellar.princeton.edu/home/hf8585/TMPredictor/survival_tm/train_models/tm_survival.ipynb#Y105sdnNjb2RlLXJlbW90ZQ%3D%3D?line=8'>9</a>\u001b[0m     model \u001b[39m=\u001b[39m DeepSurvivalMachines(k \u001b[39m=\u001b[39m param[\u001b[39m'\u001b[39m\u001b[39mk\u001b[39m\u001b[39m'\u001b[39m],\n\u001b[1;32m     <a href='vscode-notebook-cell://ssh-remote%2Bstellar.princeton.edu/home/hf8585/TMPredictor/survival_tm/train_models/tm_survival.ipynb#Y105sdnNjb2RlLXJlbW90ZQ%3D%3D?line=9'>10</a>\u001b[0m                                  distribution \u001b[39m=\u001b[39m param[\u001b[39m'\u001b[39m\u001b[39mdistribution\u001b[39m\u001b[39m'\u001b[39m],\n\u001b[1;32m     <a href='vscode-notebook-cell://ssh-remote%2Bstellar.princeton.edu/home/hf8585/TMPredictor/survival_tm/train_models/tm_survival.ipynb#Y105sdnNjb2RlLXJlbW90ZQ%3D%3D?line=10'>11</a>\u001b[0m                                  layers \u001b[39m=\u001b[39m layers, \n\u001b[1;32m     <a href='vscode-notebook-cell://ssh-remote%2Bstellar.princeton.edu/home/hf8585/TMPredictor/survival_tm/train_models/tm_survival.ipynb#Y105sdnNjb2RlLXJlbW90ZQ%3D%3D?line=11'>12</a>\u001b[0m                                  activation \u001b[39m=\u001b[39m param[\u001b[39m'\u001b[39m\u001b[39mactivation\u001b[39m\u001b[39m'\u001b[39m])\n\u001b[1;32m     <a href='vscode-notebook-cell://ssh-remote%2Bstellar.princeton.edu/home/hf8585/TMPredictor/survival_tm/train_models/tm_survival.ipynb#Y105sdnNjb2RlLXJlbW90ZQ%3D%3D?line=12'>13</a>\u001b[0m     \u001b[39m# The fit method is called to train the model\u001b[39;00m\n\u001b[0;32m---> <a href='vscode-notebook-cell://ssh-remote%2Bstellar.princeton.edu/home/hf8585/TMPredictor/survival_tm/train_models/tm_survival.ipynb#Y105sdnNjb2RlLXJlbW90ZQ%3D%3D?line=13'>14</a>\u001b[0m     _, train_loss, valid_loss \u001b[39m=\u001b[39m model\u001b[39m.\u001b[39mfit(x_train, t_train, e_train, iters \u001b[39m=\u001b[39m \u001b[39m100\u001b[39m, val_data\u001b[39m=\u001b[39m(x_valid, t_valid, e_valid),\n\u001b[1;32m     <a href='vscode-notebook-cell://ssh-remote%2Bstellar.princeton.edu/home/hf8585/TMPredictor/survival_tm/train_models/tm_survival.ipynb#Y105sdnNjb2RlLXJlbW90ZQ%3D%3D?line=14'>15</a>\u001b[0m               batch_size\u001b[39m=\u001b[39m\u001b[39m10000\u001b[39m, learning_rate \u001b[39m=\u001b[39m param[\u001b[39m'\u001b[39m\u001b[39mlearning_rate\u001b[39m\u001b[39m'\u001b[39m])\n\u001b[1;32m     <a href='vscode-notebook-cell://ssh-remote%2Bstellar.princeton.edu/home/hf8585/TMPredictor/survival_tm/train_models/tm_survival.ipynb#Y105sdnNjb2RlLXJlbW90ZQ%3D%3D?line=15'>16</a>\u001b[0m     models\u001b[39m.\u001b[39mappend([[model\u001b[39m.\u001b[39mcompute_nll(x_valid, t_valid, e_valid), model]])\n\u001b[1;32m     <a href='vscode-notebook-cell://ssh-remote%2Bstellar.princeton.edu/home/hf8585/TMPredictor/survival_tm/train_models/tm_survival.ipynb#Y105sdnNjb2RlLXJlbW90ZQ%3D%3D?line=16'>17</a>\u001b[0m best_model \u001b[39m=\u001b[39m \u001b[39mmin\u001b[39m(models)\n",
      "File \u001b[0;32m/projects/EKOLEMEN/survival_tm/train_models/auton-survival/auton_survival/models/dsm/__init__.py:261\u001b[0m, in \u001b[0;36mDSMBase.fit\u001b[0;34m(self, x, t, e, vsize, val_data, iters, learning_rate, batch_size, elbo, optimizer)\u001b[0m\n\u001b[1;32m    258\u001b[0m   inputdim \u001b[39m=\u001b[39m x_train\u001b[39m.\u001b[39mshape[\u001b[39m-\u001b[39m\u001b[39m1\u001b[39m]\n\u001b[1;32m    260\u001b[0m maxrisk \u001b[39m=\u001b[39m \u001b[39mint\u001b[39m(np\u001b[39m.\u001b[39mnanmax(e_train\u001b[39m.\u001b[39mcpu()\u001b[39m.\u001b[39mnumpy()))\n\u001b[0;32m--> 261\u001b[0m model \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_gen_torch_model(inputdim, optimizer, risks\u001b[39m=\u001b[39mmaxrisk)\n\u001b[1;32m    262\u001b[0m \u001b[39m# model, _ = train_dsm(model,\u001b[39;00m\n\u001b[1;32m    263\u001b[0m model, _, train_loss, valid_loss \u001b[39m=\u001b[39m train_dsm(model,\n\u001b[1;32m    264\u001b[0m                      x_train, t_train, e_train,\n\u001b[1;32m    265\u001b[0m                      x_val, t_val, e_val,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    269\u001b[0m                      bs\u001b[39m=\u001b[39mbatch_size,\n\u001b[1;32m    270\u001b[0m                      random_seed\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mrandom_seed)\n",
      "File \u001b[0;32m/projects/EKOLEMEN/survival_tm/train_models/auton-survival/auton_survival/models/dsm/__init__.py:203\u001b[0m, in \u001b[0;36mDSMBase._gen_torch_model\u001b[0;34m(self, inputdim, optimizer, risks)\u001b[0m\n\u001b[1;32m    200\u001b[0m np\u001b[39m.\u001b[39mrandom\u001b[39m.\u001b[39mseed(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mrandom_seed)\n\u001b[1;32m    201\u001b[0m torch\u001b[39m.\u001b[39mmanual_seed(\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mrandom_seed)\n\u001b[0;32m--> 203\u001b[0m \u001b[39mreturn\u001b[39;00m DeepSurvivalMachinesTorch(inputdim,\n\u001b[1;32m    204\u001b[0m                                  k\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mk,\n\u001b[1;32m    205\u001b[0m                                  layers\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mlayers,\n\u001b[1;32m    206\u001b[0m                                  dist\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mdist,\n\u001b[1;32m    207\u001b[0m                                  temp\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mtemp,\n\u001b[1;32m    208\u001b[0m                                  discount\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mdiscount,\n\u001b[1;32m    209\u001b[0m                                  optimizer\u001b[39m=\u001b[39moptimizer,\n\u001b[1;32m    210\u001b[0m                                  risks\u001b[39m=\u001b[39mrisks, \n\u001b[1;32m    211\u001b[0m                                  activation\u001b[39m=\u001b[39m\u001b[39mself\u001b[39m\u001b[39m.\u001b[39mactivation)\n",
      "File \u001b[0;32m/projects/EKOLEMEN/survival_tm/train_models/auton-survival/auton_survival/models/dsm/dsm_torch.py:255\u001b[0m, in \u001b[0;36mDeepSurvivalMachinesTorch.__init__\u001b[0;34m(self, inputdim, k, layers, dist, temp, discount, optimizer, risks, activation)\u001b[0m\n\u001b[1;32m    251\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mlayers \u001b[39m=\u001b[39m layers\n\u001b[1;32m    254\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mlen\u001b[39m(layers) \u001b[39m==\u001b[39m \u001b[39m0\u001b[39m: lastdim \u001b[39m=\u001b[39m inputdim\n\u001b[0;32m--> 255\u001b[0m \u001b[39melse\u001b[39;00m: lastdim \u001b[39m=\u001b[39m layers[\u001b[39m-\u001b[39m\u001b[39m1\u001b[39m]\n\u001b[1;32m    257\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_init_dsm_layers(lastdim)\n\u001b[1;32m    258\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39membedding \u001b[39m=\u001b[39m create_representation(inputdim, layers, \u001b[39m'\u001b[39m\u001b[39mReLU6\u001b[39m\u001b[39m'\u001b[39m)\n",
      "\u001b[0;31mKeyError\u001b[0m: -1"
     ]
    }
   ],
   "source": [
    "models = []\n",
    "for param in params:\n",
    "    layers = dict()\n",
    "    layers['lin1'] = param['lin1']\n",
    "    layers['lin2'] = param['lin2']\n",
    "    layers['lin3'] = param['lin3']\n",
    "    layers['lin4'] = param['lin4']\n",
    "    layers['lin5'] = param['lin5']\n",
    "    model = SurvivalModel('dsm', random_seed=0, k = param['k'],\n",
    "                                 distribution = param['distribution'],\n",
    "                                 layers = layers, \n",
    "                                 activation = param['activation'])\n",
    "    # The fit method is called to train the model\n",
    "    _, train_loss, valid_loss = model.fit(x_train, t_train, e_train, iters = 100, val_data=(x_valid, t_valid, e_valid),\n",
    "              batch_size=10000, learning_rate = param['learning_rate'])\n",
    "    models.append([[model.compute_nll(x_valid, t_valid, e_valid), model]])\n",
    "best_model = min(models)\n",
    "model = best_model[0][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "a61450b2-9492-4a84-98aa-198e476278dd",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def objective(trial):\n",
    "    activation = trial.suggest_categorical('activation', ['ReLU6', 'ReLU', 'SeLU', 'Tanh'])\n",
    "    dist = trial.suggest_categorical('distribution', ['LogNormal', 'Weibull'])\n",
    "    k = trial.suggest_int('k', 2, 20)\n",
    "    lr = trial.suggest_float('learning_rate', 1e-7, 1e-1, log=True)\n",
    "    \n",
    "    l1 = trial.suggest_int('lin1', 25, 250)\n",
    "    l2 = trial.suggest_int('lin2', 20, 80)\n",
    "    l3 = trial.suggest_int('lin3', 25, 250)\n",
    "    l4 = trial.suggest_int('lin4', 25, 250)\n",
    "    l5 = trial.suggest_int('lin5', 25, 250)\n",
    "    \n",
    "    layers = dict()\n",
    "    layers['lin1'] = l1\n",
    "    layers['lin2'] = l2\n",
    "    layers['lin3'] = l3\n",
    "    layers['lin4'] = l4\n",
    "    layers['lin5'] = l5\n",
    "    \n",
    "    model = DeepSurvivalMachines(k = k,\n",
    "                                 distribution = dist,\n",
    "                                 layers = layers, \n",
    "                                 activation = activation)\n",
    "    # The fit method is called to train the model\n",
    "    _, train_loss, valid_loss = model.fit(x_train, t_train, e_train, iters = 50, val_data=(x_valid, t_valid, e_valid),\n",
    "              batch_size=10000, learning_rate = lr)\n",
    "    \n",
    "    err = model.compute_nll(x_valid, t_valid, e_valid)\n",
    "    return err"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "09872d63-80d1-4053-abd2-4c4d3c2e1448",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[I 2024-02-16 11:12:59,930] A new study created in memory with name: no-name-6d06c71c-69d0-4334-a6e7-09de971e3d05\n"
     ]
    }
   ],
   "source": [
    "study = optuna.create_study()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "b4cbc2de-32f8-48b8-bfbb-a8a4971c55ee",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▍       | 2394/10000 [01:05<03:27, 36.73it/s]\n",
      " 28%|██▊       | 14/50 [04:55<12:39, 21.11s/it]\n",
      "[W 2024-02-16 11:19:01,047] Trial 0 failed with parameters: {'activation': 'SeLU', 'distribution': 'Weibull', 'k': 10, 'learning_rate': 9.91234317076541e-06, 'lin1': 223, 'lin2': 79, 'lin3': 145, 'lin4': 218, 'lin5': 231} because of the following error: KeyboardInterrupt().\n",
      "Traceback (most recent call last):\n",
      "  File \"/home/ar0535/.conda/envs/torch-env/lib/python3.11/site-packages/optuna/study/_optimize.py\", line 200, in _run_trial\n",
      "    value_or_values = func(trial)\n",
      "                      ^^^^^^^^^^^\n",
      "  File \"/tmp/ipykernel_1592352/2711386965.py\", line 25, in objective\n",
      "    _, train_loss, valid_loss = model.fit(x_train, t_train, e_train, iters = 50, val_data=(x_valid, t_valid, e_valid),\n",
      "                                ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/home/ar0535/auton-survival/auton_survival/models/dsm/__init__.py\", line 263, in fit\n",
      "    model, _, train_loss, valid_loss = train_dsm(model,\n",
      "                                       ^^^^^^^^^^^^^^^^\n",
      "  File \"/home/ar0535/auton-survival/auton_survival/models/dsm/utilities.py\", line 187, in train_dsm\n",
      "    valid_loss += conditional_loss(model,\n",
      "                  ^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/home/ar0535/auton-survival/auton_survival/models/dsm/losses.py\", line 287, in conditional_loss\n",
      "    return _conditional_weibull_loss(model, x, t, e, elbo, risk)\n",
      "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/home/ar0535/auton-survival/auton_survival/models/dsm/losses.py\", line 238, in _conditional_weibull_loss\n",
      "    shape, scale, logits = model.forward(x, risk)\n",
      "                           ^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/home/ar0535/auton-survival/auton_survival/models/dsm/dsm_torch.py\", line 275, in forward\n",
      "    xrep = self.embedding(x)\n",
      "           ^^^^^^^^^^^^^^^^^\n",
      "  File \"/home/ar0535/.conda/envs/torch-env/lib/python3.11/site-packages/torch/nn/modules/module.py\", line 1501, in _call_impl\n",
      "    return forward_call(*args, **kwargs)\n",
      "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/home/ar0535/auton-survival/auton_survival/models/dsm/dsm_torch.py\", line 132, in forward\n",
      "    x_profs = self.pool2(self.act(self.conv2(x_profs)))\n",
      "              ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/home/ar0535/.conda/envs/torch-env/lib/python3.11/site-packages/torch/nn/modules/module.py\", line 1501, in _call_impl\n",
      "    return forward_call(*args, **kwargs)\n",
      "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/home/ar0535/.conda/envs/torch-env/lib/python3.11/site-packages/torch/nn/modules/pooling.py\", line 166, in forward\n",
      "    return F.max_pool2d(input, self.kernel_size, self.stride,\n",
      "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/home/ar0535/.conda/envs/torch-env/lib/python3.11/site-packages/torch/_jit_internal.py\", line 484, in fn\n",
      "    return if_false(*args, **kwargs)\n",
      "           ^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "  File \"/home/ar0535/.conda/envs/torch-env/lib/python3.11/site-packages/torch/nn/functional.py\", line 782, in _max_pool2d\n",
      "    return torch.max_pool2d(input, kernel_size, stride, padding, dilation, ceil_mode)\n",
      "           ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n",
      "KeyboardInterrupt\n",
      "[W 2024-02-16 11:19:01,053] Trial 0 failed with value None.\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[15], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m study\u001b[38;5;241m.\u001b[39moptimize(objective, n_trials\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m1\u001b[39m)\n",
      "File \u001b[0;32m~/.conda/envs/torch-env/lib/python3.11/site-packages/optuna/study/study.py:451\u001b[0m, in \u001b[0;36mStudy.optimize\u001b[0;34m(self, func, n_trials, timeout, n_jobs, catch, callbacks, gc_after_trial, show_progress_bar)\u001b[0m\n\u001b[1;32m    348\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21moptimize\u001b[39m(\n\u001b[1;32m    349\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m    350\u001b[0m     func: ObjectiveFuncType,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    357\u001b[0m     show_progress_bar: \u001b[38;5;28mbool\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[1;32m    358\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    359\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Optimize an objective function.\u001b[39;00m\n\u001b[1;32m    360\u001b[0m \n\u001b[1;32m    361\u001b[0m \u001b[38;5;124;03m    Optimization is done by choosing a suitable set of hyperparameter values from a given\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    449\u001b[0m \u001b[38;5;124;03m            If nested invocation of this method occurs.\u001b[39;00m\n\u001b[1;32m    450\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m--> 451\u001b[0m     _optimize(\n\u001b[1;32m    452\u001b[0m         study\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m    453\u001b[0m         func\u001b[38;5;241m=\u001b[39mfunc,\n\u001b[1;32m    454\u001b[0m         n_trials\u001b[38;5;241m=\u001b[39mn_trials,\n\u001b[1;32m    455\u001b[0m         timeout\u001b[38;5;241m=\u001b[39mtimeout,\n\u001b[1;32m    456\u001b[0m         n_jobs\u001b[38;5;241m=\u001b[39mn_jobs,\n\u001b[1;32m    457\u001b[0m         catch\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mtuple\u001b[39m(catch) \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(catch, Iterable) \u001b[38;5;28;01melse\u001b[39;00m (catch,),\n\u001b[1;32m    458\u001b[0m         callbacks\u001b[38;5;241m=\u001b[39mcallbacks,\n\u001b[1;32m    459\u001b[0m         gc_after_trial\u001b[38;5;241m=\u001b[39mgc_after_trial,\n\u001b[1;32m    460\u001b[0m         show_progress_bar\u001b[38;5;241m=\u001b[39mshow_progress_bar,\n\u001b[1;32m    461\u001b[0m     )\n",
      "File \u001b[0;32m~/.conda/envs/torch-env/lib/python3.11/site-packages/optuna/study/_optimize.py:66\u001b[0m, in \u001b[0;36m_optimize\u001b[0;34m(study, func, n_trials, timeout, n_jobs, catch, callbacks, gc_after_trial, show_progress_bar)\u001b[0m\n\u001b[1;32m     64\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m     65\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m n_jobs \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m1\u001b[39m:\n\u001b[0;32m---> 66\u001b[0m         _optimize_sequential(\n\u001b[1;32m     67\u001b[0m             study,\n\u001b[1;32m     68\u001b[0m             func,\n\u001b[1;32m     69\u001b[0m             n_trials,\n\u001b[1;32m     70\u001b[0m             timeout,\n\u001b[1;32m     71\u001b[0m             catch,\n\u001b[1;32m     72\u001b[0m             callbacks,\n\u001b[1;32m     73\u001b[0m             gc_after_trial,\n\u001b[1;32m     74\u001b[0m             reseed_sampler_rng\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[1;32m     75\u001b[0m             time_start\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[1;32m     76\u001b[0m             progress_bar\u001b[38;5;241m=\u001b[39mprogress_bar,\n\u001b[1;32m     77\u001b[0m         )\n\u001b[1;32m     78\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m     79\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m n_jobs \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m:\n",
      "File \u001b[0;32m~/.conda/envs/torch-env/lib/python3.11/site-packages/optuna/study/_optimize.py:163\u001b[0m, in \u001b[0;36m_optimize_sequential\u001b[0;34m(study, func, n_trials, timeout, catch, callbacks, gc_after_trial, reseed_sampler_rng, time_start, progress_bar)\u001b[0m\n\u001b[1;32m    160\u001b[0m         \u001b[38;5;28;01mbreak\u001b[39;00m\n\u001b[1;32m    162\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 163\u001b[0m     frozen_trial \u001b[38;5;241m=\u001b[39m _run_trial(study, func, catch)\n\u001b[1;32m    164\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m    165\u001b[0m     \u001b[38;5;66;03m# The following line mitigates memory problems that can be occurred in some\u001b[39;00m\n\u001b[1;32m    166\u001b[0m     \u001b[38;5;66;03m# environments (e.g., services that use computing containers such as GitHub Actions).\u001b[39;00m\n\u001b[1;32m    167\u001b[0m     \u001b[38;5;66;03m# Please refer to the following PR for further details:\u001b[39;00m\n\u001b[1;32m    168\u001b[0m     \u001b[38;5;66;03m# https://github.com/optuna/optuna/pull/325.\u001b[39;00m\n\u001b[1;32m    169\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m gc_after_trial:\n",
      "File \u001b[0;32m~/.conda/envs/torch-env/lib/python3.11/site-packages/optuna/study/_optimize.py:251\u001b[0m, in \u001b[0;36m_run_trial\u001b[0;34m(study, func, catch)\u001b[0m\n\u001b[1;32m    244\u001b[0m         \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28;01mFalse\u001b[39;00m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mShould not reach.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    246\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (\n\u001b[1;32m    247\u001b[0m     frozen_trial\u001b[38;5;241m.\u001b[39mstate \u001b[38;5;241m==\u001b[39m TrialState\u001b[38;5;241m.\u001b[39mFAIL\n\u001b[1;32m    248\u001b[0m     \u001b[38;5;129;01mand\u001b[39;00m func_err \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m    249\u001b[0m     \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(func_err, catch)\n\u001b[1;32m    250\u001b[0m ):\n\u001b[0;32m--> 251\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m func_err\n\u001b[1;32m    252\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m frozen_trial\n",
      "File \u001b[0;32m~/.conda/envs/torch-env/lib/python3.11/site-packages/optuna/study/_optimize.py:200\u001b[0m, in \u001b[0;36m_run_trial\u001b[0;34m(study, func, catch)\u001b[0m\n\u001b[1;32m    198\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m get_heartbeat_thread(trial\u001b[38;5;241m.\u001b[39m_trial_id, study\u001b[38;5;241m.\u001b[39m_storage):\n\u001b[1;32m    199\u001b[0m     \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 200\u001b[0m         value_or_values \u001b[38;5;241m=\u001b[39m func(trial)\n\u001b[1;32m    201\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m exceptions\u001b[38;5;241m.\u001b[39mTrialPruned \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    202\u001b[0m         \u001b[38;5;66;03m# TODO(mamu): Handle multi-objective cases.\u001b[39;00m\n\u001b[1;32m    203\u001b[0m         state \u001b[38;5;241m=\u001b[39m TrialState\u001b[38;5;241m.\u001b[39mPRUNED\n",
      "Cell \u001b[0;32mIn[13], line 25\u001b[0m, in \u001b[0;36mobjective\u001b[0;34m(trial)\u001b[0m\n\u001b[1;32m     20\u001b[0m model \u001b[38;5;241m=\u001b[39m DeepSurvivalMachines(k \u001b[38;5;241m=\u001b[39m k,\n\u001b[1;32m     21\u001b[0m                              distribution \u001b[38;5;241m=\u001b[39m dist,\n\u001b[1;32m     22\u001b[0m                              layers \u001b[38;5;241m=\u001b[39m layers, \n\u001b[1;32m     23\u001b[0m                              activation \u001b[38;5;241m=\u001b[39m activation)\n\u001b[1;32m     24\u001b[0m \u001b[38;5;66;03m# The fit method is called to train the model\u001b[39;00m\n\u001b[0;32m---> 25\u001b[0m _, train_loss, valid_loss \u001b[38;5;241m=\u001b[39m model\u001b[38;5;241m.\u001b[39mfit(x_train, t_train, e_train, iters \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m50\u001b[39m, val_data\u001b[38;5;241m=\u001b[39m(x_valid, t_valid, e_valid),\n\u001b[1;32m     26\u001b[0m           batch_size\u001b[38;5;241m=\u001b[39m\u001b[38;5;241m10000\u001b[39m, learning_rate \u001b[38;5;241m=\u001b[39m lr)\n\u001b[1;32m     28\u001b[0m err \u001b[38;5;241m=\u001b[39m model\u001b[38;5;241m.\u001b[39mcompute_nll(x_valid, t_valid, e_valid)\n\u001b[1;32m     29\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m err\n",
      "File \u001b[0;32m~/auton-survival/auton_survival/models/dsm/__init__.py:263\u001b[0m, in \u001b[0;36mDSMBase.fit\u001b[0;34m(self, x, t, e, vsize, val_data, iters, learning_rate, batch_size, elbo, optimizer)\u001b[0m\n\u001b[1;32m    261\u001b[0m model \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_gen_torch_model(inputdim, optimizer, risks\u001b[38;5;241m=\u001b[39mmaxrisk)\n\u001b[1;32m    262\u001b[0m \u001b[38;5;66;03m# model, _ = train_dsm(model,\u001b[39;00m\n\u001b[0;32m--> 263\u001b[0m model, _, train_loss, valid_loss \u001b[38;5;241m=\u001b[39m train_dsm(model,\n\u001b[1;32m    264\u001b[0m                      x_train, t_train, e_train,\n\u001b[1;32m    265\u001b[0m                      x_val, t_val, e_val,\n\u001b[1;32m    266\u001b[0m                      n_iter\u001b[38;5;241m=\u001b[39miters,\n\u001b[1;32m    267\u001b[0m                      lr\u001b[38;5;241m=\u001b[39mlearning_rate,\n\u001b[1;32m    268\u001b[0m                      elbo\u001b[38;5;241m=\u001b[39melbo,\n\u001b[1;32m    269\u001b[0m                      bs\u001b[38;5;241m=\u001b[39mbatch_size,\n\u001b[1;32m    270\u001b[0m                      random_seed\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mrandom_seed)\n\u001b[1;32m    272\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtorch_model \u001b[38;5;241m=\u001b[39m model\u001b[38;5;241m.\u001b[39meval()\n\u001b[1;32m    273\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfitted \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n",
      "File \u001b[0;32m~/auton-survival/auton_survival/models/dsm/utilities.py:187\u001b[0m, in \u001b[0;36mtrain_dsm\u001b[0;34m(model, x_train, t_train, e_train, x_valid, t_valid, e_valid, n_iter, lr, elbo, bs, random_seed)\u001b[0m\n\u001b[1;32m    185\u001b[0m valid_loss \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m\n\u001b[1;32m    186\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m r \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mrange\u001b[39m(model\u001b[38;5;241m.\u001b[39mrisks):\n\u001b[0;32m--> 187\u001b[0m   valid_loss \u001b[38;5;241m+\u001b[39m\u001b[38;5;241m=\u001b[39m conditional_loss(model,\n\u001b[1;32m    188\u001b[0m                                  x_valid,\n\u001b[1;32m    189\u001b[0m                                  t_valid_,\n\u001b[1;32m    190\u001b[0m                                  e_valid_,\n\u001b[1;32m    191\u001b[0m                                  elbo\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[1;32m    192\u001b[0m                                  risk\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mstr\u001b[39m(r\u001b[38;5;241m+\u001b[39m\u001b[38;5;241m1\u001b[39m))\n\u001b[1;32m    194\u001b[0m valid_loss \u001b[38;5;241m=\u001b[39m valid_loss\u001b[38;5;241m.\u001b[39mdetach()\u001b[38;5;241m.\u001b[39mcpu()\u001b[38;5;241m.\u001b[39mnumpy()\n\u001b[1;32m    195\u001b[0m costs\u001b[38;5;241m.\u001b[39mappend(\u001b[38;5;28mfloat\u001b[39m(valid_loss))\n",
      "File \u001b[0;32m~/auton-survival/auton_survival/models/dsm/losses.py:287\u001b[0m, in \u001b[0;36mconditional_loss\u001b[0;34m(model, x, t, e, elbo, risk)\u001b[0m\n\u001b[1;32m    284\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mconditional_loss\u001b[39m(model, x, t, e, elbo\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, risk\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m1\u001b[39m\u001b[38;5;124m'\u001b[39m):\n\u001b[1;32m    286\u001b[0m   \u001b[38;5;28;01mif\u001b[39;00m model\u001b[38;5;241m.\u001b[39mdist \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mWeibull\u001b[39m\u001b[38;5;124m'\u001b[39m:\n\u001b[0;32m--> 287\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m _conditional_weibull_loss(model, x, t, e, elbo, risk)\n\u001b[1;32m    288\u001b[0m   \u001b[38;5;28;01melif\u001b[39;00m model\u001b[38;5;241m.\u001b[39mdist \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mLogNormal\u001b[39m\u001b[38;5;124m'\u001b[39m:\n\u001b[1;32m    289\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m _conditional_lognormal_loss(model, x, t, e, elbo, risk)\n",
      "File \u001b[0;32m~/auton-survival/auton_survival/models/dsm/losses.py:238\u001b[0m, in \u001b[0;36m_conditional_weibull_loss\u001b[0;34m(model, x, t, e, elbo, risk)\u001b[0m\n\u001b[1;32m    235\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_conditional_weibull_loss\u001b[39m(model, x, t, e, elbo\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m, risk\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m1\u001b[39m\u001b[38;5;124m'\u001b[39m):\n\u001b[1;32m    237\u001b[0m   alpha \u001b[38;5;241m=\u001b[39m model\u001b[38;5;241m.\u001b[39mdiscount\n\u001b[0;32m--> 238\u001b[0m   shape, scale, logits \u001b[38;5;241m=\u001b[39m model\u001b[38;5;241m.\u001b[39mforward(x, risk)\n\u001b[1;32m    240\u001b[0m   k_ \u001b[38;5;241m=\u001b[39m shape\n\u001b[1;32m    241\u001b[0m   b_ \u001b[38;5;241m=\u001b[39m scale\n",
      "File \u001b[0;32m~/auton-survival/auton_survival/models/dsm/dsm_torch.py:275\u001b[0m, in \u001b[0;36mDeepSurvivalMachinesTorch.forward\u001b[0;34m(self, x, risk)\u001b[0m\n\u001b[1;32m    267\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, x, risk\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m1\u001b[39m\u001b[38;5;124m'\u001b[39m):\n\u001b[1;32m    268\u001b[0m \u001b[38;5;250m  \u001b[39m\u001b[38;5;124;03m\"\"\"The forward function that is called when data is passed through DSM.\u001b[39;00m\n\u001b[1;32m    269\u001b[0m \n\u001b[1;32m    270\u001b[0m \u001b[38;5;124;03m  Args:\u001b[39;00m\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    273\u001b[0m \n\u001b[1;32m    274\u001b[0m \u001b[38;5;124;03m  \"\"\"\u001b[39;00m\n\u001b[0;32m--> 275\u001b[0m   xrep \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39membedding(x)\n\u001b[1;32m    276\u001b[0m   dim \u001b[38;5;241m=\u001b[39m x\u001b[38;5;241m.\u001b[39mshape[\u001b[38;5;241m0\u001b[39m]\n\u001b[1;32m    277\u001b[0m   \u001b[38;5;28;01mreturn\u001b[39;00m(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mact(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mshapeg[risk](xrep))\u001b[38;5;241m+\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mshape[risk]\u001b[38;5;241m.\u001b[39mexpand(dim, \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m),\n\u001b[1;32m    278\u001b[0m          \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mact(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mscaleg[risk](xrep))\u001b[38;5;241m+\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mscale[risk]\u001b[38;5;241m.\u001b[39mexpand(dim, \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m),\n\u001b[1;32m    279\u001b[0m          \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mgate[risk](xrep)\u001b[38;5;241m/\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtemp)\n",
      "File \u001b[0;32m~/.conda/envs/torch-env/lib/python3.11/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1496\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1497\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1498\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1499\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1500\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m   1502\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m   1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n",
      "File \u001b[0;32m~/auton-survival/auton_survival/models/dsm/dsm_torch.py:132\u001b[0m, in \u001b[0;36mProfDeepNet.forward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m    129\u001b[0m x_profs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpool1(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mact(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconv1(x_profs)))\n\u001b[1;32m    131\u001b[0m \u001b[38;5;66;03m# Shape is now (batch, 8, 2, 5)\u001b[39;00m\n\u001b[0;32m--> 132\u001b[0m x_profs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpool2(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mact(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconv2(x_profs)))\n\u001b[1;32m    134\u001b[0m \u001b[38;5;66;03m# Shape is now (batch, 80)\u001b[39;00m\n\u001b[1;32m    135\u001b[0m x_profs \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mflatten(x_profs, \u001b[38;5;241m1\u001b[39m)\n",
      "File \u001b[0;32m~/.conda/envs/torch-env/lib/python3.11/site-packages/torch/nn/modules/module.py:1501\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1496\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1497\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1498\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1499\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1500\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1501\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m   1502\u001b[0m \u001b[38;5;66;03m# Do not call functions when jit is used\u001b[39;00m\n\u001b[1;32m   1503\u001b[0m full_backward_hooks, non_full_backward_hooks \u001b[38;5;241m=\u001b[39m [], []\n",
      "File \u001b[0;32m~/.conda/envs/torch-env/lib/python3.11/site-packages/torch/nn/modules/pooling.py:166\u001b[0m, in \u001b[0;36mMaxPool2d.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    165\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m: Tensor):\n\u001b[0;32m--> 166\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m F\u001b[38;5;241m.\u001b[39mmax_pool2d(\u001b[38;5;28minput\u001b[39m, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mkernel_size, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstride,\n\u001b[1;32m    167\u001b[0m                         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpadding, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdilation, ceil_mode\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mceil_mode,\n\u001b[1;32m    168\u001b[0m                         return_indices\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mreturn_indices)\n",
      "File \u001b[0;32m~/.conda/envs/torch-env/lib/python3.11/site-packages/torch/_jit_internal.py:484\u001b[0m, in \u001b[0;36mboolean_dispatch.<locals>.fn\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    482\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m if_true(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m    483\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 484\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m if_false(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[0;32m~/.conda/envs/torch-env/lib/python3.11/site-packages/torch/nn/functional.py:782\u001b[0m, in \u001b[0;36m_max_pool2d\u001b[0;34m(input, kernel_size, stride, padding, dilation, ceil_mode, return_indices)\u001b[0m\n\u001b[1;32m    780\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m stride \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    781\u001b[0m     stride \u001b[38;5;241m=\u001b[39m torch\u001b[38;5;241m.\u001b[39mjit\u001b[38;5;241m.\u001b[39mannotate(List[\u001b[38;5;28mint\u001b[39m], [])\n\u001b[0;32m--> 782\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mmax_pool2d(\u001b[38;5;28minput\u001b[39m, kernel_size, stride, padding, dilation, ceil_mode)\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "study.optimize(objective, n_trials=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "30aa9fa7-ff67-4adf-a7c1-6f325b5f6ee1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "best_params = study.best_params"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1b0b5d37-8e2a-4847-89ff-5ad288eadb16",
   "metadata": {},
   "source": [
    "### Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5955bc21-cca9-4228-ad86-10e0b1dcba2c",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'model' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[1], line 3\u001b[0m\n\u001b[1;32m      1\u001b[0m times \u001b[38;5;241m=\u001b[39m [\u001b[38;5;241m50\u001b[39m, \u001b[38;5;241m100\u001b[39m, \u001b[38;5;241m200\u001b[39m, \u001b[38;5;241m300\u001b[39m, \u001b[38;5;241m400\u001b[39m, \u001b[38;5;241m500\u001b[39m]\n\u001b[0;32m----> 3\u001b[0m out_risk \u001b[38;5;241m=\u001b[39m model\u001b[38;5;241m.\u001b[39mpredict_risk(x_test, times)\n\u001b[1;32m      4\u001b[0m out_survival \u001b[38;5;241m=\u001b[39m model\u001b[38;5;241m.\u001b[39mpredict_survival(x_test, times)\n",
      "\u001b[0;31mNameError\u001b[0m: name 'model' is not defined"
     ]
    }
   ],
   "source": [
    "times = [50, 100, 200, 300, 400, 500]\n",
    "\n",
    "out_risk = model.predict_risk(x_test, times)\n",
    "out_survival = model.predict_survival(x_test, times)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c871d80a-d2bb-4120-9d29-7b405124e8d2",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "cis = []\n",
    "brs = []\n",
    "\n",
    "et_train = np.array([(e_train[i], t_train[i]) for i in range(len(e_train))],\n",
    "                 dtype = [('e', bool), ('t', float)])\n",
    "et_test = np.array([(e_test[i], t_test[i]) for i in range(len(e_test))],\n",
    "                 dtype = [('e', bool), ('t', float)])\n",
    "et_val = np.array([(e_valid[i], t_valid[i]) for i in range(len(e_valid))],\n",
    "                 dtype = [('e', bool), ('t', float)])\n",
    "\n",
    "roc_auc = []\n",
    "for i, _ in enumerate(times):\n",
    "    roc_auc.append(cumulative_dynamic_auc(et_train, et_test, out_risk[:, i], times[i])[0])\n",
    "for t in enumerate(times):\n",
    "    print(f\"For {t[1]} prediction,\")\n",
    "    print(\"ROC AUC \", roc_auc[t[0]][0], \"\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ee26dcfc-a0b4-420a-805f-ef6aa22204f7",
   "metadata": {},
   "source": [
    "# Hyperparam Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7ad13b29-f083-48c5-91e5-7741fdef621f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "with open('/home/ar0535/best_tm_model.pkl', 'rb') as f:\n",
    "    model_data = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "92a3fd7f-421d-494f-99dc-327c70cd9b87",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "model = model_data['models'][0][0][1]\n",
    "train_loss = model_data['train_loss']\n",
    "valid_loss = model_data['valid_loss']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "7d4262e2-e73b-46c5-a73b-6fdb171951a0",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<bound method Module.parameters of DeepSurvivalMachinesTorch(\n",
       "  (act): Tanh()\n",
       "  (shape): ParameterDict(  (1): Parameter containing: [torch.DoubleTensor of size 3])\n",
       "  (scale): ParameterDict(  (1): Parameter containing: [torch.DoubleTensor of size 3])\n",
       "  (gate): ModuleDict(\n",
       "    (1): Sequential(\n",
       "      (0): Linear(in_features=120, out_features=3, bias=False)\n",
       "    )\n",
       "  )\n",
       "  (scaleg): ModuleDict(\n",
       "    (1): Sequential(\n",
       "      (0): Linear(in_features=120, out_features=3, bias=True)\n",
       "    )\n",
       "  )\n",
       "  (shapeg): ModuleDict(\n",
       "    (1): Sequential(\n",
       "      (0): Linear(in_features=120, out_features=3, bias=True)\n",
       "    )\n",
       "  )\n",
       "  (embedding): ProfDeepNet(\n",
       "    (conv1): Conv2d(1, 4, kernel_size=(5, 5), stride=(1, 1), padding=same)\n",
       "    (conv2): Conv2d(4, 8, kernel_size=(5, 5), stride=(1, 1), padding=same)\n",
       "    (pool1): MaxPool2d(kernel_size=(1, 3), stride=(1, 3), padding=0, dilation=1, ceil_mode=False)\n",
       "    (pool2): MaxPool2d(kernel_size=(2, 2), stride=(2, 2), padding=0, dilation=1, ceil_mode=False)\n",
       "    (l1): Linear(in_features=80, out_features=100, bias=True)\n",
       "    (l2): Linear(in_features=100, out_features=60, bias=True)\n",
       "    (l3): Linear(in_features=74, out_features=175, bias=True)\n",
       "    (l4): Linear(in_features=175, out_features=225, bias=True)\n",
       "    (l5): Linear(in_features=225, out_features=120, bias=True)\n",
       "  )\n",
       ")>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.torch_model.parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "e1146bec-f8ff-4aaf-89eb-5a5ac4c19bba",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "total_params = sum(p.numel() for p in model.torch_model.parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "856fbbe7-5fb7-4ce5-8a75-9ab0b34a9aca",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "96009"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "9463c85c-99c5-42cb-ac93-a602e7b7e67b",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiwAAAGdCAYAAAAxCSikAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy81sbWrAAAACXBIWXMAAA9hAAAPYQGoP6dpAABNTElEQVR4nO3deXxU5b3H8c9km+wDgawQVllkEXFjUVFUgqgUxApuLIpaKliRay8Xd+tValvrcrG1tioiJSI77mAV0BJcKKAiIEjYQsKeTNaZJHPuHycMhJCQhCRnZvJ9v17zmnnOPDP85jgwX5/znOfYDMMwEBEREfFhQVYXICIiInImCiwiIiLi8xRYRERExOcpsIiIiIjPU2ARERERn6fAIiIiIj5PgUVERER8ngKLiIiI+LwQqwtoKB6Ph/379xMTE4PNZrO6HBEREakFwzDIz88nJSWFoKDqx1ECJrDs37+f1NRUq8sQERGReti7dy9t27at9vmACSwxMTGA+YFjY2MtrkZERERqw+l0kpqa6v0dr07ABJbjh4FiY2MVWERERPzMmaZzaNKtiIiI+DwFFhEREfF5CiwiIiLi8wJmDkttlJeXU1paanUZfi00NJTg4GCryxARkWam2QSWgoIC9u3bh2EYVpfi12w2G23btiU6OtrqUkREpBlpFoGlvLycffv2ERkZSXx8vBaWqyfDMDh06BD79u2jS5cuGmkREZEm0ywCS2lpKYZhEB8fT0REhNXl+LX4+Hh27dpFaWmpAouIiDSZZjXpViMrZ0/7UERErNCsAouIiIj4JwUWERER8XkKLM3MlVdeydSpU60uQ0REpE6axaRbf3SmuSLjx49n9uzZdX7fxYsXExoaWs+qRERErKHA4qOys7O9j+fPn8/jjz/Otm3bvNtOPduptLS0VkEkLi6u4YoUEZHAZxiweQl8vxDGvA1B1pwh2iwPCRmGQZG7zJJbbReuS0pK8t4cDgc2m83bLikpoUWLFrz77rtceeWVhIeHM3fuXI4cOcKtt95K27ZtiYyMpHfv3qSnp1d631MPCXXo0IFnn32Wu+66i5iYGNq1a8drr73WkLtbRET81aFtMGcELLwTtn0Am96xrJRmOcJSXFpOj8c/seTP/vF3Q4kMa5jdPn36dJ5//nnefPNN7HY7JSUlXHjhhUyfPp3Y2Fg++OADxo4dS6dOnejXr1+17/P888/z9NNP8/DDD7Nw4UJ+/etfM2jQILp3794gdYqIiJ9x5cPq52DdX8FTBiHhcNmD0GuUZSU1y8ASKKZOncqoUZW/PA899JD38f3338/HH3/MggULagws1113Hffddx9ghqAXXniBVatWKbCIiDQ3hgE/LIIVj0J+xdSEbtfBtTOhZQdLS2uWgSUiNJgffzfUsj+7oVx00UWV2uXl5fz+979n/vz5ZGVl4XK5cLlcREVF1fg+5513nvfx8UNPBw8ebLA6RUTEDxz4ET78Lez+0my37AjD/gBd06ytq0KzDCw2m63BDstY6dQg8vzzz/PCCy/w4osv0rt3b6Kiopg6dSput7vG9zl1sq7NZsPj8TR4vSIi4oNKnLDq9/DVq2CUQ0gEDPovGHA/hIZbXZ2X//9qi9cXX3zBiBEjuOOOOwDweDxs376dc8891+LKRETE5xgGfPcurHwMCg6Y284dDkOfhRbtrK3tNBRYAsg555zDokWLWLt2LS1btuTPf/4zOTk5CiwiIlJZzg/m4Z89a812XGe47g9wzjXW1lUDBZYA8thjj5GZmcnQoUOJjIzk3nvvZeTIkeTl5VldmoiI+ILiXFg1E77+u3n4JzQSBv0WBkyGELvV1dXIZtR2YRAf53Q6cTgc5OXlERsbW+m5kpISMjMz6dixI+HhvnM8zh9pX4qI+CGPB757B1Y+DoWHzG09RsLQZ8DR1tLSavr9PplGWERERAJZ9nfw4UOw9yuz3bqrefZP58HW1lVHCiwiIiKByFVgHv5Z9xcwPBAaBVdOh36/hpAwq6urMwUWERGRQLN9Jbw/DfL2mO2eo8zDP7Ep1tZ1FhRYREREAkXBQfj4f8zVagEc7eCGP0OXIdbW1QAUWERERPydYcCGt2HFY1CSC7Yg6H8fDH4Ywmpe7dxfKLCIiIj4s8M74L0HTiypn9wHhr8MKedbWlZDU2ARERHxR2Vu+PdLsOaPUO4y11QZ/Aj0mwTBgffzHnifSEREJNDt+Qre+w0c2mq2z7kGrv8ztGxvbV2NKMjqAqTxXHnllUydOtXb7tChAy+++GKNr7HZbCxdurRR6xIRkXoqyTPP/nkjzQwrUfFw0+tw+8KADiugERafNXz4cIqLi/n000+rPJeRkcHAgQNZv349F1xwQa3f85tvvqlyhWcREfETPy43r/9TkGO2+94BQ56GyDhr62oidRphmTlzJhdffDExMTEkJCQwcuRItm3bVuNrVq1ahc1mq3LbunVrpX6LFi2iR48e2O12evTowZIlS+r+aQLIxIkT+eyzz9i9e3eV59544w3OP//8OoUVgPj4eCIjIxuqRBERaQp5WZB+G7w71gwrcZ1h/Hsw4pVmE1agjoFl9erVTJ48mXXr1rFy5UrKyspIS0ujsLDwjK/dtm0b2dnZ3luXLl28z2VkZDBmzBjGjh3Lpk2bGDt2LKNHj+arr76q+ycKEDfccAMJCQnMnj270vaioiLmz5/PyJEjufXWW2nbti2RkZH07t2b9PT0Gt/z1ENC27dvZ9CgQYSHh9OjRw9WrlzZCJ9ERETqxVMOX70Gr/SDbR9AUIh5ocJfr4WOg6yursnV6ZDQxx9/XKn95ptvkpCQwPr16xk0qOadl5CQQIsWLU773IsvvsiQIUOYMWMGADNmzGD16tW8+OKLZ/wRrhfDgNKihn/f2giNBJvtjN1CQkIYN24cs2fP5vHHH8dW8ZoFCxbgdru5++67SU9PZ/r06cTGxvLBBx8wduxYOnXqRL9+/c74/h6Ph1GjRtG6dWvWrVuH0+msNN9FREQslJcFS34Fu74w220vgeEvQWIPa+uy0FnNYcnLywMgLu7MQ1J9+/alpKSEHj168OijjzJ48ImLLmVkZPDggw9W6j906NAaJ4i6XC5cLpe37XQ6a194aRE8a9HyxA/vr/UiPnfddRd//OMfWbVqlXd/vfHGG4waNYo2bdrw0EMPefvef//9fPzxxyxYsKBWgeXTTz9ly5Yt7Nq1i7ZtzSt1PvvsswwbNqweH0pERBrMj8th+f3mAnChUTDkKbhoIgQ17/Nk6v3pDcNg2rRpXHbZZfTq1avafsnJybz22mssWrSIxYsX061bN66++mrWrFnj7ZOTk0NiYmKl1yUmJpKTk1Pt+86cOROHw+G9paam1vej+Kzu3bszcOBA3njjDQB+/vlnvvjiC+666y7Ky8t55plnOO+882jVqhXR0dGsWLGCPXv21Oq9t2zZQrt27bxhBWDAgAGN8jlERKQW3IWw/DfmXJWSXEjpC5O+gEvuafZhBc5ihGXKlCl89913fPnllzX269atG926dfO2BwwYwN69e/nTn/5U6TCS7ZTDJIZhVNl2shkzZjBt2jRv2+l01j60hEaaIx1WCK3bpNeJEycyZcoUXnnlFd58803at2/P1VdfzR//+EdeeOEFXnzxRXr37k1UVBRTp07F7XbX6n0Nw6iyrab9LSIijWj/Rlh0NxzZDtjgsqlw5cN+eVXlxlKvwHL//fezfPly1qxZU+n/0Gurf//+zJ0719tOSkqqMppy8ODBKqMuJ7Pb7djt9jr/2YA5h8RPrq0wevRoHnjgAebNm8dbb73FPffcg81m44svvmDEiBHccccdgDknZfv27Zx77rm1et8ePXqwZ88e9u/fT0qKeXgsIyOj0T6HiIichscDGbPgX78DTynEpMCovzXLSbVnUqcxJsMwmDJlCosXL+azzz6jY8eO9fpDN2zYQHJysrc9YMCAKmeorFixgoEDB9br/QNJdHQ0Y8aM4eGHH2b//v1MmDABgHPOOYeVK1eydu1atmzZwq9+9asaD6Gd6pprrqFbt26MGzeOTZs28cUXX/DII4800qcQEZEqnNkw90ZY+ZgZVs4dDr/+t8JKNeo0wjJ58mTmzZvHsmXLiImJ8f5AOhwOIiIiAPNQTVZWFnPmzAHMM4A6dOhAz549cbvdzJ07l0WLFrFo0SLv+z7wwAMMGjSI5557jhEjRrBs2TI+/fTTMx5uai4mTpzI66+/TlpaGu3atQPgscceIzMzk6FDhxIZGcm9997LyJEjvROhzyQoKIglS5YwceJELrnkEjp06MDLL7/Mtdde25gfRUREALZ+CMsmQ/FRc6rAtTPhgvG1Oou0ubIZp5vMUF3nanbkm2++6f0//wkTJrBr1y5WrVoFwB/+8Adee+01srKyiIiIoGfPnsyYMYPrrruu0nssXLiQRx99lJ07d9K5c2eeeeYZRo0aVesP4nQ6cTgc5OXlERsbW+m5kpISMjMz6dixI+Hh4bV+T6lK+1JE5Cy4i2DFo/Dt62Y76Txzaf34rtbWZaGafr9PVqfA4ssUWJqG9qWISD3lfA8LJ8LhihXiB0yBqx+HkHrOxwwQtQ0supaQiIhIY/J44KtX4dMnoNwN0Ylw46vQ+SqrK/MrCiwiIiKNpeAgLP017Ki4kG3XYTBiFkS1trYuP6TAIiIi0hh+WmGGlaLDEBIOaf8LF9+tibX1pMAiIiLSkMpLYeXjsO4vZjuxF9z0D0io3TpZcnrNKrAEyPxiS2kfiojUID8H3h0Pe9eZ7X6/hmuehFCdpHC2mkVgCQ4OBsDtdnvXi5H6Ob70//F9KiIiFXZnwILxUHAA7LFw49+g+3Vnfp3USrMILCEhIURGRnLo0CFCQ0MJ0kWk6sXj8XDo0CEiIyMJCWkWXx0RkTMzDPjqb7DiEfCUQUIPGDMXWnW2urKA0ix+dWw2G8nJyWRmZrJ7926ry/FrQUFBtGvXThdKFBEB8wrL7z0A3y8w271ugl/8n99cr86fNIvAAhAWFkaXLl1qfTVjOb2wsDCNUImIABz5GeaPhYObwRYMQ5+BfpN0FlAjaTaBBczRAa3OKiIiZ23bR7D4V+DKg6gEuHk2dLjU6qoCWrMKLCIiImfFUw6rZsKaP5rt1H5w81sQm2xtXc2AAouIiEhtFB2FxfecWLX2knsh7RkICbO2rmZCgUVERORMsjfB/Dsgdw+ERMDwl6DPGKuralYUWERERGqycR68/yCUlUDLDuYpy0m9ra6q2VFgEREROZ0yF3w8A7593Wx3SYNRr0FES2vraqYUWERERE6VlwXvjoOsbwEbXPk/MOi/Qcs6WEaBRURE5GSZX8DCO6HwEIQ7YNQ/oGua1VU1ewosIiIiYC6x//Vr5mEgoxwSe8OYORDXyerKBAUWERERKHPDh/8F/5ljts8bAze8CGGRlpYlJyiwiIhI81ZwCN4dC3sywBYEQ34HA6ZoiX0fo8AiIiLNV/Z38M5tkLcX7LHwyzegyxCrq5LTUGAREZHmafNSWPprKC2CuM5w6zsQ39XqqqQaCiwiItK8eDyw+vew+jmz3fkqc2RF66v4NAUWERFpPlwFsHQSbHnPbA+YAtc8BcH6OfR1+i8kIiLNw7Hd5nyVAz9AcBjc8AL0vcPqqqSWFFhERCTw7fq3eSZQ0RGISjCvB9Sun9VVSR0osIiISGBbPxs++C/wlEFyH7hlHjjaWl2V1JECi4iIBKbyUvjkYXP1WoCeo2DEK1oMzk8psIiISOApOgoLxkPmGrN91WNw+X9pMTg/psAiIiKB5eAWSL8Fju2CsGgY9Rp0v97qquQsKbCIiEjg2PYRLLob3AXQoj3cmg6JPa2uShqAAouIiPg/w4C1/wcrHwcM6HA53PwWRLWyujJpIAosIiLi3zzl8PEM+PpvZvuiiTDsOQgOtbYuaVAKLCIi4r9Ki81DQFvfN9tDn4UBk62tSRqFAouIiPinwiPm5Np9X5sr1974N+g1yuqqpJEosIiIiP85mgn//CUc2QHhDrglHTpcanVV0ogUWERExL9k/QfmjYbCQ+BIhdsXQkJ3q6uSRqbAIiIi/uOnFeaCcKVFkNQbblsAsclWVyVNQIFFRET8w/rZ8P40MMqh81Uweg7YY6yuSpqIAouIiPg2w4DPn4U1fzDb598Ow1/SacvNTFBdOs+cOZOLL76YmJgYEhISGDlyJNu2bavxNYsXL2bIkCHEx8cTGxvLgAED+OSTTyr1mT17NjabrcqtpKSk7p9IREQCR3kpLL3vRFi5Yrp5AUOFlWanToFl9erVTJ48mXXr1rFy5UrKyspIS0ujsLCw2tesWbOGIUOG8OGHH7J+/XoGDx7M8OHD2bBhQ6V+sbGxZGdnV7qFh4fX71OJiIj/c+XDP2+GTfPAFmyOqgx+WBcwbKZshmEY9X3xoUOHSEhIYPXq1QwaNKjWr+vZsydjxozh8ccfB8wRlqlTp5Kbm1vfUnA6nTgcDvLy8oiNja33+4iIiA/IzzFPW875HkIj4ebZ0HWo1VVJI6jt73edRlhOlZeXB0BcXFytX+PxeMjPz6/ymoKCAtq3b0/btm254YYbqozAnMrlcuF0OivdREQkABzaBv+4xgwrUfEw4QOFFal/YDEMg2nTpnHZZZfRq1evWr/u+eefp7CwkNGjR3u3de/endmzZ7N8+XLS09MJDw/n0ksvZfv27dW+z8yZM3E4HN5bampqfT+KiIj4it1r4fUhkLcX4jrDxJXQ5gKrqxIfUO9DQpMnT+aDDz7gyy+/pG3btrV6TXp6OnfffTfLli3jmmuuqbafx+PhggsuYNCgQbz88sun7eNyuXC5XN620+kkNTVVh4RERPzV5iWw+F4od0PbS+DWd3S15WagtoeE6nVa8/3338/y5ctZs2ZNrcPK/PnzmThxIgsWLKgxrAAEBQVx8cUX1zjCYrfbsdvtdapbRER8VMZf4JMZ5uPuN8BN/4DQCGtrEp9Sp0NChmEwZcoUFi9ezGeffUbHjh1r9br09HQmTJjAvHnzuP7662v152zcuJHkZK1eKCIS0AwDPp95Iqxccq+5IJzCipyiTiMskydPZt68eSxbtoyYmBhycnIAcDgcRESYX64ZM2aQlZXFnDlzADOsjBs3jpdeeon+/ft7XxMREYHD4QDgqaeeon///nTp0gWn08nLL7/Mxo0beeWVVxrsg4qIiI8xDFjxKGTMMttXPQqXP6TTluW06jTC8te//pW8vDyuvPJKkpOTvbf58+d7+2RnZ7Nnzx5v+29/+xtlZWVMnjy50mseeOABb5/c3Fzuvfdezj33XNLS0sjKymLNmjVccsklDfARRUTE53jK4f2pJ8LKtc/BoN8qrEi1zmodFl+idVhERPxEeSks/TV8vwBsQTD8ZbhgrNVViUUaddKtiIhIvZS5YMGdsO0DCAqBUa9Br5usrkr8gAKLiIg0DXchvHM77Pwcgu3m5Npu11pdlfgJBRYREWl8JXkwbwzsyYDQKLg1HTpdYXVV4kcUWEREpHEVHoG5oyB7I9gdcMdCSNVJFVI3CiwiItJ48nNgzgg4tBUiW8PYJZB8ntVViR9SYBERkcaRuwfe+gUcy4SYZBi3HOK7Wl2V+CkFFhERaXiHd8CcX4AzC1q0h3HLIK52q6OLnI4Ci4iINKycH+DtkVB4CFp3NcNKbIrVVYmfU2AREZGGs2+9OcG2JBeSesPYpRDV2uqqJAAosIiISMPY9aV56rK7ANpeArcvgIgWVlclAUKBRUREzt72lTD/DigrgY6D4JZ0sEdbXZUEEAUWERE5Oz8ug4UTwVMKXa+Fm9+C0HCrq5IAo8AiIiL1t2k+LJ0Ehgd6jjKvDRQcanVVEoCCrC5ARET81H/ehiW/MsPK+XfATf9QWJFGo8AiIiJ1983rsHwKYMBFE+EX/wdBwVZXJQFMgUVEROpm3avwwTTzcb9fw/XPQ5B+TqRx6RsmIiK19++X4OPp5uNLH4BrZ4LNZm1N0ixo0q2IiNTO6j/C5/9rPr5iOlw5Q2FFmowCi4iI1Mww4PNnYc0fzPZVj8Kg31pbkzQ7CiwiIlI9w4BPnzAPBQEMeRou/Y21NUmzpMAiIiKnZxjw8Qz46q9m+9rnoP8ka2uSZkuBRUREqvJ44MP/gm/fMNs3vAAX3WVtTdKsKbCIiEhlnnJ47zewYS5ggxGzoO8dVlclzZwCi4iInFBeBsvug+/mgy0IRr4KfcZYXZWIAouIiFQoL4XF98DmJWALNpfa7zXK6qpEAAUWEREBKHPDwjth6/sQFAo3z4Zzb7C6KhEvBRYRkeautATeHQfbP4FgO4yZC13TrK5KpBIFFhGR5sxdBPNvh58/g5AIuHUedL7K6qpEqlBgERFprtyFMG8M7PoCQqPgtvnQ8XKrqxI5LQUWEZHmyJUP/7wZ9mRAWAzcsRDa9be6KpFqKbCIiDQ3JU4zrOxdB3YHjF0MbS+yuiqRGimwiIg0JyV5MPeXsO9rCHfA2KXQ5gKrqxI5IwUWEZHmoiQP3h4FWd9CeAsYtwxSzre6KpFaUWAREWkOinPh7Rth/38goqUZVpL7WF2VSK0psIiIBLriYxVhZQNExMH45ZDU2+qqROpEgUVEJJAVHYW3R0L2JohsBeOWQ1Ivq6sSqTMFFhGRQFV0FOb8AnK+h8jWMP49SOxhdVUi9aLAIiISiAoPw5wRcOAHiEoww0pCd6urEqk3BRYRkUBTcMgcWTn4I0Qnwvj3Ib6r1VWJnJWgunSeOXMmF198MTExMSQkJDBy5Ei2bdt2xtetXr2aCy+8kPDwcDp16sSrr75apc+iRYvo0aMHdrudHj16sGTJkrqUJiIiAAUH4a3hFWElCSZ8oLAiAaFOgWX16tVMnjyZdevWsXLlSsrKykhLS6OwsLDa12RmZnLddddx+eWXs2HDBh5++GF+85vfsGjRIm+fjIwMxowZw9ixY9m0aRNjx45l9OjRfPXVV/X/ZCIizU3+AZh9AxzaAjEpcOeH0LqL1VWJNAibYRhGfV986NAhEhISWL16NYMGDTptn+nTp7N8+XK2bNni3TZp0iQ2bdpERkYGAGPGjMHpdPLRRx95+1x77bW0bNmS9PT0WtXidDpxOBzk5eURGxtb348kIuKf8nPMkZXDP0FsG3POSqvOVlclcka1/f2u0wjLqfLy8gCIi4urtk9GRgZpaWmVtg0dOpRvv/2W0tLSGvusXbu22vd1uVw4nc5KNxGRZsm5H2ZfXxFW2sKE9xVWJODUO7AYhsG0adO47LLL6NWr+nP6c3JySExMrLQtMTGRsrIyDh8+XGOfnJycat935syZOBwO7y01NbW+H0VExH/lZZlh5cgOcLSDOz+AuE5WVyXS4OodWKZMmcJ3331Xq0M2NputUvv4UaiTt5+uz6nbTjZjxgzy8vK8t71799alfBER/5e3zwwrR3dCi3bmyErLDlZXJdIo6nVa8/3338/y5ctZs2YNbdu2rbFvUlJSlZGSgwcPEhISQqtWrWrsc+qoy8nsdjt2u70+5YuI+L/cvfDWDXBslxlSxr8PLTTSLIGrTiMshmEwZcoUFi9ezGeffUbHjh3P+JoBAwawcuXKSttWrFjBRRddRGhoaI19Bg4cWJfyRESah9w9MPu6irDS0Tx1WWFFAlydAsvkyZOZO3cu8+bNIyYmhpycHHJyciguLvb2mTFjBuPGjfO2J02axO7du5k2bRpbtmzhjTfe4PXXX+ehhx7y9nnggQdYsWIFzz33HFu3buW5557j008/ZerUqWf/CUVEAsmx3fDm9WZoietknrrsqHmkWyQQ1Om05urmlLz55ptMmDABgAkTJrBr1y5WrVrlfX716tU8+OCDbN68mZSUFKZPn86kSZMqvcfChQt59NFH2blzJ507d+aZZ55h1KhRtf4gOq1ZRALesV3mOit5e6HVOeapy7EpVlclclZq+/t9Vuuw+BIFFhEJaEczzXVWvGHlfYhNtroqkbNW299vXUtIRMTXHd0Js4eDcx+06mKeDRSTZHVVIk1KgUVExJcd+dkcWXFmQeuu5shKTPVnUIoEKgUWERFfdeRnc85K/n5o3c0cWYlOsLoqEUsosIiI+KIjP5uLwuVnQ3x3c4Ktwoo0YwosIiK+5vB2c2SlIAfiz60IK/FWVyViqbO6+KGIiDSwQz+dCCsJPRRWRCpohEVExFcc+slcbr/gACT0hPHLIaq11VWJ+AQFFhERX3BomzmyUngQEnvBuOUQ1crqqkR8hg4JiYhY7eBWc4Jt4UFI7K2wInIaGmEREbHSgR/NdVaKDkNSRViJjLO6KhGfoxEWERGrVAor5ymsiNRAgUVExAoHNpsTbIsOQ3IfGLdMYUWkBjokJCLS1HK+h7d+AcVHIfl8GLcUIlpaXZWIT9MIi4hIU8r+7kRYSemrsCJSSwosIiJNZf9GmHM8rFwAY5cqrIjUkg4JiYg0hf0bYM5IKMmFNhfB2MUQ7rC6KhG/oREWEZHGlrUe5owww0rbixVWROpBIywiIo1p33p4+0Zw5UFqP7h9IYTHWl2ViN9RYBERaSx7v4G5o8DlhHYD4PYFYI+xuioRv6TAIiLSGPZ+DW+PAnc+tL8UbnsX7NFWVyXitxRYREQa2p51MPcmcBdAh8vhtvkQFmV1VSJ+TYFFRKQh7V4Lc38JpYXQcRDcOh/CIq2uSsTv6SwhEZGGsuvLE2Gl05UKKyINSCMsIiINIfMLmDcaSoug02C4NR1CI6yuSiRgaIRFRORs7VwN/7zZDCudr1ZYEWkECiwiImfj58/NkZWyYuiSBrfMU1gRaQQKLCIi9bXjX5B+C5SVQJehMGYuhIZbXZVIQNIcFhGR+tj+KbxzG5S7oOswGP0WhNitrkokYGmERUSkrn5aAe/caoaV7jfA6DkKKyKNTCMsIiJ1se1jeHcslLvh3OHwyzchONTqqkQCnkZYRERqa+uHMP8OM6z0GKGwItKEFFhERGpj81JzZMVTCj1vhJteV1gRaUIKLCIiZ/LdAlh4F3jKoPfNMOofCisiTUyBRUSkJhvnweJ7wCiH82+HG/8GwZr+J9LUFFhERKqzfjYsvQ8w4MI74RezICjY6qpEmiUFFhGR0/nqNXjvAcCAS34FN7wAQfonU8Qq+tsnInKqtf8HH/3WfDzwfhj2HNhs1tYk0szpQKyIyMnW/Ak+e9p8fPlDcNWjCisiPkCBRUQEwDBg1e9h9e/N9uBH4Ir/trYmEfFSYBERMQz411Pw5Qtm+5on4bIHLS1JRCqr8xyWNWvWMHz4cFJSUrDZbCxdurTG/hMmTMBms1W59ezZ09tn9uzZp+1TUlJS5w8kIlInhgGfPHIirAydqbAi4oPqHFgKCwvp06cPs2bNqlX/l156iezsbO9t7969xMXFcfPNN1fqFxsbW6lfdnY24eG6TLuINCKPBz58CNa9Yravfx4G3GdtTSJyWnU+JDRs2DCGDRtW6/4OhwOHw+FtL126lGPHjnHnnXdW6mez2UhKSqprOSIi9ePxwPsPwH/mADb4xctwwTirqxKRajT5ac2vv/4611xzDe3bt6+0vaCggPbt29O2bVtuuOEGNmzYUOP7uFwunE5npZuISK14ymHZfWZYsQXBja8qrIj4uCYNLNnZ2Xz00UfcfffdlbZ3796d2bNns3z5ctLT0wkPD+fSSy9l+/bt1b7XzJkzvaM3DoeD1NTUxi5fRAJBeam51P6mdLAFw03/gD63WF2ViJyBzTAMo94vttlYsmQJI0eOrFX/mTNn8vzzz7N//37CwsKq7efxeLjgggsYNGgQL7/88mn7uFwuXC6Xt+10OklNTSUvL4/Y2Ng6fQ4RaSbK3LDoLtjyHgSFws1vwrnDra5KpFlzOp04HI4z/n432WnNhmHwxhtvMHbs2BrDCkBQUBAXX3xxjSMsdrsdu93e0GWKSKAqLYEF4+GnjyE4DEa/Dd2utboqEamlJjsktHr1anbs2MHEiRPP2NcwDDZu3EhycnITVCYiAc9dCO/caoaVkHC49R2FFRE/U+cRloKCAnbs2OFtZ2ZmsnHjRuLi4mjXrh0zZswgKyuLOXPmVHrd66+/Tr9+/ejVq1eV93zqqafo378/Xbp0wel08vLLL7Nx40ZeeeWVenwkEZGTFOfCvDGwdx2ERpphpdMVVlclInVU58Dy7bffMnjwYG972rRpAIwfP57Zs2eTnZ3Nnj17Kr0mLy+PRYsW8dJLL532PXNzc7n33nvJycnB4XDQt29f1qxZwyWXXFLX8kRETig4CHNHQc73EO6A2xZAu35WVyUi9XBWk259SW0n7YhIM5G7F+aMgKM/Q1QCjF0CSVVHeEXEWj436VZEpMkc3g5zRoJzHzhSYdwyaNXZ6qpE5CwosIhIYMneBG+PgqLD0LorjF0KjjZWVyUiZ0mBRUQCx+615gRblxOS+8AdiyGqtdVViUgDUGARkcCw/VOYfweUFUP7S+HWdHOirYgEBAUWEfF/PyyGxfeCpxS6DIXRb0FohNVViUgDavKLH4qINKj1s2HhXWZY6XUT3PJPhRWRAKTAIiL+698vwXsPAAZceCeM+jsEh1pdlYg0Ah0SEhH/Yxjwr9/Bl38225c9CFc/ATabtXWJSKNRYBER/+LxwIcPwbevm+1rnjQDi4gENAUWEfEf5aWw9D74/l3ABjf8GS66y+qqRKQJKLCIiH8oLYYFd8JPH0FQCNz4N+j9S6urEpEmosAiIr6vxAnpt8LuLyEkHEa/DV3TrK5KRJqQAouI+LbCw/DPX8L+DRAWA7fNhw6XWl2ViDQxBRYR8V3HdpnXBTr6M0S2gjsWQUpfq6sSEQsosIiIb8reBP+8GQoOgKOdGVbiu1pdlYhYRIFFRHzPz5/D/LHgzofE3nD7AohNtroqEbGQAouI+JbvF8KSSeZS+x0uN5fa10UMRZo9Lc0vIr5j7SxYNNEMKz1vNA8DKayICBphERFf4PHAyscgY5bZ7vdrGPosBOn/qUTEpMAiItYqc8Oy++D7BWZ7yO9g4G90XSARqUSBRUSsU+KEd8fCzlXm6rUj/gJ9xlhdlYj4IAUWEbFG/gFzQbic7yA0Csa8DedcbXVVIuKjFFhEpOkd3gFzR0HuboiKN09b1oJwIlIDBRYRaVr71sO8m6HoCLTsCGMXQ1wnq6sSER+nwCIiTWf7Snh3HJQWmSMqty2A6HirqxIRP6DAIiJNY8M/Yfn9YJRD56th9BywR1tdlYj4CQUWEWlchgFfPA+fPW22+9wKv/g/CA61ti4R8SsKLCLSeDzl8NF0+ObvZvuyB+HqJ7TGiojUmQKLiDQOdyEsvhe2vg/Y4NrfQ/9JVlclIn5KgUVEGp5zP6TfAtmbIDgMbvwb9BpldVUi4scUWESkYWVvgnm3QP5+iGwNt8yDdv2srkpE/JwCi4g0nK0fmldbLi2C+O5w23xo2cHqqkQkACiwiMjZMwzzSssrHgMM6HwV3Dwbwh1WVyYiAUKBRUTOTnkpfPBf8J+3zPZFE2HYHyBY/7yISMPRvygiUn/Fx+Dd8ZC5GmxBMPRZ6DdJpy2LSINTYBGR+jm6E+aNgcM/mVdb/uUb0O1aq6sSkQClwCIidbd7LbxzOxQfhdg25uTapN5WVyUiAUyBRUTqZtM75jWByt3mBQxvfQdikqyuSkQCnAKLiNSOxwOfPwNf/Mlsn/sLc0G4sEhr6xKRZiGori9Ys2YNw4cPJyUlBZvNxtKlS2vsv2rVKmw2W5Xb1q1bK/VbtGgRPXr0wG6306NHD5YsWVLX0kSksZQWw6K7ToSVy6bBzW8prIhIk6lzYCksLKRPnz7MmjWrTq/btm0b2dnZ3luXLl28z2VkZDBmzBjGjh3Lpk2bGDt2LKNHj+arr76qa3ki0tAKDsLsG2DzEggKhRF/gWuegKA6//MhIlJvNsMwjHq/2GZjyZIljBw5sto+q1atYvDgwRw7dowWLVqcts+YMWNwOp189NFH3m3XXnstLVu2JD09vVa1OJ1OHA4HeXl5xMbG1uVjiEh1Dmw2zwTK2wsRLWHMXOhwmdVViUgAqe3vd5P9L1Lfvn1JTk7m6quv5vPPP6/0XEZGBmlpaZW2DR06lLVr11b7fi6XC6fTWekmIg1o+0p4fagZVuI6w93/UlgREcs0emBJTk7mtddeY9GiRSxevJhu3bpx9dVXs2bNGm+fnJwcEhMTK70uMTGRnJycat935syZOBwO7y01NbXRPoNIs2IY8OULMG80uPOhw+Vw96fQqrPVlYlIM9boZwl169aNbt26edsDBgxg7969/OlPf2LQoEHe7bZTVsY0DKPKtpPNmDGDadOmedtOp1OhReRsuQth2WRzvgrABePhuj9BSJi1dYlIs2fJac39+/dn7ty53nZSUlKV0ZSDBw9WGXU5md1ux263N1qNIs3O0UxzMbiDm83Jtdf9AS66y+qqRESAJpzDcrINGzaQnJzsbQ8YMICVK1dW6rNixQoGDhzY1KWJNE87/gWvXWmGlehEmPC+woqI+JQ6j7AUFBSwY8cObzszM5ONGzcSFxdHu3btmDFjBllZWcyZMweAF198kQ4dOtCzZ0/cbjdz585l0aJFLFq0yPseDzzwAIMGDeK5555jxIgRLFu2jE8//ZQvv/yyAT6iiFTLMODfL8G/ngLDA20ugjFvQ2yK1ZWJiFRS58Dy7bffMnjwYG/7+DyS8ePHM3v2bLKzs9mzZ4/3ebfbzUMPPURWVhYRERH07NmTDz74gOuuu87bZ+DAgbzzzjs8+uijPPbYY3Tu3Jn58+fTr1+/s/lsIlKTKvNVxlXMV9GhVhHxPWe1Dosv0TosInVwNBPm3wEHfoCgEBhWMV+lhonuIiKNoba/37qWkEhz8/NnsOBOKMmFqAQYPQfaD7C6KhGRGimwiDQXmq8iIn5MgUWkOXAXwrIpsHmx2e57B1z/Z81XERG/ocAiEuiO7TLXVzk+X+Xa38PFd2u+ioj4FQUWkUD282ew8C4oPqb5KiLi1xRYRAKRYcDal+HTJyvmq1wIo98GRxurKxMRqRcFFpFA4yqA934DP1Qsznj+HXD98xAabm1dIiJnQYFFJJBkfwcL74QjOzRfRUQCigKLSCAwDPj677DiESh3Q0wK/PINzVcRkYChwCLi74qOmqcsb/vAbHcdBiP/ApFx1tYlItKAFFhE/NnuDFh0Nzj3QXAYDHka+v1Kh4BEJOAosIj4I085fPFnWPWseRZQXGfzEFDK+VZXJiLSKBRYRPyNMxsW3wO7vjDb540xzwKyx1hbl4hII1JgEfEn21fCkl9B0REIjTKDyvm3Wl2ViEijU2AR8QdlbvOihRmzzHZib7j5TWjdxdq6RESaiAKLiK87utNcXn//BrN9ya9gyO+0EJyINCsKLCK+7PuF8N5UcOdDeAvzdOXu11tdlYhIk1NgEfFF7kL4aDpseNtstxsAo/4OLVKtrUtExCIKLCK+5sBmWHAnHN4G2GDQb+GK6RCsv64i0nzpX0ARX+HxwDf/gJWPQVkJRCfBTX+HjoOsrkxExHIKLCK+IHcPLJsMmWvM9jlD4MZXIaq1tXWJiPgIBRYRKxkG/GcOfPKIObE2JAKGPAUX3wNBQVZXJyLiMxRYRKzizIbl98OOlWY7tR+M/Cu06mxtXSIiPkiBRaSpGQZ89y589FsoyYNgO1z1KAyYDEHBVlcnIuKTFFhEmlLBIXh/Kmx932yn9IWRr0JCd0vLEhHxdQosIk1l81L4YJp5HaCgUPNU5cse1OnKIiK1oH8pRRpb0VH48Lfww0KzndjLnKuSfJ61dYmI+BEFFpHGtO1jeO83UHAAbMHmiMoV0yEkzOrKRET8igKLSGMoyYOPH4aNc812667mXJW2F1pbl4iIn1JgEWloP38Oy6aAcx9gM8/+uepRCI2wujIREb+lwCLSUFwFsPJx+PZ1s92yozlXpf0Aa+sSEQkACiwiDeGnT+DDh8wl9gEuuReueRLCoiwtS0QkUCiwiJyNvH3w0fQT66o4UmHELOh0paVliYgEGgUWkfooL4V1f4VVv4fSQggKgf73mWcA2aOtrk5EJOAosIjU1Z518P40OLjZbLcbANf/GRJ7WFuXiEgAU2ARqa3CI/Dp47Ch4lTliDgY8js4/3ZdWVlEpJEpsIicicdjrqey8gkoPmpuu2AcXPMURMZZW5uISDOhwCJSkwObzcM/e9eZ7YSecMOfoV1/a+sSEWlmFFhETsdVAKtmmhNrjXIIjYLBM6DfJAgOtbo6EZFmR4FF5GSGAVveg4//B5xZ5rZzh8O1vwdHW2trExFpxuo8U3DNmjUMHz6clJQUbDYbS5curbH/4sWLGTJkCPHx8cTGxjJgwAA++eSTSn1mz56NzWarcispKalreSL1d2wXzBsN7441w0qL9nDbAhgzV2FFRMRidQ4shYWF9OnTh1mzZtWq/5o1axgyZAgffvgh69evZ/DgwQwfPpwNGzZU6hcbG0t2dnalW3h4eF3LE6m70hJY80d4pR9sXwFBoXD5Q3DfOuiaZnV1IiJCPQ4JDRs2jGHDhtW6/4svvlip/eyzz7Js2TLee+89+vbt691us9lISkqqazki9WcY8MMi+NdTJ5bU73C5uaZKfFdraxMRkUqafA6Lx+MhPz+fuLjKp4MWFBTQvn17ysvLOf/883n66acrBZpTuVwuXC6Xt+10OhutZglAe9bBJ49A1rdmOybZXFOl981gs1lbm4iIVNHkq109//zzFBYWMnr0aO+27t27M3v2bJYvX056ejrh4eFceumlbN++vdr3mTlzJg6Hw3tLTU1tivLF3x3dCfPHwhtDzbASGgWDH4H7/wPnjVZYERHxUTbDMIx6v9hmY8mSJYwcObJW/dPT07n77rtZtmwZ11xzTbX9PB4PF1xwAYMGDeLll18+bZ/TjbCkpqaSl5dHbGxsnT6HNANFR815Kl//HTylYAuCvmNh8MMQo0ORIiJWcTqdOByOM/5+N9khofnz5zNx4kQWLFhQY1gBCAoK4uKLL65xhMVut2O32xu6TAk0ZW745u+w+g9QkmtuO+ca8/BPYk9LSxMRkdprksCSnp7OXXfdRXp6Otdff/0Z+xuGwcaNG+ndu3cTVFez+d/s4aDTRVrPJLomRmPTIQP/YBiwZbm5nP6xTHNbQk9IexrOudra2kREpM7qHFgKCgrYsWOHt52ZmcnGjRuJi4ujXbt2zJgxg6ysLObMmQOYYWXcuHG89NJL9O/fn5ycHAAiIiJwOBwAPPXUU/Tv358uXbrgdDp5+eWX2bhxI6+88kpDfMaz8ua/d7E1J5/nV/5E+1aRpPVIJK1nEhe0a0lwkMKLT9r3rTmh9vhy+tGJ5jyVvndAULC1tYmISL3UeQ7LqlWrGDx4cJXt48ePZ/bs2UyYMIFdu3axatUqAK688kpWr15dbX+ABx98kMWLF5OTk4PD4aBv3748+eSTDBgwoNZ11fYYWF0YhsG73+5lxeYDfLHjMO4yj/e5VlFhXHNuImk9E7n0nNaEh+qH0HLHdsGnT8HmxWY7JAIu/Q0M/A3Yoy0tTURETq+2v99nNenWlzRGYDlZoauMNT8dYsWPB/jXlgM4S8q8z0WGBXNF13jSeiZyVbdEHJG61kyTKs6FL56Hr16Fcjdgg/Nvh6segdgUq6sTEZEaKLA0otJyD19nHmXF5hxW/HiA7LwTlxAIDrLRv1McaT2SGNIjkZQWEY1aS7PmLoJv3zDDSvFRc1vHKyDtfyH5PGtrExGRWlFgaSKGYfBDlpMVP+awYvMBth3Ir/R87zYO0nokMqRnIt0SYzRptyG4C82g8u+XoPCQua11NzOodBmitVRERPyIAotFdh0uZOWPB1jxYw7f7j7GyXu3ZWQo57VtQZ+2DvqktuC8ti2Ij9Gp2bXmLoRv/gH/fhmKDpvbWrSHQQ9Bn9sgWBcfFxHxNwosPuBwgYt/bTnAyh8P8MX2w7hOmrR7XJsWEZznDTAOerdxEBOuOTCVuArMoLL2/04ElZYdYNBv4bwxEKz9JSLirxRYfIy7zMPWHCeb9uWxaW8um/bmsuNQAafufZsNOsdH06dtC/qkOujTtgXdk2OwhzTDs5Bc+ebKtBmzoOiIua1lx4qgMlpBRUQkACiw+IECVxnf78vju325bNqXy6a9eWTlFlfpFxYcxLnJMd7DSOenOujUOpqgQF0HxpUPX78Ga2edmEwb1wkG/bd5cUId+hERCRgKLH7qUL6rIsCYIzHf7cvlWFFplX7R9hB6tzEPJZ2f6uC8ti1IdoT796TeEid8/TfIeAWKj5nbWp1jjqj0+qWCiohIAFJgCRCGYbD3aDGb9uWysSLA/JDlpLi0vErf+Bi7eSipYk5Mn7Yt/GNNmJI8+Oo189DP8ev9tOoCV/w39LpJq9OKiAQwBZYAVlbuYfvBAnMuTMVIzLYD+ZR7qv6n7NAq0hte+qS2oGdKrO+syluSB1/9rSKo5JnbWnc1D/30GqWgIiLSDCiwNDPF7nJ+zM5j494Th5J2HSmq0i8kyEa3JHM+zPkVIeachOimvS5S7h4zqPxnDric5rbW3cwRlZ43KqiIiDQjCizCsUI332Xl8d3e3IpDSnkcLnBV6RcZFkzvNg7OTzUDTJ/UFqQ0xnyYvd/Aulfgx+VgVBzSij8Xrvgt9BipoCIi0gwpsEgVhmGQnVfCxr3Hz0rK5ft9eRS6q86HaR1t5/yK06rPaj5MeRlsfd+cSLvv6xPbOw2GAZOh89UQFHQWn0pERPyZAovUSrnH4OdDBWaIqQgyW7PzKTvNfJiOraNOTOhNbUGP5Brmw5Q4YcPb5gUJc/eY24LDoPdoGHAfJPZsxE8lIiL+QoFF6q2ktJzN+53eALNp7+nnwwQH2eiSEE2vNg56pcTSu62DnpFOwv/zd1j/FrgrrqsU2QouvhsumggxiU38aURExJfV9vdbC1tIFeGhwVzYviUXtm/p3ZZb5GbTvpPnw+RyuMDN1px8tubk8/N/ttM65EPOD/oabGYGPhbVidw+d5MwcBxR0TFWfRwREQkAGmGRejEMgwO5hRz4eiHxP/yDlPzvvc+tKe/NG+XDWO05D4MgbDbo1DqK3m0c5mhMGwc9U2J1zSQREdEIizSioqPYNqWTtO5VkvIqz0850nsipe42nJ+VR0iWkx+y8shxlvDzoUJ+PlTI0o37vW/TqXUUPds46JEcS4+UWHokx+rq1SIicloaYZHaMQzY9aW5dsqPy6C84vToWsxPOZTv4oesPH7IyuP7ivv9eSWn7RsfY68UYHqkxNKhVVTTrhMjIiJNRpNupWEUHISN/zSDytGdJ7Yn9oaLJ0KfWyA0os5ve6TAxfdZefyY7eTH/U5+zHaSebiwytWrASJCg+meHFMpyHRPiiUiTOu2iIj4OwUWqT9POfz8GayfDT99DJ4yc3tYNPT+JVwwHlL6QgMvLFfkLmNrTr43wPy438nWHCclpZ4qfYNs5mnWPVJOHFLqmRJL62gdUhIR8ScKLFJ3uXtgwz9hw1xw7juxve0lcME4c9l8e3STllTuMcg8XFhpJObH/c7TrtgLkBhrp2dFiOmZEkvPFAepcRH+fRVrEZEApsAitVPmhp8+Mg/57PgXUPF1iGgJfW6FvmMhsYelJZ7OwfwStmSbozGb9+fx434nmUdOf0gpJjykIsA4vCMx5yREExqsFXZFRKymwCI1O7wD/vMWbEqHwkMntnccZB7y6X4DhIZbV189FLrK2JJtjsJsznKyOTuPn3IKcJdXPaQUFhJEt0RzXkzPNmaI6ZYUS7RdJ86JiDQlBRapqvgYbHkfNr0Du788sT06Ec6/HfreAa06W1dfI3CXedhxsMAMMfvz2LzfyZb9TvJdZaft36ZFBN2SYuiaGEO3pGi6JsbQOT66+ksQiIjIWVFgEVOJE7Z9CD8sNifSekrN7bYgOGcIXDgeuqRBcPNZxM3jMdh7rKjicFLFIaVsJwecp58XE2SDDq2j6JZ4PMiY9x1aRRKiw0oiImdFgaU5cxfCto9g8xLYvvLEmikACT2h143Q5zZwtLGuRh+UW+TmpwMFbDuQz085+Ww7kM+2nHzyiktP2z8sOIjOCdF0TTRHYrpVhJk2LSII0roxIiK1osDS3JQWw/YV5kjKT59AWfGJ51p3hZ6joNcoiO9mXY1+yDAMDuW7vOHlpwP5bDtQwPYD+RS5y0/7mqiwYLpUBJiuScfvo4mPtutsJRGRUyiwNAdlLvPMns2LzREVd8GJ51p2NANKz1GQ2LPB10xp7jweg6zc4ooAY47IbM3JZ+ehwtNO8gVoGRnqPaTUrSLIdEmMwRHRfA7HiYicSoElUJWXws5V5kjK1g/AlXfiOUequVZKr1GQfL5CigXKyj3sOlLETwfMAPNTxajMriOFeKr5m5bsCK80N6ZbYgznJERrJV8RaRYUWAJJidMMKds/MUNK8bETz8UkmyGl5yhoe5FCio8qKS1nx8ECc0Qm58SoTHXXVLLZoG3LCLokxNAlIZpzEqLpUhFkdOq1iAQSBRZ/ZhhweLsZULavgN0ZJ87uAYiKhx4jzZGU1P4QpDNV/JWzpJTtB/LZllNQMSrj5KcDBRwtdFf7mhRHOOckxtA1IZouidGck2AGGR1aEhF/pMDib0qLzashb19hTprN3V35+bjO5unH3YZBh8sgSIcLAtmRApc5InOwgB0H8tl+sIDtBws4lH/6U6/BvCxBl4rw0iUxmnPio+kUH03r6DBN9hURn6XA4g+O7TYDyvaVkLmm8pk9wWFmMOmSZt4CbEE3qZ/cIjc7KsLL9gMFbD+Yz/YDBeQ4T39oCSA2PITOCdF0jo+mU3wUnePNx+1bReryBCJiOQUWX1ReCnvWVRzqWQmHtlZ+PrbNiYDScVCTX2hQ/JezpJQdBwvYURFifjpQwM7DBew7Vnza6ysBBAfZaB8XSaf4aDofDzIJUXRqHU3LqLCm/QAi0mwpsPgCTzkc2Ax7MszDPTtXgct54nlbMKT2gy5DoOtQSOihSbPSoEpKy8k8XMjOQ4X8fKiAnYcK+LnicXXryADERYXROT6Kjq2j6NA6io6tzPsOraJ09pKINKja/n7rdIOGVOaCrP/AnrXmRNm9X1c+7RggsrUZULoMgc5XmVdFFmkk4aHBnJscy7nJlf8RMAyDA04XPx8qqAgyZoj5+WAB+/NKOFro5mihm292Havynkmx4XRoHWmGmYog07F1FO3iInXNJRFpNBphORslTjOUHA8oWesrL4MPEBYNqZdAu4FmQEnpq7N6xKcVucu8ASbzcCG7DheSeaSIXYcLq71MAZiDgymOCDq0jqRDq6hKgSY1LgJ7iMKMiFSlQ0KNIf/AiXCyJwMO/ADGKauaRsVDuwHQfqB5n9gLgjWQJYHhWKGbzCNmiDk5yOw6XFjtFbDBDDNJseGkxkXSPi6SdnGRtGtVcR8XSVyUzmQSaa4UWBrKxnRz/smetXB0Z9XnW3YwR0/aDzDvW3XWPBRpdgzD4Eih2wwxhwvZdaSQXYeLyDxcyO4jhRTWMF8GzOsvpcZF0v6kENOulXmYqU2LCMJCNCopEqgaLbCsWbOGP/7xj6xfv57s7GyWLFnCyJEja3zN6tWrmTZtGps3byYlJYX//u//ZtKkSZX6LFq0iMcee4yff/6Zzp0788wzz3DjjTfWuq5GCyx/vQwOfF/RsJnX5Wk3oCKgDIDYlIb7s0QCkGEYHC10s+dokXk7Yt7vPlrE3qNF5DhLqj2TCSDIBsmOCNq0jKBNC/OW0uJ4O5yUFhFEhmkUU8RfNdqk28LCQvr06cOdd97JTTfddMb+mZmZXHfdddxzzz3MnTuXf//739x3333Ex8d7X5+RkcGYMWN4+umnufHGG1myZAmjR4/myy+/pF+/fnUtsWH1vQPys81DPKn9IKKFtfWI+BmbzUaraDutou30bVd1knlJaTlZucXeILPnaBG7j5hhZs/RIoorns/KLT7Nu5taRoaaIaYizLRtad4f36bF80T831kdErLZbGccYZk+fTrLly9ny5Yt3m2TJk1i06ZNZGRkADBmzBicTicfffSRt8+1115Ly5YtSU9Pr1UtPnlas4icFcMwOFzgZs/RQrJyS8g6Vsz+ivCyP7eYrGPFNc6dOS4sJIg2LSJIdoST5AivuI8gxduOoGVkqEKNiAV85rTmjIwM0tLSKm0bOnQor7/+OqWlpYSGhpKRkcGDDz5Ypc+LL75Y7fu6XC5crhNn5Didzmr7ioh/stlsxMfYiY+xc2H70/dxlpR6w8v+3GL25RazP7fEu+1AfgnuMg+ZFfNrqhMWEmQGmdgTgaZywAmndZSdoCCFGhErNHpgycnJITExsdK2xMREysrKOHz4MMnJydX2ycnJqfZ9Z86cyVNPPdUoNYuI/4gNDyU2KZTuSaf/P7PScg85eSXsO1ZMjrOY7LwScvJKKt0fLnDhLvOw+4h5OKo6IUE2EmPDSYy1V9yHe9tJseEkVDyOtodotEakgTXJTLVT/+IePwp18vbT9anpL/yMGTOYNm2at+10OklNTW2IckUkgIQGB5EaF0lqXGS1fdxlHg44S8hxHg8yVYPNwfwSyjzGGefTAESGBVcEmKrBJjE2nMSYcOJj7Fo1WKQOGj2wJCUlVRkpOXjwICEhIbRq1arGPqeOupzMbrdjt9sbvmARaXbCQs4casrKPRwqcJGdV8JBpxliDuS7OOAs4aDTRY6zhAPOEvJLyihyl7PzcCE7azgEBWawaRUdRqsoO60r7ltFh9Eq+tR2GHGRYYToYpXSjDV6YBkwYADvvfdepW0rVqzgoosuIjQ01Ntn5cqVleaxrFixgoEDBzZ2eSIitRISHESyI4JkR0SN/YrcZZUCzKmPD+SbYcdV5qHIXU7R0WL2Hq15xOa4lpGh5hlXUWG0jrbTMiqUFhFhOCJCcUSG0iIiFEdEKC0iw2gRaT7W5RIkUNQ5sBQUFLBjxw5vOzMzk40bNxIXF0e7du2YMWMGWVlZzJkzBzDPCJo1axbTpk3jnnvuISMjg9dff73S2T8PPPAAgwYN4rnnnmPEiBEsW7aMTz/9lC+//LIBPqKISNOJDAuhQ+sQOrSOqraPYRgUuss5UuDicIGbIwUujhS6T7QrHh8pcHOk0MXRQjceA44VlXKsqJQd1b5zVfaQIG94aRERhsP7ONS7PbYi6Jz6OFQjOuJD6nxa86pVqxg8eHCV7ePHj2f27NlMmDCBXbt2sWrVKu9zq1ev5sEHH/QuHDd9+vQqC8ctXLiQRx99lJ07d3oXjhs1alSt69JpzSISqMo9BrlF5gUpD1eEmCMVwcZZXEpukZu84lJyi0vJKyr1Pi73nN1C5pFhwSeCTHjlMGPeQrzbYiNCiQkPISY8lGh7CDH2EJ1RJbWipflFRJoxwzAocJWRWxFg8opLvY9zi90ngs1Jz+cVl+IsLq3V2ja1EW0PqQgxJwWZisexFdvNbWbYiQ4PISI0mIiwYCJDQwgPCyIyzNwWrPATsHxmHRYREWl6NputIgiEUtfzJ8s9BvkllYOMGWbKqoQbZ8lJQaekjHxXGe4y86KwBa4yClxlZOed/ecJCw4iIiyYiNBgIsOCCT8ebI4/Pmm7eQsy70OCsB9vh5jP2b3PnehnDwnyvlbhyDcpsIiISCXBQbaKibth9Xq9q6zcDC8lZeSXlJ5yf8p214ntBa4yit3llJSWU1xxO34MwF3uwV3sIa+4tAE/6emFBtuwhwQTFhJEWHCQeX/KY3vFrWqf4Ep9woKDCA22ERoSRGiwuS00+PjN5n3t8W1hITbCgoMJDbFV6hccZCMkKIggW9VlQJoLBRYREWlQ9pBg7NHBtI4+u6UnDMPAVeah2F1OUWm5N8wUuSsCjbus4t5DkbvM+5yrzENJaTklpR5KyspxHX9cWl7RNrcf3+Yq9eAu93j/3NJyg9LyMnDVUJyFQoKOBxgbIcFBldrBwWaw8W4LthEcFESwzQyiQTZzW5DNfD7YZiOo4j44uOI+6PjzJ70myOx316Udazz9v1E/tyV/qoiIyBnYbDbvYZqql81sWOUeA9dJIcZdZoYYd5kHV9nx+8rbT9fnRNvsW1ZueLeVlnsordKu2Fbx2tJyD6VlJ/qdTpnHoMxjVOSp8kbeM5UN75OiwCIiImKV4CAbkWEh1PMoWKMwDIPScoNyj0GZx1Nxb7ZLyyu3y07qV3Zqu9yg3DDweMz7cs+Jm8cwKPfgfb7MU7nfqa9Jig23bH8osIiIiPggm81GWMjx+SpaAFCrAomIiIjPU2ARERERn6fAIiIiIj5PgUVERER8ngKLiIiI+DwFFhEREfF5CiwiIiLi8xRYRERExOcpsIiIiIjPU2ARERERn6fAIiIiIj5PgUVERER8ngKLiIiI+LyAuVqzYRgAOJ1OiysRERGR2jr+u338d7w6ARNY8vPzAUhNTbW4EhEREamr/Px8HA5Htc/bjDNFGj/h8XjYv38/MTEx2Gy2Bntfp9NJamoqe/fuJTY2tsHeV7RvG5P2bePQfm082reNx9f3rWEY5Ofnk5KSQlBQ9TNVAmaEJSgoiLZt2zba+8fGxvrkf+hAoH3beLRvG4f2a+PRvm08vrxvaxpZOU6TbkVERMTnKbCIiIiIz1NgOQO73c4TTzyB3W63upSAo33beLRvG4f2a+PRvm08gbJvA2bSrYiIiAQujbCIiIiIz1NgEREREZ+nwCIiIiI+T4FFREREfJ4Cyxn85S9/oWPHjoSHh3PhhRfyxRdfWF2S33vyySex2WyVbklJSVaX5XfWrFnD8OHDSUlJwWazsXTp0krPG4bBk08+SUpKChEREVx55ZVs3rzZmmL9zJn27YQJE6p8h/v3729NsX5k5syZXHzxxcTExJCQkMDIkSPZtm1bpT763tZPbfatv39vFVhqMH/+fKZOncojjzzChg0buPzyyxk2bBh79uyxujS/17NnT7Kzs72377//3uqS/E5hYSF9+vRh1qxZp33+D3/4A3/+85+ZNWsW33zzDUlJSQwZMsR73S2p3pn2LcC1115b6Tv84YcfNmGF/mn16tVMnjyZdevWsXLlSsrKykhLS6OwsNDbR9/b+qnNvgU//94aUq1LLrnEmDRpUqVt3bt3N/7nf/7HoooCwxNPPGH06dPH6jICCmAsWbLE2/Z4PEZSUpLx+9//3rutpKTEcDgcxquvvmpBhf7r1H1rGIYxfvx4Y8SIEZbUE0gOHjxoAMbq1asNw9D3tiGdum8Nw/+/txphqYbb7Wb9+vWkpaVV2p6WlsbatWstqipwbN++nZSUFDp27Mgtt9zCzp07rS4poGRmZpKTk1Pp+2u327niiiv0/W0gq1atIiEhga5du3LPPfdw8OBBq0vyO3l5eQDExcUB+t42pFP37XH+/L1VYKnG4cOHKS8vJzExsdL2xMREcnJyLKoqMPTr1485c+bwySef8Pe//52cnBwGDhzIkSNHrC4tYBz/jur72ziGDRvGP//5Tz777DOef/55vvnmG6666ipcLpfVpfkNwzCYNm0al112Gb169QL0vW0op9u34P/f24C5WnNjsdlsldqGYVTZJnUzbNgw7+PevXszYMAAOnfuzFtvvcW0adMsrCzw6PvbOMaMGeN93KtXLy666CLat2/PBx98wKhRoyyszH9MmTKF7777ji+//LLKc/renp3q9q2/f281wlKN1q1bExwcXCXVHzx4sEr6l7MTFRVF79692b59u9WlBIzjZ13p+9s0kpOTad++vb7DtXT//fezfPlyPv/8c9q2bevdru/t2atu356Ov31vFViqERYWxoUXXsjKlSsrbV+5ciUDBw60qKrA5HK52LJlC8nJyVaXEjA6duxIUlJSpe+v2+1m9erV+v42giNHjrB37159h8/AMAymTJnC4sWL+eyzz+jYsWOl5/W9rb8z7dvT8bfvrQ4J1WDatGmMHTuWiy66iAEDBvDaa6+xZ88eJk2aZHVpfu2hhx5i+PDhtGvXjoMHD/K///u/OJ1Oxo8fb3VpfqWgoIAdO3Z425mZmWzcuJG4uDjatWvH1KlTefbZZ+nSpQtdunTh2WefJTIykttuu83Cqv1DTfs2Li6OJ598kptuuonk5GR27drFww8/TOvWrbnxxhstrNr3TZ48mXnz5rFs2TJiYmK8IykOh4OIiAhsNpu+t/V0pn1bUFDg/99bC89Q8guvvPKK0b59eyMsLMy44IILKp0iJvUzZswYIzk52QgNDTVSUlKMUaNGGZs3b7a6LL/z+eefG0CV2/jx4w3DME8RfeKJJ4ykpCTDbrcbgwYNMr7//ntri/YTNe3boqIiIy0tzYiPjzdCQ0ONdu3aGePHjzf27Nljddk+73T7FDDefPNNbx99b+vnTPs2EL63NsMwjKYMSCIiIiJ1pTksIiIi4vMUWERERMTnKbCIiIiIz1NgEREREZ+nwCIiIiI+T4FFREREfJ4Ci4iIiPg8BRYRERHxeQosIiIi4vMUWERERMTnKbCIiIiIz1NgEREREZ/3/wi5H+fwQy6gAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(train_loss, label='Train')\n",
    "plt.plot(valid_loss, label='Valid')\n",
    "plt.legend();"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "109c6194-268e-4729-b3f7-58c48f5164fa",
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'activation': 'SeLU',\n",
       " 'distribution': 'LogNormal',\n",
       " 'k': 3,\n",
       " 'learning_rate': 0.003925619631692259,\n",
       " 'lin1': 102,\n",
       " 'lin2': 60,\n",
       " 'lin3': 173,\n",
       " 'lin4': 226,\n",
       " 'lin5': 117}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "study.best_trials[0].params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4853ec4f-18fc-46b4-a65d-b14f147d0084",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}