analysis/training-stats-combined.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding = \"sceptr-tiny-autoencoder\"\n",
    "# embedding = \"aaprop-autoencoder\"\n",
    "# embedding = \"atchley-no-reduction\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "import os\n",
    "\n",
    "os.chdir(\"C:/Users/rcwyuen/OneDrive/Studies/UCL/Publications/TCR-Embeddings\")\n",
    "sys.path.append(\"C:/Users/rcwyuen/OneDrive/Studies/UCL/Publications/TCR-Embeddings\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "\n",
    "dir = Path.cwd() / f\"results/{embedding}/\"\n",
    "\n",
    "if not dir.exists():\n",
    "    raise FileNotFoundError\n",
    "\n",
    "folders = list(dir.glob(\"kfold-*\"))\n",
    "\n",
    "data = {}\n",
    "\n",
    "for folder in folders:\n",
    "    data[str(folder.relative_to(dir))] = {\n",
    "        \"train\": [\n",
    "            i.relative_to(dir / folder)\n",
    "            for i in folder.glob(\"Epoch */train-records.csv\")\n",
    "        ],\n",
    "        \"test\": [\n",
    "            i.relative_to(dir / folder) for i in folder.glob(\"Epoch */test-records.csv\")\n",
    "        ],\n",
    "        # \"eval\": [i.relative_to(dir / folder) for i in folder.glob(\"Epoch */eval-records.csv\")],\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "\n",
    "\n",
    "def plot_loss_result(category):\n",
    "    for kfold, results in data.items():\n",
    "        plotting = []\n",
    "        for train_record in results[category]:\n",
    "            df = pd.read_csv(Path.cwd() / dir / kfold / train_record)\n",
    "            plotting.append(\n",
    "                (int(str(train_record.parent).replace(\"Epoch \", \"\")), df[\"loss\"].mean())\n",
    "            )\n",
    "        plt.plot(*zip(*sorted(plotting)), marker=\".\", label=kfold)\n",
    "\n",
    "    plt.title(f\"{category.title()} Loss\")\n",
    "    plt.xlabel(\"Epoch\")\n",
    "    plt.ylabel(\"BCE Loss\")\n",
    "    plt.legend()\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "plot_loss_result(\"train\")\n",
    "plot_loss_result(\"test\")\n",
    "# plot_loss_result(\"eval\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_auc_score, roc_curve\n",
    "import numpy as np\n",
    "\n",
    "\n",
    "def plot_auc_result(category):\n",
    "    best_aucs = []\n",
    "    fig, axs = plt.subplots(\n",
    "        ncols=2,\n",
    "        figsize=[\n",
    "            plt.rcParams[\"figure.figsize\"][0] * 2,\n",
    "            plt.rcParams[\"figure.figsize\"][1],\n",
    "        ],\n",
    "    )\n",
    "\n",
    "    for kfold, results in data.items():\n",
    "        plotting = []\n",
    "        for train_record in results[category]:\n",
    "            df = pd.read_csv(Path.cwd() / dir / kfold / train_record)\n",
    "            plotting.append(\n",
    "                (\n",
    "                    int(str(train_record.parent).replace(\"Epoch \", \"\")),\n",
    "                    roc_auc_score(df[\"actual\"], df[\"pred\"]),\n",
    "                )\n",
    "            )\n",
    "        axs[0].plot(*zip(*sorted(plotting)), marker=\".\", label=kfold)\n",
    "\n",
    "        plotting.sort(key=lambda x: -x[1])\n",
    "        best_epoch = plotting[0][0]\n",
    "        best_aucs.append(plotting[0][1])\n",
    "        df = pd.read_csv(\n",
    "            Path.cwd() / dir / kfold / f\"Epoch {best_epoch}/{category}-records.csv\"\n",
    "        )\n",
    "        fpr, tpr, thresholds = roc_curve(df[\"actual\"], df[\"pred\"])\n",
    "        axs[1].plot(fpr, tpr, label=f\"{kfold}: AUC {round(best_aucs[-1], 3)}\")\n",
    "\n",
    "    axs[0].set_title(f\"{category.title()} AUC\")\n",
    "    axs[0].set_xlabel(\"Epoch\")\n",
    "    axs[0].set_ylabel(\"AUROC\")\n",
    "    axs[0].legend()\n",
    "\n",
    "    axs[1].plot([0, 1], [0, 1], \"k--\", label=\"Random Predictions Curve\")\n",
    "    axs[1].set_title(f\"Best Performing Epoch {category.title()} AUC\")\n",
    "    axs[1].set_xlabel(\"False Positive Rate\")\n",
    "    axs[1].set_ylabel(\"True Positive Rate\")\n",
    "    axs[1].legend()\n",
    "    plt.show()\n",
    "    print(f\"K-Fold CV {category.title()} AUC: {np.mean(best_aucs)}\")\n",
    "\n",
    "\n",
    "plot_auc_result(\"train\")\n",
    "# plot_auc_result(\"eval\")\n",
    "plot_auc_result(\"test\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "\n",
    "def plot_acc_result(category):\n",
    "    fig, axs = plt.subplots(\n",
    "        ncols=2,\n",
    "        figsize=[\n",
    "            plt.rcParams[\"figure.figsize\"][0] * 2,\n",
    "            plt.rcParams[\"figure.figsize\"][1],\n",
    "        ],\n",
    "    )\n",
    "    for kfold, results in data.items():\n",
    "        optim_threshold = []\n",
    "        optim_accuracy = []\n",
    "        for train_record in results[category]:\n",
    "            df = pd.read_csv(Path.cwd() / dir / kfold / train_record)\n",
    "            fpr, tpr, thresholds = roc_curve(df[\"actual\"], df[\"pred\"])\n",
    "            P = np.sum(df[\"actual\"])\n",
    "            N = len(df) - P\n",
    "            accuracies = []\n",
    "            for i in range(len(thresholds)):\n",
    "                TP = tpr[i] * P\n",
    "                TN = (1 - fpr[i]) * N\n",
    "                accuracy = (TP + TN) / (P + N)\n",
    "                accuracies.append(accuracy)\n",
    "\n",
    "            max_accuracy_idx = np.argmax(accuracies)\n",
    "\n",
    "            optim_threshold.append(\n",
    "                (\n",
    "                    int(str(train_record.parent).replace(\"Epoch \", \"\")),\n",
    "                    thresholds[max_accuracy_idx],\n",
    "                )\n",
    "            )\n",
    "\n",
    "            optim_accuracy.append(\n",
    "                (\n",
    "                    int(str(train_record.parent).replace(\"Epoch \", \"\")),\n",
    "                    accuracies[max_accuracy_idx],\n",
    "                )\n",
    "            )\n",
    "\n",
    "        axs[0].plot(*zip(*sorted(optim_accuracy)), label=kfold, marker=\".\")\n",
    "        axs[1].plot(*zip(*sorted(optim_threshold)), label=kfold, marker=\".\")\n",
    "\n",
    "    axs[0].set_title(f\"{category.title()} Accuracy\")\n",
    "    axs[0].set_xlabel(\"Epoch\")\n",
    "    axs[0].set_ylabel(\"Accuracy\")\n",
    "    axs[0].legend()\n",
    "\n",
    "    axs[1].set_title(f\"{category.title()} Optimal Threshold\")\n",
    "    axs[1].set_xlabel(\"Epoch\")\n",
    "    axs[1].set_ylabel(\"Optimal Threshold\")\n",
    "    axs[1].legend()\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "plot_acc_result(\"train\")\n",
    "# plot_acc_result(\"eval\")\n",
    "plot_acc_result(\"test\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tcr-cancer",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}