model_creation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import csv\n",
    "import tqdm\n",
    "import json\n",
    "import re\n",
    "import ast\n",
    "import os\n",
    "import torch\n",
    "\n",
    "from sentence_transformers import SentenceTransformer\n",
    "from sklearn.cluster import KMeans\n",
    "import numpy as np\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# K-means Clustering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/50 [00:00<?, ?it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      "  2%|▏         | 1/50 [00:00<00:33,  1.45it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      "  4%|▍         | 2/50 [00:01<00:38,  1.25it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      "  6%|▌         | 3/50 [00:02<00:31,  1.49it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      "  8%|▊         | 4/50 [00:02<00:20,  2.22it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      " 10%|█         | 5/50 [00:02<00:24,  1.86it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      " 12%|█▏        | 6/50 [00:03<00:21,  2.02it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      " 14%|█▍        | 7/50 [00:04<00:24,  1.73it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      " 16%|█▌        | 8/50 [00:04<00:22,  1.84it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      " 18%|█▊        | 9/50 [00:05<00:26,  1.54it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      " 20%|██        | 10/50 [00:05<00:19,  2.08it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      " 22%|██▏       | 11/50 [00:06<00:23,  1.67it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      " 24%|██▍       | 12/50 [00:07<00:23,  1.61it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      "c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      " 28%|██▊       | 14/50 [00:07<00:18,  1.98it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      " 30%|███       | 15/50 [00:08<00:17,  2.04it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      " 32%|███▏      | 16/50 [00:08<00:17,  1.96it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      " 34%|███▍      | 17/50 [00:09<00:15,  2.19it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      " 36%|███▌      | 18/50 [00:10<00:18,  1.70it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      " 38%|███▊      | 19/50 [00:10<00:15,  1.97it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      " 40%|████      | 20/50 [00:11<00:16,  1.77it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      " 42%|████▏     | 21/50 [00:11<00:13,  2.09it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      " 44%|████▍     | 22/50 [00:11<00:10,  2.65it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      " 46%|████▌     | 23/50 [00:11<00:09,  2.93it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      " 48%|████▊     | 24/50 [00:11<00:08,  3.25it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      " 50%|█████     | 25/50 [00:12<00:07,  3.55it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      " 52%|█████▏    | 26/50 [00:13<00:11,  2.06it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      "c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      " 56%|█████▌    | 28/50 [00:13<00:07,  2.93it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      " 58%|█████▊    | 29/50 [00:13<00:07,  2.91it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      " 60%|██████    | 30/50 [00:14<00:09,  2.13it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      " 62%|██████▏   | 31/50 [00:14<00:07,  2.56it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      " 64%|██████▍   | 32/50 [00:14<00:05,  3.04it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      " 66%|██████▌   | 33/50 [00:15<00:08,  2.02it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      " 68%|██████▊   | 34/50 [00:16<00:06,  2.46it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      " 70%|███████   | 35/50 [00:16<00:05,  3.00it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      " 72%|███████▏  | 36/50 [00:17<00:07,  1.93it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      " 74%|███████▍  | 37/50 [00:17<00:07,  1.80it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      " 76%|███████▌  | 38/50 [00:18<00:05,  2.31it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      "c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      " 80%|████████  | 40/50 [00:18<00:02,  3.76it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      "c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      " 84%|████████▍ | 42/50 [00:19<00:02,  2.79it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      " 86%|████████▌ | 43/50 [00:20<00:03,  2.08it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      "c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      " 90%|█████████ | 45/50 [00:20<00:01,  2.58it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      " 92%|█████████▏| 46/50 [00:20<00:01,  2.56it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      " 94%|█████████▍| 47/50 [00:21<00:01,  2.25it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      " 96%|█████████▌| 48/50 [00:22<00:01,  1.97it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      " 98%|█████████▊| 49/50 [00:23<00:00,  1.62it/s]c:\\Users\\hsawhney\\anaconda3\\envs\\ml-project-2\\lib\\site-packages\\sklearn\\cluster\\_kmeans.py:1429: UserWarning: KMeans is known to have a memory leak on Windows with MKL, when there are less chunks than available threads. You can avoid it by setting the environment variable OMP_NUM_THREADS=1.\n",
      "  warnings.warn(\n",
      "100%|██████████| 50/50 [00:23<00:00,  2.14it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                 SubmissionID  SubmissionYear  \\\n",
      "0             /10.1101/188524            2017   \n",
      "1             /10.1101/188524            2017   \n",
      "2             /10.1101/783175            2019   \n",
      "3  /10.1101/2020.06.30.176537            2020   \n",
      "4  /10.1101/2020.06.30.176537            2020   \n",
      "\n",
      "                                     SubmissionTitle  \\\n",
      "0  characterizing highly dynamic conformational s...   \n",
      "1  characterizing highly dynamic conformational s...   \n",
      "2  dynamic reconfiguration fragmentation and inte...   \n",
      "3  attenuated subcomponent vaccine design targeti...   \n",
      "4  attenuated subcomponent vaccine design targeti...   \n",
      "\n",
      "                                  SubmissionAbstract firstName middleName  \\\n",
      "0  biomacromolecules carry out complicated functi...     eitan        NaN   \n",
      "1  biomacromolecules carry out complicated functi...  antonino        NaN   \n",
      "2  general anesthetics are routinely used to indu...    corson         N.   \n",
      "3  the novel coronavirus disease covid19 caused b...    onyeka         S.   \n",
      "4  the novel coronavirus disease covid19 caused b...   rebecca   Chinyelu   \n",
      "\n",
      "       lastName  isFirstAuthor  city        country  ...  \\\n",
      "0        lerner              1   NaN  United States  ...   \n",
      "1    ingargiola              0   NaN  United States  ...   \n",
      "2   areshenkoff              0   NaN         Canada  ...   \n",
      "3   chukwudozie              1   NaN        Nigeria  ...   \n",
      "4  chukwuanukwu              0   NaN        Nigeria  ...   \n",
      "\n",
      "                      doi             authorID  \\\n",
      "0       10.1063/1.5004606          35205792300   \n",
      "1       10.1063/1.5004606          23973120100   \n",
      "2  10.1093/cercor/bhaa085  0000-0001-9072-9185   \n",
      "3    10.1155/2020/2837670  0000-0002-0814-6798   \n",
      "4    10.1155/2020/2837670  0000-0001-5023-9560   \n",
      "\n",
      "                            authorPublicationHistory match  \\\n",
      "0  [{'title': 'rapid and specific detection of na...     1   \n",
      "1  [{'title': 'erratum monte carlo diffusionenhan...     1   \n",
      "2  [{'title': 'distinct patterns of connectivity ...     1   \n",
      "3  [{'title': 'the relevance of bioinformatics ap...     1   \n",
      "4  [{'title': 'the role of immuneinflammatory mar...     1   \n",
      "\n",
      "                        fullName  \\\n",
      "0                   eitan lerner   \n",
      "1            antonino ingargiola   \n",
      "2          corson n. areshenkoff   \n",
      "3          onyeka s. chukwudozie   \n",
      "4  rebecca chinyelu chukwuanukwu   \n",
      "\n",
      "                                     title_embedding  \\\n",
      "0  [0.000631210277788341, -0.06552567332983017, -...   \n",
      "1  [0.000631210277788341, -0.06552567332983017, -...   \n",
      "2  [-0.03589945659041405, 0.03250805288553238, -0...   \n",
      "3  [0.018001500517129898, -0.033130768686532974, ...   \n",
      "4  [0.018001500517129898, -0.033130768686532974, ...   \n",
      "\n",
      "                                  abstract_embedding  \\\n",
      "0  [0.007256368640810251, -0.03036230057477951, -...   \n",
      "1  [0.007256368640810251, -0.03036230057477951, -...   \n",
      "2  [-0.018003256991505623, 0.04315100610256195, -...   \n",
      "3  [-0.00747875077649951, -0.04243578389286995, 0...   \n",
      "4  [-0.00747875077649951, -0.04243578389286995, 0...   \n",
      "\n",
      "                                       doi_embedding  \\\n",
      "0  [0.007630346342921257, -0.03242341801524162, -...   \n",
      "1  [0.007630346342921257, -0.03242341801524162, -...   \n",
      "2  [-0.0036318886559456587, -0.04171673208475113,...   \n",
      "3  [-0.014353901147842407, -0.019061246886849403,...   \n",
      "4  [-0.014353901147842407, -0.019061246886849403,...   \n",
      "\n",
      "                  authorPublicationHistory_embedding  \\\n",
      "0  [{'title_embedding': [0.00456725899130106, -0....   \n",
      "1  [{'title_embedding': [-0.011444708332419395, -...   \n",
      "2  [{'title_embedding': [-0.05066244304180145, 0....   \n",
      "3  [{'title_embedding': [-0.027665186673402786, 0...   \n",
      "4  [{'title_embedding': [0.02183222956955433, -0....   \n",
      "\n",
      "                                      cluster_filter  \n",
      "0  [{'title': 'rapid and specific detection of na...  \n",
      "1  [{'title': 'erratum monte carlo diffusionenhan...  \n",
      "2  [{'title': 'distinct patterns of connectivity ...  \n",
      "3  [{'title': 'attenuated subcomponent vaccine de...  \n",
      "4  [{'title': 'neutrophil extracellular traps cha...  \n",
      "\n",
      "[5 rows x 22 columns]\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 50 entries, 0 to 49\n",
      "Data columns (total 22 columns):\n",
      " #   Column                              Non-Null Count  Dtype  \n",
      "---  ------                              --------------  -----  \n",
      " 0   SubmissionID                        50 non-null     object \n",
      " 1   SubmissionYear                      50 non-null     int64  \n",
      " 2   SubmissionTitle                     50 non-null     object \n",
      " 3   SubmissionAbstract                  50 non-null     object \n",
      " 4   firstName                           50 non-null     object \n",
      " 5   middleName                          13 non-null     object \n",
      " 6   lastName                            50 non-null     object \n",
      " 7   isFirstAuthor                       50 non-null     int64  \n",
      " 8   city                                0 non-null      float64\n",
      " 9   country                             50 non-null     object \n",
      " 10  countryCode                         50 non-null     object \n",
      " 11  institution                         50 non-null     object \n",
      " 12  doi                                 50 non-null     object \n",
      " 13  authorID                            50 non-null     object \n",
      " 14  authorPublicationHistory            50 non-null     object \n",
      " 15  match                               50 non-null     int64  \n",
      " 16  fullName                            50 non-null     object \n",
      " 17  title_embedding                     50 non-null     object \n",
      " 18  abstract_embedding                  50 non-null     object \n",
      " 19  doi_embedding                       50 non-null     object \n",
      " 20  authorPublicationHistory_embedding  50 non-null     object \n",
      " 21  cluster_filter                      50 non-null     object \n",
      "dtypes: float64(1), int64(3), object(18)\n",
      "memory usage: 8.7+ KB\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "model = SentenceTransformer('all-MiniLM-L6-v2')\n",
    "if os.path.exists('./data/cluster_filtering.csv'):\n",
    "    embeddings_dataset = pd.read_csv('./data/cluster_filtering.csv')\n",
    "else:\n",
    "    embeddings_dataset = pd.read_csv('./data/mini_dataset_v3.csv')\n",
    "    embeddings_dataset['cluster_filter'] = None\n",
    "    i = 0\n",
    "    same = 0\n",
    "    total = 0\n",
    "    for index, row in tqdm.tqdm(embeddings_dataset.iterrows(), total=embeddings_dataset.shape[0]):\n",
    "        embeddings = []\n",
    "        title_embedding = model.encode(row['SubmissionTitle'])\n",
    "        abstract_embedding = model.encode(row['SubmissionAbstract'])\n",
    "        s_id = row['SubmissionID']\n",
    "\n",
    "        \n",
    "        embeddings.append(np.concatenate((title_embedding, abstract_embedding), axis=None))\n",
    "        \n",
    "        row['authorPublicationHistory_embedding'] = ast.literal_eval(row['authorPublicationHistory_embedding'])\n",
    "        embeddings_dataset.at[index, 'authorPublicationHistory_embedding'] = row['authorPublicationHistory_embedding']\n",
    "        \n",
    "        row['authorPublicationHistory'] = ast.literal_eval(row['authorPublicationHistory'])\n",
    "        embeddings_dataset.at[index, 'authorPublicationHistory'] = row['authorPublicationHistory']\n",
    "        for authorWorks in row['authorPublicationHistory']:\n",
    "            authorTitleEmbedding = model.encode(authorWorks['title'])\n",
    "            authorAbstractEmbedding = model.encode(authorWorks['abstract'])\n",
    "            \n",
    "            embeddings.append(np.concatenate((authorTitleEmbedding, authorAbstractEmbedding), axis=None))            \n",
    "            \n",
    "        if len(embeddings) == 1:\n",
    "            k = 0\n",
    "        if len(embeddings) == 2:\n",
    "            k = 1\n",
    "        if len(embeddings) >= 3:\n",
    "            k = 2\n",
    "        \n",
    "        kmeans = KMeans(n_clusters=k, random_state=42)\n",
    "        clusters = kmeans.fit_predict(embeddings)\n",
    "        \n",
    "        submission_cluster = clusters[0]\n",
    "        clusters = clusters[1:]\n",
    "        clusters_authors = []\n",
    "        for i in range(len(clusters)):\n",
    "            if clusters[i] == submission_cluster:\n",
    "                clusters_authors.append(row['authorPublicationHistory'][i])\n",
    "                if row['authorPublicationHistory'][i]['doi'] == row['doi']:\n",
    "                    same += 1\n",
    "        \n",
    "        embeddings_dataset.at[index, 'cluster_filter'] = clusters_authors\n",
    "        #all_embeddings.append(embeddings)\n",
    "        \n",
    "        i += 1\n",
    "        total += 1\n",
    "\n",
    "embeddings_dataset.to_csv('data/cluster_filtering.csv', index=False)\n",
    "print(embeddings_dataset.head())\n",
    "print(embeddings_dataset.info())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TF-IDF Cosine Similarity "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 50/50 [00:00<00:00, 1450.07it/s]\n",
      "100%|██████████| 50/50 [00:00<00:00, 63.68it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                 SubmissionID  SubmissionYear  \\\n",
      "0             /10.1101/188524            2017   \n",
      "1             /10.1101/188524            2017   \n",
      "2             /10.1101/783175            2019   \n",
      "3  /10.1101/2020.06.30.176537            2020   \n",
      "4  /10.1101/2020.06.30.176537            2020   \n",
      "\n",
      "                                     SubmissionTitle  \\\n",
      "0  characterizing highly dynamic conformational s...   \n",
      "1  characterizing highly dynamic conformational s...   \n",
      "2  dynamic reconfiguration fragmentation and inte...   \n",
      "3  attenuated subcomponent vaccine design targeti...   \n",
      "4  attenuated subcomponent vaccine design targeti...   \n",
      "\n",
      "                                  SubmissionAbstract firstName middleName  \\\n",
      "0  biomacromolecules carry out complicated functi...     eitan        NaN   \n",
      "1  biomacromolecules carry out complicated functi...  antonino        NaN   \n",
      "2  general anesthetics are routinely used to indu...    corson         N.   \n",
      "3  the novel coronavirus disease covid19 caused b...    onyeka         S.   \n",
      "4  the novel coronavirus disease covid19 caused b...   rebecca   Chinyelu   \n",
      "\n",
      "       lastName  isFirstAuthor  city        country  ...             authorID  \\\n",
      "0        lerner              1   NaN  United States  ...          35205792300   \n",
      "1    ingargiola              0   NaN  United States  ...          23973120100   \n",
      "2   areshenkoff              0   NaN         Canada  ...  0000-0001-9072-9185   \n",
      "3   chukwudozie              1   NaN        Nigeria  ...  0000-0002-0814-6798   \n",
      "4  chukwuanukwu              0   NaN        Nigeria  ...  0000-0001-5023-9560   \n",
      "\n",
      "                            authorPublicationHistory match  \\\n",
      "0  [{'title': 'rapid and specific detection of na...     1   \n",
      "1  [{'title': 'erratum monte carlo diffusionenhan...     1   \n",
      "2  [{'title': 'distinct patterns of connectivity ...     1   \n",
      "3  [{'title': 'the relevance of bioinformatics ap...     1   \n",
      "4  [{'title': 'the role of immuneinflammatory mar...     1   \n",
      "\n",
      "                        fullName  \\\n",
      "0                   eitan lerner   \n",
      "1            antonino ingargiola   \n",
      "2          corson n. areshenkoff   \n",
      "3          onyeka s. chukwudozie   \n",
      "4  rebecca chinyelu chukwuanukwu   \n",
      "\n",
      "                                     title_embedding  \\\n",
      "0  [0.000631210277788341, -0.06552567332983017, -...   \n",
      "1  [0.000631210277788341, -0.06552567332983017, -...   \n",
      "2  [-0.03589945659041405, 0.03250805288553238, -0...   \n",
      "3  [0.018001500517129898, -0.033130768686532974, ...   \n",
      "4  [0.018001500517129898, -0.033130768686532974, ...   \n",
      "\n",
      "                                  abstract_embedding  \\\n",
      "0  [0.007256368640810251, -0.03036230057477951, -...   \n",
      "1  [0.007256368640810251, -0.03036230057477951, -...   \n",
      "2  [-0.018003256991505623, 0.04315100610256195, -...   \n",
      "3  [-0.00747875077649951, -0.04243578389286995, 0...   \n",
      "4  [-0.00747875077649951, -0.04243578389286995, 0...   \n",
      "\n",
      "                                       doi_embedding  \\\n",
      "0  [0.007630346342921257, -0.03242341801524162, -...   \n",
      "1  [0.007630346342921257, -0.03242341801524162, -...   \n",
      "2  [-0.0036318886559456587, -0.04171673208475113,...   \n",
      "3  [-0.014353901147842407, -0.019061246886849403,...   \n",
      "4  [-0.014353901147842407, -0.019061246886849403,...   \n",
      "\n",
      "                  authorPublicationHistory_embedding  \\\n",
      "0  [{'title_embedding': [0.00456725899130106, -0....   \n",
      "1  [{'title_embedding': [-0.011444708332419395, -...   \n",
      "2  [{'title_embedding': [-0.05066244304180145, 0....   \n",
      "3  [{'title_embedding': [-0.027665186673402786, 0...   \n",
      "4  [{'title_embedding': [0.02183222956955433, -0....   \n",
      "\n",
      "                                      cluster_filter  \\\n",
      "0  [{'title': 'rapid and specific detection of na...   \n",
      "1  [{'title': 'erratum monte carlo diffusionenhan...   \n",
      "2  [{'title': 'distinct patterns of connectivity ...   \n",
      "3  [{'title': 'attenuated subcomponent vaccine de...   \n",
      "4  [{'title': 'neutrophil extracellular traps cha...   \n",
      "\n",
      "                             tfidf_cosine_similarity  \n",
      "0  [{'title': 'characterizing highly dynamic conf...  \n",
      "1  [{'title': 'characterizing highly dynamic conf...  \n",
      "2  [{'title': 'dynamic reconfiguration fragmentat...  \n",
      "3  [{'title': 'attenuated subcomponent vaccine de...  \n",
      "4  [{'title': 'attenuated subcomponent vaccine de...  \n",
      "\n",
      "[5 rows x 23 columns]\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 50 entries, 0 to 49\n",
      "Data columns (total 23 columns):\n",
      " #   Column                              Non-Null Count  Dtype  \n",
      "---  ------                              --------------  -----  \n",
      " 0   SubmissionID                        50 non-null     object \n",
      " 1   SubmissionYear                      50 non-null     int64  \n",
      " 2   SubmissionTitle                     50 non-null     object \n",
      " 3   SubmissionAbstract                  50 non-null     object \n",
      " 4   firstName                           50 non-null     object \n",
      " 5   middleName                          13 non-null     object \n",
      " 6   lastName                            50 non-null     object \n",
      " 7   isFirstAuthor                       50 non-null     int64  \n",
      " 8   city                                0 non-null      float64\n",
      " 9   country                             50 non-null     object \n",
      " 10  countryCode                         50 non-null     object \n",
      " 11  institution                         50 non-null     object \n",
      " 12  doi                                 50 non-null     object \n",
      " 13  authorID                            50 non-null     object \n",
      " 14  authorPublicationHistory            50 non-null     object \n",
      " 15  match                               50 non-null     int64  \n",
      " 16  fullName                            50 non-null     object \n",
      " 17  title_embedding                     50 non-null     object \n",
      " 18  abstract_embedding                  50 non-null     object \n",
      " 19  doi_embedding                       50 non-null     object \n",
      " 20  authorPublicationHistory_embedding  50 non-null     object \n",
      " 21  cluster_filter                      50 non-null     object \n",
      " 22  tfidf_cosine_similarity             50 non-null     object \n",
      "dtypes: float64(1), int64(3), object(19)\n",
      "memory usage: 9.1+ KB\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "dataset = pd.read_csv('data/cluster_filtering.csv')\n",
    "# Compute the TF-IDF matrix\n",
    "vectorizer = TfidfVectorizer()\n",
    "dataset['tfidf_cosine_similarity'] = None\n",
    "# Combine all texts for fitting the vectorizer\n",
    "all_texts = []\n",
    "for index, row in tqdm.tqdm(dataset.iterrows(), total=dataset.shape[0]):\n",
    "    all_texts.append(row['SubmissionTitle'])\n",
    "    all_texts.append(row['SubmissionAbstract'])\n",
    "    row['cluster_filter'] = ast.literal_eval(row['cluster_filter'])\n",
    "    for authorWorks in row['cluster_filter']:\n",
    "        all_texts.append(authorWorks['title'])\n",
    "        all_texts.append(authorWorks['abstract'])\n",
    "\n",
    "vectorizer.fit(all_texts)\n",
    "\n",
    "for index, row in tqdm.tqdm(dataset.iterrows(), total=dataset.shape[0]):\n",
    "    all_cosine_similarities = {}\n",
    "    doi = row['doi']\n",
    "    # Transform the title and abstract\n",
    "    titletfidf = vectorizer.transform([row['SubmissionTitle']])\n",
    "    abstracttfidf = vectorizer.transform([row['SubmissionAbstract']])\n",
    "    \n",
    "    row['cluster_filter'] = ast.literal_eval(row['cluster_filter'])\n",
    "    dataset.at[index, 'cluster_filter'] = row['cluster_filter']\n",
    "    \n",
    "    for authorWorks in row['cluster_filter']:\n",
    "        authDOI = authorWorks['doi']\n",
    "        authorTitleTfidf = vectorizer.transform([authorWorks['title']])\n",
    "        authorAbstractTfidf = vectorizer.transform([authorWorks['abstract']])\n",
    "        \n",
    "        # Compute the cosine similarity\n",
    "        title_cosine_similarity = cosine_similarity(titletfidf, authorTitleTfidf)\n",
    "        abstract_cosine_similarity = cosine_similarity(abstracttfidf, authorAbstractTfidf)\n",
    "        \n",
    "        all_cosine_similarities[authDOI] = 0.3 * title_cosine_similarity + 0.7 * abstract_cosine_similarity\n",
    "\n",
    "    # Keep the top k most similar works\n",
    "    if len(all_cosine_similarities) == 1:\n",
    "        k = 1\n",
    "    elif len(all_cosine_similarities) < 4:\n",
    "        k = 2\n",
    "    else:\n",
    "        k = 3\n",
    "    \n",
    "    # Sort the list of cosine similarities\n",
    "    all_cosine_similarities = dict(sorted(all_cosine_similarities.items(), key=lambda item: item[1], reverse=True))\n",
    "    # Get the top k most similar works\n",
    "    top_k = dict(list(all_cosine_similarities.items())[:k])\n",
    "    \n",
    "    # get the auth objects of the top k\n",
    "    top_k_auths = []\n",
    "    for key in top_k:\n",
    "        for auth in row['cluster_filter']:\n",
    "            if auth['doi'] == key:\n",
    "                top_k_auths.append(auth)\n",
    "                break\n",
    "    \n",
    "    dataset.at[index, 'tfidf_cosine_similarity'] = top_k_auths\n",
    "dataset.to_csv('data/tfidf_cosine_similarity.csv', index=False)\n",
    "print(dataset.head())\n",
    "print(dataset.info())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# LLM Based Prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 50/50 [00:06<00:00,  8.10it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                 SubmissionID  SubmissionYear  \\\n",
      "0             /10.1101/188524            2017   \n",
      "1             /10.1101/188524            2017   \n",
      "2             /10.1101/783175            2019   \n",
      "3  /10.1101/2020.06.30.176537            2020   \n",
      "4  /10.1101/2020.06.30.176537            2020   \n",
      "\n",
      "                                     SubmissionTitle  \\\n",
      "0  characterizing highly dynamic conformational s...   \n",
      "1  characterizing highly dynamic conformational s...   \n",
      "2  dynamic reconfiguration fragmentation and inte...   \n",
      "3  attenuated subcomponent vaccine design targeti...   \n",
      "4  attenuated subcomponent vaccine design targeti...   \n",
      "\n",
      "                                  SubmissionAbstract firstName middleName  \\\n",
      "0  biomacromolecules carry out complicated functi...     eitan        NaN   \n",
      "1  biomacromolecules carry out complicated functi...  antonino        NaN   \n",
      "2  general anesthetics are routinely used to indu...    corson         N.   \n",
      "3  the novel coronavirus disease covid19 caused b...    onyeka         S.   \n",
      "4  the novel coronavirus disease covid19 caused b...   rebecca   Chinyelu   \n",
      "\n",
      "       lastName  isFirstAuthor  city        country  ...  \\\n",
      "0        lerner              1   NaN  United States  ...   \n",
      "1    ingargiola              0   NaN  United States  ...   \n",
      "2   areshenkoff              0   NaN         Canada  ...   \n",
      "3   chukwudozie              1   NaN        Nigeria  ...   \n",
      "4  chukwuanukwu              0   NaN        Nigeria  ...   \n",
      "\n",
      "                            authorPublicationHistory match  \\\n",
      "0  [{'title': 'rapid and specific detection of na...     1   \n",
      "1  [{'title': 'erratum monte carlo diffusionenhan...     1   \n",
      "2  [{'title': 'distinct patterns of connectivity ...     1   \n",
      "3  [{'title': 'the relevance of bioinformatics ap...     1   \n",
      "4  [{'title': 'the role of immuneinflammatory mar...     1   \n",
      "\n",
      "                        fullName  \\\n",
      "0                   eitan lerner   \n",
      "1            antonino ingargiola   \n",
      "2          corson n. areshenkoff   \n",
      "3          onyeka s. chukwudozie   \n",
      "4  rebecca chinyelu chukwuanukwu   \n",
      "\n",
      "                                     title_embedding  \\\n",
      "0  [0.000631210277788341, -0.06552567332983017, -...   \n",
      "1  [0.000631210277788341, -0.06552567332983017, -...   \n",
      "2  [-0.03589945659041405, 0.03250805288553238, -0...   \n",
      "3  [0.018001500517129898, -0.033130768686532974, ...   \n",
      "4  [0.018001500517129898, -0.033130768686532974, ...   \n",
      "\n",
      "                                  abstract_embedding  \\\n",
      "0  [0.007256368640810251, -0.03036230057477951, -...   \n",
      "1  [0.007256368640810251, -0.03036230057477951, -...   \n",
      "2  [-0.018003256991505623, 0.04315100610256195, -...   \n",
      "3  [-0.00747875077649951, -0.04243578389286995, 0...   \n",
      "4  [-0.00747875077649951, -0.04243578389286995, 0...   \n",
      "\n",
      "                                       doi_embedding  \\\n",
      "0  [0.007630346342921257, -0.03242341801524162, -...   \n",
      "1  [0.007630346342921257, -0.03242341801524162, -...   \n",
      "2  [-0.0036318886559456587, -0.04171673208475113,...   \n",
      "3  [-0.014353901147842407, -0.019061246886849403,...   \n",
      "4  [-0.014353901147842407, -0.019061246886849403,...   \n",
      "\n",
      "                  authorPublicationHistory_embedding  \\\n",
      "0  [{'title_embedding': [0.00456725899130106, -0....   \n",
      "1  [{'title_embedding': [-0.011444708332419395, -...   \n",
      "2  [{'title_embedding': [-0.05066244304180145, 0....   \n",
      "3  [{'title_embedding': [-0.027665186673402786, 0...   \n",
      "4  [{'title_embedding': [0.02183222956955433, -0....   \n",
      "\n",
      "                                      cluster_filter  \\\n",
      "0  [{'title': 'rapid and specific detection of na...   \n",
      "1  [{'title': 'erratum monte carlo diffusionenhan...   \n",
      "2  [{'title': 'distinct patterns of connectivity ...   \n",
      "3  [{'title': 'attenuated subcomponent vaccine de...   \n",
      "4  [{'title': 'neutrophil extracellular traps cha...   \n",
      "\n",
      "                             tfidf_cosine_similarity predicted_published_work  \n",
      "0  [{'title': 'characterizing highly dynamic conf...        10.1063/1.5004606  \n",
      "1  [{'title': 'characterizing highly dynamic conf...        10.1063/1.5004606  \n",
      "2  [{'title': 'dynamic reconfiguration fragmentat...   10.1093/cercor/bhaa085  \n",
      "3  [{'title': 'attenuated subcomponent vaccine de...     10.1155/2020/2837670  \n",
      "4  [{'title': 'attenuated subcomponent vaccine de...     10.1155/2020/2837670  \n",
      "\n",
      "[5 rows x 24 columns]\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 50 entries, 0 to 49\n",
      "Data columns (total 24 columns):\n",
      " #   Column                              Non-Null Count  Dtype  \n",
      "---  ------                              --------------  -----  \n",
      " 0   SubmissionID                        50 non-null     object \n",
      " 1   SubmissionYear                      50 non-null     int64  \n",
      " 2   SubmissionTitle                     50 non-null     object \n",
      " 3   SubmissionAbstract                  50 non-null     object \n",
      " 4   firstName                           50 non-null     object \n",
      " 5   middleName                          13 non-null     object \n",
      " 6   lastName                            50 non-null     object \n",
      " 7   isFirstAuthor                       50 non-null     int64  \n",
      " 8   city                                0 non-null      float64\n",
      " 9   country                             50 non-null     object \n",
      " 10  countryCode                         50 non-null     object \n",
      " 11  institution                         50 non-null     object \n",
      " 12  doi                                 50 non-null     object \n",
      " 13  authorID                            50 non-null     object \n",
      " 14  authorPublicationHistory            50 non-null     object \n",
      " 15  match                               50 non-null     int64  \n",
      " 16  fullName                            50 non-null     object \n",
      " 17  title_embedding                     50 non-null     object \n",
      " 18  abstract_embedding                  50 non-null     object \n",
      " 19  doi_embedding                       50 non-null     object \n",
      " 20  authorPublicationHistory_embedding  50 non-null     object \n",
      " 21  cluster_filter                      50 non-null     object \n",
      " 22  tfidf_cosine_similarity             50 non-null     object \n",
      " 23  predicted_published_work            50 non-null     object \n",
      "dtypes: float64(1), int64(3), object(20)\n",
      "memory usage: 9.5+ KB\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "model = SentenceTransformer('all-mpnet-base-v2')\n",
    "dataset['predicted_published_work'] = None\n",
    "for index, row in tqdm.tqdm(dataset.iterrows(), total=dataset.shape[0]):\n",
    "    title_embedding = model.encode(row['SubmissionTitle'])\n",
    "    abstract_embedding = model.encode(row['SubmissionAbstract'])\n",
    "    doi = row['doi']\n",
    "    # row['tfidf_cosine_similarity'] = ast.literal_eval(row['tfidf_cosine_similarity'])\n",
    "    # dataset.at[index, 'tfidf_cosine_similarity'] = row['tfidf_cosine_similarity']\n",
    "    \n",
    "    author_all_works_similarity = {}\n",
    "    for authorWorks in row['tfidf_cosine_similarity']:\n",
    "        authorTitleEmbedding = model.encode(authorWorks['title'])\n",
    "        authorAbstractEmbedding = model.encode(authorWorks['abstract'])\n",
    "        \n",
    "        # Convert NumPy arrays to PyTorch tensors and add an extra dimension\n",
    "        title_embedding_tensor = torch.tensor(title_embedding).unsqueeze(0)\n",
    "        authorTitleEmbedding_tensor = torch.tensor(authorTitleEmbedding).unsqueeze(0)\n",
    "        abstract_embedding_tensor = torch.tensor(abstract_embedding).unsqueeze(0)\n",
    "        authorAbstractEmbedding_tensor = torch.tensor(authorAbstractEmbedding).unsqueeze(0)\n",
    "        \n",
    "        # Compute the cosine similarity\n",
    "        title_cosine_similarity = torch.nn.functional.cosine_similarity(title_embedding_tensor, authorTitleEmbedding_tensor)\n",
    "        abstract_cosine_similarity = torch.nn.functional.cosine_similarity(abstract_embedding_tensor, authorAbstractEmbedding_tensor)\n",
    "        author_all_works_similarity[authorWorks['doi']] = 0.3 * title_cosine_similarity.item() + 0.7 * abstract_cosine_similarity.item()\n",
    "        \n",
    "    # Sort the list of cosine similarities\n",
    "    author_all_works_similarity = dict(sorted(author_all_works_similarity.items(), key=lambda item: item[1], reverse=True))\n",
    "    # Get the top most similar work\n",
    "    top_work = list(author_all_works_similarity.items())[0]\n",
    "    dataset.at[index, 'predicted_published_work'] = top_work[0]\n",
    "\n",
    "dataset.to_csv('data/predicted_published_work.csv', index=False)\n",
    "print(dataset.head())\n",
    "print(dataset.info())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluation Metrics for Our Implementation\n",
      "Accuracy: 1.0\n",
      "Precision: 1.0\n",
      "Recall: 1.0\n",
      "F1 Score: 1.0\n"
     ]
    }
   ],
   "source": [
    "y_true = dataset['doi']\n",
    "y_pred = dataset['predicted_published_work']\n",
    "\n",
    "\n",
    "accuracy = accuracy_score(y_true, y_pred)\n",
    "precision = precision_score(y_true, y_pred, average='macro')\n",
    "recall = recall_score(y_true, y_pred, average='macro')\n",
    "f1 = f1_score(y_true, y_pred, average='macro')\n",
    "\n",
    "print(\"Evaluation Metrics for Our Implementation\")\n",
    "print(f'Accuracy: {accuracy}')\n",
    "print(f'Precision: {precision}')\n",
    "print(f'Recall: {recall}')\n",
    "print(f'F1 Score: {f1}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "ml-project-2",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}