CRF_model.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"CRF_model.ipynb","provenance":[],"collapsed_sections":[],"mount_file_id":"1ZuHLviEFsPOGZL7q9txcTlmCP-LjJVxL","authorship_tag":"ABX9TyOI4NOVrdoZn6T3CtoMBp0Q"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"}},"cells":[{"cell_type":"markdown","source":["## **Importing Library**"],"metadata":{"id":"KEmbDqchjGUD"}},{"cell_type":"code","execution_count":3,"metadata":{"id":"01z7QozUHZVl","executionInfo":{"status":"ok","timestamp":1661124437026,"user_tz":-60,"elapsed":2695,"user":{"displayName":"Antra Tripathi","userId":"14282966453087221057"}}},"outputs":[],"source":["#importing all the needed libraries\n","import pandas as pd       \n","import nltk\n","import sklearn\n","import sklearn_crfsuite\n","import scipy.stats\n","import math, string, re\n","import csv\n","import json\n","\n","from sklearn.metrics import make_scorer\n","from sklearn.metrics import accuracy_score\n","from sklearn.model_selection import cross_val_score\n","from sklearn.model_selection import RandomizedSearchCV\n","from sklearn_crfsuite import scorers\n","from sklearn_crfsuite import metrics\n","from itertools import chain\n","from sklearn.preprocessing import MultiLabelBinarizer\n","from scipy.special import softmax\n","import numpy as np\n","\n","from sklearn.metrics import classification_report"]},{"cell_type":"markdown","source":["## **Data Import**"],"metadata":{"id":"qMO22IVvjOPX"}},{"cell_type":"markdown","source":["**File paths:**"],"metadata":{"id":"_XzEN0ZMjWit"}},{"cell_type":"code","source":["DATA_PATH = '/content/drive/MyDrive/MscThesis/public_data_PP/gimpel_pos'\n","\n","train_path = DATA_PATH + '/input_data/train_data.tsv'\n","dev_path = DATA_PATH + '/input_data/dev_data.tsv'\n","\n","# Path of dev reference file\n","dev_ref_path ='/content/drive/MyDrive/MscThesis/dev_reference_labels/Gimpel-POS_answers.jsonlines'"],"metadata":{"id":"AVoYwCzrHy0x","executionInfo":{"status":"ok","timestamp":1661124441988,"user_tz":-60,"elapsed":234,"user":{"displayName":"Antra Tripathi","userId":"14282966453087221057"}}},"execution_count":4,"outputs":[]},{"cell_type":"code","source":["def read_csv_data(path):\n","  '''\n","  User define function to read the .csv file from the given path\n","  '''\n","  data = []\n","  with open(path, 'r') as f:\n","    tsvreader = csv.reader(f, delimiter=\"\\t\")\n","    for line in tsvreader:\n","      data.append(line)\n","    # to remove the header\n","  return data[1:]\n","\n","def load_dictionary(filepath):\n","    with open(filepath, 'r') as f:\n","        dictionary = json.load(f)\n","    return dictionary\n","\n","def get_dev(path):\n","  # submissions are in a dictionary format\n","    hard_lable=[]\n","    soft_lable=[]\n","    reference_dictionary = load_dictionary(path)\n","    for document, doc_contents in reference_dictionary.items():      \n","        for item_id, contents in doc_contents.items():\n","            hard_lable.append(contents['gold'])\n","            soft_lable.append(contents['soft'])\n","\n","    return hard_lable, soft_lable"],"metadata":{"id":"uJJCZoeYJBTB","executionInfo":{"status":"ok","timestamp":1661124443438,"user_tz":-60,"elapsed":242,"user":{"displayName":"Antra Tripathi","userId":"14282966453087221057"}}},"execution_count":5,"outputs":[]},{"cell_type":"code","source":["# the train and dev data from csv\n","train_data = read_csv_data(train_path)\n","train_df = pd.read_csv(train_path, sep='\\t')\n","\n","dev_df = pd.read_csv(dev_path, sep='\\t')\n","h, s = get_dev(dev_ref_path)\n","dev_df['Hard']=h\n","dev_df['Soft']=s"],"metadata":{"id":"Xejz96FEHyxP","executionInfo":{"status":"ok","timestamp":1661124480922,"user_tz":-60,"elapsed":1465,"user":{"displayName":"Antra Tripathi","userId":"14282966453087221057"}}},"execution_count":6,"outputs":[]},{"cell_type":"code","source":["train_df.head(5)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":206},"id":"ZmqeUkQ5HyvC","executionInfo":{"status":"ok","timestamp":1661124480924,"user_tz":-60,"elapsed":6,"user":{"displayName":"Antra Tripathi","userId":"14282966453087221057"}},"outputId":"6d33555d-2801-4435-b87e-bb2d649b02a4"},"execution_count":7,"outputs":[{"output_type":"execute_result","data":{"text/plain":["   Twitt_ID  Token_Id_in_Dataset    Token  \\\n","0         0                    0        I   \n","1         0                    1  predict   \n","2         0                    2        I   \n","3         0                    3    won't   \n","4         0                    4      win   \n","\n","                                         Annotations  \n","0  ,,,,,,,PRON,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,...  \n","1  VERB,VERB,VERB,VERB,,VERB,VERB,VERB,VERB,NOUN,...  \n","2  PRON,PRON,PRON,PRON,,PRON,PRON,PRON,PRON,,PRON...  \n","3  ,,,,,,,,,,,,,,,,,VERB,,,,,,,,,,,,,,,,,,,,,,,,,...  \n","4  ,,,,,,,,,,,,,,,,,,VERB,,,,,,,,,,,,,,,,,,,,,,,,...  "],"text/html":["\n","  <div id=\"df-267564e2-fdac-49c9-b6d5-9b087b00df41\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>Twitt_ID</th>\n","      <th>Token_Id_in_Dataset</th>\n","      <th>Token</th>\n","      <th>Annotations</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>I</td>\n","      <td>,,,,,,,PRON,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,...</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>predict</td>\n","      <td>VERB,VERB,VERB,VERB,,VERB,VERB,VERB,VERB,NOUN,...</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>0</td>\n","      <td>2</td>\n","      <td>I</td>\n","      <td>PRON,PRON,PRON,PRON,,PRON,PRON,PRON,PRON,,PRON...</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>0</td>\n","      <td>3</td>\n","      <td>won't</td>\n","      <td>,,,,,,,,,,,,,,,,,VERB,,,,,,,,,,,,,,,,,,,,,,,,,...</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>0</td>\n","      <td>4</td>\n","      <td>win</td>\n","      <td>,,,,,,,,,,,,,,,,,,VERB,,,,,,,,,,,,,,,,,,,,,,,,...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-267564e2-fdac-49c9-b6d5-9b087b00df41')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-267564e2-fdac-49c9-b6d5-9b087b00df41 button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-267564e2-fdac-49c9-b6d5-9b087b00df41');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "]},"metadata":{},"execution_count":7}]},{"cell_type":"code","source":["dev_df.head(5)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":206},"id":"ep2PBhUEj1HP","executionInfo":{"status":"ok","timestamp":1661124482756,"user_tz":-60,"elapsed":222,"user":{"displayName":"Antra Tripathi","userId":"14282966453087221057"}},"outputId":"3868d8e9-6181-49ab-ff15-076c5a9d86a4"},"execution_count":8,"outputs":[{"output_type":"execute_result","data":{"text/plain":["   Twitt_ID  Token_Id_in_Dataset Token  Hard  \\\n","0         0                    0    If     1   \n","1         0                    1   you     7   \n","2         0                    2   can    10   \n","3         0                    3   see    10   \n","4         0                    4  only     0   \n","\n","                                                Soft  \n","0  [0.037740984641406065, 0.2788702527442295, 0.0...  \n","1  [0.0062730078597622975, 0.0062730078597622975,...  \n","2  [0.006273007859762292, 0.006273007859762292, 0...  \n","3  [0.014855213984839731, 0.014855213984839731, 0...  \n","4  [0.11545587797864408, 0.04247384387074182, 0.3...  "],"text/html":["\n","  <div id=\"df-f0d51b89-3cd2-48bc-9334-ce04e10b4938\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>Twitt_ID</th>\n","      <th>Token_Id_in_Dataset</th>\n","      <th>Token</th>\n","      <th>Hard</th>\n","      <th>Soft</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>If</td>\n","      <td>1</td>\n","      <td>[0.037740984641406065, 0.2788702527442295, 0.0...</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>you</td>\n","      <td>7</td>\n","      <td>[0.0062730078597622975, 0.0062730078597622975,...</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>0</td>\n","      <td>2</td>\n","      <td>can</td>\n","      <td>10</td>\n","      <td>[0.006273007859762292, 0.006273007859762292, 0...</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>0</td>\n","      <td>3</td>\n","      <td>see</td>\n","      <td>10</td>\n","      <td>[0.014855213984839731, 0.014855213984839731, 0...</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>0</td>\n","      <td>4</td>\n","      <td>only</td>\n","      <td>0</td>\n","      <td>[0.11545587797864408, 0.04247384387074182, 0.3...</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-f0d51b89-3cd2-48bc-9334-ce04e10b4938')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-f0d51b89-3cd2-48bc-9334-ce04e10b4938 button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-f0d51b89-3cd2-48bc-9334-ce04e10b4938');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "]},"metadata":{},"execution_count":8}]},{"cell_type":"markdown","source":["### **Data Processing:**\n"],"metadata":{"id":"MyeWE66Aj4-Y"}},{"cell_type":"markdown","source":["#### **Generating Hard label and Soft Distribution for training data**\n"],"metadata":{"id":"5Wuvj0c0vHvd"}},{"cell_type":"code","source":["def get_soft(string_format):\n","    annotator_dict = {ann_idx:labels_dict[annotation] for ann_idx, annotation in enumerate(string_format.split(',')) if annotation != \"\"}\n","    ann_labs = list(annotator_dict.values())\n","    distr = [ann_labs.count(l) for l in range(len(labels_dict))]\n","    return distr, softmax(distr).tolist()\n","\n","labels_dict = {'ADJ': 0, 'ADP': 1, 'ADV': 2, 'CCONJ': 3, 'DET': 4,'NOUN': 5, 'NUM': 6, 'PART': 8,'PRON': 7,'PUNCT': 9,'VERB': 10, 'X': 11}\n","\n","train_softs = []\n","train_distr = []\n","for line in train_data:\n","    if line:\n","        distr, soft = get_soft(line[-1])\n","        train_softs.append(soft)\n","        train_distr.append(distr)\n","\n","#appending soft distribution to the training df\n","train_df['Distr']= train_distr\n","train_df['Soft']= train_softs\n","train_df['Hard_label']=''"],"metadata":{"id":"KT0ps8FqHysw","executionInfo":{"status":"ok","timestamp":1661124648227,"user_tz":-60,"elapsed":778,"user":{"displayName":"Antra Tripathi","userId":"14282966453087221057"}}},"execution_count":9,"outputs":[]},{"cell_type":"code","source":["#converting numerical label to text label\n","for i in range(len(train_df)):\n","  train_df['Hard_label'].iloc[i] = list(labels_dict.keys())[np.argmax(train_df['Soft'].iloc[i])]\n","  "],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"xO48D_yGJU21","executionInfo":{"status":"ok","timestamp":1661124677263,"user_tz":-60,"elapsed":4813,"user":{"displayName":"Antra Tripathi","userId":"14282966453087221057"}},"outputId":"d3cf2d9f-7e71-47a4-bc3d-5c0668208d2d"},"execution_count":11,"outputs":[{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.7/dist-packages/pandas/core/indexing.py:1732: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame\n","\n","See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","  self._setitem_single_block(indexer, value, name)\n"]}]},{"cell_type":"markdown","source":["#### **Training Dataframe after processing:**"],"metadata":{"id":"eUneZuafvyFd"}},{"cell_type":"code","source":["train_df.head(5)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":250},"id":"9viBi_j_JUzf","executionInfo":{"status":"ok","timestamp":1661124710782,"user_tz":-60,"elapsed":303,"user":{"displayName":"Antra Tripathi","userId":"14282966453087221057"}},"outputId":"1b7b355e-aaad-4206-b6a5-eb782fbe95db"},"execution_count":12,"outputs":[{"output_type":"execute_result","data":{"text/plain":["   Twitt_ID  Token_Id_in_Dataset    Token  \\\n","0         0                    0        I   \n","1         0                    1  predict   \n","2         0                    2        I   \n","3         0                    3    won't   \n","4         0                    4      win   \n","\n","                                         Annotations  \\\n","0  ,,,,,,,PRON,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,...   \n","1  VERB,VERB,VERB,VERB,,VERB,VERB,VERB,VERB,NOUN,...   \n","2  PRON,PRON,PRON,PRON,,PRON,PRON,PRON,PRON,,PRON...   \n","3  ,,,,,,,,,,,,,,,,,VERB,,,,,,,,,,,,,,,,,,,,,,,,,...   \n","4  ,,,,,,,,,,,,,,,,,,VERB,,,,,,,,,,,,,,,,,,,,,,,,...   \n","\n","                                    Distr  \\\n","0    [0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0]   \n","1  [2, 1, 0, 0, 0, 4, 0, 0, 0, 0, 134, 0]   \n","2  [0, 0, 0, 0, 1, 9, 0, 118, 1, 1, 0, 0]   \n","3    [0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0]   \n","4    [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0]   \n","\n","                                                Soft Hard_label  \n","0  [0.0062730078597622975, 0.0062730078597622975,...       PART  \n","1  [4.7111658015535965e-58, 1.733141042341547e-58...       VERB  \n","2  [5.665668176358939e-52, 5.665668176358939e-52,...       PART  \n","3  [0.014855213984839731, 0.014855213984839731, 0...       VERB  \n","4  [0.006273007859762292, 0.006273007859762292, 0...       VERB  "],"text/html":["\n","  <div id=\"df-26ac543d-00cb-4641-8eb2-c68bd0fe4db6\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>Twitt_ID</th>\n","      <th>Token_Id_in_Dataset</th>\n","      <th>Token</th>\n","      <th>Annotations</th>\n","      <th>Distr</th>\n","      <th>Soft</th>\n","      <th>Hard_label</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>I</td>\n","      <td>,,,,,,,PRON,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,...</td>\n","      <td>[0, 0, 0, 0, 0, 0, 0, 5, 0, 0, 0, 0]</td>\n","      <td>[0.0062730078597622975, 0.0062730078597622975,...</td>\n","      <td>PART</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>predict</td>\n","      <td>VERB,VERB,VERB,VERB,,VERB,VERB,VERB,VERB,NOUN,...</td>\n","      <td>[2, 1, 0, 0, 0, 4, 0, 0, 0, 0, 134, 0]</td>\n","      <td>[4.7111658015535965e-58, 1.733141042341547e-58...</td>\n","      <td>VERB</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>0</td>\n","      <td>2</td>\n","      <td>I</td>\n","      <td>PRON,PRON,PRON,PRON,,PRON,PRON,PRON,PRON,,PRON...</td>\n","      <td>[0, 0, 0, 0, 1, 9, 0, 118, 1, 1, 0, 0]</td>\n","      <td>[5.665668176358939e-52, 5.665668176358939e-52,...</td>\n","      <td>PART</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>0</td>\n","      <td>3</td>\n","      <td>won't</td>\n","      <td>,,,,,,,,,,,,,,,,,VERB,,,,,,,,,,,,,,,,,,,,,,,,,...</td>\n","      <td>[0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 4, 0]</td>\n","      <td>[0.014855213984839731, 0.014855213984839731, 0...</td>\n","      <td>VERB</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>0</td>\n","      <td>4</td>\n","      <td>win</td>\n","      <td>,,,,,,,,,,,,,,,,,,VERB,,,,,,,,,,,,,,,,,,,,,,,,...</td>\n","      <td>[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 5, 0]</td>\n","      <td>[0.006273007859762292, 0.006273007859762292, 0...</td>\n","      <td>VERB</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-26ac543d-00cb-4641-8eb2-c68bd0fe4db6')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-26ac543d-00cb-4641-8eb2-c68bd0fe4db6 button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-26ac543d-00cb-4641-8eb2-c68bd0fe4db6');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "]},"metadata":{},"execution_count":12}]},{"cell_type":"markdown","source":["#### **Dev Data Processing**"],"metadata":{"id":"stMdVgYpwKgM"}},{"cell_type":"code","source":["dev_df['Hard_label']=''\n","#converting numerical lable to text label\n","for i in range(len(dev_df)):\n","  dev_df['Hard_label'].iloc[i] = list(labels_dict.keys())[dev_df['Hard'].iloc[i]]"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"3ZjioCSjZ623","executionInfo":{"status":"ok","timestamp":1661124818579,"user_tz":-60,"elapsed":1572,"user":{"displayName":"Antra Tripathi","userId":"14282966453087221057"}},"outputId":"7320cf38-f3ae-4e80-cbfe-9eac9fcc4470"},"execution_count":13,"outputs":[{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.7/dist-packages/pandas/core/indexing.py:1732: SettingWithCopyWarning: \n","A value is trying to be set on a copy of a slice from a DataFrame\n","\n","See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n","  self._setitem_single_block(indexer, value, name)\n"]}]},{"cell_type":"markdown","source":["####**Dev Dataframe after processing**"],"metadata":{"id":"bIqWPfOKwWI-"}},{"cell_type":"code","source":["dev_df.head(5)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":206},"id":"sRc2_gR1aSIy","executionInfo":{"status":"ok","timestamp":1661085621455,"user_tz":-60,"elapsed":332,"user":{"displayName":"Antra Tripathi","userId":"14282966453087221057"}},"outputId":"3b6eb990-bb8a-45f4-dcef-02211b4e37b1"},"execution_count":107,"outputs":[{"output_type":"execute_result","data":{"text/plain":["   Twitt_ID  Token_Id_in_Dataset Token  Hard  \\\n","0         0                    0    If     1   \n","1         0                    1   you     7   \n","2         0                    2   can    10   \n","3         0                    3   see    10   \n","4         0                    4  only     0   \n","\n","                                                Soft Hard_lable  \n","0  [0.037740984641406065, 0.2788702527442295, 0.0...        ADP  \n","1  [0.0062730078597622975, 0.0062730078597622975,...       PART  \n","2  [0.006273007859762292, 0.006273007859762292, 0...       VERB  \n","3  [0.014855213984839731, 0.014855213984839731, 0...       VERB  \n","4  [0.11545587797864408, 0.04247384387074182, 0.3...        ADJ  "],"text/html":["\n","  <div id=\"df-bd5ab0ec-667b-467b-99ee-41a6335c0638\">\n","    <div class=\"colab-df-container\">\n","      <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>Twitt_ID</th>\n","      <th>Token_Id_in_Dataset</th>\n","      <th>Token</th>\n","      <th>Hard</th>\n","      <th>Soft</th>\n","      <th>Hard_lable</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>0</td>\n","      <td>0</td>\n","      <td>If</td>\n","      <td>1</td>\n","      <td>[0.037740984641406065, 0.2788702527442295, 0.0...</td>\n","      <td>ADP</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>0</td>\n","      <td>1</td>\n","      <td>you</td>\n","      <td>7</td>\n","      <td>[0.0062730078597622975, 0.0062730078597622975,...</td>\n","      <td>PART</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>0</td>\n","      <td>2</td>\n","      <td>can</td>\n","      <td>10</td>\n","      <td>[0.006273007859762292, 0.006273007859762292, 0...</td>\n","      <td>VERB</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>0</td>\n","      <td>3</td>\n","      <td>see</td>\n","      <td>10</td>\n","      <td>[0.014855213984839731, 0.014855213984839731, 0...</td>\n","      <td>VERB</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>0</td>\n","      <td>4</td>\n","      <td>only</td>\n","      <td>0</td>\n","      <td>[0.11545587797864408, 0.04247384387074182, 0.3...</td>\n","      <td>ADJ</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-bd5ab0ec-667b-467b-99ee-41a6335c0638')\"\n","              title=\"Convert this dataframe to an interactive table.\"\n","              style=\"display:none;\">\n","        \n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","       width=\"24px\">\n","    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n","    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n","  </svg>\n","      </button>\n","      \n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      flex-wrap:wrap;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","      <script>\n","        const buttonEl =\n","          document.querySelector('#df-bd5ab0ec-667b-467b-99ee-41a6335c0638 button.colab-df-convert');\n","        buttonEl.style.display =\n","          google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","        async function convertToInteractive(key) {\n","          const element = document.querySelector('#df-bd5ab0ec-667b-467b-99ee-41a6335c0638');\n","          const dataTable =\n","            await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                     [key], {});\n","          if (!dataTable) return;\n","\n","          const docLinkHtml = 'Like what you see? Visit the ' +\n","            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","            + ' to learn more about interactive tables.';\n","          element.innerHTML = '';\n","          dataTable['output_type'] = 'display_data';\n","          await google.colab.output.renderOutput(dataTable, element);\n","          const docLink = document.createElement('div');\n","          docLink.innerHTML = docLinkHtml;\n","          element.appendChild(docLink);\n","        }\n","      </script>\n","    </div>\n","  </div>\n","  "]},"metadata":{},"execution_count":107}]},{"cell_type":"markdown","source":["### Feature Generation :"],"metadata":{"id":"1GHwczKlkGs5"}},{"cell_type":"code","source":["def format_data(data):\n","  \"\"\"\n","  Input: dataframe\n","  Function: Concatenate the token and token label as a pair\n","  Output: return the list containing token pair\n","  \"\"\"\n","  flag, start= 0,0\n","  end=0\n","  sentences=[]\n","  for i in range(len(data)):\n","    if data['Twitt_ID'][i]!= flag:\n","      end= i-1\n","      txt = list(data['Token'][start:end])\n","      txt_lable= list(data['Hard_label'][start:end])\n","\n","      sent=[]\n","      for j in range(len(txt)):\n","        sent.append([txt[j], txt_lable[j]])      \n","      sentences.append(sent)\n","      start=i\n","      flag=flag+1\n","\n","  return sentences"],"metadata":{"id":"lZoH9mYUKkr6","executionInfo":{"status":"ok","timestamp":1661125297117,"user_tz":-60,"elapsed":2,"user":{"displayName":"Antra Tripathi","userId":"14282966453087221057"}}},"execution_count":18,"outputs":[]},{"cell_type":"code","source":["def word2features(sent, i):\n","    word = sent[i][0]\n","    # State feature: current token\n","    features = {\n","        'bias': 1.0,\n","        'word': word,\n","        'len(word)': len(word),\n","        'word[:4]': word[:4],\n","        'word[:3]': word[:3],\n","        'word[:2]': word[:2],\n","        'word[-3:]': word[-3:],\n","        'word[-2:]': word[-2:],\n","        'word[-4:]': word[-4:],\n","        'word.lower()': word.lower(),\n","        'word.stemmed': re.sub(r'(.{2,}?)([aeiougyn]+$)',r'\\1', word.lower()),\n","        'word.ispunctuation': (word in string.punctuation),\n","        'word.isdigit()': word.isdigit(),\n","    }\n","\n","    # Transition Feature: token one position before\n","    if i > 0:\n","      word1 = sent[i-1][0]\n","      features.update({          \n","            '-1:word': word1,\n","            '-1:len(word)': len(word1),\n","            '-1:word.lower()': word1.lower(),\n","            '-1:word.stemmed': re.sub(r'(.{2,}?)([aeiougyn]+$)',r'\\1', word1.lower()),\n","            '-1:word[:3]': word1[:3],\n","            '-1:word[:2]': word1[:2],\n","            '-1:word[-3:]': word1[-3:],\n","            '-1:word[-2:]': word1[-2:],\n","            '-1:word.isdigit()': word1.isdigit(),\n","            '-1:word.ispunctuation': (word1 in string.punctuation),})\n","          \n","    else:\n","      features['BOS'] = True #beginning of sentence feature\n","\n","    # Transition Feature: token two position before\n","    if i > 1:\n","        word2 = sent[i-2][0]\n","        features.update({\n","            '-2:word': word2,\n","            '-2:len(word)': len(word2),\n","            '-2:word.lower()': word2.lower(),\n","            '-2:word[:3]': word2[:3],\n","            '-2:word[:2]': word2[:2],\n","            '-2:word[-3:]': word2[-3:],\n","            '-2:word[-2:]': word2[-2:],\n","            '-2:word.isdigit()': word2.isdigit(),\n","            '-2:word.ispunctuation': (word2 in string.punctuation),})\n","\n","    # Transition Feature: token one position ahead\n","    if i < len(sent)-1:\n","        word1 = sent[i+1][0]\n","        features.update({\n","            '+1:word': word1,\n","            '+1:len(word)': len(word1),\n","            '+1:word.lower()': word1.lower(),\n","            '+1:word[:3]': word1[:3],\n","            '+1:word[:2]': word1[:2],\n","            '+1:word[-3:]': word1[-3:],\n","            '+1:word[-2:]': word1[-2:],\n","            '+1:word.isdigit()': word1.isdigit(),\n","            '+1:word.ispunctuation': (word1 in string.punctuation),\n","        })\n","\n","    else:\n","        features['EOS'] = True # end of sentence feature\n","\n","    # Transition Feature: token two position ahead\n","    if i < len(sent) - 2:\n","      word2 = sent[i+2][0]\n","      features.update({\n","              '+2:word': word2,\n","              '+2:len(word)': len(word2),\n","              '+2:word.lower()': word2.lower(),\n","              '+2:word.stemmed': re.sub(r'(.{2,}?)([aeiougyn]+$)',r'\\1', word2.lower()),\n","              '+2:word[:3]': word2[:3],\n","              '+2:word[:2]': word2[:2],\n","              '+2:word[-3:]': word2[-3:],\n","              '+2:word[-2:]': word2[-2:],\n","              '+2:word.isdigit()': word2.isdigit(),\n","              '+2:word.ispunctuation': (word2 in string.punctuation),\n","          })\n","\n","    return features\n","\n","def sent2features(sent):\n","    return [word2features(sent, i) for i in range(len(sent))]\n","\n","def sent2labels(sent):\n","    return [word[1] for word in sent]\n","\n","def sent2tokens(sent):\n","    return [word[0] for word in sent]\n"],"metadata":{"id":"qau2_fK4Q7fU","executionInfo":{"status":"ok","timestamp":1661125277381,"user_tz":-60,"elapsed":2,"user":{"displayName":"Antra Tripathi","userId":"14282966453087221057"}}},"execution_count":15,"outputs":[]},{"cell_type":"code","source":["#formatting the data into sentences\n","train_sents = format_data(train_df)\n","dev_sents = format_data(dev_df)"],"metadata":{"id":"AACLnLb0P3Zq","executionInfo":{"status":"ok","timestamp":1661125301546,"user_tz":-60,"elapsed":218,"user":{"displayName":"Antra Tripathi","userId":"14282966453087221057"}}},"execution_count":19,"outputs":[]},{"cell_type":"code","source":["train_sents[1]"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"9HVQBYiQ0baq","executionInfo":{"status":"ok","timestamp":1661125924587,"user_tz":-60,"elapsed":223,"user":{"displayName":"Antra Tripathi","userId":"14282966453087221057"}},"outputId":"e03d8246-a73b-4b08-e6c4-1fd666d30b64"},"execution_count":26,"outputs":[{"output_type":"execute_result","data":{"text/plain":["[['RT', 'X'],\n"," ['@USER', 'NOUN'],\n"," [':', 'PUNCT'],\n"," ['wat', 'PRON'],\n"," ['muhfuckaz', 'NOUN'],\n"," ['wearin', 'VERB'],\n"," ['4', 'NUM'],\n"," ['the', 'DET'],\n"," ['lingerie', 'ADJ'],\n"," ['party', 'NOUN']]"]},"metadata":{},"execution_count":26}]},{"cell_type":"code","source":["print('Train data: ', len(train_sents), '| Dev Data', len(dev_sents))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"ga5NVcr3KkoX","executionInfo":{"status":"ok","timestamp":1661125305027,"user_tz":-60,"elapsed":233,"user":{"displayName":"Antra Tripathi","userId":"14282966453087221057"}},"outputId":"03247839-641d-45fc-809e-5588b9d84189"},"execution_count":20,"outputs":[{"output_type":"stream","name":"stdout","text":["Train data:  599 | Dev Data 199\n"]}]},{"cell_type":"code","source":["#extracting features from all the sentences\n","Xtrain = [sent2features(s) for s in train_sents]\n","ytrain = [sent2labels(s) for s in train_sents]\n","\n","Xtest = [sent2features(s) for s in dev_sents]\n","ytest = [sent2labels(s) for s in dev_sents]"],"metadata":{"id":"90bJJHEzKkj3","executionInfo":{"status":"ok","timestamp":1661125309023,"user_tz":-60,"elapsed":526,"user":{"displayName":"Antra Tripathi","userId":"14282966453087221057"}}},"execution_count":21,"outputs":[]},{"cell_type":"markdown","source":["## **CRF Training** "],"metadata":{"id":"18iaEDnKisSe"}},{"cell_type":"markdown","source":["### **Using Hard Labels**"],"metadata":{"id":"EnqP4VCuzScR"}},{"cell_type":"code","source":["c1=[0, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4]\n","c2= [0, 0.1, 0.15, 0.2,0.25, 0.3, 0.35, 0.4]"],"metadata":{"id":"uYCGRczkBw8Q","executionInfo":{"status":"ok","timestamp":1661096012243,"user_tz":-60,"elapsed":2,"user":{"displayName":"Antra Tripathi","userId":"14282966453087221057"}}},"execution_count":159,"outputs":[]},{"cell_type":"code","source":["print(\"training tagger...\")\n","for i in c1:\n","  for j in c2:\n","    crf = sklearn_crfsuite.CRF(algorithm = 'lbfgs',\n","                               c1 = i,\n","                               c2 = j,\n","                               max_iterations = 100,\n","                               all_possible_transitions=True)    \n","    \n","    try:\n","      crf.fit(Xtrain, ytrain)\n","    except AttributeError:\n","      pass\n","\n","    ypred = crf.predict(Xtrain)\n","    #obtaining metrics such as accuracy, etc. on the train set\n","    labels = list(crf.classes_)\n","    print('C1: ', i, '| C2: ', j)\n","    print('F1 score train set = {}'.format(metrics.flat_f1_score(ytrain, ypred, average='weighted', labels=labels)))\n","    print('Accuracy  train set = {}\\n'.format(metrics.flat_accuracy_score(ytrain, ypred)))\n","    \n","    ypred_d = crf.predict(Xtest)\n","    print('F1 score test set = {}'.format(metrics.flat_f1_score(ytest, ypred_d,average='weighted', labels=labels)))\n","    print('Accuracy on test set = {}'.format(metrics.flat_accuracy_score(ytest, ypred_d)))\n","\n","    print('----------------------------------')\n","    "],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Y4ArLZzpB4z7","executionInfo":{"status":"ok","timestamp":1661097218117,"user_tz":-60,"elapsed":337159,"user":{"displayName":"Antra Tripathi","userId":"14282966453087221057"}},"outputId":"af3adbd2-1bd7-45b1-ee39-fa2794d83eea"},"execution_count":162,"outputs":[{"output_type":"stream","name":"stdout","text":["training tagger...\n","C1:  0 | C2:  0\n","F1 score train set = 0.995591457610168\n","Accuracy  train set = 0.9955872809863725\n","\n","F1 score test set = 0.7333302657889865\n","Accuracy on test set = 0.747693399574166\n","----------------------------------\n","C1:  0 | C2:  0.1\n","F1 score train set = 0.9902597247754735\n","Accuracy  train set = 0.9902660609993511\n","\n","F1 score test set = 0.7385793576696729\n","Accuracy on test set = 0.7562100780695529\n","----------------------------------\n","C1:  0 | C2:  0.15\n","F1 score train set = 0.992721763223498\n","Accuracy  train set = 0.9927319922128488\n","\n","F1 score test set = 0.7341394003554922\n","Accuracy on test set = 0.7526614620298083\n","----------------------------------\n","C1:  0 | C2:  0.2\n","F1 score train set = 0.9880482727874919\n","Accuracy  train set = 0.9880597014925373\n","\n","F1 score test set = 0.7368323640157265\n","Accuracy on test set = 0.752306600425834\n","----------------------------------\n","C1:  0 | C2:  0.25\n","F1 score train set = 0.9933723837582755\n","Accuracy  train set = 0.9933809214795587\n","\n","F1 score test set = 0.7367223961206715\n","Accuracy on test set = 0.7533711852377573\n","----------------------------------\n","C1:  0 | C2:  0.3\n","F1 score train set = 0.9868566342190318\n","Accuracy  train set = 0.9868916288124594\n","\n","F1 score test set = 0.7329826587667411\n","Accuracy on test set = 0.7508871540099361\n","----------------------------------\n","C1:  0 | C2:  0.35\n","F1 score train set = 0.9894741559960682\n","Accuracy  train set = 0.9894873458792992\n","\n","F1 score test set = 0.738571343355772\n","Accuracy on test set = 0.7558552164655784\n","----------------------------------\n","C1:  0 | C2:  0.4\n","F1 score train set = 0.9873892936176026\n","Accuracy  train set = 0.9874107722258274\n","\n","F1 score test set = 0.7351522374327863\n","Accuracy on test set = 0.7526614620298083\n","----------------------------------\n","C1:  0.1 | C2:  0\n","F1 score train set = 0.9998702048940353\n","Accuracy  train set = 0.999870214146658\n","\n","F1 score test set = 0.7426493930574359\n","Accuracy on test set = 0.7579843860894251\n","----------------------------------\n","C1:  0.1 | C2:  0.1\n","F1 score train set = 0.9998702048940353\n","Accuracy  train set = 0.999870214146658\n","\n","F1 score test set = 0.7466178932033133\n","Accuracy on test set = 0.7633073101490419\n","----------------------------------\n","C1:  0.1 | C2:  0.15\n","F1 score train set = 0.9997403868320949\n","Accuracy  train set = 0.999740428293316\n","\n","F1 score test set = 0.7454859787070381\n","Accuracy on test set = 0.7625975869410929\n","----------------------------------\n","C1:  0.1 | C2:  0.2\n","F1 score train set = 0.9997403868320949\n","Accuracy  train set = 0.999740428293316\n","\n","F1 score test set = 0.7465428881646895\n","Accuracy on test set = 0.7636621717530163\n","----------------------------------\n","C1:  0.1 | C2:  0.25\n","F1 score train set = 0.9996105228233084\n","Accuracy  train set = 0.999610642439974\n","\n","F1 score test set = 0.7460542996961236\n","Accuracy on test set = 0.7633073101490419\n","----------------------------------\n","C1:  0.1 | C2:  0.3\n","F1 score train set = 0.9996105228233084\n","Accuracy  train set = 0.999610642439974\n","\n","F1 score test set = 0.7477079506741422\n","Accuracy on test set = 0.7647267565649397\n","----------------------------------\n","C1:  0.1 | C2:  0.35\n","F1 score train set = 0.9996105228233084\n","Accuracy  train set = 0.999610642439974\n","\n","F1 score test set = 0.7465573717215561\n","Accuracy on test set = 0.7640170333569908\n","----------------------------------\n","C1:  0.1 | C2:  0.4\n","F1 score train set = 0.9992203182941876\n","Accuracy  train set = 0.9992212848799481\n","\n","F1 score test set = 0.7468468309996399\n","Accuracy on test set = 0.7643718949609652\n","----------------------------------\n","C1:  0.15 | C2:  0\n","F1 score train set = 0.9998702048940353\n","Accuracy  train set = 0.999870214146658\n","\n","F1 score test set = 0.7420709110173319\n","Accuracy on test set = 0.7569198012775018\n","----------------------------------\n","C1:  0.15 | C2:  0.1\n","F1 score train set = 0.9997403868320949\n","Accuracy  train set = 0.999740428293316\n","\n","F1 score test set = 0.7471555177324456\n","Accuracy on test set = 0.7629524485450674\n","----------------------------------\n","C1:  0.15 | C2:  0.15\n","F1 score train set = 0.9996105228233084\n","Accuracy  train set = 0.999610642439974\n","\n","F1 score test set = 0.7470189114215053\n","Accuracy on test set = 0.7636621717530163\n","----------------------------------\n","C1:  0.15 | C2:  0.2\n","F1 score train set = 0.9996105228233084\n","Accuracy  train set = 0.999610642439974\n","\n","F1 score test set = 0.7458954282479169\n","Accuracy on test set = 0.7629524485450674\n","----------------------------------\n","C1:  0.15 | C2:  0.25\n","F1 score train set = 0.9996105228233084\n","Accuracy  train set = 0.999610642439974\n","\n","F1 score test set = 0.7456447432752681\n","Accuracy on test set = 0.7629524485450674\n","----------------------------------\n","C1:  0.15 | C2:  0.3\n","F1 score train set = 0.9994807602610388\n","Accuracy  train set = 0.9994808565866321\n","\n","F1 score test set = 0.7442887733032912\n","Accuracy on test set = 0.7615330021291696\n","----------------------------------\n","C1:  0.15 | C2:  0.35\n","F1 score train set = 0.999220146983573\n","Accuracy  train set = 0.9992212848799481\n","\n","F1 score test set = 0.7443326107334483\n","Accuracy on test set = 0.7618878637331441\n","----------------------------------\n","C1:  0.15 | C2:  0.4\n","F1 score train set = 0.9990902965219984\n","Accuracy  train set = 0.9990914990266061\n","\n","F1 score test set = 0.7463257271471553\n","Accuracy on test set = 0.7640170333569908\n","----------------------------------\n","C1:  0.2 | C2:  0\n","F1 score train set = 0.9998702048940353\n","Accuracy  train set = 0.999870214146658\n","\n","F1 score test set = 0.741832537317778\n","Accuracy on test set = 0.7565649396735273\n","----------------------------------\n","C1:  0.2 | C2:  0.1\n","F1 score train set = 0.9996105228233084\n","Accuracy  train set = 0.999610642439974\n","\n","F1 score test set = 0.7449067697476318\n","Accuracy on test set = 0.7608232789212207\n","----------------------------------\n","C1:  0.2 | C2:  0.15\n","F1 score train set = 0.9996105228233084\n","Accuracy  train set = 0.999610642439974\n","\n","F1 score test set = 0.7458120985381237\n","Accuracy on test set = 0.7622427253371186\n","----------------------------------\n","C1:  0.2 | C2:  0.2\n","F1 score train set = 0.9994807602610388\n","Accuracy  train set = 0.9994808565866321\n","\n","F1 score test set = 0.744754819663493\n","Accuracy on test set = 0.7615330021291696\n","----------------------------------\n","C1:  0.2 | C2:  0.25\n","F1 score train set = 0.9992205089258626\n","Accuracy  train set = 0.9992212848799481\n","\n","F1 score test set = 0.7441277183939912\n","Accuracy on test set = 0.7611781405251952\n","----------------------------------\n","C1:  0.2 | C2:  0.3\n","F1 score train set = 0.998701150465487\n","Accuracy  train set = 0.9987021414665801\n","\n","F1 score test set = 0.7451278638467629\n","Accuracy on test set = 0.7622427253371186\n","----------------------------------\n","C1:  0.2 | C2:  0.35\n","F1 score train set = 0.9984413325244665\n","Accuracy  train set = 0.9984425697598962\n","\n","F1 score test set = 0.7462355347908098\n","Accuracy on test set = 0.7636621717530163\n","----------------------------------\n","C1:  0.2 | C2:  0.4\n","F1 score train set = 0.99766199348845\n","Accuracy  train set = 0.9976638546398443\n","\n","F1 score test set = 0.7445147731174628\n","Accuracy on test set = 0.7618878637331441\n","----------------------------------\n","C1:  0.25 | C2:  0\n","F1 score train set = 0.9996105228233084\n","Accuracy  train set = 0.999610642439974\n","\n","F1 score test set = 0.740895518011771\n","Accuracy on test set = 0.7558552164655784\n","----------------------------------\n","C1:  0.25 | C2:  0.1\n","F1 score train set = 0.9994807602610388\n","Accuracy  train set = 0.9994808565866321\n","\n","F1 score test set = 0.7479061747621223\n","Accuracy on test set = 0.7636621717530163\n","----------------------------------\n","C1:  0.25 | C2:  0.15\n","F1 score train set = 0.9994807602610388\n","Accuracy  train set = 0.9994808565866321\n","\n","F1 score test set = 0.745702747096477\n","Accuracy on test set = 0.7622427253371186\n","----------------------------------\n","C1:  0.25 | C2:  0.2\n","F1 score train set = 0.998831061930988\n","Accuracy  train set = 0.9988319273199221\n","\n","F1 score test set = 0.7471026853369045\n","Accuracy on test set = 0.7636621717530163\n","----------------------------------\n","C1:  0.25 | C2:  0.25\n","F1 score train set = 0.9981814370919134\n","Accuracy  train set = 0.9981829980532122\n","\n","F1 score test set = 0.7456303444822329\n","Accuracy on test set = 0.7622427253371186\n","----------------------------------\n","C1:  0.25 | C2:  0.3\n","F1 score train set = 0.9976618738203253\n","Accuracy  train set = 0.9976638546398443\n","\n","F1 score test set = 0.7446326048853757\n","Accuracy on test set = 0.7615330021291696\n","----------------------------------\n","C1:  0.25 | C2:  0.35\n","F1 score train set = 0.9968834795152941\n","Accuracy  train set = 0.9968851395197923\n","\n","F1 score test set = 0.7447493701417457\n","Accuracy on test set = 0.7618878637331441\n","----------------------------------\n","C1:  0.25 | C2:  0.4\n","F1 score train set = 0.9966235102931831\n","Accuracy  train set = 0.9966255678131084\n","\n","F1 score test set = 0.7449563078375405\n","Accuracy on test set = 0.7622427253371186\n","----------------------------------\n","C1:  0.3 | C2:  0\n","F1 score train set = 0.9994807270599864\n","Accuracy  train set = 0.9994808565866321\n","\n","F1 score test set = 0.7421465719883658\n","Accuracy on test set = 0.7576295244854506\n","----------------------------------\n","C1:  0.3 | C2:  0.1\n","F1 score train set = 0.998830963652355\n","Accuracy  train set = 0.9988319273199221\n","\n","F1 score test set = 0.7460882196476767\n","Accuracy on test set = 0.7618878637331441\n","----------------------------------\n","C1:  0.3 | C2:  0.15\n","F1 score train set = 0.9985714177815591\n","Accuracy  train set = 0.9985723556132382\n","\n","F1 score test set = 0.7461931546381615\n","Accuracy on test set = 0.7622427253371186\n","----------------------------------\n","C1:  0.3 | C2:  0.2\n","F1 score train set = 0.9974025686196025\n","Accuracy  train set = 0.9974042829331603\n","\n","F1 score test set = 0.7454058290091485\n","Accuracy on test set = 0.7618878637331441\n","----------------------------------\n","C1:  0.3 | C2:  0.25\n","F1 score train set = 0.9970123701657905\n","Accuracy  train set = 0.9970149253731343\n","\n","F1 score test set = 0.7452087960058312\n","Accuracy on test set = 0.7618878637331441\n","----------------------------------\n","C1:  0.3 | C2:  0.3\n","F1 score train set = 0.9963637508480718\n","Accuracy  train set = 0.9963659961064244\n","\n","F1 score test set = 0.7452932542005292\n","Accuracy on test set = 0.7618878637331441\n","----------------------------------\n","C1:  0.3 | C2:  0.35\n","F1 score train set = 0.9950639717865132\n","Accuracy  train set = 0.9950681375730045\n","\n","F1 score test set = 0.7435265358914184\n","Accuracy on test set = 0.7608232789212207\n","----------------------------------\n","C1:  0.3 | C2:  0.4\n","F1 score train set = 0.9938940704655325\n","Accuracy  train set = 0.9939000648929267\n","\n","F1 score test set = 0.743284997522394\n","Accuracy on test set = 0.7604684173172462\n","----------------------------------\n","C1:  0.35 | C2:  0\n","F1 score train set = 0.9992208730689035\n","Accuracy  train set = 0.9992212848799481\n","\n","F1 score test set = 0.739393468323785\n","Accuracy on test set = 0.7555003548616039\n","----------------------------------\n","C1:  0.35 | C2:  0.1\n","F1 score train set = 0.9974031474160984\n","Accuracy  train set = 0.9974042829331603\n","\n","F1 score test set = 0.7447184250463197\n","Accuracy on test set = 0.7604684173172462\n","----------------------------------\n","C1:  0.35 | C2:  0.15\n","F1 score train set = 0.9970128500119361\n","Accuracy  train set = 0.9970149253731343\n","\n","F1 score test set = 0.7464532937687844\n","Accuracy on test set = 0.7625975869410929\n","----------------------------------\n","C1:  0.35 | C2:  0.2\n","F1 score train set = 0.9963631082557377\n","Accuracy  train set = 0.9963659961064244\n","\n","F1 score test set = 0.7447487354690883\n","Accuracy on test set = 0.7615330021291696\n","----------------------------------\n","C1:  0.35 | C2:  0.25\n","F1 score train set = 0.9950647548117962\n","Accuracy  train set = 0.9950681375730045\n","\n","F1 score test set = 0.7449595883420509\n","Accuracy on test set = 0.7615330021291696\n","----------------------------------\n","C1:  0.35 | C2:  0.3\n","F1 score train set = 0.9934997336394553\n","Accuracy  train set = 0.9935107073329007\n","\n","F1 score test set = 0.7455311924427748\n","Accuracy on test set = 0.7622427253371186\n","----------------------------------\n","C1:  0.35 | C2:  0.35\n","F1 score train set = 0.9920697143616792\n","Accuracy  train set = 0.9920830629461389\n","\n","F1 score test set = 0.743822144329933\n","Accuracy on test set = 0.7604684173172462\n","----------------------------------\n","C1:  0.35 | C2:  0.4\n","F1 score train set = 0.9910273320992589\n","Accuracy  train set = 0.991044776119403\n","\n","F1 score test set = 0.7443075285877073\n","Accuracy on test set = 0.7615330021291696\n","----------------------------------\n","C1:  0.4 | C2:  0\n","F1 score train set = 0.9983125757183251\n","Accuracy  train set = 0.9983127839065542\n","\n","F1 score test set = 0.7403336386479038\n","Accuracy on test set = 0.7558552164655784\n","----------------------------------\n","C1:  0.4 | C2:  0.1\n","F1 score train set = 0.9958437189859058\n","Accuracy  train set = 0.9958468526930565\n","\n","F1 score test set = 0.7454292130492889\n","Accuracy on test set = 0.7608232789212207\n","----------------------------------\n","C1:  0.4 | C2:  0.15\n","F1 score train set = 0.9942836785829756\n","Accuracy  train set = 0.9942894224529526\n","\n","F1 score test set = 0.7451852089070546\n","Accuracy on test set = 0.7611781405251952\n","----------------------------------\n","C1:  0.4 | C2:  0.2\n","F1 score train set = 0.9928522348851605\n","Accuracy  train set = 0.9928617780661908\n","\n","F1 score test set = 0.7447701957598113\n","Accuracy on test set = 0.7611781405251952\n","----------------------------------\n","C1:  0.4 | C2:  0.25\n","F1 score train set = 0.9914183542185928\n","Accuracy  train set = 0.9914341336794289\n","\n","F1 score test set = 0.7463811021073125\n","Accuracy on test set = 0.7629524485450674\n","----------------------------------\n","C1:  0.4 | C2:  0.3\n","F1 score train set = 0.9897266820378148\n","Accuracy  train set = 0.9897469175859831\n","\n","F1 score test set = 0.7433630790994464\n","Accuracy on test set = 0.7601135557132718\n","----------------------------------\n","C1:  0.4 | C2:  0.35\n","F1 score train set = 0.9881657270465883\n","Accuracy  train set = 0.9881894873458793\n","\n","F1 score test set = 0.7425096720736643\n","Accuracy on test set = 0.7597586941092974\n","----------------------------------\n","C1:  0.4 | C2:  0.4\n","F1 score train set = 0.9865982731330265\n","Accuracy  train set = 0.9866320571057755\n","\n","F1 score test set = 0.7450964293280041\n","Accuracy on test set = 0.7618878637331441\n","----------------------------------\n"]}]},{"cell_type":"markdown","source":["#### Traing on best hyperparameter"],"metadata":{"id":"nZXQCz3tyaP3"}},{"cell_type":"code","source":["crf = sklearn_crfsuite.CRF(algorithm = 'lbfgs',\n","                               c1 = 0.25,\n","                               c2 = 0.1,\n","                               max_iterations = 100,\n","                               all_possible_transitions=True)    \n","    \n","try:\n","  crf.fit(Xtrain, ytrain)\n","except AttributeError:\n","  pass"],"metadata":{"id":"DR7PSYERykrc","executionInfo":{"status":"ok","timestamp":1661125522367,"user_tz":-60,"elapsed":3828,"user":{"displayName":"Antra Tripathi","userId":"14282966453087221057"}}},"execution_count":22,"outputs":[]},{"cell_type":"markdown","source":["#### **Training Evaluation**"],"metadata":{"id":"t6pnIj2fi1ds"}},{"cell_type":"code","source":["#obtaining metrics such as accuracy, etc. on the train set\n","labels = list(crf.classes_)\n","#labels.remove('X')\n","\n","ypred = crf.predict(Xtrain)\n","print('F1 score on the train set = {}\\n'.format(metrics.flat_f1_score(ytrain, ypred, average='weighted', labels=labels)))\n","print('Accuracy on the train set = {}\\n'.format(metrics.flat_accuracy_score(ytrain, ypred)))\n","\n","print('----------------------------------')\n","yt, yp=[], []\n","for i in range(len(ytrain)):\n","  yt.extend(ytrain[i])\n","  yp.extend(ypred[i])\n","\n","print(classification_report(yt, yp))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"YFiJEpxDbVoM","executionInfo":{"status":"ok","timestamp":1661125527980,"user_tz":-60,"elapsed":630,"user":{"displayName":"Antra Tripathi","userId":"14282966453087221057"}},"outputId":"597d1b98-b221-4981-9414-0399f8f668f2"},"execution_count":23,"outputs":[{"output_type":"stream","name":"stdout","text":["F1 score on the train set = 0.9994807602610388\n","\n","Accuracy on the train set = 0.9994808565866321\n","\n","----------------------------------\n","              precision    recall  f1-score   support\n","\n","         ADJ       1.00      1.00      1.00       561\n","         ADP       1.00      1.00      1.00       787\n","         ADV       1.00      1.00      1.00       239\n","       CCONJ       1.00      1.00      1.00       138\n","         DET       1.00      1.00      1.00       281\n","        NOUN       1.00      1.00      1.00      2057\n","         NUM       1.00      1.00      1.00        40\n","        PART       1.00      1.00      1.00       830\n","        PRON       1.00      1.00      1.00       295\n","       PUNCT       1.00      1.00      1.00       893\n","        VERB       1.00      1.00      1.00      1263\n","           X       1.00      1.00      1.00       321\n","\n","    accuracy                           1.00      7705\n","   macro avg       1.00      1.00      1.00      7705\n","weighted avg       1.00      1.00      1.00      7705\n","\n"]}]},{"cell_type":"markdown","source":["#### **Devset Evaluation**"],"metadata":{"id":"H-KUOk6si7ar"}},{"cell_type":"code","source":["#obtaining metrics such as accuracy, etc. on the test set\n","ypred_d = crf.predict(Xtest)\n","print('F1 score on the test set = {}\\n'.format(metrics.flat_f1_score(ytest, ypred_d,\n","                      average='weighted', labels=labels)))\n","print('Accuracy on the test set = {}\\n'.format(metrics.flat_accuracy_score(ytest, ypred_d)))\n","\n","print('----------------------------------')\n","yt, yp=[], []\n","for i in range(len(ytest)):\n","  yt.extend(ytest[i])\n","  yp.extend(ypred_d[i])\n","\n","print(classification_report(yt, yp))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"7a_uaFP2eCef","executionInfo":{"status":"ok","timestamp":1661125533387,"user_tz":-60,"elapsed":206,"user":{"displayName":"Antra Tripathi","userId":"14282966453087221057"}},"outputId":"28dcdda1-3410-4254-e188-80dab28dcb8e"},"execution_count":24,"outputs":[{"output_type":"stream","name":"stdout","text":["F1 score on the test set = 0.7479061747621223\n","\n","Accuracy on the test set = 0.7636621717530163\n","\n","----------------------------------\n","              precision    recall  f1-score   support\n","\n","         ADJ       0.54      0.54      0.54       165\n","         ADP       0.76      0.87      0.81       238\n","         ADV       0.83      0.40      0.54       150\n","       CCONJ       0.81      0.85      0.83        52\n","         DET       1.00      0.53      0.69       195\n","        NOUN       0.77      0.89      0.82       748\n","         NUM       0.60      0.26      0.36        47\n","        PART       0.64      0.93      0.76       211\n","        PRON       0.66      0.36      0.46       138\n","       PUNCT       0.79      0.98      0.87       278\n","        VERB       0.85      0.88      0.86       452\n","           X       0.81      0.40      0.53       144\n","\n","    accuracy                           0.76      2818\n","   macro avg       0.76      0.66      0.67      2818\n","weighted avg       0.77      0.76      0.75      2818\n","\n"]}]},{"cell_type":"markdown","source":["#### **Transation Summary**"],"metadata":{"id":"Ucqc93bkil4g"}},{"cell_type":"code","source":["#obtaining the most likely and the least likely transitions \n","from collections import Counter\n","\n","def print_transitions(transition_features):\n","    for (label_from, label_to), weight in transition_features:\n","        print(\"%-6s -> %-7s %0.6f\" % (label_from, label_to, weight))\n","\n","print(\"Top 10 likely transitions - \\n\")\n","print_transitions(Counter(crf.transition_features_).most_common(10))\n","\n","print(\"\\nTop 10 unlikely transitions - \\n\")\n","print_transitions(Counter(crf.transition_features_).most_common()[-10:])"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"GKMQwrmweTZD","executionInfo":{"status":"ok","timestamp":1661087427340,"user_tz":-60,"elapsed":366,"user":{"displayName":"Antra Tripathi","userId":"14282966453087221057"}},"outputId":"8cc3b45d-7ce9-4ecb-87e8-decd3eaeee9b"},"execution_count":155,"outputs":[{"output_type":"stream","name":"stdout","text":["Top 10 likely transitions - \n","\n","X      -> X       1.249807\n","PRON   -> PUNCT   1.141814\n","ADJ    -> NOUN    0.946512\n","PART   -> VERB    0.943096\n","ADP    -> DET     0.752923\n","CCONJ  -> PART    0.733016\n","NOUN   -> CCONJ   0.671694\n","VERB   -> DET     0.644489\n","ADV    -> VERB    0.634640\n","VERB   -> ADV     0.628565\n","\n","Top 10 unlikely transitions - \n","\n","ADP    -> X       -0.577970\n","ADJ    -> X       -0.596109\n","NOUN   -> ADJ     -0.611641\n","ADJ    -> PART    -0.635384\n","PUNCT  -> ADJ     -0.645086\n","PART   -> X       -0.646575\n","X      -> ADP     -0.770626\n","ADV    -> X       -0.772202\n","NOUN   -> DET     -0.928222\n","VERB   -> X       -1.084295\n"]}]},{"cell_type":"markdown","source":["### **Using Soft Distribution**"],"metadata":{"id":"5rO7rcHRzoH5"}},{"cell_type":"code","source":["def format_data_soft(data):\n","  \"\"\"\n","  Input: dataframe\n","  Function: Concatenate the token and token label as a pair\n","  Output: return the list containing token pair\n","  \"\"\"\n","  flag, start= 0,0\n","  end=0\n","  sentences=[]\n","  for i in range(len(data)):\n","    if data['Twitt_ID'][i]!= flag:\n","      end= i-1\n","      txt = list(data['Token'][start:end])\n","      txt_label= list(data['Hard_label'][start:end])\n","      soft_label= list(data['Soft'][start:end])\n","\n","      sent=[]\n","      for j in range(len(txt)):\n","        sent.append([txt[j], txt_label[j], soft_label[j]])      \n","      sentences.append(sent)\n","      start=i\n","      flag=flag+1\n","\n","  return sentences"],"metadata":{"id":"fDzws9k80D4e","executionInfo":{"status":"ok","timestamp":1661126098305,"user_tz":-60,"elapsed":2,"user":{"displayName":"Antra Tripathi","userId":"14282966453087221057"}}},"execution_count":27,"outputs":[]},{"cell_type":"code","source":["#formatting the data into sentences\n","train_sents = format_data_soft(train_df)\n","dev_sents = format_data_soft(dev_df)"],"metadata":{"id":"mnpmvN9LzwdA","executionInfo":{"status":"ok","timestamp":1661126157547,"user_tz":-60,"elapsed":290,"user":{"displayName":"Antra Tripathi","userId":"14282966453087221057"}}},"execution_count":30,"outputs":[]},{"cell_type":"code","source":["def word2features_soft(sent, i):\n","    word = sent[i][0]\n","    distr = np.array(sent[i][2]) #soft_label distribution\n","\n","    features = {\n","        'bias': 1.0,\n","        'word': word,\n","        'len(word)': len(word),\n","        'word[:4]': word[:4],\n","        'word[:3]': word[:3],\n","        'word[:2]': word[:2],\n","        'word[-3:]': word[-3:],\n","        'word[-2:]': word[-2:],\n","        'word[-4:]': word[-4:],\n","        'word.lower()': word.lower(),\n","        'word.stemmed': re.sub(r'(.{2,}?)([aeiougyn]+$)',r'\\1', word.lower()),\n","        'word.ispunctuation': (word in string.punctuation),\n","        'word.isdigit()': word.isdigit(),\n","        'var': np.var(distr), #added variance of soft label of word in feature\n","        'std': np.std(distr)  #added st.dev of soft label of word in feature\n","    }\n","\n","    if i > 0:\n","      word1 = sent[i-1][0]\n","      distr1 = np.array(sent[i-1][2])\n","      features.update({          \n","            '-1:word': word1,\n","            '-1:len(word)': len(word1),\n","            '-1:word.lower()': word1.lower(),\n","            '-1:word.stemmed': re.sub(r'(.{2,}?)([aeiougyn]+$)',r'\\1', word1.lower()),\n","            '-1:word[:3]': word1[:3],\n","            '-1:word[:2]': word1[:2],\n","            '-1:word[-3:]': word1[-3:],\n","            '-1:word[-2:]': word1[-2:],\n","            '-1:word.isdigit()': word1.isdigit(),\n","            '-1:word.ispunctuation': (word1 in string.punctuation),\n","            '-1:var': np.var(distr1), #added variance of soft label of word in feature\n","            '-1:std': np.std(distr1), #added st.dev of soft label of word in feature         \n","            })\n","      \n","          \n","    else:\n","      features['BOS'] = True\n","\n","    if i > 1:\n","        word2 = sent[i-2][0]\n","        distr2 = np.array(sent[i-2][2])\n","        features.update({\n","            '-2:word': word2,\n","            '-2:len(word)': len(word2),\n","            '-2:word.lower()': word2.lower(),\n","            '-2:word[:3]': word2[:3],\n","            '-2:word[:2]': word2[:2],\n","            '-2:word[-3:]': word2[-3:],\n","            '-2:word[-2:]': word2[-2:],\n","            '-2:word.isdigit()': word2.isdigit(),\n","            '-2:word.ispunctuation': (word2 in string.punctuation),\n","            '-2:var': np.var(distr2), #added variance of soft label of word in feature\n","            '-2:std': np.std(distr2), #added st.dev of soft label of word in feature\n","            })\n","\n","    if i < len(sent)-1:\n","        word1 = sent[i+1][0]\n","        distr1 = np.array(sent[i+1][2])\n","        features.update({\n","            '+1:word': word1,\n","            '+1:len(word)': len(word1),\n","            '+1:word.lower()': word1.lower(),\n","            '+1:word[:3]': word1[:3],\n","            '+1:word[:2]': word1[:2],\n","            '+1:word[-3:]': word1[-3:],\n","            '+1:word[-2:]': word1[-2:],\n","            '+1:word.isdigit()': word1.isdigit(),\n","            '+1:word.ispunctuation': (word1 in string.punctuation),\n","            '+1:var': np.var(distr1), #added variance of soft label of word in feature\n","            '+1:std': np.std(distr1), #added st.dev of soft label of word in feature\n","        })\n","\n","    else:\n","        features['EOS'] = True\n","\n","    if i < len(sent) - 2:\n","      word2 = sent[i+2][0]\n","      distr2 = np.array(sent[i+2][2])\n","      features.update({\n","              '+2:word': word2,\n","              '+2:len(word)': len(word2),\n","              '+2:word.lower()': word2.lower(),\n","              '+2:word.stemmed': re.sub(r'(.{2,}?)([aeiougyn]+$)',r'\\1', word2.lower()),\n","              '+2:word[:3]': word2[:3],\n","              '+2:word[:2]': word2[:2],\n","              '+2:word[-3:]': word2[-3:],\n","              '+2:word[-2:]': word2[-2:],\n","              '+2:word.isdigit()': word2.isdigit(),\n","              '+2:word.ispunctuation': (word2 in string.punctuation),\n","              '+2:var': np.var(distr2), #added variance of soft label of word in feature\n","              '+2:std': np.std(distr2), #added st.dev of soft label of word in feature\n","          })\n","\n","    return features\n","\n","def sent2features(sent):\n","    return [word2features_soft(sent, i) for i in range(len(sent))]\n","\n","def sent2labels(sent):\n","    return [word[1] for word in sent]\n","\n","def sent2tokens(sent):\n","    return [word[0] for word in sent]\n"],"metadata":{"executionInfo":{"status":"ok","timestamp":1661130530694,"user_tz":-60,"elapsed":210,"user":{"displayName":"Antra Tripathi","userId":"14282966453087221057"}},"id":"I8ykxfkQC35j"},"execution_count":49,"outputs":[]},{"cell_type":"code","source":["#extracting features from all the sentences\n","Xtrain = [sent2features(s) for s in train_sents]\n","ytrain = [sent2labels(s) for s in train_sents]\n","\n","Xtest = [sent2features(s) for s in dev_sents]\n","ytest = [sent2labels(s) for s in dev_sents]"],"metadata":{"id":"uJ4J3skKzv-T","executionInfo":{"status":"ok","timestamp":1661130537008,"user_tz":-60,"elapsed":3030,"user":{"displayName":"Antra Tripathi","userId":"14282966453087221057"}}},"execution_count":50,"outputs":[]},{"cell_type":"code","source":["c1=[0, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4]\n","c2= [0, 0.1, 0.15, 0.2,0.25, 0.3, 0.35, 0.4]"],"metadata":{"id":"dmZsQtoQJhy_","executionInfo":{"status":"ok","timestamp":1661131461723,"user_tz":-60,"elapsed":226,"user":{"displayName":"Antra Tripathi","userId":"14282966453087221057"}}},"execution_count":53,"outputs":[]},{"cell_type":"code","source":["print(\"training tagger...\")\n","for i in c1:\n","  for j in c2:\n","    crf = sklearn_crfsuite.CRF(algorithm = 'lbfgs',\n","                               c1 = i,\n","                               c2 = j,\n","                               max_iterations = 100,\n","                               all_possible_transitions=True)    \n","    \n","    try:\n","      crf.fit(Xtrain, ytrain)\n","    except AttributeError:\n","      pass\n","\n","    ypred = crf.predict(Xtrain)\n","    #obtaining metrics such as accuracy, etc. on the train set\n","    labels = list(crf.classes_)\n","    print('C1: ', i, '| C2: ', j)\n","    print('F1 score train set = {}'.format(metrics.flat_f1_score(ytrain, ypred, average='weighted', labels=labels)))\n","    print('Accuracy  train set = {}\\n'.format(metrics.flat_accuracy_score(ytrain, ypred)))\n","    \n","    ypred_d = crf.predict(Xtest)\n","    print('F1 score test set = {}'.format(metrics.flat_f1_score(ytest, ypred_d,average='weighted', labels=labels)))\n","    print('Accuracy on test set = {}'.format(metrics.flat_accuracy_score(ytest, ypred_d)))\n","\n","    print('----------------------------------')\n","    "],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"IHs7YL7FJn6w","executionInfo":{"status":"ok","timestamp":1661131773682,"user_tz":-60,"elapsed":300561,"user":{"displayName":"Antra Tripathi","userId":"14282966453087221057"}},"outputId":"75b2b58a-77a1-42f6-a417-b5c611290246"},"execution_count":54,"outputs":[{"output_type":"stream","name":"stdout","text":["training tagger...\n","C1:  0 | C2:  0\n","F1 score train set = 0.9940237668613786\n","Accuracy  train set = 0.9940298507462687\n","\n","F1 score test set = 0.7350477287440113\n","Accuracy on test set = 0.7512420156139106\n","----------------------------------\n","C1:  0 | C2:  0.1\n","F1 score train set = 0.9920822673238652\n","Accuracy  train set = 0.9920830629461389\n","\n","F1 score test set = 0.7362883189987486\n","Accuracy on test set = 0.7544357700496807\n","----------------------------------\n","C1:  0 | C2:  0.15\n","F1 score train set = 0.991561789113483\n","Accuracy  train set = 0.9915639195327709\n","\n","F1 score test set = 0.7340524912565379\n","Accuracy on test set = 0.748403122782115\n","----------------------------------\n","C1:  0 | C2:  0.2\n","F1 score train set = 0.9937685294264134\n","Accuracy  train set = 0.9937702790395847\n","\n","F1 score test set = 0.7351534808587241\n","Accuracy on test set = 0.7505322924059616\n","----------------------------------\n","C1:  0 | C2:  0.25\n","F1 score train set = 0.9950622994368954\n","Accuracy  train set = 0.9950681375730045\n","\n","F1 score test set = 0.7348142691484197\n","Accuracy on test set = 0.7512420156139106\n","----------------------------------\n","C1:  0 | C2:  0.3\n","F1 score train set = 0.99142949680522\n","Accuracy  train set = 0.9914341336794289\n","\n","F1 score test set = 0.7385127299570302\n","Accuracy on test set = 0.7537260468417317\n","----------------------------------\n","C1:  0 | C2:  0.35\n","F1 score train set = 0.9909230096615995\n","Accuracy  train set = 0.990914990266061\n","\n","F1 score test set = 0.7343427936413626\n","Accuracy on test set = 0.7512420156139106\n","----------------------------------\n","C1:  0 | C2:  0.4\n","F1 score train set = 0.9918186711708913\n","Accuracy  train set = 0.9918234912394549\n","\n","F1 score test set = 0.7351112557712087\n","Accuracy on test set = 0.751596877217885\n","----------------------------------\n","C1:  0.1 | C2:  0\n","F1 score train set = 0.9998702048940353\n","Accuracy  train set = 0.999870214146658\n","\n","F1 score test set = 0.7391114414426088\n","Accuracy on test set = 0.7540809084457062\n","----------------------------------\n","C1:  0.1 | C2:  0.1\n","F1 score train set = 0.9998702048940353\n","Accuracy  train set = 0.999870214146658\n","\n","F1 score test set = 0.7452212711978806\n","Accuracy on test set = 0.7615330021291696\n","----------------------------------\n","C1:  0.1 | C2:  0.15\n","F1 score train set = 0.9998702048940353\n","Accuracy  train set = 0.999870214146658\n","\n","F1 score test set = 0.7452551160323285\n","Accuracy on test set = 0.7615330021291696\n","----------------------------------\n","C1:  0.1 | C2:  0.2\n","F1 score train set = 0.9998702048940353\n","Accuracy  train set = 0.999870214146658\n","\n","F1 score test set = 0.74661626126354\n","Accuracy on test set = 0.7633073101490419\n","----------------------------------\n","C1:  0.1 | C2:  0.25\n","F1 score train set = 0.9997403407821529\n","Accuracy  train set = 0.999740428293316\n","\n","F1 score test set = 0.7455081555292191\n","Accuracy on test set = 0.7622427253371186\n","----------------------------------\n","C1:  0.1 | C2:  0.3\n","F1 score train set = 0.9996105228233084\n","Accuracy  train set = 0.999610642439974\n","\n","F1 score test set = 0.7455766409740304\n","Accuracy on test set = 0.7625975869410929\n","----------------------------------\n","C1:  0.1 | C2:  0.35\n","F1 score train set = 0.9996105228233084\n","Accuracy  train set = 0.999610642439974\n","\n","F1 score test set = 0.7443435656659432\n","Accuracy on test set = 0.7611781405251952\n","----------------------------------\n","C1:  0.1 | C2:  0.4\n","F1 score train set = 0.999220146983573\n","Accuracy  train set = 0.9992212848799481\n","\n","F1 score test set = 0.7457229566112343\n","Accuracy on test set = 0.7625975869410929\n","----------------------------------\n","C1:  0.15 | C2:  0\n","F1 score train set = 0.9998702048940353\n","Accuracy  train set = 0.999870214146658\n","\n","F1 score test set = 0.7386509758823533\n","Accuracy on test set = 0.7533711852377573\n","----------------------------------\n","C1:  0.15 | C2:  0.1\n","F1 score train set = 0.9998702048940353\n","Accuracy  train set = 0.999870214146658\n","\n","F1 score test set = 0.7442879686180349\n","Accuracy on test set = 0.7597586941092974\n","----------------------------------\n","C1:  0.15 | C2:  0.15\n","F1 score train set = 0.9997403407821529\n","Accuracy  train set = 0.999740428293316\n","\n","F1 score test set = 0.7428518535048391\n","Accuracy on test set = 0.7590489709013485\n","----------------------------------\n","C1:  0.15 | C2:  0.2\n","F1 score train set = 0.9996105228233084\n","Accuracy  train set = 0.999610642439974\n","\n","F1 score test set = 0.7435030582681155\n","Accuracy on test set = 0.7604684173172462\n","----------------------------------\n","C1:  0.15 | C2:  0.25\n","F1 score train set = 0.9996105228233084\n","Accuracy  train set = 0.999610642439974\n","\n","F1 score test set = 0.7450344763361354\n","Accuracy on test set = 0.7618878637331441\n","----------------------------------\n","C1:  0.15 | C2:  0.3\n","F1 score train set = 0.9994807602610388\n","Accuracy  train set = 0.9994808565866321\n","\n","F1 score test set = 0.7464021424883712\n","Accuracy on test set = 0.7636621717530163\n","----------------------------------\n","C1:  0.15 | C2:  0.35\n","F1 score train set = 0.999220146983573\n","Accuracy  train set = 0.9992212848799481\n","\n","F1 score test set = 0.7435545684473013\n","Accuracy on test set = 0.7608232789212207\n","----------------------------------\n","C1:  0.15 | C2:  0.4\n","F1 score train set = 0.9989603846222092\n","Accuracy  train set = 0.9989617131732641\n","\n","F1 score test set = 0.7441252612347778\n","Accuracy on test set = 0.7615330021291696\n","----------------------------------\n","C1:  0.2 | C2:  0\n","F1 score train set = 0.9998702048940353\n","Accuracy  train set = 0.999870214146658\n","\n","F1 score test set = 0.7417703187485735\n","Accuracy on test set = 0.7565649396735273\n","----------------------------------\n","C1:  0.2 | C2:  0.1\n","F1 score train set = 0.9997403407821529\n","Accuracy  train set = 0.999740428293316\n","\n","F1 score test set = 0.742790356219876\n","Accuracy on test set = 0.7583392476933996\n","----------------------------------\n","C1:  0.2 | C2:  0.15\n","F1 score train set = 0.9996105228233084\n","Accuracy  train set = 0.999610642439974\n","\n","F1 score test set = 0.7437726931951859\n","Accuracy on test set = 0.7597586941092974\n","----------------------------------\n","C1:  0.2 | C2:  0.2\n","F1 score train set = 0.9994807602610388\n","Accuracy  train set = 0.9994808565866321\n","\n","F1 score test set = 0.7434021750537494\n","Accuracy on test set = 0.759403832505323\n","----------------------------------\n","C1:  0.2 | C2:  0.25\n","F1 score train set = 0.9993509097994642\n","Accuracy  train set = 0.9993510707332901\n","\n","F1 score test set = 0.7421416235216196\n","Accuracy on test set = 0.758694109297374\n","----------------------------------\n","C1:  0.2 | C2:  0.3\n","F1 score train set = 0.9988306947017198\n","Accuracy  train set = 0.9988319273199221\n","\n","F1 score test set = 0.7437668092371561\n","Accuracy on test set = 0.7604684173172462\n","----------------------------------\n","C1:  0.2 | C2:  0.35\n","F1 score train set = 0.9985708762452191\n","Accuracy  train set = 0.9985723556132382\n","\n","F1 score test set = 0.7442076509618953\n","Accuracy on test set = 0.7611781405251952\n","----------------------------------\n","C1:  0.2 | C2:  0.4\n","F1 score train set = 0.9976627586127832\n","Accuracy  train set = 0.9976638546398443\n","\n","F1 score test set = 0.7438251509165552\n","Accuracy on test set = 0.7608232789212207\n","----------------------------------\n","C1:  0.25 | C2:  0\n","F1 score train set = 0.9997403407821529\n","Accuracy  train set = 0.999740428293316\n","\n","F1 score test set = 0.7402609165089704\n","Accuracy on test set = 0.7547906316536551\n","----------------------------------\n","C1:  0.25 | C2:  0.1\n","F1 score train set = 0.9996105782198832\n","Accuracy  train set = 0.999610642439974\n","\n","F1 score test set = 0.7419833425834459\n","Accuracy on test set = 0.7572746628814763\n","----------------------------------\n","C1:  0.25 | C2:  0.15\n","F1 score train set = 0.9992212847820554\n","Accuracy  train set = 0.9992212848799481\n","\n","F1 score test set = 0.7426120483148715\n","Accuracy on test set = 0.758694109297374\n","----------------------------------\n","C1:  0.25 | C2:  0.2\n","F1 score train set = 0.9990906272304725\n","Accuracy  train set = 0.9990914990266061\n","\n","F1 score test set = 0.7431213317481173\n","Accuracy on test set = 0.7590489709013485\n","----------------------------------\n","C1:  0.25 | C2:  0.25\n","F1 score train set = 0.9985711775718727\n","Accuracy  train set = 0.9985723556132382\n","\n","F1 score test set = 0.7434775703392841\n","Accuracy on test set = 0.7601135557132718\n","----------------------------------\n","C1:  0.25 | C2:  0.3\n","F1 score train set = 0.9977921747765442\n","Accuracy  train set = 0.9977936404931862\n","\n","F1 score test set = 0.7421683229436836\n","Accuracy on test set = 0.7590489709013485\n","----------------------------------\n","C1:  0.25 | C2:  0.35\n","F1 score train set = 0.9971439099932122\n","Accuracy  train set = 0.9971447112264763\n","\n","F1 score test set = 0.7432422387732619\n","Accuracy on test set = 0.7601135557132718\n","----------------------------------\n","C1:  0.25 | C2:  0.4\n","F1 score train set = 0.9967544630263722\n","Accuracy  train set = 0.9967553536664504\n","\n","F1 score test set = 0.744353168569792\n","Accuracy on test set = 0.7611781405251952\n","----------------------------------\n","C1:  0.3 | C2:  0\n","F1 score train set = 0.999610478323734\n","Accuracy  train set = 0.999610642439974\n","\n","F1 score test set = 0.7425217181304818\n","Accuracy on test set = 0.7565649396735273\n","----------------------------------\n","C1:  0.3 | C2:  0.1\n","F1 score train set = 0.9993509645053851\n","Accuracy  train set = 0.9993510707332901\n","\n","F1 score test set = 0.7400070492231443\n","Accuracy on test set = 0.7551454932576295\n","----------------------------------\n","C1:  0.3 | C2:  0.15\n","F1 score train set = 0.9988314257596211\n","Accuracy  train set = 0.9988319273199221\n","\n","F1 score test set = 0.7424554327194895\n","Accuracy on test set = 0.7579843860894251\n","----------------------------------\n","C1:  0.3 | C2:  0.2\n","F1 score train set = 0.99779268938098\n","Accuracy  train set = 0.9977936404931862\n","\n","F1 score test set = 0.7438530301846675\n","Accuracy on test set = 0.7597586941092974\n","----------------------------------\n","C1:  0.3 | C2:  0.25\n","F1 score train set = 0.9971442277595359\n","Accuracy  train set = 0.9971447112264763\n","\n","F1 score test set = 0.7434453171159439\n","Accuracy on test set = 0.759403832505323\n","----------------------------------\n","C1:  0.3 | C2:  0.3\n","F1 score train set = 0.996365598448457\n","Accuracy  train set = 0.9963659961064244\n","\n","F1 score test set = 0.7436965365706522\n","Accuracy on test set = 0.7601135557132718\n","----------------------------------\n","C1:  0.3 | C2:  0.35\n","F1 score train set = 0.995586783820657\n","Accuracy  train set = 0.9955872809863725\n","\n","F1 score test set = 0.742844211783889\n","Accuracy on test set = 0.759403832505323\n","----------------------------------\n","C1:  0.3 | C2:  0.4\n","F1 score train set = 0.994545458869495\n","Accuracy  train set = 0.9945489941596366\n","\n","F1 score test set = 0.7428719979761182\n","Accuracy on test set = 0.759403832505323\n","----------------------------------\n","C1:  0.35 | C2:  0\n","F1 score train set = 0.9996105449984903\n","Accuracy  train set = 0.999610642439974\n","\n","F1 score test set = 0.7400372013676872\n","Accuracy on test set = 0.7544357700496807\n","----------------------------------\n","C1:  0.35 | C2:  0.1\n","F1 score train set = 0.9984415492160557\n","Accuracy  train set = 0.9984425697598962\n","\n","F1 score test set = 0.7444023265922961\n","Accuracy on test set = 0.7590489709013485\n","----------------------------------\n","C1:  0.35 | C2:  0.15\n","F1 score train set = 0.9971437720720273\n","Accuracy  train set = 0.9971447112264763\n","\n","F1 score test set = 0.7414393605034656\n","Accuracy on test set = 0.7565649396735273\n","----------------------------------\n","C1:  0.35 | C2:  0.2\n","F1 score train set = 0.9961056555252097\n","Accuracy  train set = 0.9961064243997404\n","\n","F1 score test set = 0.7418591231822972\n","Accuracy on test set = 0.7576295244854506\n","----------------------------------\n","C1:  0.35 | C2:  0.25\n","F1 score train set = 0.995196539893416\n","Accuracy  train set = 0.9951979234263465\n","\n","F1 score test set = 0.7432405032398297\n","Accuracy on test set = 0.7590489709013485\n","----------------------------------\n","C1:  0.35 | C2:  0.3\n","F1 score train set = 0.9941557659208934\n","Accuracy  train set = 0.9941596365996106\n","\n","F1 score test set = 0.7426087456755517\n","Accuracy on test set = 0.758694109297374\n","----------------------------------\n","C1:  0.35 | C2:  0.35\n","F1 score train set = 0.9932399009829922\n","Accuracy  train set = 0.9932511356262167\n","\n","F1 score test set = 0.7427032429640255\n","Accuracy on test set = 0.7590489709013485\n","----------------------------------\n","C1:  0.35 | C2:  0.4\n","F1 score train set = 0.9916805507005847\n","Accuracy  train set = 0.9916937053861129\n","\n","F1 score test set = 0.7424533499323752\n","Accuracy on test set = 0.7590489709013485\n","----------------------------------\n","C1:  0.4 | C2:  0\n","F1 score train set = 0.9984425032095854\n","Accuracy  train set = 0.9984425697598962\n","\n","F1 score test set = 0.7371100123101986\n","Accuracy on test set = 0.7519517388218595\n","----------------------------------\n","C1:  0.4 | C2:  0.1\n","F1 score train set = 0.9964950213535146\n","Accuracy  train set = 0.9964957819597664\n","\n","F1 score test set = 0.7430635754004508\n","Accuracy on test set = 0.7583392476933996\n","----------------------------------\n","C1:  0.4 | C2:  0.15\n","F1 score train set = 0.9949362303894826\n","Accuracy  train set = 0.9949383517196626\n","\n","F1 score test set = 0.7432718876439164\n","Accuracy on test set = 0.758694109297374\n","----------------------------------\n","C1:  0.4 | C2:  0.2\n","F1 score train set = 0.9938950724657395\n","Accuracy  train set = 0.9939000648929267\n","\n","F1 score test set = 0.7437994109407524\n","Accuracy on test set = 0.7590489709013485\n","----------------------------------\n","C1:  0.4 | C2:  0.25\n","F1 score train set = 0.9928544407220783\n","Accuracy  train set = 0.9928617780661908\n","\n","F1 score test set = 0.7407394006374227\n","Accuracy on test set = 0.7562100780695529\n","----------------------------------\n","C1:  0.4 | C2:  0.3\n","F1 score train set = 0.9906415466562526\n","Accuracy  train set = 0.990655418559377\n","\n","F1 score test set = 0.7415568946467292\n","Accuracy on test set = 0.7576295244854506\n","----------------------------------\n","C1:  0.4 | C2:  0.35\n","F1 score train set = 0.9901261406546746\n","Accuracy  train set = 0.9901362751460091\n","\n","F1 score test set = 0.7419847572726964\n","Accuracy on test set = 0.7579843860894251\n","----------------------------------\n","C1:  0.4 | C2:  0.4\n","F1 score train set = 0.988430887312062\n","Accuracy  train set = 0.9884490590525633\n","\n","F1 score test set = 0.743057549014681\n","Accuracy on test set = 0.7590489709013485\n","----------------------------------\n"]}]},{"cell_type":"code","source":["crf = sklearn_crfsuite.CRF(algorithm = 'lbfgs',\n","                               c1 = 0.25,\n","                               c2 = 0.1,\n","                               max_iterations = 100,\n","                               all_possible_transitions=True)    \n","    \n","try:\n","  crf.fit(Xtrain, ytrain)\n","except AttributeError:\n","  pass"],"metadata":{"id":"SCChC2kh4kCQ","executionInfo":{"status":"ok","timestamp":1661130541326,"user_tz":-60,"elapsed":4331,"user":{"displayName":"Antra Tripathi","userId":"14282966453087221057"}}},"execution_count":51,"outputs":[]},{"cell_type":"code","source":["#obtaining metrics such as accuracy, etc. on the train set\n","labels = list(crf.classes_)\n","#labels.remove('X')\n","\n","ypred = crf.predict(Xtrain)\n","print('F1 score on the train set = {}\\n'.format(metrics.flat_f1_score(ytrain, ypred, average='weighted', labels=labels)))\n","print('Accuracy on the train set = {}\\n'.format(metrics.flat_accuracy_score(ytrain, ypred)))\n","\n","print('----------------------------------')\n","yt, yp=[], []\n","for i in range(len(ytrain)):\n","  yt.extend(ytrain[i])\n","  yp.extend(ypred[i])\n","\n","print(classification_report(yt, yp))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"j87yvlRT4j-4","executionInfo":{"status":"ok","timestamp":1661130431769,"user_tz":-60,"elapsed":1411,"user":{"displayName":"Antra Tripathi","userId":"14282966453087221057"}},"outputId":"3ee5e836-155f-4986-d642-92b1ee3c4502"},"execution_count":47,"outputs":[{"output_type":"stream","name":"stdout","text":["F1 score on the train set = 0.9996105782198832\n","\n","Accuracy on the train set = 0.999610642439974\n","\n","----------------------------------\n","              precision    recall  f1-score   support\n","\n","         ADJ       1.00      1.00      1.00       561\n","         ADP       1.00      1.00      1.00       787\n","         ADV       1.00      1.00      1.00       239\n","       CCONJ       1.00      1.00      1.00       138\n","         DET       1.00      1.00      1.00       281\n","        NOUN       1.00      1.00      1.00      2057\n","         NUM       1.00      1.00      1.00        40\n","        PART       1.00      1.00      1.00       830\n","        PRON       1.00      1.00      1.00       295\n","       PUNCT       1.00      1.00      1.00       893\n","        VERB       1.00      1.00      1.00      1263\n","           X       1.00      1.00      1.00       321\n","\n","    accuracy                           1.00      7705\n","   macro avg       1.00      1.00      1.00      7705\n","weighted avg       1.00      1.00      1.00      7705\n","\n"]}]},{"cell_type":"code","source":["#obtaining metrics such as accuracy, etc. on the test set\n","ypred_d = crf.predict(Xtest)\n","print('F1 score on the test set = {}\\n'.format(metrics.flat_f1_score(ytest, ypred_d,\n","                      average='weighted', labels=labels)))\n","print('Accuracy on the test set = {}\\n'.format(metrics.flat_accuracy_score(ytest, ypred_d)))\n","\n","print('----------------------------------')\n","yt, yp=[], []\n","for i in range(len(ytest)):\n","  yt.extend(ytest[i])\n","  yp.extend(ypred_d[i])\n","\n","print(classification_report(yt, yp))"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Ov2EyfY34j8t","executionInfo":{"status":"ok","timestamp":1661130546050,"user_tz":-60,"elapsed":191,"user":{"displayName":"Antra Tripathi","userId":"14282966453087221057"}},"outputId":"f60e535c-4361-4718-c737-ab80cab40fa7"},"execution_count":52,"outputs":[{"output_type":"stream","name":"stdout","text":["F1 score on the test set = 0.7419833425834459\n","\n","Accuracy on the test set = 0.7572746628814763\n","\n","----------------------------------\n","              precision    recall  f1-score   support\n","\n","         ADJ       0.50      0.53      0.51       165\n","         ADP       0.76      0.86      0.81       238\n","         ADV       0.81      0.39      0.52       150\n","       CCONJ       0.81      0.83      0.82        52\n","         DET       0.99      0.53      0.69       195\n","        NOUN       0.76      0.88      0.82       748\n","         NUM       0.60      0.26      0.36        47\n","        PART       0.65      0.94      0.77       211\n","        PRON       0.62      0.36      0.45       138\n","       PUNCT       0.79      0.98      0.87       278\n","        VERB       0.84      0.87      0.85       452\n","           X       0.83      0.40      0.54       144\n","\n","    accuracy                           0.76      2818\n","   macro avg       0.75      0.65      0.67      2818\n","weighted avg       0.77      0.76      0.74      2818\n","\n"]}]},{"cell_type":"markdown","source":["## **References**\n","1.   scikit-learn. 2022. User guide: contents. [online] Available at: <https://scikit-learn.org/stable/user_guide.html> [Accessed 21 August 2022].\n","\n","2.   Sklearn-crfsuite.readthedocs.io. 2022. sklearn-crfsuite — sklearn-crfsuite 0.3 documentation. [online] Available at: <https://sklearn-crfsuite.readthedocs.io/en/latest/> [Accessed 21 August 2022].\n","\n","\n"],"metadata":{"id":"IFwavI0zh0bn"}},{"cell_type":"markdown","source":["## **Installation**"],"metadata":{"id":"i1xERl03hsHJ"}},{"cell_type":"code","source":["!pip install sklearn_crfsuite"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"BIcFxk7gHmxq","executionInfo":{"status":"ok","timestamp":1661124414982,"user_tz":-60,"elapsed":4760,"user":{"displayName":"Antra Tripathi","userId":"14282966453087221057"}},"outputId":"f466d941-c8c7-4ade-8e6d-87f89667fdd6"},"execution_count":1,"outputs":[{"output_type":"stream","name":"stdout","text":["Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","Collecting sklearn_crfsuite\n","  Downloading sklearn_crfsuite-0.3.6-py2.py3-none-any.whl (12 kB)\n","Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sklearn_crfsuite) (1.15.0)\n","Requirement already satisfied: tqdm>=2.0 in /usr/local/lib/python3.7/dist-packages (from sklearn_crfsuite) (4.64.0)\n","Requirement already satisfied: tabulate in /usr/local/lib/python3.7/dist-packages (from sklearn_crfsuite) (0.8.10)\n","Collecting python-crfsuite>=0.8.3\n","  Downloading python_crfsuite-0.9.8-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (965 kB)\n","\u001b[K     |████████████████████████████████| 965 kB 28.3 MB/s \n","\u001b[?25hInstalling collected packages: python-crfsuite, sklearn-crfsuite\n","Successfully installed python-crfsuite-0.9.8 sklearn-crfsuite-0.3.6\n"]}]},{"cell_type":"code","source":["!pip install scikit-learn==0.22.2 --user"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"Ga5t3fo1HvLy","executionInfo":{"status":"ok","timestamp":1661124427301,"user_tz":-60,"elapsed":10542,"user":{"displayName":"Antra Tripathi","userId":"14282966453087221057"}},"outputId":"378a0d12-4aea-419e-a4de-378832710f88"},"execution_count":2,"outputs":[{"output_type":"stream","name":"stdout","text":["Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","Collecting scikit-learn==0.22.2\n","  Downloading scikit_learn-0.22.2-cp37-cp37m-manylinux1_x86_64.whl (7.1 MB)\n","\u001b[K     |████████████████████████████████| 7.1 MB 24.3 MB/s \n","\u001b[?25hRequirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.1.0)\n","Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.7.3)\n","Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.22.2) (1.21.6)\n","Installing collected packages: scikit-learn\n","\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n","yellowbrick 1.4 requires scikit-learn>=1.0.0, but you have scikit-learn 0.22.2 which is incompatible.\n","imbalanced-learn 0.8.1 requires scikit-learn>=0.24, but you have scikit-learn 0.22.2 which is incompatible.\u001b[0m\n","Successfully installed scikit-learn-0.22.2\n"]}]},{"cell_type":"code","source":[""],"metadata":{"id":"2L-LjwE9edbO"},"execution_count":null,"outputs":[]}]}