Week4.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "76246c96",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Text Cleaning Week 4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "04cdef51",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import string\n",
    "from nltk.corpus import stopwords\n",
    "from collections import Counter \n",
    "from nltk.stem import PorterStemmer\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from bs4 import BeautifulSoup\n",
    "from spellchecker import SpellChecker "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "a1ad5d01",
   "metadata": {},
   "outputs": [],
   "source": [
    "file_path = 'twcs.csv'\n",
    "data_set = pd.read_csv('twcs.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7af3537e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>author_id</th>\n",
       "      <th>inbound</th>\n",
       "      <th>created_at</th>\n",
       "      <th>text</th>\n",
       "      <th>response_tweet_id</th>\n",
       "      <th>in_response_to_tweet_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>sprintcare</td>\n",
       "      <td>False</td>\n",
       "      <td>Tue Oct 31 22:10:47 +0000 2017</td>\n",
       "      <td>@115712 I understand. I would like to assist y...</td>\n",
       "      <td>2</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>115712</td>\n",
       "      <td>True</td>\n",
       "      <td>Tue Oct 31 22:11:45 +0000 2017</td>\n",
       "      <td>@sprintcare and how do you propose we do that</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>115712</td>\n",
       "      <td>True</td>\n",
       "      <td>Tue Oct 31 22:08:27 +0000 2017</td>\n",
       "      <td>@sprintcare I have sent several private messag...</td>\n",
       "      <td>1</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>sprintcare</td>\n",
       "      <td>False</td>\n",
       "      <td>Tue Oct 31 21:54:49 +0000 2017</td>\n",
       "      <td>@115712 Please send us a Private Message so th...</td>\n",
       "      <td>3</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>115712</td>\n",
       "      <td>True</td>\n",
       "      <td>Tue Oct 31 21:49:35 +0000 2017</td>\n",
       "      <td>@sprintcare I did.</td>\n",
       "      <td>4</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>sprintcare</td>\n",
       "      <td>False</td>\n",
       "      <td>Tue Oct 31 21:46:24 +0000 2017</td>\n",
       "      <td>@115712 Can you please send us a private messa...</td>\n",
       "      <td>5,7</td>\n",
       "      <td>8.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>8</td>\n",
       "      <td>115712</td>\n",
       "      <td>True</td>\n",
       "      <td>Tue Oct 31 21:45:10 +0000 2017</td>\n",
       "      <td>@sprintcare is the worst customer service</td>\n",
       "      <td>9,6,10</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>11</td>\n",
       "      <td>sprintcare</td>\n",
       "      <td>False</td>\n",
       "      <td>Tue Oct 31 22:10:35 +0000 2017</td>\n",
       "      <td>@115713 This is saddening to hear. Please shoo...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>12.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>12</td>\n",
       "      <td>115713</td>\n",
       "      <td>True</td>\n",
       "      <td>Tue Oct 31 22:04:47 +0000 2017</td>\n",
       "      <td>@sprintcare You gonna magically change your co...</td>\n",
       "      <td>11,13,14</td>\n",
       "      <td>15.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>15</td>\n",
       "      <td>sprintcare</td>\n",
       "      <td>False</td>\n",
       "      <td>Tue Oct 31 20:03:31 +0000 2017</td>\n",
       "      <td>@115713 We understand your concerns and we'd l...</td>\n",
       "      <td>12</td>\n",
       "      <td>16.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   tweet_id   author_id  inbound                      created_at  \\\n",
       "0         1  sprintcare    False  Tue Oct 31 22:10:47 +0000 2017   \n",
       "1         2      115712     True  Tue Oct 31 22:11:45 +0000 2017   \n",
       "2         3      115712     True  Tue Oct 31 22:08:27 +0000 2017   \n",
       "3         4  sprintcare    False  Tue Oct 31 21:54:49 +0000 2017   \n",
       "4         5      115712     True  Tue Oct 31 21:49:35 +0000 2017   \n",
       "5         6  sprintcare    False  Tue Oct 31 21:46:24 +0000 2017   \n",
       "6         8      115712     True  Tue Oct 31 21:45:10 +0000 2017   \n",
       "7        11  sprintcare    False  Tue Oct 31 22:10:35 +0000 2017   \n",
       "8        12      115713     True  Tue Oct 31 22:04:47 +0000 2017   \n",
       "9        15  sprintcare    False  Tue Oct 31 20:03:31 +0000 2017   \n",
       "\n",
       "                                                text response_tweet_id  \\\n",
       "0  @115712 I understand. I would like to assist y...                 2   \n",
       "1      @sprintcare and how do you propose we do that               NaN   \n",
       "2  @sprintcare I have sent several private messag...                 1   \n",
       "3  @115712 Please send us a Private Message so th...                 3   \n",
       "4                                 @sprintcare I did.                 4   \n",
       "5  @115712 Can you please send us a private messa...               5,7   \n",
       "6          @sprintcare is the worst customer service            9,6,10   \n",
       "7  @115713 This is saddening to hear. Please shoo...               NaN   \n",
       "8  @sprintcare You gonna magically change your co...          11,13,14   \n",
       "9  @115713 We understand your concerns and we'd l...                12   \n",
       "\n",
       "   in_response_to_tweet_id  \n",
       "0                      3.0  \n",
       "1                      1.0  \n",
       "2                      4.0  \n",
       "3                      5.0  \n",
       "4                      6.0  \n",
       "5                      8.0  \n",
       "6                      NaN  \n",
       "7                     12.0  \n",
       "8                     15.0  \n",
       "9                     16.0  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_set.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2eb0a12a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lower casing\n",
    "data_set['text_lower'] = data_set['text'].str.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "8a5b8f53",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "         tweet_id   author_id  inbound                      created_at  \\\n",
      "0               1  sprintcare    False  Tue Oct 31 22:10:47 +0000 2017   \n",
      "1               2      115712     True  Tue Oct 31 22:11:45 +0000 2017   \n",
      "2               3      115712     True  Tue Oct 31 22:08:27 +0000 2017   \n",
      "3               4  sprintcare    False  Tue Oct 31 21:54:49 +0000 2017   \n",
      "4               5      115712     True  Tue Oct 31 21:49:35 +0000 2017   \n",
      "...           ...         ...      ...                             ...   \n",
      "2811769   2987947  sprintcare    False  Wed Nov 22 08:43:51 +0000 2017   \n",
      "2811770   2987948      823869     True  Wed Nov 22 08:35:16 +0000 2017   \n",
      "2811771   2812240      121673     True  Thu Nov 23 04:13:07 +0000 2017   \n",
      "2811772   2987949      AldiUK    False  Wed Nov 22 08:31:24 +0000 2017   \n",
      "2811773   2987950      823870     True  Tue Nov 21 22:01:04 +0000 2017   \n",
      "\n",
      "                                                      text response_tweet_id  \\\n",
      "0        @115712 I understand. I would like to assist y...                 2   \n",
      "1            @sprintcare and how do you propose we do that               NaN   \n",
      "2        @sprintcare I have sent several private messag...                 1   \n",
      "3        @115712 Please send us a Private Message so th...                 3   \n",
      "4                                       @sprintcare I did.                 4   \n",
      "...                                                    ...               ...   \n",
      "2811769  @823869 Hey, we'd be happy to look into this f...               NaN   \n",
      "2811770  @115714 wtf!? I’ve been having really shitty s...           2987947   \n",
      "2811771  @143549 @sprintcare You have to go to https://...               NaN   \n",
      "2811772  @823870 Sounds delicious, Sarah! 😋 https://t.c...               NaN   \n",
      "2811773  @AldiUK  warm sloe gin mince pies with ice cre...   2987951,2987949   \n",
      "\n",
      "         in_response_to_tweet_id  \\\n",
      "0                            3.0   \n",
      "1                            1.0   \n",
      "2                            4.0   \n",
      "3                            5.0   \n",
      "4                            6.0   \n",
      "...                          ...   \n",
      "2811769                2987948.0   \n",
      "2811770                      NaN   \n",
      "2811771                2812239.0   \n",
      "2811772                2987950.0   \n",
      "2811773                      NaN   \n",
      "\n",
      "                                                text_lower  \n",
      "0        @115712 i understand. i would like to assist y...  \n",
      "1            @sprintcare and how do you propose we do that  \n",
      "2        @sprintcare i have sent several private messag...  \n",
      "3        @115712 please send us a private message so th...  \n",
      "4                                       @sprintcare i did.  \n",
      "...                                                    ...  \n",
      "2811769  @823869 hey, we'd be happy to look into this f...  \n",
      "2811770  @115714 wtf!? i’ve been having really shitty s...  \n",
      "2811771  @143549 @sprintcare you have to go to https://...  \n",
      "2811772  @823870 sounds delicious, sarah! 😋 https://t.c...  \n",
      "2811773  @aldiuk  warm sloe gin mince pies with ice cre...  \n",
      "\n",
      "[2811774 rows x 8 columns]\n"
     ]
    }
   ],
   "source": [
    "print(data_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "bfe21cb5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Removal of punctuation \n",
    "data_set['no_punct'] = data_set['text_lower'].apply(lambda text: text.translate(str.maketrans('', '', string.punctuation)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "9421b185",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "         tweet_id   author_id  inbound                      created_at  \\\n",
      "0               1  sprintcare    False  Tue Oct 31 22:10:47 +0000 2017   \n",
      "1               2      115712     True  Tue Oct 31 22:11:45 +0000 2017   \n",
      "2               3      115712     True  Tue Oct 31 22:08:27 +0000 2017   \n",
      "3               4  sprintcare    False  Tue Oct 31 21:54:49 +0000 2017   \n",
      "4               5      115712     True  Tue Oct 31 21:49:35 +0000 2017   \n",
      "...           ...         ...      ...                             ...   \n",
      "2811769   2987947  sprintcare    False  Wed Nov 22 08:43:51 +0000 2017   \n",
      "2811770   2987948      823869     True  Wed Nov 22 08:35:16 +0000 2017   \n",
      "2811771   2812240      121673     True  Thu Nov 23 04:13:07 +0000 2017   \n",
      "2811772   2987949      AldiUK    False  Wed Nov 22 08:31:24 +0000 2017   \n",
      "2811773   2987950      823870     True  Tue Nov 21 22:01:04 +0000 2017   \n",
      "\n",
      "                                                      text response_tweet_id  \\\n",
      "0        @115712 I understand. I would like to assist y...                 2   \n",
      "1            @sprintcare and how do you propose we do that               NaN   \n",
      "2        @sprintcare I have sent several private messag...                 1   \n",
      "3        @115712 Please send us a Private Message so th...                 3   \n",
      "4                                       @sprintcare I did.                 4   \n",
      "...                                                    ...               ...   \n",
      "2811769  @823869 Hey, we'd be happy to look into this f...               NaN   \n",
      "2811770  @115714 wtf!? I’ve been having really shitty s...           2987947   \n",
      "2811771  @143549 @sprintcare You have to go to https://...               NaN   \n",
      "2811772  @823870 Sounds delicious, Sarah! 😋 https://t.c...               NaN   \n",
      "2811773  @AldiUK  warm sloe gin mince pies with ice cre...   2987951,2987949   \n",
      "\n",
      "         in_response_to_tweet_id  \\\n",
      "0                            3.0   \n",
      "1                            1.0   \n",
      "2                            4.0   \n",
      "3                            5.0   \n",
      "4                            6.0   \n",
      "...                          ...   \n",
      "2811769                2987948.0   \n",
      "2811770                      NaN   \n",
      "2811771                2812239.0   \n",
      "2811772                2987950.0   \n",
      "2811773                      NaN   \n",
      "\n",
      "                                                text_lower  \\\n",
      "0        @115712 i understand. i would like to assist y...   \n",
      "1            @sprintcare and how do you propose we do that   \n",
      "2        @sprintcare i have sent several private messag...   \n",
      "3        @115712 please send us a private message so th...   \n",
      "4                                       @sprintcare i did.   \n",
      "...                                                    ...   \n",
      "2811769  @823869 hey, we'd be happy to look into this f...   \n",
      "2811770  @115714 wtf!? i’ve been having really shitty s...   \n",
      "2811771  @143549 @sprintcare you have to go to https://...   \n",
      "2811772  @823870 sounds delicious, sarah! 😋 https://t.c...   \n",
      "2811773  @aldiuk  warm sloe gin mince pies with ice cre...   \n",
      "\n",
      "                                                  no_punct  \n",
      "0        115712 i understand i would like to assist you...  \n",
      "1             sprintcare and how do you propose we do that  \n",
      "2        sprintcare i have sent several private message...  \n",
      "3        115712 please send us a private message so tha...  \n",
      "4                                         sprintcare i did  \n",
      "...                                                    ...  \n",
      "2811769  823869 hey wed be happy to look into this for ...  \n",
      "2811770  115714 wtf i’ve been having really shitty serv...  \n",
      "2811771  143549 sprintcare you have to go to httpstcov2...  \n",
      "2811772  823870 sounds delicious sarah 😋 httpstco7uqpwy...  \n",
      "2811773  aldiuk  warm sloe gin mince pies with ice crea...  \n",
      "\n",
      "[2811774 rows x 9 columns]\n"
     ]
    }
   ],
   "source": [
    "print(data_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "0da6d632",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Stemming (reduce words to their base form)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "f9ab9ac2",
   "metadata": {},
   "outputs": [],
   "source": [
    "stemmer = PorterStemmer()\n",
    "data_set['text'] = data_set['text'].apply(lambda x: ' '.join([stemmer.stem(word) for word in x.split()]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "5bd1334a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Lemmatization (combines similar meaning words as one word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "efeb3ed1",
   "metadata": {},
   "outputs": [],
   "source": [
    "lemmatizer = WordNetLemmatizer()\n",
    "data_set['text'] = data_set['text'].apply(lambda x: ' '.join([lemmatizer.lemmatize(word) for word in x.split()]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "69dcc294",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(data_set.head(10))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4222bbee",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Removal of Emoticons (punctuation marks, letters, and numbers)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c761ac75",
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_emoticons(text):\n",
    "    emoticon_pattern = re.compile(u'(' + u'|'.join(k for k in EMOTICONS) + u')')\n",
    "    return emoticon_pattern.sub(r'', text)\n",
    "\n",
    "data_set['text'] = data_set['text'].apply(remove_emoticons)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b61d713a",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(data_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a1499b31",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove URLs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10764b63",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_set['text'] = data_set['text'].str.replace(r'http\\S+|www\\S+|https\\S+', '', case=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dcf0a69f",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(data_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ccb1ccc1",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove HTML Tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "93c33b38",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_set['text'] = data_set['text'].apply(lambda x: BeautifulSoup(x, 'html.parser').get_text())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0c51b40b",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(data_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "77bf624c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Spelling Corrector "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d270d6d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "spell = SpellChecker()\n",
    "data_set['text'] = data_set['text'].apply(lambda x: ' '.join([spell.correction(word) for word in x.split()]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c2ce2998",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(data_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "56abc432",
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO: fix the outputs so it prints the updated data set "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}