Data_Pre_Processing.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "6p4j_4f29reu",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8d70b178-f4a4-422c-d821-ecf1f01b3b74"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: tensorflow_text in /usr/local/lib/python3.7/dist-packages (2.9.0)\n",
            "Requirement already satisfied: tensorflow<2.10,>=2.9.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow_text) (2.9.1)\n",
            "Requirement already satisfied: tensorflow-hub>=0.8.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow_text) (0.12.0)\n",
            "Requirement already satisfied: h5py>=2.9.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.10,>=2.9.0->tensorflow_text) (3.1.0)\n",
            "Requirement already satisfied: tensorboard<2.10,>=2.9 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.10,>=2.9.0->tensorflow_text) (2.9.1)\n",
            "Requirement already satisfied: six>=1.12.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.10,>=2.9.0->tensorflow_text) (1.15.0)\n",
            "Requirement already satisfied: flatbuffers<2,>=1.12 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.10,>=2.9.0->tensorflow_text) (1.12)\n",
            "Requirement already satisfied: grpcio<2.0,>=1.24.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.10,>=2.9.0->tensorflow_text) (1.46.3)\n",
            "Requirement already satisfied: protobuf<3.20,>=3.9.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.10,>=2.9.0->tensorflow_text) (3.17.3)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.10,>=2.9.0->tensorflow_text) (57.4.0)\n",
            "Requirement already satisfied: opt-einsum>=2.3.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.10,>=2.9.0->tensorflow_text) (3.3.0)\n",
            "Requirement already satisfied: tensorflow-io-gcs-filesystem>=0.23.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.10,>=2.9.0->tensorflow_text) (0.26.0)\n",
            "Requirement already satisfied: absl-py>=1.0.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.10,>=2.9.0->tensorflow_text) (1.1.0)\n",
            "Requirement already satisfied: wrapt>=1.11.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.10,>=2.9.0->tensorflow_text) (1.14.1)\n",
            "Requirement already satisfied: numpy>=1.20 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.10,>=2.9.0->tensorflow_text) (1.21.6)\n",
            "Requirement already satisfied: termcolor>=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.10,>=2.9.0->tensorflow_text) (1.1.0)\n",
            "Requirement already satisfied: typing-extensions>=3.6.6 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.10,>=2.9.0->tensorflow_text) (4.1.1)\n",
            "Requirement already satisfied: google-pasta>=0.1.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.10,>=2.9.0->tensorflow_text) (0.2.0)\n",
            "Requirement already satisfied: astunparse>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.10,>=2.9.0->tensorflow_text) (1.6.3)\n",
            "Requirement already satisfied: libclang>=13.0.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.10,>=2.9.0->tensorflow_text) (14.0.1)\n",
            "Requirement already satisfied: keras<2.10.0,>=2.9.0rc0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.10,>=2.9.0->tensorflow_text) (2.9.0)\n",
            "Requirement already satisfied: keras-preprocessing>=1.1.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.10,>=2.9.0->tensorflow_text) (1.1.2)\n",
            "Requirement already satisfied: tensorflow-estimator<2.10.0,>=2.9.0rc0 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.10,>=2.9.0->tensorflow_text) (2.9.0)\n",
            "Requirement already satisfied: gast<=0.4.0,>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.10,>=2.9.0->tensorflow_text) (0.4.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from tensorflow<2.10,>=2.9.0->tensorflow_text) (21.3)\n",
            "Requirement already satisfied: wheel<1.0,>=0.23.0 in /usr/local/lib/python3.7/dist-packages (from astunparse>=1.6.0->tensorflow<2.10,>=2.9.0->tensorflow_text) (0.37.1)\n",
            "Requirement already satisfied: cached-property in /usr/local/lib/python3.7/dist-packages (from h5py>=2.9.0->tensorflow<2.10,>=2.9.0->tensorflow_text) (1.5.2)\n",
            "Requirement already satisfied: requests<3,>=2.21.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.10,>=2.9->tensorflow<2.10,>=2.9.0->tensorflow_text) (2.23.0)\n",
            "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.10,>=2.9->tensorflow<2.10,>=2.9.0->tensorflow_text) (0.6.1)\n",
            "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.10,>=2.9->tensorflow<2.10,>=2.9.0->tensorflow_text) (0.4.6)\n",
            "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.10,>=2.9->tensorflow<2.10,>=2.9.0->tensorflow_text) (1.8.1)\n",
            "Requirement already satisfied: werkzeug>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.10,>=2.9->tensorflow<2.10,>=2.9.0->tensorflow_text) (1.0.1)\n",
            "Requirement already satisfied: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.10,>=2.9->tensorflow<2.10,>=2.9.0->tensorflow_text) (3.3.7)\n",
            "Requirement already satisfied: google-auth<3,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard<2.10,>=2.9->tensorflow<2.10,>=2.9.0->tensorflow_text) (1.35.0)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.10,>=2.9->tensorflow<2.10,>=2.9.0->tensorflow_text) (0.2.8)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.10,>=2.9->tensorflow<2.10,>=2.9.0->tensorflow_text) (4.8)\n",
            "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<3,>=1.6.3->tensorboard<2.10,>=2.9->tensorflow<2.10,>=2.9.0->tensorflow_text) (4.2.4)\n",
            "Requirement already satisfied: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.10,>=2.9->tensorflow<2.10,>=2.9.0->tensorflow_text) (1.3.1)\n",
            "Requirement already satisfied: importlib-metadata>=4.4 in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard<2.10,>=2.9->tensorflow<2.10,>=2.9.0->tensorflow_text) (4.11.4)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata>=4.4->markdown>=2.6.8->tensorboard<2.10,>=2.9->tensorflow<2.10,>=2.9.0->tensorflow_text) (3.8.0)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3,>=1.6.3->tensorboard<2.10,>=2.9->tensorflow<2.10,>=2.9.0->tensorflow_text) (0.4.8)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.10,>=2.9->tensorflow<2.10,>=2.9.0->tensorflow_text) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.10,>=2.9->tensorflow<2.10,>=2.9.0->tensorflow_text) (3.0.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.10,>=2.9->tensorflow<2.10,>=2.9.0->tensorflow_text) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard<2.10,>=2.9->tensorflow<2.10,>=2.9.0->tensorflow_text) (2022.6.15)\n",
            "Requirement already satisfied: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard<2.10,>=2.9->tensorflow<2.10,>=2.9.0->tensorflow_text) (3.2.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->tensorflow<2.10,>=2.9.0->tensorflow_text) (3.0.9)\n"
          ]
        }
      ],
      "source": [
        "!pip install tensorflow_text"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "P7TJjyP7A1Gx"
      },
      "outputs": [],
      "source": [
        "!pip install -q transformers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8UAdlAZL4kwU",
        "outputId": "ed2a83d6-6905-4081-f020-574c22d7b563"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n",
            "[nltk_data]   Package wordnet is already up-to-date!\n",
            "[nltk_data] Downloading package omw-1.4 to /root/nltk_data...\n",
            "[nltk_data]   Package omw-1.4 is already up-to-date!\n"
          ]
        }
      ],
      "source": [
        "import tensorflow as tf\n",
        "import matplotlib.pyplot as plt\n",
        "from tensorflow.keras.preprocessing.text import Tokenizer\n",
        "from tensorflow.keras.preprocessing.sequence import pad_sequences\n",
        "import keras\n",
        "import csv\n",
        "import pandas as pd\n",
        "import random\n",
        "import numpy as np\n",
        "from keras import regularizers\n",
        "import re\n",
        "from tqdm import tqdm_notebook\n",
        "from nltk.corpus import stopwords\n",
        "from nltk import WordNetLemmatizer\n",
        "import nltk\n",
        "nltk.download('wordnet')\n",
        "nltk.download('omw-1.4')\n",
        "import itertools\n",
        "from string import ascii_lowercase\n",
        "from functools import reduce\n",
        "import seaborn as sns\n",
        "from sklearn.model_selection import train_test_split\n",
        "from transformers import BertTokenizer, TFBertForSequenceClassification\n",
        "from transformers import InputExample, InputFeatures"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "axLOfMVm7cYf",
        "outputId": "5e5c55f8-546f-47c9-9083-20e10aa6ad07"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "G8VmC6mL7j1T"
      },
      "outputs": [],
      "source": [
        "path='/content/drive/MyDrive/Twitter Sentiment Analysis/Dataset.csv'\n",
        "df=pd.read_csv(path, encoding='latin')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DgAxR85mBSpO"
      },
      "outputs": [],
      "source": [
        "df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_t-d5PAlDdff"
      },
      "outputs": [],
      "source": [
        "df.columns"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "MxKNU3dxD0gn"
      },
      "outputs": [],
      "source": [
        "df.isnull().sum()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DUdoMNkSHBam"
      },
      "outputs": [],
      "source": [
        "df.drop(['id', 'date', 'DK', 'username'], axis=1, inplace=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "9dbgnxlvHffQ"
      },
      "outputs": [],
      "source": [
        "df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_1XP5krGHgfg"
      },
      "outputs": [],
      "source": [
        "df[df['sentiment']==4]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "GMeo1UWbIseA"
      },
      "outputs": [],
      "source": [
        "df.info()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "kbOnBYH7IGHt"
      },
      "outputs": [],
      "source": [
        "df['sentiment']=df['sentiment'].map({4:1,0:0})"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Iq9U2W8pIYOT"
      },
      "outputs": [],
      "source": [
        "df.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2o51ENK2IcdI"
      },
      "outputs": [],
      "source": [
        "lens=df['tweet'].str.len()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0f8VOdecKpcI"
      },
      "outputs": [],
      "source": [
        "resize = np.arange(0, 200,20)\n",
        "lens.hist(color='lightblue', figsize=(10, 6), bins=resize,width=20)\n",
        "plt.title('Length Distribution')\n",
        "plt.xlabel('Length of the Tweets')\n",
        "plt.ylabel('Number of Tweets')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WkVN9wbqLBRy"
      },
      "outputs": [],
      "source": [
        "label=df['sentiment'].value_counts().index\n",
        "val=df['sentiment'].value_counts().values"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "VVeCc7U0N5Dq"
      },
      "outputs": [],
      "source": [
        "plt.pie(val, labels=label, autopct='%1.2f%%')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fbNA0686N5qu"
      },
      "outputs": [],
      "source": [
        "RE_PATTERNS = {\n",
        "    \n",
        "    \" are not \" : [\"aren't\"],\n",
        "    \" cannot \" : [\"can't\"],\n",
        "    \" cannot have \": [\"can't've\"],\n",
        "    \" because \": [\"cause\"],\n",
        "    \" could have \": [\"could've\"],\n",
        "    \" could not \": [\"couldn't\"],\n",
        "    \" could not have \": [\"couldn't've\"],\n",
        "    \" did not \": [\"didn't\"],\n",
        "    \" does not \": [\"doesn't\"],\n",
        "    \" do not \": [\"don't\"],\n",
        "    \" had not \": [\"hadn't\"],\n",
        "    \" had not have \": [\"hadn't've\"],\n",
        "    \" has not \": [\"hasn't\"],\n",
        "    \" have not \": [\"haven't\"],\n",
        "    \" he would \": [\"he'd\"],\n",
        "    \" he would have \": [\"he'd've\"],\n",
        "    \" he will \": [\"he'll\"],\n",
        "    \" he is \": [\"he's\"],\n",
        "    \" how did \": [\"how'd\"],\n",
        "    \" how will \": [\"how'll\"],\n",
        "    \" how is \": [\"how's\"],\n",
        "    \" i would \": [\"i'd\"],\n",
        "    \" i will \": [\"i'll\"],\n",
        "    \" i am \": [\"i'm\"],\n",
        "    \" i have \": [\"i've\"],\n",
        "    \" is not \": [\"isn't\"],\n",
        "    \" it would \": [\"it'd\"],\n",
        "    \" it will \": [\"it'll\"],\n",
        "    \" it is \": [\"it's\"],\n",
        "    \" let us \": [\"let's\"],\n",
        "    \" madam \": [\"ma'am\"],\n",
        "    \" may not \": [\"mayn't\"],\n",
        "    \" might have \": [\"might've\"],\n",
        "    \" might not \": [\"mightn't\"],\n",
        "    \" must have \": [\"must've\"],\n",
        "    \" must not \": [\"mustn't\"],\n",
        "    \" need not \": [\"needn't\"],\n",
        "    \" ought not \": [\"oughtn't\"],\n",
        "    \" shall not \": [\"shan't\"],\n",
        "    \" shall not \": [\"sha'n't\"],\n",
        "    \" she would \": [\"she'd\"],\n",
        "    \" she will \": [\"she'll\"],\n",
        "    \" she is \": [\"she's\"],\n",
        "    \" should have \": [\"should've\"],\n",
        "    \" should not \": [\"shouldn't\"],\n",
        "    \" that would \": [\"that'd\"],\n",
        "    \" that is \": [\"that's\"],\n",
        "    \" there had \": [\"there'd\"],\n",
        "    \" there is \": [\"there's\"],\n",
        "    \" they would \": [\"they'd\"],\n",
        "    \" they will \": [\"they'll\"],\n",
        "    \" they are \": [\"they're\"],\n",
        "    \" they have \": [\"they've\"],\n",
        "    \" was not \":[ \"wasn't\"],\n",
        "    \" we would \": [\"we'd\"],\n",
        "    \" we will \": [\"we'll\"],\n",
        "    \" we are \": [\"we're\"],\n",
        "    \" we have \": [\"we've\"],\n",
        "    \" were not \": [\"weren't\"],\n",
        "    \" what will \": [\"what'll\"],\n",
        "    \" what are \": [\"what're\"],\n",
        "    \" what is \": [\"what's\"],\n",
        "    \" what have \":[ \"what've\"],\n",
        "    \" where did \": [\"where'd\"],\n",
        "    \" where is \": [\"where's\"],\n",
        "    \" who will \": [\"who'll\"],\n",
        "    \" who is \": [\"who's\"],\n",
        "    \" will not \": [\"won't\"],\n",
        "    \" would not \": [\"wouldn't\"],\n",
        "    \" you would \": [\"you'd\"],\n",
        "    \" you will \": [\"you'll\"],\n",
        "    \" you are \": [\"you're\"],\n",
        "    ' american ':\n",
        "        [\n",
        "            'amerikan'\n",
        "        ],\n",
        "\n",
        "    ' adolf ':\n",
        "        [\n",
        "            'adolf'\n",
        "        ],\n",
        "\n",
        "\n",
        "    ' hitler ':\n",
        "        [\n",
        "            'hitler'\n",
        "        ],\n",
        "\n",
        "    ' fuck':\n",
        "        [\n",
        "            '(f)(u|[^a-z0-9 ])(c|[^a-z0-9 ])(k|[^a-z0-9 ])([^ ])*',\n",
        "            '(f)([^a-z]*)(u)([^a-z]*)(c)([^a-z]*)(k)',\n",
        "            ' f[!@#\\$%\\^\\&\\*]*u[!@#\\$%\\^&\\*]*k', 'f u u c',\n",
        "            '(f)(c|[^a-z ])(u|[^a-z ])(k)', r'f\\*',\n",
        "            'feck ', ' fux ', 'f\\*\\*', \n",
        "            'f\\-ing', 'f\\.u\\.', 'f###', ' fu ', 'f@ck', 'f u c k', 'f uck', 'f ck','fuk', 'wtf','fucck','f cking'\n",
        "        ],\n",
        "\n",
        "    ' ass ':\n",
        "        [\n",
        "            '[^a-z]ass ', '[^a-z]azz ', 'arrse', ' arse ', '@\\$\\$'\n",
        "                                                           '[^a-z]anus', ' a\\*s\\*s', '[^a-z]ass[^a-z ]',\n",
        "            'a[@#\\$%\\^&\\*][@#\\$%\\^&\\*]', '[^a-z]anal ', 'a s s'\n",
        "        ],\n",
        "\n",
        "    ' asshole ':\n",
        "        [\n",
        "            ' a[s|z]*wipe', 'a[s|z]*[w]*h[o|0]+[l]*e', '@\\$\\$hole', 'ass hole'\n",
        "        ],\n",
        "\n",
        "    ' bitch ':\n",
        "        [\n",
        "            'b[w]*i[t]*ch', 'b!tch',\n",
        "            'bi\\+ch', 'b!\\+ch', '(b)([^a-z]*)(i)([^a-z]*)(t)([^a-z]*)(c)([^a-z]*)(h)',\n",
        "            'biatch', 'bi\\*\\*h', 'bytch', 'b i t c h','beetch'\n",
        "        ],\n",
        "\n",
        "    ' bastard ':\n",
        "        [\n",
        "            'ba[s|z]+t[e|a]+rd'\n",
        "        ],\n",
        "\n",
        "    ' transgender':\n",
        "        [\n",
        "            'transgender','trans gender'\n",
        "        ],\n",
        "\n",
        "    ' gay ':\n",
        "        [\n",
        "            'gay'\n",
        "        ],\n",
        "\n",
        "    ' cock ':\n",
        "        [\n",
        "            '[^a-z]cock', 'c0ck', '[^a-z]cok ', 'c0k', '[^a-z]cok[^aeiou]', ' cawk',\n",
        "            '(c)([^a-z ])(o)([^a-z ]*)(c)([^a-z ]*)(k)', 'c o c k'\n",
        "        ],\n",
        "\n",
        "    ' dick ':\n",
        "        [\n",
        "            ' dick[^aeiou]', 'deek', 'd i c k','diick '\n",
        "        ],\n",
        "\n",
        "    ' suck ':\n",
        "        [\n",
        "            'sucker', '(s)([^a-z ]*)(u)([^a-z ]*)(c)([^a-z ]*)(k)', 'sucks', '5uck', 's u c k'\n",
        "        ],\n",
        "\n",
        "    ' cunt ':\n",
        "        [\n",
        "            'cunt', 'c u n t'\n",
        "        ],\n",
        "\n",
        "    ' bullshit ':\n",
        "        [\n",
        "            'bullsh\\*t', 'bull\\$hit','bs'\n",
        "        ],\n",
        "\n",
        "    ' homosexual':\n",
        "        [\n",
        "            'homo sexual','homosex'\n",
        "        ],\n",
        "\n",
        "    ' jerk ':\n",
        "        [\n",
        "            'jerk'\n",
        "        ],\n",
        "\n",
        "    ' idiot ':\n",
        "        [\n",
        "            'i[d]+io[t]+', '(i)([^a-z ]*)(d)([^a-z ]*)(i)([^a-z ]*)(o)([^a-z ]*)(t)', 'idiots', 'i d i o t'\n",
        "        ],\n",
        "\n",
        "    ' dumb ':\n",
        "        [\n",
        "            '(d)([^a-z ]*)(u)([^a-z ]*)(m)([^a-z ]*)(b)'\n",
        "        ],\n",
        "\n",
        "    ' shit ':\n",
        "        [\n",
        "            'shitty', '(s)([^a-z ]*)(h)([^a-z ]*)(i)([^a-z ]*)(t)', 'shite', '\\$hit', 's h i t'\n",
        "        ],\n",
        "\n",
        "    ' shithole ':\n",
        "        [\n",
        "            'shythole','shit hole'\n",
        "        ],\n",
        "\n",
        "    ' retard ':\n",
        "        [\n",
        "            'returd', 'retad', 'retard', 'wiktard', 'wikitud'\n",
        "        ],\n",
        "\n",
        "    ' rape ':\n",
        "        [\n",
        "            ' raped'\n",
        "        ],\n",
        "\n",
        "    ' dumbass':\n",
        "        [\n",
        "            'dumb ass', 'dubass'\n",
        "        ],\n",
        "\n",
        "    ' asshead':\n",
        "        [\n",
        "            'butthead', 'ass head'\n",
        "        ],\n",
        "\n",
        "    ' sex ':\n",
        "        [\n",
        "            's3x', 'sexuality',\n",
        "        ],\n",
        "\n",
        "\n",
        "    ' nigger ':\n",
        "        [\n",
        "            'nigger', 'ni[g]+a', ' nigr ', 'negrito', 'niguh', 'n3gr', 'n i g g e r'\n",
        "        ],\n",
        "\n",
        "    ' shut the fuck up':\n",
        "        [\n",
        "            'stfu'\n",
        "        ],\n",
        "\n",
        "    ' pussy ':\n",
        "        [\n",
        "            'pussy[^c]', 'pusy', 'pussi[^l]', 'pusses'\n",
        "        ],\n",
        "\n",
        "    ' faggot ':\n",
        "        [\n",
        "            'faggot', ' fa[g]+[s]*[^a-z ]', 'fagot', 'f a g g o t', 'faggit',\n",
        "            '(f)([^a-z ]*)(a)([^a-z ]*)([g]+)([^a-z ]*)(o)([^a-z ]*)(t)', 'fau[g]+ot', 'fae[g]+ot',\n",
        "        ],\n",
        "\n",
        "    ' motherfucker':\n",
        "        [\n",
        "            ' motha ', ' motha f', ' mother f', 'motherucker', 'mother fucker'\n",
        "        ],\n",
        "\n",
        "    ' whore ':\n",
        "        [\n",
        "            'wh\\*\\*\\*', 'w h o r e'\n",
        "        ],\n",
        "    ' though ': ['tho'],\n",
        "    #' picture ': ['pic', 'pics'],\n",
        "    ' soo ': ['so'],\n",
        "    ' should ':['shoulda'],\n",
        "    \" aint \": [\"am not\"],\n",
        "}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "sOsnmknMOAoq"
      },
      "outputs": [],
      "source": [
        "#Text Normalization\n",
        "\n",
        "def clean_text(text,remove_repeat_text=True, remove_patterns_text=True, is_lower=True):\n",
        "  \n",
        "  if is_lower:\n",
        "    text=text.lower()\n",
        "  \n",
        "\n",
        "  if remove_patterns_text:\n",
        "    for target, patterns in RE_PATTERNS.items():\n",
        "      for pat in patterns:\n",
        "        text=str(text).replace(pat, target)\n",
        "\n",
        "  if remove_repeat_text:\n",
        "    text = re.sub(r'(.)\\1{2,}', r'\\1', text) \n",
        "\n",
        "  \n",
        "  text = re.sub('@[^\\s]+','',text)\n",
        "  text = re.sub(r'http\\S+', '', text, flags=re.MULTILINE)\n",
        "  text = str(text).replace(\"\\n\", \" \")\n",
        "  text = re.sub(r'[^\\w\\s]',' ',text)\n",
        "  text = re.sub('[0-9]',\"\",text)\n",
        "  text = re.sub(\" +\", \" \", text)\n",
        "  text = re.sub(\"([^\\x00-\\x7F])+\",\" \",text)\n",
        "  \n",
        "  return text "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "m20T0754OUJz"
      },
      "outputs": [],
      "source": [
        "tweets = [] \n",
        "for line in tqdm_notebook(df['tweet'], total=1600000): \n",
        "    tweets.append(clean_text(line))\n",
        "# test=[]\n",
        "# test.append(clean_text(\"@switchfoot http://twitpic.com/2y1zl - Awww, that's a bummer. You shoulda got David Carr of Third Day to do it. ;D\"))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "bTzwEdlBOe_T"
      },
      "outputs": [],
      "source": [
        "#Lemmatization\n",
        "\n",
        "lemmatizer= WordNetLemmatizer()\n",
        "def lemma(text, lemmatization=True):\n",
        "  output=''\n",
        "  if lemmatization:\n",
        "    text=text.split(' ')\n",
        "    for word in text:\n",
        "      word1 = lemmatizer.lemmatize(word, pos = \"n\") #noun \n",
        "      word2 = lemmatizer.lemmatize(word1, pos = \"v\") #verb\n",
        "      word3 = lemmatizer.lemmatize(word2, pos = \"a\") #adjective\n",
        "      word4 = lemmatizer.lemmatize(word3, pos = \"r\") #adverb\n",
        "      output=output + \" \" + word4\n",
        "  else:\n",
        "    output=text\n",
        "  \n",
        "  return str(output.strip())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qGth3xNMPdhn"
      },
      "outputs": [],
      "source": [
        "lema_tweets=[]\n",
        "for line in tweets:\n",
        "    lema_tweets.append(lemma(line))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1TA1kID2PpFq"
      },
      "outputs": [],
      "source": [
        "#Stopwords Removal\n",
        "\n",
        "stopword_list=[]\n",
        "def iter_all_strings():\n",
        "    for size in itertools.count(1):\n",
        "        for s in itertools.product(ascii_lowercase, repeat=size):\n",
        "            yield \"\".join(s)\n",
        "\n",
        "dual_alpha_list=[]\n",
        "for s in iter_all_strings():\n",
        "    dual_alpha_list.append(s)\n",
        "    if s == 'zz':\n",
        "        break\n",
        "\n",
        "dual_alpha_list.remove('i')\n",
        "dual_alpha_list.remove('a')\n",
        "dual_alpha_list.remove('am')\n",
        "dual_alpha_list.remove('an')\n",
        "dual_alpha_list.remove('as')\n",
        "dual_alpha_list.remove('at')\n",
        "dual_alpha_list.remove('be')\n",
        "dual_alpha_list.remove('by')\n",
        "dual_alpha_list.remove('do')\n",
        "dual_alpha_list.remove('go')\n",
        "dual_alpha_list.remove('he')\n",
        "dual_alpha_list.remove('hi')\n",
        "dual_alpha_list.remove('if')\n",
        "dual_alpha_list.remove('is')\n",
        "dual_alpha_list.remove('in')\n",
        "dual_alpha_list.remove('me')\n",
        "dual_alpha_list.remove('my')\n",
        "dual_alpha_list.remove('no')\n",
        "dual_alpha_list.remove('of')\n",
        "dual_alpha_list.remove('on')\n",
        "dual_alpha_list.remove('or')\n",
        "dual_alpha_list.remove('ok')\n",
        "dual_alpha_list.remove('so')\n",
        "dual_alpha_list.remove('to')\n",
        "dual_alpha_list.remove('up')\n",
        "dual_alpha_list.remove('us')\n",
        "dual_alpha_list.remove('we')\n",
        "\n",
        "\n",
        "for letter in dual_alpha_list:\n",
        "    stopword_list.append(letter)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dsz1QFi_Ptyw"
      },
      "outputs": [],
      "source": [
        "print(len(stopword_list))\n",
        "print(len(lema_tweets))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "KGes_dhFPwpL"
      },
      "outputs": [],
      "source": [
        "# Function that reads the entire dataset and adds words that are not already present in STOP_WORDS into a list of potential_stopwords.\n",
        "\n",
        "def search_stopwords(data, search_stop=True):\n",
        "  output=\"\"\n",
        "  if search_stop:\n",
        "    data=data.split(\" \")\n",
        "    for word in data:\n",
        "      if not word in stopword_list:\n",
        "        output=output+\" \"+word \n",
        "  else:\n",
        "    output=data\n",
        "  return str(output.strip())\n",
        "\n",
        "# Calling the search_stopwords function, and saving words into a list (potential_stopwords).\n",
        "#TRAIN\n",
        "potential_stopwords = []\n",
        "for line in tqdm_notebook(lema_tweets, total=1600000): \n",
        "    potential_stopwords.append(search_stopwords(line))\n",
        "print(len(potential_stopwords))\n",
        "print(potential_stopwords)\n",
        "\n",
        "\n",
        "\n",
        "# Combining all the sentences present in potential_stopwords into 4 different strings.\n",
        "# Creation of each new string has a different function, this facilitates faster concatination of sentences. \n",
        "def string_combine_a(stopword):\n",
        "  final_a=\"\"\n",
        "  for item in range(400000):\n",
        "    final_a=final_a+\" \"+stopword[item]\n",
        "  return final_a\n",
        "\n",
        "def string_combine_b(stopword):\n",
        "  final_b=\"\"\n",
        "  for item in range(400001,800000):\n",
        "    final_b=final_b+\" \"+stopword[item]\n",
        "  return final_b\n",
        "\n",
        "def string_combine_c(stopword):\n",
        "  final_c=\"\"\n",
        "  for item in range(800001,1200000):\n",
        "    final_c=final_c+\" \"+stopword[item]\n",
        "  return final_c\n",
        "\n",
        "def string_combine_d(stopword):\n",
        "  final_d=\"\"\n",
        "  for item in range(1200001,1600000):\n",
        "    final_d=final_d+\" \"+stopword[item]\n",
        "  return final_d\n",
        "\n",
        "total_string_potential_a=string_combine_a(potential_stopwords)\n",
        "total_string_potential_b=string_combine_b(potential_stopwords)\n",
        "total_string_potential_c=string_combine_c(potential_stopwords)\n",
        "total_string_potential_d=string_combine_d(potential_stopwords)\n",
        "\n",
        "# Counting the number of words in each of the 4 strings and saving it in a dictionary.\n",
        "def word_count(str):\n",
        "    counts = dict()\n",
        "    words = str.split()\n",
        "\n",
        "    for word in words:\n",
        "        if word in counts:\n",
        "            counts[word] += 1\n",
        "        else:\n",
        "            counts[word] = 1\n",
        "    return counts\n",
        "\n",
        "total_string_potential_a_dict=word_count(total_string_potential_a)\n",
        "total_string_potential_b_dict=word_count(total_string_potential_b)\n",
        "total_string_potential_c_dict=word_count(total_string_potential_c)\n",
        "total_string_potential_d_dict=word_count(total_string_potential_d)\n",
        "\n",
        "#Converting Dictionaries to Dataframe.\n",
        "total_string_potential_a_df = pd.DataFrame(list(total_string_potential_a_dict.items()),columns = ['Word','Count'])\n",
        "total_string_potential_b_df = pd.DataFrame(list(total_string_potential_b_dict.items()),columns = ['Word','Count'])\n",
        "total_string_potential_c_df = pd.DataFrame(list(total_string_potential_c_dict.items()),columns = ['Word','Count'])\n",
        "total_string_potential_d_df = pd.DataFrame(list(total_string_potential_d_dict.items()),columns = ['Word','Count'])\n",
        "\n",
        "#Getting Dataframe output in descending order.\n",
        "top50_potential_stopwords_a=total_string_potential_a_df.sort_values(by=['Count'],ascending=False).head(50)\n",
        "top50_potential_stopwords_b=total_string_potential_b_df.sort_values(by=['Count'],ascending=False).head(50)\n",
        "top50_potential_stopwords_c=total_string_potential_c_df.sort_values(by=['Count'],ascending=False).head(50)\n",
        "top50_potential_stopwords_d=total_string_potential_d_df.sort_values(by=['Count'],ascending=False).head(50)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "hB4u5-1gRxBT"
      },
      "outputs": [],
      "source": [
        "#Looking for common terms in all top 50 dataframes.\n",
        "common_potential_stopwords=list(reduce(set.intersection,map(set,[top50_potential_stopwords_a.Word,top50_potential_stopwords_b.Word,top50_potential_stopwords_c.Word,top50_potential_stopwords_d.Word])))\n",
        "print(common_potential_stopwords)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "uhr9lsNzSSI0"
      },
      "outputs": [],
      "source": [
        "#Retaining certain words and removing others from the above list.\n",
        "#potential_stopwords_top500 = ['which', 'catch', 'since', 'full', 'pretty', 'exam', 'damn', 'minute', 'his', 'about', 'him', 'lol', 'send', 'then', 'rest', 'think', 'night', 'right', 'some', 'next', 'one', 'way', 'finish', 'yet', 'though', 'home', 'well', 'thing', 'more', 'eat', 'hair', 'totally', 'keep', 'already', 'can', 'be', 'awesome', 'sun', 'update', 'suck', 'meet', 'ok', 'any', 'soon', 'at', 'so', 'seem', 'feel', 'hey', 'month', 'girl', 'else', 'for', 'pick', 'stuff', 'link', 'saturday', 'com', 'idea', 'while', 'friday', 'concert', 'try', 'dinner', 'wonder', 'both', 'maybe', 'have', 'tweet', 'ago', 'gotta', 'money', 'find', 'gonna', 'dream', 'sister', 'watch', 'part', 'back', 'hear', 'these', 'least', 'world', 'early', 'worry', 'off', 'follower', 'through', 'haha', 'use', 'once', 'shoot', 'run', 'tho', 'dog', 'around', 'or', 'another', 'name', 'online', 'call', 'dad', 'excite', 'glad', 'leave', 'very', 'haven', 'tell', 'omg', 'should', 'know', 'do', 'didn', 'live', 'they', 'believe', 'enjoy', 'plurk', 'not', 'start', 'come', 'super', 'till', 'weather', 'okay', 'ever', 'the', 'bullshit', 'why', 'follow', 'by', 'eye', 'people', 'day', 'someone', 'something', 'problem', 'move', 'i', 'where', 'old', 'face', 'guy', 'sure', 'whole', 'outside', 'in', 'happy', 'pic', 'again', 'anyone', 'mean', 'last', 'read', 'hard', 'heart', 'go', 'own', 'stop', 'thank', 'trip', 'no', 'coffee', 'cause', 'brother', 'check', 'never', 'up', 'fuck', 'beach', 'god', 'everything', 'tomorrow', 'final', 'twitpic', 'he', 'boy', 'birthday', 'forget', 'but', 'luck', 'must', 'lunch', 'shop', 'post', 'them', 'cool', 'week', 'than', 'sunday', 'enough', 'also', 'sound', 'party', 'plan', 'nothing', 'probably', 'big', 'she', 'game', 'guess', 'only', 'rain', 'bite', 'two', 'myself', 'hi', 'reply', 'there', 'tonight', 'hot', 'talk', 'many', 'family', 'life', 'how', 'song', 'we', 'open', 'hopefully', 'finally', 'fan', 'nice', 'crazy', 'isn', 'yes', 'year', 'aww', 'time', 'that', 'wish', 'doe', 'really', 'before', 'play', 'summer', 'first', 'little', 'half', 'hour', 'see', 'saw', 'hate', 'new', 'book', 'if', 'real', 'lose', 'down', 'baby', 'work', 'kid', 'their', 'kinda', 'just', 'hit', 'room', 'hug', 'music', 'quot', 'want', 'yesterday', 'who', 'same', 'could', 'into', 'sit', 'wow', 'show', 'hop', 'man', 'all', 'actually', 'when', 'anything', 'long', 'give', 'don', 'best', 'busy', 'you', 'her', 'forward', 'shit', 'wait', 'study', 'tire', 'bring', 'thats', 'bed', 'turn', 'close', 'change', 'even', 'amaze', 'always', 'buy', 'cant', 'every', 'dude', 'school', 'to', 'dont', 'most', 'late', 'hand', 'lucky', 'yay', 'this', 'your', 'out', 'mom', 'here', 'love', 'may', 'those', 'doesn', 'win', 'bad', 'place', 'of', 'too', 'phone', 'miss', 'monday', 'over', 'picture', 'much', 'drink', 'stay', 'other', 'bore', 'happen', 'might', 'today', 'would', 'help', 'make', 'fun', 'news', 'will', 'soo', 'still', 'movie', 'far', 'take', 'until', 'mind', 'get', 'from', 'video', 'ready', 'wanna', 'ill', 'free', 'after', 'such', 'everyone', 'listen', 'put', 'break', 'http', 'hope', 'like', 'ask', 'hang', 'food', 'sorry', 'with', 'clean', 'a', 'away', 'need', 'job', 'and', 'spend', 'wake', 'an', 'say', 'mine', 'drive', 'me', 'weekend', 'wear', 'sleep', 'my', 'hell', 'yeah', 'almost', 'good', 'end', 'let', 'head', 'lot', 'great', 'twitter', 'class', 'car', 'please', 'remember', 'walk', 'few', 'because', 'on', 'what', 'look', 'our', 'write', 'now', 'house', 'morning', 'friend', 'amp']\n",
        "potential_stopwords = ['age', 'which', 'since', 'his', 'about', 'him', 'then' , 'night', 'next', 'thing', 'at', 'so', 'hey', 'girl', 'for', 'com', 'tweet', 'sister', 'name', 'dad', 'should', 'they', 'the', 'where', 'brother', 'he', 'boy', 'must', 'them', 'week', 'than', 'she', 'hi', 'reply', 'there', 'man', 'her', 'thats', 'mom', 'here', 'those', 'will', 'http', 'and', 'twitter', 'amp']\n",
        "#Adding above retrived words into the stopwords list.\n",
        "for word in potential_stopwords:\n",
        "    stopword_list.append(word)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "8nODheasSUak"
      },
      "outputs": [],
      "source": [
        "def remove_stopwords(text, remove_stop=True):\n",
        "  output = \"\"\n",
        "  if remove_stop:\n",
        "    text=text.split(\" \")\n",
        "    for word in text:\n",
        "      if word not in stopword_list:\n",
        "        output=output + \" \" + word\n",
        "  else :\n",
        "    output=text\n",
        "\n",
        "  return str(output.strip())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1Mr8YZkHSYw0"
      },
      "outputs": [],
      "source": [
        "Removing Stopwords from Train Data\n",
        "processed_tweets = [] \n",
        "for line in tqdm_notebook(lema_tweets, total=1600000): \n",
        "    processed_tweets.append(remove_stopwords(line))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "zcCv3u2QZ0yh"
      },
      "outputs": [],
      "source": [
        "processed_tweets[:10]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eHDnARtoeyrk"
      },
      "outputs": [],
      "source": [
        "df.head(10)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_lJiKI8Sp8s9"
      },
      "outputs": [],
      "source": [
        "df.to_csv('final_dataframe.csv')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YJ-LmdGxtxi-"
      },
      "outputs": [],
      "source": [
        "df.head()"
      ]
    }
  ],
  "metadata": {
    "accelerator": "TPU",
    "colab": {
      "collapsed_sections": [],
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}