actual-preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "__1Nxb4gRIeN"
   },
   "source": [
    "Done so far :\n",
    "\n",
    "\n",
    "*   Lemmatization\n",
    "*   Stop Words Removal\n",
    "\n",
    "Verify :\n",
    "\n",
    "* Normalization - removing accents, etc.\n",
    "* Dates replaced with strings\n",
    "* Case-folding\n",
    "* Removed HTML entity codes\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "8qZ-TMc9SVHl"
   },
   "outputs": [],
   "source": [
    "import nltk\n",
    "import numpy as np\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk.corpus import wordnet\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import RegexpTokenizer\n",
    "import re\n",
    "import wordninja \n",
    "\n",
    "####### After importing nltk, run the following only once ######\n",
    "# nltk.download('averaged_perceptron_tagger')\n",
    "# nltk.download('wordnet')\n",
    "# nltk.download('stopwords')\n",
    "### pip install wordninja ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_htmlcodes(document):\n",
    "    \n",
    "    '''Removes HTML entity codes such as &amp from document and returns the clean document'''\n",
    "    \n",
    "    replacement = {\n",
    "                    \"&ampnbsp\": ' ',\n",
    "                    \"&ampamp\": '&',\n",
    "                    \"&ampquot\": '\\'',\n",
    "                    \"&ampldquo\": '\\\"',\n",
    "                    \"&amprdquo\": '\\\"',\n",
    "                    \"&amplsquo\": '\\'',\n",
    "                    \"&amprsquo\": '\\'',\n",
    "                    \"&amphellip\": '...',\n",
    "                    \"&ampndash\": '-',\n",
    "                    \"&ampmdash\": '-'\n",
    "                  }\n",
    "    \n",
    "    for str in replacement:\n",
    "        document = document.replace(str, replacement[str])\n",
    "        \n",
    "    return document"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LpQkEVQOSVHr"
   },
   "outputs": [],
   "source": [
    "def get_wordnet_pos(word):\n",
    "    \n",
    "    '''Returns the tag of usage of word depending on context'''\n",
    "    \n",
    "    tag=nltk.pos_tag([word])[0][1][0].upper()\n",
    "    tag_dict={\"J\": wordnet.ADJ, \n",
    "              \"N\": wordnet.NOUN,\n",
    "              \"V\": wordnet.VERB,\n",
    "              \"R\": wordnet.ADV}\n",
    "    return tag_dict.get(tag,wordnet.NOUN)\n",
    "\n",
    "def lemma_stop(str):\n",
    "    \n",
    "    lemmatizer = WordNetLemmatizer()\n",
    "    tokenizer = RegexpTokenizer('\\w+|\\$]\\d\\[+|\\S+,-')\n",
    "    tokenized = tokenizer.tokenize(str)\n",
    "    lemmatized = [lemmatizer.lemmatize(w,get_wordnet_pos(w)) for w in tokenized]\n",
    "    stop_words = set(stopwords.words('english'))\n",
    "    filtered_sentence = [w for w in lemmatized if w.lower() not in stop_words]\n",
    "    after_lemma_stop = ' '.join(w for w in filtered_sentence)\n",
    "    return filtered_sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def is_not_credible (text):\n",
    "    \n",
    "    '''Returns true if text has no special characters, else returns false'''\n",
    "    \n",
    "    match = re.search(r'[!@#?&{}()]', text)\n",
    "    \n",
    "    if match:\n",
    "        return True\n",
    "    else:\n",
    "        return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def scrub_words(text):\n",
    "    \n",
    "    '''Removes special characters from text and returns a clean string'''\n",
    "    \n",
    "    text = re.sub('[!@#?&{}()]', '', text)\n",
    "    text=re.sub(r'[^\\x00-\\x7F]',\" \",text)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_document (document_string):\n",
    "    \n",
    "    '''Cleans document_string by splitting very long strings and identifying garbage JSON and HTML and discarding'''\n",
    "\n",
    "    \n",
    "    cleaned_doc = document_string\n",
    "    for word in document_string.split():\n",
    "                if is_not_credible(word):\n",
    "                    temp= scrub_words(word)\n",
    "                    split=wordninja.split(temp)\n",
    "                    if len(split)>7:\n",
    "                          cleaned_doc = cleaned_doc.replace(word,'')\n",
    "                    else:\n",
    "                        replace_with=' '.join(word for word in split)\n",
    "                        cleaned_doc = cleaned_doc.replace(word, replace_with)\n",
    "    return cleaned_doc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def replace_dates(documentString, docID):\n",
    "    \n",
    "    '''Replaces dates of the format MM/DD and MM/DD/YYYY with DDmmmYYYY inside documentString'''\n",
    "    \n",
    "    from datetime import datetime\n",
    "    count_dates = []\n",
    "    \n",
    "    regEx = '(([0-9]+(/)[0-9]+(/)[0-9]+)|([0-9]+(/)[0-9]+))'\n",
    "    iterator = re.finditer(regEx, documentString)\n",
    "    listOfDates = [(m.start(0), m.end(0)) for m in iterator]\n",
    "    tmp = []\n",
    "    replace_with = []\n",
    "    for indices in listOfDates:\n",
    "        date = documentString[indices[0]:indices[1]]\n",
    "        tmp.append(date)\n",
    "        count = date.count('/')\n",
    "        newDate = ''\n",
    "        if count == 2:\n",
    "            check_year = date[-3]\n",
    "            \n",
    "            if check_year == '/':\n",
    "                YY = date[-2:]\n",
    "                \n",
    "                if int(YY) <= 19:\n",
    "                    proper_date = date[:-2] + '20' + YY\n",
    "                    date = date.replace(date,proper_date)\n",
    "                else:\n",
    "                    proper_date = date[:-2] + '19' + YY\n",
    "                    date = date.replace(YY,('19'+YY))\n",
    "                    \n",
    "            try:\n",
    "                newDate = datetime.strptime(date, '%m/%d/%Y').strftime('%d %b %Y')\n",
    "            except ValueError as ve:\n",
    "                newDate = date\n",
    "        else:\n",
    "            try:\n",
    "                newDate = datetime.strptime(date, '%m/%d').strftime('%d %b')\n",
    "            except ValueError as ve:\n",
    "                newDate = date\n",
    "                \n",
    "        count_dates.append([docID, date])\n",
    "        newDate = newDate.replace(' ', '')\n",
    "        replace_with.append(newDate)\n",
    "        \n",
    "    for i in range(len(tmp)):\n",
    "        documentString = documentString.replace(tmp[i], replace_with[i])\n",
    "    \n",
    "    return documentString"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "################################################\n",
    "## ------------ PREPROCESSING --------------- ##\n",
    "##              run only once                 ##\n",
    "################################################\n",
    "\n",
    "def preprocess(data):\n",
    "    \n",
    "    '''\n",
    "    Performs the following on data:\n",
    "        1. Lemmatization\n",
    "        2. Stop Words Removal\n",
    "        3. Normalization - removing accents, etc.\n",
    "        4. Replacing dates with strings\n",
    "        5. Case-folding\n",
    "        6. Removed HTML entity codes\n",
    "    '''\n",
    "    \n",
    "    import time\n",
    "    from tqdm import tqdm_notebook\n",
    "    \n",
    "    start = time.time()\n",
    "\n",
    "    titles = []\n",
    "    contents = []\n",
    "    lower = len(data) // 2\n",
    "    upper = lower + 3000\n",
    "\n",
    "    for i in tqdm_notebook(range(lower, upper)):\n",
    "\n",
    "        if data[i][4] == None or data[i][1] == None or data[i][0] == None:\n",
    "            continue\n",
    "\n",
    "        # actually modifying the document\n",
    "        data[i][4] = remove_htmlcodes(data[i][4])\n",
    "        data[i][1] = remove_htmlcodes(data[i][1])\n",
    "        data[i][4] = clean_document(data[i][4])\n",
    "        data[i][1] = clean_document(data[i][1])\n",
    "\n",
    "        # not actually modifying the document\n",
    "        modifiedContent = replace_dates(data[i][4], data[i][0])\n",
    "        modifiedContent = lemma_stop((modifiedContent))\n",
    "        modifiedTitle = replace_dates(data[i][1], data[i][0])\n",
    "        modifiedTitle = lemma_stop((modifiedTitle))\n",
    "\n",
    "        # case-folding\n",
    "        for j in range(len(modifiedContent)):\n",
    "            modifiedContent[j] = modifiedContent[j].lower()\n",
    "        for j in range(len(modifiedTitle)):\n",
    "            modifiedTitle[j] = modifiedTitle[j].lower()\n",
    "\n",
    "        titles.append(modifiedTitle)\n",
    "        contents.append(modifiedContent)\n",
    "\n",
    "    # filet = \"/home/nihaljain/3-1/CS F469/Assignment-1/mod_titles\"\n",
    "    # filec = \"/home/nihaljain/3-1/CS F469/Assignment-1/mod_contents\"\n",
    "    # filed = \"/home/nihaljain/3-1/CS F469/Assignment-1/mod_data\"\n",
    "\n",
    "    # np.save(filet, titles)\n",
    "    # np.save(filec, contents)\n",
    "    # np.save(filed, data)\n",
    "\n",
    "    print(time.time() - start)  # 110.26236414909363\n",
    "    \n",
    "    return data, contents, titles\n",
    "\n",
    "    # --------------------OPTIONALLY------------------------\n",
    "\n",
    "    # contents = []\n",
    "    # titles = []\n",
    "    # data = []\n",
    "\n",
    "    # filet = \"/home/nihaljain/3-1/CS F469/Assignment-1/mod_titles\"\n",
    "    # filec = \"/home/nihaljain/3-1/CS F469/Assignment-1/mod_contents\"\n",
    "    # filed = \"/home/nihaljain/3-1/CS F469/Assignment-1/mod_data\"\n",
    "\n",
    "    # titles = np.load(filet + \".npy\", allow_pickle = True)\n",
    "    # contents = np.load(filec + \".npy\", allow_pickle = True)\n",
    "    # data = np.load(filed + \".npy\", allow_pickle = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#-----------NOTE------------\n",
    "# len(contents) != len(data) // 2\n",
    "\n",
    "def remove_accents(contents, titles):\n",
    "    \n",
    "    '''Removes accents from all strings in contents and titles'''\n",
    "    \n",
    "    import unidecode\n",
    "    import pickle\n",
    "\n",
    "    for i in range(len(contents)):\n",
    "        for j in range(len(contents[i])):\n",
    "            contents[i][j] = unidecode.unidecode(contents[i][j])\n",
    "        for j in range(len(titles[i])):\n",
    "            titles[i][j] = unidecode.unidecode(titles[i][j])\n",
    "    \n",
    "    # OPTIONALLY TO SAVE PERSISTENTLY #\n",
    "    \n",
    "    # with open('modified_contents_ascii.pickle', 'wb') as handle:\n",
    "    #    pickle.dump(contents, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "    # with open('modified_titles_ascii.pickle', 'wb') as handle:\n",
    "    #    pickle.dump(titles, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "    \n",
    "    return contents, titles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "def construct_corpus(contents, titles, data):\n",
    "    \n",
    "    '''Constructs global corpus trie and several document tries using contents, titles and data'''\n",
    "\n",
    "    import pickle\n",
    "    import time\n",
    "    from tqdm import tqdm_notebook\n",
    "\n",
    "    max_tf = {}\n",
    "    start = time.time()\n",
    "    j = 0\n",
    "    lower = len(data) // 2\n",
    "    upper = len(data) // 2 + 3000\n",
    "\n",
    "\n",
    "    for i in tqdm_notebook(range(lower, upper)):\n",
    "\n",
    "        if data[i][4] == None or data[i][1] == None or data[i][0] == None:\n",
    "            continue\n",
    "\n",
    "        for w in contents[j]:\n",
    "            collection.add_document(w, 0, get_docID[i])\n",
    "            documentRoot[get_docID[i]].add(w, 0)\n",
    "\n",
    "            if get_docID[i] in max_tf:\n",
    "                max_tf[get_docID[i]] = max(documentRoot[get_docID[i]].count_words(w, 0), max_tf[get_docID[i]])\n",
    "            else:\n",
    "                max_tf[get_docID[i]] = documentRoot[get_docID[i]].count_words(w, 0)\n",
    "\n",
    "        for w in titles[j]:\n",
    "            collection.add_title(w, 0, get_docID[i])\n",
    "\n",
    "        j += 1\n",
    "\n",
    "    print(time.time() - start)  #39.19705152511597\n",
    "\n",
    "    return collection, documentRoot, max_tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "88d82b0a03cf45c5902853b8eacc1c9f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "124.62871313095093\n",
      "204135 2963 2963\n",
      "here\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d19e2d8eb8c4440c845c9d99aa56b8cb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "15.387139797210693\n"
     ]
    }
   ],
   "source": [
    "if __name__ == '__main__':\n",
    "    \n",
    "    import trie\n",
    "    import pickle\n",
    "    \n",
    "    # loading data.npy\n",
    "    # data.npy is a 2D array containing the dataset information as\n",
    "    # data[i][0] : docID of ith document\n",
    "    # data[i][1] : title of ith document\n",
    "    # data[i][4] : content of ith document\n",
    "    data = np.load('data.npy',allow_pickle = True)\n",
    "    \n",
    "    # preprocessing\n",
    "    data, contents, titles = preprocess(data)\n",
    "    contents, titles = remove_accents(contents, titles)  \n",
    "    \n",
    "    # print(len(data), len(contents), len(titles))\n",
    "    \n",
    "    # constructing the tries\n",
    "    \n",
    "    getReference = {}\n",
    "    get_docID = {}\n",
    "    get_index = {}\n",
    "\n",
    "    for i in range(0, len(data)) :\n",
    "        get_docID[i] = int(data[i][0])\n",
    "        get_index[int(data[i][0])] = i\n",
    "        \n",
    "    documentRoot = {}\n",
    "    collection = trie.CollectionNode()\n",
    "\n",
    "    # initializing the root for N documents\n",
    "    \n",
    "    lower = len(data) // 2\n",
    "    upper = lower + 3000\n",
    "    \n",
    "    for i in range(lower, upper):\n",
    "        newDocument = trie.Node()\n",
    "        documentRoot[get_docID[i]] = newDocument\n",
    "        \n",
    "    collection, documentRoot, max_tf = construct_corpus(contents, titles, data)\n",
    "    \n",
    "    # saving to pickle files\n",
    "    \n",
    "    # with open('collection.pickle', 'wb') as handle:\n",
    "    #    pickle.dump(collection, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "    # with open('documentRoot.pickle', 'wb') as handle:\n",
    "    #    pickle.dump(documentRoot, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "    # with open('max_tf.pickle', 'wb') as handle:\n",
    "    #    pickle.dump(max_tf, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "    # reading from pickle files\n",
    "\n",
    "    # with open('collection.pickle', 'rb') as handle:\n",
    "    #     collection = pickle.load(handle)\n",
    "    # with open('documentRoot.pickle', 'rb') as handle:\n",
    "    #     documentRoot = pickle.load(handle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import math\n",
    "# import queue\n",
    "\n",
    "# documentLength = {}\n",
    "# N = len(documentRoot)\n",
    "\n",
    "# for i in tqdm(range(len(documentRoot))):\n",
    "    \n",
    "#     docID = get_docID[i]\n",
    "#     length = 0\n",
    "#     document = documentRoot[i]\n",
    "#     q = queue.Queue()\n",
    "#     q.put([document, ''])\n",
    "\n",
    "#     while q.qsize() > 0:\n",
    "\n",
    "#         current = q.get()\n",
    "#         reference = current[0]\n",
    "#         word = current[1]\n",
    "\n",
    "#         if reference.words > 0:\n",
    "#             df = len(collection.get_doc_list(word, 0))\n",
    "#             idf = math.log10(N/df)\n",
    "#             # print(word, reference.words, df)\n",
    "#             length += (reference.words * idf) ** 2\n",
    "\n",
    "#         for i in range(256):\n",
    "#             if reference.children[i] is not None:\n",
    "#                 new_word = word + chr(i)\n",
    "#                 q.put([reference.children[i], new_word])\n",
    "\n",
    "#     # print(length**0.5)\n",
    "#     documentLength[docID] = length**0.5"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "preprocessing.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}