preprocessing-Copy1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "__1Nxb4gRIeN"
   },
   "source": [
    "Done so far :\n",
    "\n",
    "\n",
    "*   Lemmatization\n",
    "*   Stop Words Removal\n",
    "\n",
    "Verify :\n",
    "\n",
    "* Normalization - removing accents, etc.\n",
    "* Dates replaced with strings\n",
    "* Case-folding\n",
    "* Removed HTML entity codes\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "8qZ-TMc9SVHl"
   },
   "outputs": [],
   "source": [
    "import nltk\n",
    "import numpy as np\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from nltk.corpus import wordnet\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.tokenize import RegexpTokenizer\n",
    "import re\n",
    "import wordninja \n",
    "\n",
    "####### After importing nltk, run the following only once ######\n",
    "# nltk.download('averaged_perceptron_tagger')\n",
    "# nltk.download('wordnet')\n",
    "# nltk.download('stopwords')\n",
    "### pip install wordninja ###"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_htmlcodes(document):\n",
    "    \n",
    "    replacement = {\n",
    "                    \"&ampnbsp\": ' ',\n",
    "                    \"&ampamp\": '&',\n",
    "                    \"&ampquot\": '\\'',\n",
    "                    \"&ampldquo\": '\\\"',\n",
    "                    \"&amprdquo\": '\\\"',\n",
    "                    \"&amplsquo\": '\\'',\n",
    "                    \"&amprsquo\": '\\'',\n",
    "                    \"&amphellip\": '...',\n",
    "                    \"&ampndash\": '-',\n",
    "                    \"&ampmdash\": '-'\n",
    "                  }\n",
    "    \n",
    "    for str in replacement:\n",
    "        document = document.replace(str, replacement[str])\n",
    "        \n",
    "    return document"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "LpQkEVQOSVHr"
   },
   "outputs": [],
   "source": [
    "def get_wordnet_pos(word):\n",
    "    \n",
    "    tag=nltk.pos_tag([word])[0][1][0].upper()\n",
    "    tag_dict={\"J\": wordnet.ADJ, \n",
    "              \"N\": wordnet.NOUN,\n",
    "              \"V\": wordnet.VERB,\n",
    "              \"R\": wordnet.ADV}\n",
    "    return tag_dict.get(tag,wordnet.NOUN)\n",
    "\n",
    "def lemma_stop(str):\n",
    "    \n",
    "    lemmatizer = WordNetLemmatizer()\n",
    "    tokenizer = RegexpTokenizer('\\w+|\\$]\\d\\[+|\\S+,-')\n",
    "    tokenized = tokenizer.tokenize(str)\n",
    "    lemmatized = [lemmatizer.lemmatize(w,get_wordnet_pos(w)) for w in tokenized]\n",
    "    stop_words = set(stopwords.words('english'))\n",
    "    filtered_sentence = [w for w in lemmatized if w.lower() not in stop_words]\n",
    "    after_lemma_stop = ' '.join(w for w in filtered_sentence)\n",
    "    return filtered_sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "0PvIwQ6ySVHv",
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# loading data.npy\n",
    "# data.npy is a 2D array containing the dataset information as\n",
    "# data[i][0] : docID of ith document\n",
    "# data[i][1] : title of ith document\n",
    "# data[i][4] : content of ith document\n",
    "\n",
    "data = np.load('data.npy',allow_pickle = True)\n",
    "# sentence = data[0][4]\n",
    "# print(sentence)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "y6aQ7ZcPCH4d",
    "outputId": "8fc93dc4-51e3-48ff-fa64-d5bbef5847a9",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# creating a map {index_in_data_npy, docID}\n",
    "\n",
    "# ex. if ith element in data has docID j,\n",
    "# get_docID[i] will return j\n",
    "\n",
    "get_docID = {}\n",
    "get_index = {}\n",
    "\n",
    "for i in range(0, len(data)) :\n",
    "    get_docID[i] = int(data[i][0])\n",
    "    get_index[int(data[i][0])] = i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def is_not_credible (text):\n",
    "    \n",
    "    match = re.search(r'[!@#?&{}()]', text)\n",
    "    \n",
    "    if match:\n",
    "        return True\n",
    "    else:\n",
    "        return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def scrub_words(text):\n",
    "    \n",
    "    text = re.sub('[!@#?&{}()]', '', text)\n",
    "    text=re.sub(r'[^\\x00-\\x7F]',\" \",text)\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_document (document_string):\n",
    "    \n",
    "    cleaned_doc = document_string\n",
    "    for word in document_string.split():\n",
    "                if is_not_credible(word):\n",
    "                    temp= scrub_words(word)\n",
    "                    split=wordninja.split(temp)\n",
    "                    if len(split)>7:\n",
    "                          cleaned_doc = cleaned_doc.replace(word,'')\n",
    "                    else:\n",
    "                        replace_with=' '.join(word for word in split)\n",
    "                        cleaned_doc = cleaned_doc.replace(word, replace_with)\n",
    "    return cleaned_doc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "count_dates=[]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "\n",
    "def replace_dates(documentString, docID):\n",
    "    \n",
    "    # regEx = '(([0-9]+(/|\\\\.|-)[0-9]+(/|\\\\.|-)[0-9]+)|([0-9]+(/|\\\\.|-)[0-9]+))'\n",
    "    regEx = '(([0-9]+(/)[0-9]+(/)[0-9]+)|([0-9]+(/)[0-9]+))'\n",
    "    iterator = re.finditer(regEx, documentString)\n",
    "    listOfDates = [(m.start(0), m.end(0)) for m in iterator]\n",
    "    tmp = []\n",
    "    replace_with = []\n",
    "    for indices in listOfDates:\n",
    "        date = documentString[indices[0]:indices[1]]\n",
    "        tmp.append(date)\n",
    "        # date = date.replace('.', '/')\n",
    "        # date = date.replace('-', '/')\n",
    "        count = date.count('/')\n",
    "        newDate = ''\n",
    "        if count == 2:\n",
    "            check_year = date[-3]\n",
    "            \n",
    "            if check_year == '/':\n",
    "                YY = date[-2:]\n",
    "                \n",
    "                if int(YY) <= 19:\n",
    "                    proper_date = date[:-2] + '20' + YY\n",
    "                    date = date.replace(date,proper_date)\n",
    "                else:\n",
    "                    proper_date = date[:-2] + '19' + YY\n",
    "                    date = date.replace(YY,('19'+YY))\n",
    "                    \n",
    "            try:\n",
    "                newDate = datetime.strptime(date, '%m/%d/%Y').strftime('%d %b %Y')\n",
    "            except ValueError as ve:\n",
    "                newDate = date\n",
    "        else:\n",
    "            try:\n",
    "                newDate = datetime.strptime(date, '%m/%d').strftime('%d %b')\n",
    "            except ValueError as ve:\n",
    "                newDate = date\n",
    "        count_dates.append([docID, date])\n",
    "        newDate = newDate.replace(' ', '')\n",
    "        replace_with.append(newDate)\n",
    "        \n",
    "    for i in range(len(tmp)):\n",
    "        documentString = documentString.replace(tmp[i], replace_with[i])\n",
    "    \n",
    "    return documentString"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(replace_dates('12/12/12 to 9/11'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creating a temporary smaller dataset\n",
    "\n",
    "subset = []\n",
    "counter = 0\n",
    "for document in data:\n",
    "    subset.append(document)\n",
    "    counter += 1\n",
    "    if counter == 1000:\n",
    "        break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1000/1000 [02:06<00:00,  7.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "126.60539221763611\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "from tqdm import tqdm\n",
    "\n",
    "start = time.time()\n",
    "\n",
    "titles = []\n",
    "contents = []\n",
    "for i in tqdm(range(len(subset))):\n",
    "    # actually modifying the document\n",
    "    subset[i][4] = remove_htmlcodes(subset[i][4])\n",
    "    subset[i][1] = remove_htmlcodes(subset[i][1])\n",
    "    subset[i][4] = clean_document(subset[i][4])\n",
    "    subset[i][1] = clean_document(subset[i][1])\n",
    "    \n",
    "    # not actually modifying the document\n",
    "    modifiedContent = replace_dates(subset[i][4], subset[i][0])\n",
    "    modifiedContent = lemma_stop((modifiedContent))\n",
    "    modifiedTitle = replace_dates(subset[i][1], subset[i][0])\n",
    "    modifiedTitle = lemma_stop((modifiedTitle))\n",
    "    \n",
    "    # case-folding\n",
    "    for i in range(len(modifiedContent)):\n",
    "        modifiedContent[i] = modifiedContent[i].lower()\n",
    "    for i in range(len(modifiedTitle)):\n",
    "        modifiedTitle[i] = modifiedTitle[i].lower()\n",
    "    \n",
    "    titles.append(modifiedTitle)\n",
    "    contents.append(modifiedContent)\n",
    "    \n",
    "print(time.time() - start)  # 110.26236414909363"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "import unidecode\n",
    "contents_temp = contents\n",
    "\n",
    "titles_temp = titles\n",
    "for i in range(1000):\n",
    "    for j in range(len(contents[i])):\n",
    "        contents[i][j] = unidecode.unidecode(contents[i][j])\n",
    "    for j in range(len(titles[i])):\n",
    "        titles[i][j] = unidecode.unidecode(titles[i][j])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "\n",
    "with open('modified_contents.pickle', 'wb') as handle:\n",
    "    pickle.dump(contents, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "with open('modified_titles.pickle', 'wb') as handle:\n",
    "    pickle.dump(titles, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "    \n",
    "\n",
    "# To read the data again\n",
    "\n",
    "# with open('modified_contents.pickle', 'rb') as handle:\n",
    "#     contents = pickle.load(handle)\n",
    "# with open('modified_titles.pickle', 'rb') as handle:\n",
    "#     titles = pickle.load(handle)\n",
    "    \n",
    "# print(unserialized_title == titles)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "fr-HGZhDrJfT"
   },
   "outputs": [],
   "source": [
    "import trie\n",
    "\n",
    "# Create map from docID of the document to an object of class Node \n",
    "# (i.e, the corresponding document trie structure)\n",
    "# ex. if the docID of the document is 1, \n",
    "# getReference[1] gives the object which is the trie structure of docID 1\n",
    "\n",
    "getReference = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "documentRoot = []\n",
    "collection = trie.CollectionNode()\n",
    "\n",
    "# initializing the root for 1000 documents\n",
    "for i in range(1000):\n",
    "    newDocument = trie.Node()\n",
    "    documentRoot.append(newDocument)\n",
    "    getReference[get_docID[i]] = newDocument"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1000/1000 [00:11<00:00, 84.34it/s] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11.858062744140625\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# creating the documents\n",
    "\n",
    "max_tf = {}\n",
    "N = 1000\n",
    "\n",
    "with open('modified_contents.pickle', 'rb') as handle:\n",
    "    contents = pickle.load(handle)\n",
    "with open('modified_titles.pickle', 'rb') as handle:\n",
    "    titles = pickle.load(handle)\n",
    "\n",
    "import time\n",
    "from tqdm import tqdm\n",
    "\n",
    "start = time.time()\n",
    "for i in tqdm(range(N)):\n",
    "    for w in contents[i]:\n",
    "        collection.add_document(w, 0, get_docID[i])\n",
    "        documentRoot[i].add(w, 0)\n",
    "        if get_docID[i] in max_tf:\n",
    "            max_tf[get_docID[i]] = max(documentRoot[i].count_words(w, 0), max_tf[get_docID[i]])\n",
    "        else:\n",
    "            max_tf[get_docID[i]] = documentRoot[i].count_words(w, 0)\n",
    "    for w in titles[i]:\n",
    "        collection.add_title(w, 0, get_docID[i])\n",
    "        \n",
    "print(time.time() - start)  #39.19705152511597"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('collection.pickle', 'wb') as handle:\n",
    "    pickle.dump(collection, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "with open('documentRoot.pickle', 'wb') as handle:\n",
    "    pickle.dump(documentRoot, handle, protocol=pickle.HIGHEST_PROTOCOL)\n",
    "    \n",
    "# reading from pickle files\n",
    "\n",
    "with open('collection.pickle', 'rb') as handle:\n",
    "    collection = pickle.load(handle)\n",
    "with open('documentRoot.pickle', 'rb') as handle:\n",
    "    documentRoot = pickle.load(handle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import math\n",
    "# import queue\n",
    "\n",
    "# documentLength = {}\n",
    "# N = len(documentRoot)\n",
    "\n",
    "# for i in tqdm(range(len(documentRoot))):\n",
    "    \n",
    "#     docID = get_docID[i]\n",
    "#     length = 0\n",
    "#     document = documentRoot[i]\n",
    "#     q = queue.Queue()\n",
    "#     q.put([document, ''])\n",
    "\n",
    "#     while q.qsize() > 0:\n",
    "\n",
    "#         current = q.get()\n",
    "#         reference = current[0]\n",
    "#         word = current[1]\n",
    "\n",
    "#         if reference.words > 0:\n",
    "#             df = len(collection.get_doc_list(word, 0))\n",
    "#             idf = math.log10(N/df)\n",
    "#             # print(word, reference.words, df)\n",
    "#             length += (reference.words * idf) ** 2\n",
    "\n",
    "#         for i in range(256):\n",
    "#             if reference.children[i] is not None:\n",
    "#                 new_word = word + chr(i)\n",
    "#                 q.put([reference.children[i], new_word])\n",
    "\n",
    "#     # print(length**0.5)\n",
    "#     documentLength[docID] = length**0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Go9w6bB9rJfg",
    "outputId": "2e5b7bc8-2e68-4de8-f81d-a142a989c902"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['19jun']\n"
     ]
    }
   ],
   "source": [
    "query = '6/19'\n",
    "final_query = replace_dates(query, -1)\n",
    "final_query = lemma_stop(final_query)\n",
    "\n",
    "for i in range(len(final_query)):\n",
    "    final_query[i] = unidecode.unidecode(final_query[i])\n",
    "    # case-folding\n",
    "    final_query[i] = final_query[i].lower()\n",
    "print(final_query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "j4Kqgk9hrJfj",
    "outputId": "04bfaa8a-8053-4ab7-e55a-cee2d47edb6f"
   },
   "outputs": [],
   "source": [
    "tf_query = {}\n",
    "for w in final_query:\n",
    "    if w not in tf_query:\n",
    "        tf_query[w] = 1\n",
    "    else:\n",
    "        tf_query[w] += 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "rBHDg-lJrJfo"
   },
   "source": [
    "***Ranked Retrieval based on TF-IDF Score :***\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "lfT-kdUqrJfp",
    "outputId": "3c95efd0-76c9-4dfe-e425-e79c089931ee",
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------------------------------------\n",
      "Term in query =  19jun\n",
      "\n",
      "df =  1\n",
      "idf =  3.0\n",
      "{}\n",
      "\n",
      "\n",
      "\n",
      "============================================\n",
      "\n",
      "doc ID =  6981\n",
      "Keywords:\n",
      "\n",
      "Amazon's Fire TV is safer for kids with latest update\n",
      "\n",
      "title score =  0\n",
      "19jun -1.6666666666666667 1\n",
      "\n",
      "\n",
      "... of the boob tube anyway, Amazon is betting that doing it in a pre-set environment will be better for everyone.Update 6/19 3:00PM: Amazon now says that the initial rollout will start within the next week, but not today.  ... \n",
      "\n",
      "tf-idf score= 5.0\n",
      "\n",
      "\n",
      "============================================\n"
     ]
    }
   ],
   "source": [
    "import queue\n",
    "\n",
    "# scores[i] stores the dot product of the tf-idf score vectors of the query and document of docID i in the corpus\n",
    "scores = {}\n",
    "title_score = {}\n",
    "\n",
    "# N is the total number of documents in the corpus\n",
    "N = len(documentRoot)\n",
    "\n",
    "# wordsInDoc[i] is a sorted list of (word, score) tuples where\n",
    "# score is the tf-idf score for the (word, <ith doc>) pair\n",
    "wordsInDoc = {}\n",
    "\n",
    "factor = {}\n",
    "\n",
    "import math\n",
    "import bisect\n",
    "\n",
    "for query_term in tf_query:\n",
    "    \n",
    "    docs_having_query_term = collection.get_doc_list(query_term, 0)\n",
    "    df = len(docs_having_query_term)\n",
    "    idf = 0\n",
    "    \n",
    "    print('-------------------------------------')\n",
    "    print('Term in query = ', query_term)\n",
    "    print()\n",
    "    \n",
    "    if df == 0:\n",
    "        idf = 0\n",
    "    else:\n",
    "        idf = math.log10(N/df)\n",
    "        \n",
    "    docs_having_query_term_in_title = collection.get_title_list(query_term,0)\n",
    "    \n",
    "    for docID in docs_having_query_term_in_title:\n",
    "        if docID in title_score:\n",
    "            title_score[docID] += idf\n",
    "        else:\n",
    "            title_score[docID] = idf\n",
    "        \n",
    "    print('df = ',df)\n",
    "    print('idf = ',idf)\n",
    "    \n",
    "    tfidf_query = tf_query[query_term] * idf\n",
    "        \n",
    "    for docID in docs_having_query_term:\n",
    "        \n",
    "        tf_doc = getReference[docID].count_words(query_term, 0)\n",
    "        tf_doc = 0.5 + 0.5*tf_doc/max_tf[docID]\n",
    "        tfidf_doc = (tf_doc)\n",
    "        \n",
    "        if docID not in scores:\n",
    "            scores[docID] = (tfidf_query * tfidf_doc)\n",
    "            wordsInDoc[docID] = []\n",
    "            bisect.insort(wordsInDoc[docID], [-tfidf_query * tfidf_doc, query_term])\n",
    "            factor[docID] = idf\n",
    "        else:\n",
    "            scores[docID] += (tfidf_query * tfidf_doc)\n",
    "            bisect.insort(wordsInDoc[docID], [-tfidf_query * tfidf_doc, query_term])\n",
    "            factor[docID] += idf\n",
    "            \n",
    "print(title_score)\n",
    "\n",
    "for docID in scores:\n",
    "    \n",
    "    #if documentLength[docID] != 0:\n",
    "    scores[docID] *= factor[docID]\n",
    "    if docID in title_score:\n",
    "        scores[docID] *= 1 + title_score[docID]\n",
    "\n",
    "sorted_scores = sorted(scores.items(), key = lambda kv : kv[1] , reverse = True)\n",
    "\n",
    "maxshow = min(10, len(scores))\n",
    "\n",
    "print('\\n\\n')\n",
    "print('============================================')\n",
    "\n",
    "for i in range(maxshow):\n",
    "    \n",
    "    print()\n",
    "    docID = sorted_scores[i][0]\n",
    "    print('doc ID = ', docID)\n",
    "    cnt = 0\n",
    "    print('Keywords:')\n",
    "    print()\n",
    "    print(subset[get_index[sorted_scores[i][0]]][1])\n",
    "    print()\n",
    "    if sorted_scores[i][0] not in title_score:\n",
    "        print('title score = ',0)\n",
    "    else:\n",
    "        print('title score = ',title_score[sorted_scores[i][0]])\n",
    "    for j in range(len(wordsInDoc[docID])):\n",
    "        print(wordsInDoc[docID][j][1], wordsInDoc[docID][j][0], end = ' ')\n",
    "        print(getReference[docID].count_words(wordsInDoc[docID][j][1], 0))\n",
    "    print()\n",
    "    print()\n",
    "    count = 0\n",
    "    found = 0\n",
    "    words_before=queue.Queue()\n",
    "    at_start = 1\n",
    "    display = \"\"\n",
    "    \n",
    "    for word in subset[get_index[docID]][4].split():\n",
    "            \n",
    "        check_with=replace_dates(word, -1)\n",
    "        check_with = check_with.lower()\n",
    "        if len(lemma_stop(check_with)) > 0:\n",
    "            check_with=lemma_stop(check_with)[0]\n",
    "        else:\n",
    "            check_with=word\n",
    "        \n",
    "        if check_with == wordsInDoc[docID][0][1]:\n",
    "            found=1\n",
    "            \n",
    "        if found == 1:\n",
    "            display = display + word + \" \"\n",
    "            count += 1\n",
    "            if count == 50:\n",
    "                break\n",
    "        if found == 0:\n",
    "            words_before.put(word)\n",
    "            if words_before.qsize()>20:\n",
    "                remove=words_before.get()\n",
    "                at_start=0\n",
    "                \n",
    "    if not at_start:\n",
    "        print('...', end = ' ')\n",
    "    while words_before.qsize() > 0:\n",
    "        print(words_before.get(), end = ' ')\n",
    "    print(display, end = ' ')\n",
    "    print('...', end = ' ')\n",
    "    print('\\n')\n",
    "    print('tf-idf score=', sorted_scores[i][1])\n",
    "    print('\\n')\n",
    "    print('============================================')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[16, '50/50'], [42, '1/2'], [47, '4/4'], [47, '4/5'], [47, '3/4'], [47, '4/5'], [67, '24/7'], [70, '50/50'], [104, '9/11'], [151, '9/11'], [6432, '9/11'], [6440, '9/11'], [6492, '9/11'], [6620, '12/8'], [6701, '24/7'], [6765, '24/7'], [6802, '9/11'], [6811, '2008/8'], [6812, '12/22'], [6919, '1/5000'], [6919, '1/11'], [6919, '1/150'], [6919, '1/750'], [6919, '1/5000'], [6977, '9/11'], [6979, '1/2'], [6981, '6/19'], [7078, '9/11'], [7127, '9/11'], [7313, '24/7'], [-1, '2/5']]\n"
     ]
    }
   ],
   "source": [
    "print((count_dates))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      The order came from the Sultan of Brunei’s nephew. It was 1994, and Prince Abdul Hakeem, then 20 years old, had inherited two things: access to a $40 billion fortune, and his family’s penchant for spending it.              The Secret Six         A half-dozen groundbreaking Ferraris were built for the Prince of Brunei.Only one made it out of the jungle aliveBy Ryan ZumMallen | Photography by Paul Barshon The order came from the Sultan of Brunei’s nephew. It was 1994, and Prince Abdul Hakeem, then 20 years old, had inherited two things: access to a $40 billion fortune, and his family’s penchant for spending it.In the ‘80s and ‘90s, the monarchs of this tiny, oil-rich nation in the South China Sea bankrolled a fleet of special custom vehicles, ordering the most coveted cars in the world by the half-dozen or more. Their notorious collection — located in nondescript concrete garages, surrounded by razor wire, and patrolled by armed Nepalese soldiers known as gurkhas — numbered in the thousands. Visitors described rows and rows of warehouses, filled to the brim with luxury cars, sometimes organized by make, model, or even color. In his online newsletter, Ferrari broker Michael Sheehan described one floor dedicated solely to black Mercedes SL500s, another to red Ferrari Testarossas, and yet another to bright yellow Bentleys and Lamborghinis. One particularly star-studded showroom included a Ferrari 288 GTO Evo, F40 LM, F50, and three McLaren F1s.Several of these cars could be found nowhere else in the worldSeveral of the cars in these warehouses were one-of-a-kind, built specifically for Brunei, and could be found nowhere else in the world. For his next order, Prince Hakeem again wanted something special: a Ferrari with a semi-automatic transmission. At the time, the technology didn’t exist on any street-legal vehicle in the world. That didn’t give him any pause. Prince Hakeem was used to getting what he wanted. According to a Fortune article published in 1999, as a teen Prince Hakeem had decided he wanted to learn to play American football, so the Royal Family paid Joe Montana and Herschel Walker to come to Brunei and teach him the game. No one was allowed to tackle the prince, who scored touchdowns every time he touched the ball. On his 18th birthday, he received $1 billion in cash.For a job like this, Prince Hakeem knew just who to turn to: legendary automotive design house Carozzeria Pininfarina, which had already completed one-off versions of Ferraris, Bentleys, Jaguars, and other historic marques for the Royal Family. It’s estimated that in less than a decade, Pininfarina had built more than 50 cars destined for Brunei.The assignment came to Paolo Garella, Pininfarina’s prototype manager. Garella’s team would take the Ferrari Testarossa, a pillar of automotive style and performance, and crank up the adrenaline. It was a tall order, but Garella had the experience, talent, and vision necessary for the job.What Garella’s team built would become one of the most mysterious and compelling supercars ever, one that would come to be known as the Ferrari FX.            Born and raised in Italy, Garella knew by age five that he would make cars his life’s work. After seeing the Ferrari Pinin concept — the first, and still the only four-door Ferrari ever produced — at the 1980 Torino Auto Show, he committed to becoming an engineer and creating one-of-a-kind designs. He quickly established an enviable career: in the late 1980s, Garella spent five years working at Goodyear, testing prototypes — like, as luck would have it, the Ferrari Testarossa — around its famous Circuit de Mireval in France. He then returned to Italy, developing hybrid and electric racecars for PTI Albatech. In 1992 Pininfarina came calling, and Garella became manager of the Prototype Manufacturing and Engineering Department, working on experimental, often top-secret projects. Photos from the ‘90s show him with the pushbroom mustache and wide, round glasses typical of the era — often, with a beaming smile. Garella was 35 when the FX project came to him, and uniquely qualified to make the prince’s dream car a reality.Pininfarina opted for the Ferrari Testarossa as the platform for the new car. With its supercar proportions and mid-engine layout, it was the premiere horse in the Ferrari stable. Having designed the original Testarossa in the early 1980s, Pininfarina was very familiar with the car. In fact, they had already been involved in a custom job based on the Testarossa, called the Ferrari Mythos, a drool-inducing prototype shown at the 1989 Tokyo Motor Show. Ferrari made it clear the Mythos was not for sale. Somehow, two cropped up in the Brunei warehouses anyhow.The only requirement, Garella says, was the semi-automatic transmissionWith the platform chosen, Garella and his team could turn to the task at hand. The only requirement, Garella says, was the semi-automatic transmission. Until then, supercars like the Testarossa almost exclusively featured five- or six-speed manuals. Semi-automatics came from the world of motorsports, where they began appearing in rally cars in the late 1980s, and Formula One cars in the early ‘90s. Instead of relying on the driver to depress the clutch, manually engage a new gear, and then release the clutch before accelerating, the semi-automatic used computers to execute shifts — with input from the driver via buttons, paddles, or a lever — in a fraction of the time. Built for quickness and nothing else, semi-automatic transmissions were far too jerky and unrefined for the road in the early 1990s. Pininfarina faced a tall order in adopting one for the FX.Pininfarina turned to British racing outfit Prodrive, a company that had made a name for itself with Subaru in the World Rally Championship in the early ‘90s. Prodrive was already in the process of gauging public interest in a semi-automatic transmission for the street that could be fitted directly to a manual gearbox buttons on the steering wheel would shift gears while an electronic control unit worked the clutch. Pininfarina saw an opportunity to collaborate, and assigned an engineer whose sole responsibility was working with Prodrive to adapt their system for the FX. Prodrive transmission was mated to the stock Ferrari six-speed manual gearbox already in the Testarossa that Garella had picked out. It wasn’t a perfect fit. While the clutch was hydraulic, engaging different gears was pneumatic. \"Prodrive had a system that was a little bit complex,\" says Garella. \"It had a number of problems and was not really straightforward.\"The transmission was going to need continued work, and in fact required constant attention from mechanics to be driveable at all. But it was the best option available at the time. The rest of the FX internals came together without much issue. Pininfarina moved the radiator from the back of the car to the front, to make room for the rear-mounted transmission’s additional electronics, and to capitalize on the increased airflow from the redesigned front end. The engine was tuned, and the exhaust was converted into a snarling straight Tubi system. The catalytic converters on the 4.9-liter V12 were removed, pushing its 420 horsepower northward. Everything else was left stock.With the engineering complete, Garella and Pininfarina turned to what they do best: the exterior design.The result was stunning, if polarizing. Though the deeply inset headlights bring to mind Camaros of the era, and quickly became dated, sculpted dual intakes bring an elegance to the aggressive slope of the front end. Massive, scooping hips presaged the Ferrari 360 Modena — designed, of course, by Pininfarina — that wouldn’t come out until 1999. At the rear, the car’s roofline flows down like cooling lava, smoothing out into a point. Quad round taillights from the Ferrari 355 punctuate a dark mesh grille. From the rear three-quarter perspective, the FX radiates aggression, and tremendous potential for speed. The transparent plastic cover over the mid-mounted engine has twelve portholes — one for each cylinder — that simultaneously invite and distort the view inside a Ferrari funhouse. A stylized \"FX\" logo on top of the engine makes it clear this is no run-of-the-mill Ferrari. It is the only such emblem on the car.Pininfarina tested the design in its on-site wind tunnel, and came away pleased. \"The car was definitely better than the Testarossa,\" says Garella. \"And I think it was a quite attractive design for the car at that time.\" Then-chairman Sergio Pininfarina personally showed the completed FX to Gianni Agnelli, the former head of Ferrari parent company Fiat, who simply said: \"We are lucky to have such clients.\"Pininfarina built the entire outer shell from carbon fiber, which is lighter than aluminum, stronger than steel, and was, at that time, unfathomably expensive. It was also difficult to work with, as very few companies had experience with the material. Pininfarina went through three suppliers before finding the quality necessary. The Pininfarina office in Germany originally constructed the prototypes, but after several mishaps and botched models, the work was brought back to the in-house design center in Cambiano, Italy. At one point, Pininfarina built an entire bonnet for the FX and sent it to Brunei, just to see how it reacted to the sun and humidity of the stifling South Asian climate an exercise that would prove prophetic The quality gradually improved. \"We worked a lot to have perfect surfaces,\" says Garella. \"At the end, [it] was the best carbon fibre body Brunei had.\"\"We are lucky to have such clients.\"Pininfarina completed the FX in late 1995, with the transmission being its crowning — but still problematic — achievement. It was unlike anything done before.Pininfarina built three FXs, and continued experimenting with the Prodrive transmission, adding it to a couple Ferrari F355s – more than a year before Ferrari made its F355 F1 the first production car with a semi-automatic transmission, in 1997. Pininfarina even put one in another Brunei project, the equally evasive Jaguar XJ220 known internally as \"The J.\" Two were built: one caught fire and was destroyed, the other’s location is simply unknown.            Delivering a car to the royal family of Brunei isn’t as simple as rolling up in a transporter. Pininfarina often loaded the cars into a massive Antanov aircraft, then sent them to Brunei unfinished and had crews complete the work on site. The cars needed to be perfect before the Royal Family saw them.Occasionally, this led to panic-inducing deadlines. The team once flew a custom Bentley B2 to Brunei the night of a birthday party for Prince Jefri, Hakeem’s father. Condensation from the flight made the car soaking wet, and many of the electronics weren’t working. The car had to be hand-dried, and delivered  to the palace in the nick of time. Hours later, though, Garella got a call that the steering wheel wouldn’t line up perfectly straight. The team pulled the wheel apart and were still working when Prince Hakeem suddenly walked outside to present the car to his father and the entire court. Garella hid behind a pillar — the Royal Family was not pleased to find their newest vehicle in pieces.Around 3AM, Brunei officials seized their passports, effectively holding them hostageAnother time, converting a Ferrari 288 GTO to right-hand drive had somehow caused the turn signal indicator to stop automatically returning. The team began repairing the problem at 5PM and as time went on, their hosts grew impatient. Around 3AM, Brunei officials seized their passports, effectively holding them hostage. Garella and the team stayed up to finish the work, and reclaimed their passports around 7AM. \"I think still that this was the longest night of my life,\" he says. \"Yes it was stressful but also fantastic. I love it, for an engineer to have the possibility to do a project with an almost unlimited budget is a lifetime dream… and I was there.\"To Garella’s relief, there weren’t any such \"difficulties\" with the FX. In early 1996, Garella and a select few associates made the 6,800-mile trek to Brunei to deliver the three custom Ferraris during the height of the Royal Family’s car-buying frenzy, he made this trip every other month The cars came off the plane ready to be presented, and Prince Hakeem was very pleased with his new toys. Satisfied with the progress of the build months prior, Hakeem had ordered three more FXs this time, based on the newly-released F512M, a lighter, faster platform that would come with an extra 20 horsepower, a lightweight crankshaft, and titanium connecting rods. Not only that, but the family decided to send the new cars to Oxfordshire, England-based Williams F1, one of the most successful teams in Formula One history, in order to better tune and refine the complicated transmission.Five of Garella’s FXs were painted the same familiar Ferrari red the other, FX No. 4, was painted a deep sapphire blue. It’s the only FX that actually made it to Williams, and the only FX that escaped the decaying garages of Brunei.Once it had possession of the car, Williams used a team from their F1 arm to build the new and improved transmission system in FX No. 4. They replaced the Prodrive components, taking special care to remove all Prodrive branding. Williams plugged in the semi-automatic unit they had been using in Formula One since 1992, and ran immense amounts of wiring through the car. They also replaced the buttons on the steering wheel with paddles. The paddles in the FX appear primitive by today’s standards: they are made of metal, with red on the left for down shifts, and green on the right for up. But it was shaping up to be a marked improvement over the first FX.The transmission was still being tuned when the Asian markets crashed.            In the summer of 1997, the free fall of the Thai baht sparked a domino effect that spread across the Asian markets, and quickly set off a global financial meltdown. Buoyed by its vast oil reserves, Brunei emerged relatively unscathed. The following year, however, a company called Amedeo Development Corporation — owned and operated by Prince Hakeem’s father, Jefri — collapsed, leaving behind $6 billion in debt. Prince Jefri’s brother, the Sultan, ordered a review of the country’s financials Vanity Fair once reported that 200 accountants were assigned to the case and found Jefri had raided some $14 billion of Brunei’s oil funds as his personal piggy bank, using Amedeo to fund extravagant public and personal projects.Jefri’s exploits during this fantastic spending spree have since become the stuff of legend. His family of four wives and at least 18 children lived lavishly across London, Paris, Los Angeles, and elsewhere. He commissioned life-sized bronze sculptures of couples having sex, just to offend his more-conservative brother Mohamed. He owned a 152-foot yacht named \"Tits,\" and called the lifeboats \"Nipple 1\" and \"Nipple 2.\" Court documents would reveal that Jefri ultimately spent some $78 million with Pininfarina alone. Some estimates pegged the total number of cars in his priceless collection at more than 2,500 — others, closer to 5,000.\"Other heads of state and wealthy people have had impressive collections of bespoke cars, but I am pretty sure the Royal Family of Brunei is unparalleled in this regard,\" says Ed Kim, vice president of industry analysis for Auto Pacific. \"Certainly nothing like what they were doing back then is happening now.\"Though Jefri was the chief financier of the collection, his brother the Sultan, who once held the title of richest man alive, certainly has his own excessive and frightening habits: he owns and regularly pilots a Boeing 747 with a gold and crystal interior, and participated with Jefri in a sex harem that included dozens of women. Some of these women were models and actresses that later claimed they were lured into joining, and sued the brothers. It would be naïve to suggest the Sultan was unaware of his family’s obscene spending. Garella personally witnessed private concerts by the likes of The Gipsy Kings and Sting on palace grounds. Michael Jackson once performed in Brunei for a fee of $17 million, in a stadium built specifically for the event.Williams made a phone call. American businessman Dick Marconi picked upStill, the Sultan has held his throne for nearly 50 years, and successfully distanced himself from the global scorn that forced Jefri to flee the country. The Sultan removed Jefri from his positions as Brunei’s minister of finance, and head of the Brunei Investment Agency BIA that housed the oil funds that Jefri had likely been using to keep Amedeo afloat. In 2000, Prince Jefri turned over his assets. Meanwhile, the prince’s groundbreaking Ferrari sat in a British workshop, suddenly with no owner.Williams made a phone call. American businessman Dick Marconi picked up. About $1.3 million and 15 years later, I’m staring at my reflection in the indigo carbon fiber weave of Ferrari FX No. 4 in Tustin, California.            \"This is Mr. Marconi’s baby,\" says Marconi Museum operations director Todd Ottosen, more than once, as he carefully lowers the entire back section of the FX. Before closing it, he makes a show of effortlessly lifting the carbon fiber engine bay up and down, with two fingers. \"Pretty cool, huh You can’t even do that with the F40.\"The Marconi Museum is located inside an unassuming warehouse in a Southern California business park and houses one of the most eclectic, star-studded car collections you could conjure up. Ferraris from every era are scattered about, along with icons like the DeLorean DMC-12 and Lamborghini Countach, and ultra-rare finds like the Cizeta V16T and Aston Martin DB AR1 Zagato. Several retired Indy and Formula One racecars are parked on the floor others, like the McLaren MP4/4 that Ayrton Senna drove during his 1988 championship season, hang vertically from the walls like 200-mile-per-hour drapes.The museum is the passion project gone wild of Dick Marconi, a co-founder of Herbalife, who has used his fortune to bring automotive brilliance to the public. Admission is free — there’s a suggested $5 donation that benefits the museum’s childrens foundation This is where Ferrari FX No. 4 has spent the majority of its life, occupying a space between the AR1 and another Pininfarina creation for the Royal Family: a forest-green convertible based on the Ferrari 456 four-seater, called the Venice like the FX, there are several of these still in Brunei, possibly dozens if you count the custom sedan and wagon versions Though it may be a relatively pedestrian existence for the FX, one of the most compelling automotive mysteries in a world full of them, the car’s allure is still strong. Rumor has it Marconi has repeatedly turned down offers to sell the car for no less than $10 million.Automotive journalist Winston Goodfellow is one of the few people to have driven the FX outside of Marconi Museum mechanics, when he reviewed the car for an issue of FORZA Magazine in 2001.In his article, Goodfellow raved that the low weight and advanced technology in the FX made it a thrill to drive: \"While this Ferrari pulls with more alacrity than any TR, what makes the FX truly extraordinary when run hard through the gears is the way the engine works in concert with the transmission: it delivers an excitement that the heavier, production TRs — or any 355 or 360 — can’t touch.\"\"One thing I will say, that does still linger, is it remains one of the coolest things I’ve driven.\"Fifteen years after that drive, the experience of the Williams transmission is still clear in his mind. \"One thing I will say, that does still linger, is it remains one of the coolest things I’ve driven,\" he remembers. Goodfellow’s impressions may provide the clearest perspective to the car’s potential, and that of the Williams semi-automatic system, that we’ll ever have.\"You could feel it going through the motions of what it did, even though it was all paddles. That gave it this perfect blend of coming ‘digital’ couple with beautiful analog sensations. It had a unique taste that was most appealing, particularly that howling exhaust.\"Some of the cars at the Marconi Museum get out more than others. The FX, for example, made a couple of appearances in local car shows during the 2000s, but has otherwise been a homebody. On a routine test drive about two years ago, the FX limped back to the museum spewing white smoke, and hasn’t run since. In total, FX No. 4 has approximately 1,800 miles on the odometer. If they still function — a big if —  it’s likely that the five other FXs have rarely or more likely, never been driven at all.Few have seen it fewer still have seen what it’s capable of. Vincent Marconi, a 26-year-old mechanic, who also happens to be Dick’s grandson, has seen the FX’s guts spewed out all over the floor. Last year he and Larry Englehart, owner of Ferrari Technology in nearby Orange, began the long journey of making sense of the transmission, and getting the car running again.\"It’s still a work in process, and it’s a slow process,\" says Vincent. \"It’s a combination of F1 and space technology, and there are no schematics.\"Vincent describes a system truly ahead of its time. The gearbox is the stock six-speed from the F512M, with small modifications to the gears and differential. The attached Prodrive semi-automatic system features fascinating technologies, like an early version of automatic rev-matching to prevent power loss between shifts. Prodrive appears to have never brought its innovative transmission to market, unable to smooth out the wrinkles.Even now, it will likely take a Williams technician visiting the car again to get the computer systems ready to run. Vincent Marconi and Englehart plan to get back around to finishing the project one day, but the museum has lots of cars that need attention and the FX isn’t very high on the priority list. Still, the payoff is enticing.\"It works bitching once everything is rocking and rolling,\" says Vincent lt function   var l = function     new py m Parent      'ferrari-photo-gallery__graphic',      '\"m-snippet thin gt Gar ella now 57, resembles his younger self, though the glasses are thinner and the mustache is grayer. He remains a force in innovative design and construction. He spent 17 years at Pininfarina, the final seven as Head of Special Projects. In recent years, he’s been the invisible hand behind two of the most celebrated one-off designs of the past decade. The first is the Pininfarina New Stratos, a modern interpretation of the famous, wedge-shaped Lancia Stratos.The other is the Ferrari P4/5 by Pininfarina, commissioned by the filmmaker Jim Glickenhaus. It’s built on a Ferrari Enzo but completely redesigned to pay homage to the legendary 1967 Ferrari 330 P3/4. Garella would also become Project Manager for the company’s associated racing program, designing the P4/5 Competizione, which was built on a Ferrari F430 Scuderia and homologated to race in the Nurburgring 24 Hours. In 2012, it placed 12th overall and won its class. Last year, the team debuted and raced the SCG 003, a new variation that shattered the lap record at The Ring and fielded two entries in the 24-hour race. All of the Glickenhaus racing cars are street-legal, and compete with New York license plates. Another one-off project is underway, but that’s all Garella can say. In an email, Glickenhaus described Garella as being \"Not fazed by problems and willing to work until they are solved Still the FX holds a special place on his long list of accomplishments. Garella often visits the US to attend events like the Pebble Beach Concours d’Elegance in Monterey, and in 2007, he looked up the Marconi Museum and made the trip down the coast to Tustin. Dick Marconi was waiting for him, and the two posed for pictures with the FX — a legend created by one and saved by the other. The car wasn’t running then, but its futuristic carbon fiber was still there as were its eye-popping design, and its embattled but ambitious transmission. Its advancements in design, construction, and technology make the FX a sort of automotive missing link.\"The FX was definitely ahead of its time for having a semi-automatic gearbox,\" says Kim.\"The Sultan and his brother really were RD for many high-end automakers. They were building custom bespoke cars that not only had unique bodies and interiors, but sometimes exotic technologies. And the money was limitless.\"            Garella maintains that the true masterpiece of the FX is its body. Not for the design, but for the materials. Today, carbon fiber technology is commonplace in supercars, and even many sports cars. BMW now uses carbon fiber reinforced plastic C FRP for both the frame and the body of their i3, as well as multiple parts of the new 7 Series luxury sedan. The FX helped set the stage for a new movement in mass production as automakers aim for weight savings to improve performance and fuel economy.\"And the money was limitless.\"\"For me personally, and I think also for Pininfarina, it was the project in which we learned most about carbon fiber and the composite process,\" he says. \"I think for sure that was the project where we brought back the maximum know-how at that point.\"Never intended for public consumption, the FX led to the adoption of many new ideas that are common practice in the cars that drive our streets every day. For now, it waits in the museum for an overhaul that may never come, in exile from those that commissioned it the product of a man who poured years into making it a reality and would use it to influence his trend-setting creations in the future. When he visited the Marconi Museum, the sight of the FX was a welcome one for Garella.\"Prince Hakeem wanted to have all the cars with semi-auto transmission,\" says Garella. \"He thought that this was the future.\"He was right.\"* *Design by James BarehamProduct by Frank BiEdited by Michael Zelenko and Chris Ziegler&lt!--tryTypekit.load( async: true })}catch(e)}// gt\n"
     ]
    }
   ],
   "source": [
    "print(subset[get_index[47]][4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[''], ['']]\n"
     ]
    }
   ],
   "source": [
    "a = [['aaaa'],['bbbb']]\n",
    "for document in a:\n",
    "    document[0] = ''\n",
    "    document = []\n",
    "print(a)"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "preprocessing.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}