WordNet Similarities and MSRPCorpus Serialization.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.corpus import wordnet as wn\n",
    "import string\n",
    "import itertools\n",
    "from nltk.corpus import wordnet_ic\n",
    "import codecs\n",
    "import pickle\n",
    "import Stemmer\n",
    "import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def load_MSRP_corpus(path, ftype):\n",
    "    \"\"\"Returns the tuples in the MSRP Corus, which has the following format\n",
    "       class   #1_ID   #2_ID   #1_String   #2_String\n",
    "    \"\"\"\n",
    "    if ftype == \"train\":\n",
    "        fpath = (path+\"msr_paraphrase_train.txt\"\n",
    "                 if path[-1] == '/' or path[-1] == '\\\\'\n",
    "                 else path+\"/msr_paraphrase_train.txt\")\n",
    "    elif ftype == \"test\":\n",
    "        fpath = (path+\"msr_paraphrase_test.txt\"\n",
    "                 if path[-1] == '/' or path[-1] == '\\\\'\n",
    "                 else path+\"/msr_paraphrase_test.txt\")\n",
    "    pairs = []\n",
    "    with codecs.open(fpath, 'r', encoding='utf-8') as fid:\n",
    "        for line in list(fid)[1:]:\n",
    "            pairs.append(line.split('\\t'))\n",
    "    return pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def word_tokenizer(text, stemmer, flat_output=True):\n",
    "    \"\"\"Returns the text divided by sentences and tokenized.\n",
    "       If flat_output is True, returns \n",
    "       The format of the results is (token, lemma, stem, tag)\"\"\"\n",
    "    # Methods for sentence splitting, tokenization and lemmatization\n",
    "    sent_detector = nltk.data.load('tokenizers/punkt/english.pickle')\n",
    "    word_detector = nltk.TreebankWordTokenizer()\n",
    "    wnl = nltk.stem.WordNetLemmatizer()\n",
    "    tags_mapping = {\"NN\":wn.NOUN, \"VB\":wn.VERB, \"JJ\":wn.ADJ, \"RB\":wn.ADV}\n",
    "    \n",
    "    # Text processing\n",
    "    output = []\n",
    "    for sent in sent_detector.tokenize(text.lower()):\n",
    "        tokens = word_detector.tokenize(sent)\n",
    "        tmp = []\n",
    "        for token, tag in nltk.pos_tag(tokens):\n",
    "            wn_pos = tags_mapping.get(tag[:2], None)\n",
    "            if wn_pos:\n",
    "                tmp.append((token, wnl.lemmatize(token, pos=wn_pos), stemmer.stemWord(token), tag))\n",
    "            else:\n",
    "                tmp.append((token, token, stemmer.stemWord(token), tag))\n",
    "        output.append(tmp)\n",
    "    \n",
    "    if flat_output:\n",
    "        return list(itertools.chain(*output))\n",
    "    return output"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Parsing training instances\n",
    "We parse both sentences in each instance of the MSPR training dataset in the format (token, lemma, POS_tag). It might be divided in sentences as well depending on the flag **flat_output** being false.\n",
    "\n",
    "tags_mapping maps Treebank POS tags to WordNet POS tags."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "stemmer = Stemmer.Stemmer('english')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "raw_data = load_MSRP_corpus(path=\"D:/Corpus/MSPC\", ftype=\"train\")\n",
    "\n",
    "parsed_texts = {}\n",
    "train_pairs = []\n",
    "train_y = []\n",
    "for _class, id1, id2, text1, text2 in raw_data:\n",
    "    if id1 not in parsed_texts:\n",
    "        parsed_texts[int(id1)] = word_tokenizer(text1, stemmer=stemmer, flat_output=True)\n",
    "    if id2 not in parsed_texts:\n",
    "        parsed_texts[int(id2)] = word_tokenizer(text2, stemmer=stemmer, flat_output=True)\n",
    "    train_pairs.append((int(id1),int(id2)))\n",
    "    train_y.append(int(_class))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Parsing test instances\n",
    "We parse both sentences in each instance of the MSPR test dataset in the format (token, lemma, POS_tag). It might be divided in sentences as well depending on the flag **flat_output** being false."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "raw_data = load_MSRP_corpus(path=\"D:/Corpus/MSPC\", ftype=\"test\")\n",
    "\n",
    "test_pairs = []\n",
    "test_y = []\n",
    "for _class, id1, id2, text1, text2 in raw_data:\n",
    "    if id1 not in parsed_texts:\n",
    "        parsed_texts[int(id1)] = word_tokenizer(text1, stemmer=stemmer, flat_output=True)\n",
    "    if id2 not in parsed_texts:\n",
    "        parsed_texts[int(id2)] = word_tokenizer(text2, stemmer=stemmer, flat_output=True)\n",
    "    test_pairs.append((int(id1),int(id2)))\n",
    "    test_y.append(int(_class))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Generating vocabulary\n",
    "The vocabulary is generated taking into account the lemma and wordnet tag (if possible).\n",
    "\n",
    "Here, we can perform several experiments like taking the lemma and regular POS tag, only taking the tokens, etc.\n",
    "\n",
    "**Notes:**\n",
    "- The code depends on the format used in parsed_texts, assuming **flat_output=True**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18003 tokens in the vocabulary of tokens\n",
      "15224 tokens in the vocabulary of lemmas\n",
      "13801 tokens in the vocabulary of stems\n",
      "18168 tokens in the vocabulary of lemmas+pos\n"
     ]
    }
   ],
   "source": [
    "tags_mapping = {\"NN\":wn.NOUN, \"VB\":wn.VERB, \"JJ\":wn.ADJ, \"RB\":wn.ADV}\n",
    "\n",
    "# itertools.chain(*parsed_texts.values()) converts all documents into one single list\n",
    "vocab_tokens = nltk.FreqDist([token\n",
    "                              for token, lemma, stem, tag\n",
    "                              in itertools.chain(*parsed_texts.values())])\n",
    "\n",
    "vocab_lemmas = nltk.FreqDist([lemma\n",
    "                              for token, lemma, stem, tag\n",
    "                              in itertools.chain(*parsed_texts.values())])\n",
    "\n",
    "vocab_stems = nltk.FreqDist([stem\n",
    "                             for token, lemma, stem, tag\n",
    "                             in itertools.chain(*parsed_texts.values())])\n",
    "\n",
    "vocab_lemmapos = nltk.FreqDist([(lemma, tags_mapping.get(tag[:2], tag))\n",
    "                                for token, lemma, stem, tag\n",
    "                                in itertools.chain(*parsed_texts.values())])\n",
    "\n",
    "print(len(vocab_tokens), \"tokens in the vocabulary of tokens\")\n",
    "print(len(vocab_lemmas), \"tokens in the vocabulary of lemmas\")\n",
    "print(len(vocab_stems), \"tokens in the vocabulary of stems\")\n",
    "print(len(vocab_lemmapos), \"tokens in the vocabulary of lemmas+pos\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "index2token = list(vocab_tokens.keys())\n",
    "token2index = dict([(token,i) for i, token in enumerate(index2token)])\n",
    "\n",
    "index2lemma = list(vocab_lemmas.keys())\n",
    "lemma2index = dict([(lemma,i) for i, lemma in enumerate(index2lemma)])\n",
    "\n",
    "index2stem = list(vocab_stems.keys())\n",
    "stem2index = dict([(stem,i) for i, stem in enumerate(index2stem)])\n",
    "\n",
    "index2lemmapos = list(vocab_lemmapos.keys())\n",
    "lemmapos2index = dict([(lemmapos,i) for i, lemmapos in enumerate(index2lemmapos)])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "#### Serializing all MSRPC and needed data structures\n",
    "We serialize the Microsoft Research Paraphrase Corpus with the format:\n",
    "\n",
    "    [parsed_texts, train_pairs, train_y, test_pairs, test_y]\n",
    "   \n",
    "The goal of serializing all of these is to make experiments with the parsed corpus used to compute the WordNet similarity measures.\n",
    "\n",
    "<parsed_texts> contains all the documents from the MSRPC tokenized. It is a dictionary where the keys are the id of the documents and the values are lists of the following tuples (token, lemma, POS tag)\n",
    "\n",
    "We also serialize the vocabularies for different features (tokens, lemmas, stems, and lemmas+pos) and the data structure needed to convert from indexes to the given feature and viceversa.\n",
    "\n",
    "    [vocab, token_to_index, index_to_token]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "date_obj = datetime.date.today()\n",
    "date_str = \"{:04d}\".format(date_obj.year) + \"{:02d}\".format(date_obj.month) + \"{:02d}\".format(date_obj.day)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pickle.dump([parsed_texts, train_pairs, train_y, test_pairs, test_y],\n",
    "            open(\"msrpc_parsed_\"+date_str+\".pickle\", \"wb\"),\n",
    "            protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pickle.dump([vocab_tokens, token2index, index2token],\n",
    "            open(\"tokens_data_\"+date_str+\".pickle\", \"wb\"),\n",
    "            protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "pickle.dump([vocab_lemmas, lemma2index, index2lemma],\n",
    "            open(\"lemmas_data_\"+date_str+\".pickle\", \"wb\"),\n",
    "            protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "pickle.dump([vocab_stems, stem2index, index2stem],\n",
    "            open(\"stems_data_\"+date_str+\".pickle\", \"wb\"),\n",
    "            protocol=pickle.HIGHEST_PROTOCOL)\n",
    "\n",
    "pickle.dump([vocab_lemmapos, lemmapos2index, index2lemmapos],\n",
    "            open(\"lemmapos_data_\"+date_str+\".pickle\", \"wb\"),\n",
    "            protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Computing some Information Content (IC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "brown_ic = wordnet_ic.ic('ic-brown.dat')\n",
    "semcor_ic = wordnet_ic.ic('ic-semcor.dat')\n",
    "bnc_ic_2000 = wordnet_ic.ic('ic-bnc.dat')\n",
    "bnc_ic_2007 = wordnet_ic.ic('ic-bnc-2007.dat')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Note:** This code is not in use!\n",
    "\n",
    "***Warning*** Computing BNC Information Content (IC) takes hours!!!\n",
    "\n",
    "There is a version of the IC for the BNC corpus. However it was computed using the 2000 version of BNC. The corpus we are computing here is the version of 2007. The similiarty measures using these two IC are different."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "reader_bnc = nltk.corpus.reader.BNCCorpusReader(root='D:/Corpus/2554/2554/download/Texts/', fileids=r'[A-K]/\\w*/\\w*\\.xml')\n",
    "bnc_ic = wn.ic(reader_bnc, False, 0.0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "def is_root(synset_x):\n",
    "    if synset_x.root_hypernyms()[0] == synset_x:\n",
    "        return True\n",
    "    return False\n",
    "\n",
    "def generate_ic_file(IC, output_filename):\n",
    "    \"\"\"Dump in output_filename the IC counts.\n",
    "    The expected format of IC is a dict \n",
    "    {'v':defaultdict, 'n':defaultdict, 'a':defaultdict, 'r':defaultdict}\"\"\"\n",
    "    with codecs.open(output_filename, 'w', encoding='utf-8') as fid:\n",
    "        # Hash code of WordNet 3.0\n",
    "        fid.write(\"wnver::eOS9lXC6GvMWznF1wkZofDdtbBU\"+\"\\n\")\n",
    "        \n",
    "        # We only stored nouns and verbs because those are the only POS tags\n",
    "        # supported by wordnet.ic() function\n",
    "        for tag_type in ['v', 'n']:#IC:\n",
    "            for key, value in IC[tag_type].items():\n",
    "                if key != 0:\n",
    "                    synset_x = wn.of2ss(of=\"{:08d}\".format(key)+tag_type)\n",
    "                    if is_root(synset_x):\n",
    "                        fid.write(str(key)+tag_type+\" \"+str(value)+\" ROOT\\n\")\n",
    "                    else:\n",
    "                        fid.write(str(key)+tag_type+\" \"+str(value)+\"\\n\")\n",
    "    print(\"Done\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "generate_ic_file(bnc_ic, \"C:/Users/MiguelAngel/AppData/Roaming/nltk_data/corpora/wordnet_ic/ic-bnc-2007.dat\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Examples applying WordNet similarity measures and using different IC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[Synset('car.n.01'), Synset('car.n.02'), Synset('car.n.03'), Synset('car.n.04'), Synset('cable_car.n.01')]\n"
     ]
    }
   ],
   "source": [
    "syns = wn.synsets(lemma=\"car\", pos='n')\n",
    "print(syns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Synset('car.n.01')"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "syns[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def nan_default(result):\n",
    "    if result:\n",
    "        return result\n",
    "    return 0.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Path Similarity:                         0.20\n",
      "Leacock-Chodorow Similarity:             2.03\n",
      "Wu-Palmer Similarity:                    0.73\n",
      "Resnik Similarity (brown_ic):            6.45\n",
      "Resnik Similarity (semcor_ic):           6.31\n",
      "Resnik Similarity (bnc_ic):              6.39\n",
      "Jiang-Conrath Similarity (brown_ic):     0.36\n",
      "Jiang-Conrath Similarity (semcor_ic):    0.27\n",
      "Jiang-Conrath Similarity (bnc_ic):       0.36\n",
      "Lin Similarity (brown_ic):               0.82\n",
      "Lin Similarity (semcor_ic):              0.77\n",
      "Lin Similarity (bnc_ic):                 0.82\n"
     ]
    }
   ],
   "source": [
    "print(\"{:40} {:.2f}\".format(\"Path Similarity:\", nan_default(wn.path_similarity(syns[0], syns[1]))))\n",
    "\n",
    "print(\"{:40} {:.2f}\".format(\"Leacock-Chodorow Similarity:\", nan_default(wn.lch_similarity(syns[0], syns[1]))))\n",
    "\n",
    "print(\"{:40} {:.2f}\".format(\"Wu-Palmer Similarity:\", nan_default(wn.wup_similarity(syns[0], syns[1]))))\n",
    "\n",
    "print(\"{:40} {:.2f}\".format(\"Resnik Similarity (brown_ic):\", nan_default(wn.res_similarity(syns[0], syns[1], brown_ic))))\n",
    "print(\"{:40} {:.2f}\".format(\"Resnik Similarity (semcor_ic):\", nan_default(wn.res_similarity(syns[0], syns[1], semcor_ic))))\n",
    "print(\"{:40} {:.2f}\".format(\"Resnik Similarity (bnc_ic):\", nan_default(wn.res_similarity(syns[0], syns[1], bnc_ic_2007))))\n",
    "\n",
    "print(\"{:40} {:.2f}\".format(\"Jiang-Conrath Similarity (brown_ic):\", nan_default(wn.jcn_similarity(syns[0], syns[1], brown_ic))))\n",
    "print(\"{:40} {:.2f}\".format(\"Jiang-Conrath Similarity (semcor_ic):\", nan_default(wn.jcn_similarity(syns[0], syns[1], semcor_ic))))\n",
    "print(\"{:40} {:.2f}\".format(\"Jiang-Conrath Similarity (bnc_ic):\", nan_default(wn.jcn_similarity(syns[0], syns[1], bnc_ic_2007))))\n",
    "\n",
    "print(\"{:40} {:.2f}\".format(\"Lin Similarity (brown_ic):\", nan_default(wn.lin_similarity(syns[0], syns[1], brown_ic))))\n",
    "print(\"{:40} {:.2f}\".format(\"Lin Similarity (semcor_ic):\", nan_default(wn.lin_similarity(syns[0], syns[1], semcor_ic))))\n",
    "print(\"{:40} {:.2f}\".format(\"Lin Similarity (bnc_ic):\", nan_default(wn.lin_similarity(syns[0], syns[1], bnc_ic_2007))))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Computing all the similarities needed in the MSRPC\n",
    "We only care about the following POS tags: NN, VB, RB, JJ. Note: The WordNet authors divide the adjectives in two types: ('a' - Adjevtive) and ('s' - Adjective Satellite)\n",
    "\n",
    "The dictionary *tags_mapping** maps Treebank POS to WordNet POS.\n",
    "\n",
    "We compute the similarities for all words in phrase1 to all words in phrase2 that have the same POS tag. Also, for each pair of words we compute the similarity between all the words synsents. During testing we can decide if we use the first synset or the max similarity between all synsets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def wordnet_similarity(s1, s2, metric=\"Path\", ic=None):\n",
    "    # Computing the Path similarity\n",
    "    # This measure ranges between 0 and 1, not normalization needed\n",
    "    if metric == \"path\":\n",
    "        if s1.pos() == s2.pos():\n",
    "            # path_similarity returns None if no connecting path could be found\n",
    "            # between s1 and s2. We convert None to 0.0\n",
    "            return nan_default(wn.path_similarity(s1, s2))\n",
    "        else:\n",
    "            return 0.0\n",
    "    \n",
    "    # Computing the Leacock-Chodorow similarity\n",
    "    # This measure range is unknown, normalization needed!!!**********\n",
    "    if metric == \"lch\":\n",
    "        if s1.pos() == s2.pos():\n",
    "            # path_similarity returns None if no connecting path could be found\n",
    "            # between s1 and s2. We convert None to 0.0\n",
    "            return nan_default(wn.lch_similarity(s1, s2))\n",
    "        else:\n",
    "            return 0.0\n",
    "    \n",
    "    # Computing the Wu-Palmer similarity\n",
    "    # This measure range is unknown, normalization needed!!!**********\n",
    "    # path_similarity returns None if no connecting path could be found\n",
    "    # between s1 and s2. We convert None to 0.0\n",
    "    if metric == \"wup\":\n",
    "        if s1.pos() == s2.pos():\n",
    "            return nan_default(wn.wup_similarity(s1, s2))\n",
    "        else:\n",
    "            return 0.0\n",
    "    \n",
    "    # Computing the Resnik similarity\n",
    "    # This measure range is unknown, normalization needed!!!**********\n",
    "    # Only comparing verbs and nouns because those are the only keys in IC\n",
    "    if metric == \"res\":\n",
    "        if s1.pos() == s2.pos() and s1.pos() in ['v', 'n']:\n",
    "            return wn.res_similarity(s1, s2, ic)\n",
    "        else:\n",
    "            return 0.0\n",
    "    \n",
    "    # Computing the Jiang-Conrath similarity\n",
    "    # This measure range is unknown, normalization needed!!!**********\n",
    "    # Only comparing verbs and nouns because those are the only keys in IC\n",
    "    if metric == \"jcn\":\n",
    "        if s1.pos() == s2.pos() and s1.pos() in ['v', 'n']:\n",
    "            return wn.jcn_similarity(s1, s2, ic)\n",
    "        else:\n",
    "            return 0.0\n",
    "    \n",
    "    # Computing the Lin similarity\n",
    "    # This measure range is unknown, normalization needed!!!**********\n",
    "    # Only comparing verbs and nouns because those are the only keys in IC\n",
    "    if metric == \"lin\":\n",
    "        try:\n",
    "            if s1 == s2:\n",
    "                return 1.0\n",
    "            if s1.pos() == s2.pos() and s1.pos() in ['v', 'n']:\n",
    "                return wn.lin_similarity(s1, s2, ic)\n",
    "            else:\n",
    "                return 0.0\n",
    "        except:\n",
    "            print(\"Synsets causing the exception\")\n",
    "            print(s1, s2)\n",
    "            raise\n",
    "    \n",
    "    print(\"No available metric selected. Raising an error.\")\n",
    "    raise\n",
    "    return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def compute_msrpc_wordnet_sims_tokens(pairs, vocab, token2index, tags_mapping,\n",
    "                                      sims_first_synset, sims_all_synsets, metric, ic):\n",
    "    \"\"\"Compute the WordNet <metric> similarity for all the pairs of tokens needed in the MSRPCorpus\n",
    "    using only the given tokens to extract the wordnet synsets\"\"\"\n",
    "    count_first = 0\n",
    "    count_max = 0\n",
    "\n",
    "    # For each pair of phrase in training dataset\n",
    "    for id1, id2 in pairs:\n",
    "        # For each token in first phrase\n",
    "        for token1, lemma1, stem1, tag1 in parsed_texts[id1]:\n",
    "            token1_id = token2index.get(token1, None)\n",
    "            if token1_id == None:\n",
    "                print(\"Error, token1 not in vocabulary\", token1)\n",
    "                raise\n",
    "            # Extract the synsets from first phrase tokens\n",
    "            syns_tk1 = wn.synsets(lemma=token1)\n",
    "            # Compare to each token in phrase 2\n",
    "            for token2, lemma2, stem2, tag2 in parsed_texts[id2]:\n",
    "                # index of token2 in the vocabulary\n",
    "                token2_id = token2index.get(token2, None)\n",
    "                if token2_id == None:\n",
    "                    print(\"Error, token2 not in vocabulary\")\n",
    "                    raise\n",
    "                # Extract the synsets from second prhase tokens\n",
    "                syns_tk2 = wn.synsets(lemma=token2)\n",
    "\n",
    "                # Validating we have not computed this similarity before\n",
    "                # *****************\n",
    "                # *** IMPORTANT *** We need to apply the same method when retrieving similarity scores***\n",
    "                # *****************\n",
    "                sim_key = (token1_id, token2_id) if token1_id > token2_id else (token2_id, token1_id)\n",
    "                # Must be different tokens\n",
    "                if sim_key[0] != sim_key[1] and len(syns_tk1)*len(syns_tk2) != 0 and sim_key not in sims_all_synsets:\n",
    "                    all_sims = []\n",
    "                    #sims_first_synset[sim_key] = 0.0\n",
    "                    #sims_all_synsets[sim_key] = 0.0\n",
    "                    for s1 in syns_tk1:\n",
    "                        for s2 in syns_tk2:\n",
    "                            # The similarity must be different of None and Zero\n",
    "                            res = wordnet_similarity(s1, s2, metric=metric, ic=ic)\n",
    "                            if res != 0.0:\n",
    "                                all_sims.append(res)\n",
    "                    if len(all_sims) > 0:\n",
    "                        sims_first_synset[sim_key] = all_sims[0]\n",
    "                        sims_all_synsets[sim_key] = max(all_sims)\n",
    "                        count_max += len(syns_tk1)*len(syns_tk2)\n",
    "                        count_first += 0 if len(syns_tk1)*len(syns_tk2) == 0 else 1\n",
    "        # For debugging. Stopping early.\n",
    "        #if count_first > 1:\n",
    "        #    break\n",
    "        # Finish debugging\n",
    "    print(count_first, \"new token pairs computed.\")\n",
    "    print(count_max, \"Wordnet similarity computed.\")\n",
    "    return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def compute_msrpc_wordnet_sims_lemmas(pairs, vocab, lemma2index, tags_mapping,\n",
    "                                      sims_first_synset, sims_all_synsets, metric, ic):\n",
    "    \"\"\"Compute the WordNet <metric> similarity for all the pairs of tokens needed in the MSRPCorpus\n",
    "    using only the lemmas (NO POS tags) of the given tokens to extract the wordnet synsets\"\"\"\n",
    "    count_first = 0\n",
    "    count_max = 0\n",
    "\n",
    "    # For each pair of phrase in training dataset\n",
    "    for id1, id2 in pairs:\n",
    "        # For each token in first phrase\n",
    "        for token1, lemma1, stem1, tag1 in parsed_texts[id1]:\n",
    "            token1_id = lemma2index.get(lemma1, None)\n",
    "            if token1_id == None:\n",
    "                print(\"Error, lemma1 not in vocabulary\", lemma1)\n",
    "                raise\n",
    "            # Extract the synsets from first phrase tokens\n",
    "            syns_tk1 = wn.synsets(lemma=lemma1)\n",
    "            # Compare to each token in phrase 2\n",
    "            for token2, lemma2, stem2, tag2 in parsed_texts[id2]:\n",
    "                # index of token2 in the vocabulary\n",
    "                token2_id = lemma2index.get(lemma2, None)\n",
    "                if token2_id == None:\n",
    "                    print(\"Error, lemma2 not in vocabulary\")\n",
    "                    raise\n",
    "                # Extract the synsets from second prhase tokens\n",
    "                syns_tk2 = wn.synsets(lemma=lemma2)\n",
    "\n",
    "                # Validating we have not computed this similarity before\n",
    "                # *****************\n",
    "                # *** IMPORTANT *** We need to apply the same method when retrieving similarity scores***\n",
    "                # *****************\n",
    "                sim_key = (token1_id, token2_id) if token1_id > token2_id else (token2_id, token1_id)\n",
    "                # Must be different tokens\n",
    "                if sim_key[0] != sim_key[1] and len(syns_tk1)*len(syns_tk2) != 0 and sim_key not in sims_all_synsets:\n",
    "                    all_sims = []\n",
    "                    #sims_first_synset[sim_key] = 0.0\n",
    "                    #sims_all_synsets[sim_key] = 0.0\n",
    "                    for s1 in syns_tk1:\n",
    "                        for s2 in syns_tk2:\n",
    "                            # The similarity must be different of None and Zero\n",
    "                            res = wordnet_similarity(s1, s2, metric=metric, ic=ic)\n",
    "                            if res != 0.0:\n",
    "                                all_sims.append(res)\n",
    "                    if len(all_sims) > 0:\n",
    "                        sims_first_synset[sim_key] = all_sims[0]\n",
    "                        sims_all_synsets[sim_key] = max(all_sims)\n",
    "                        count_max += len(syns_tk1)*len(syns_tk2)\n",
    "                        count_first += 0 if len(syns_tk1)*len(syns_tk2) == 0 else 1\n",
    "        # For debugging. Stopping early.\n",
    "        #if count_first > 1:\n",
    "        #    break\n",
    "        # Finish debugging\n",
    "    print(count_first, \"new token pairs computed.\")\n",
    "    print(count_max, \"Wordnet similarity computed.\")\n",
    "    return None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def compute_msrpc_wordnet_sims_lemmaspos(pairs, vocab, lemmapos2index, tags_mapping,\n",
    "                                          sims_first_synset, sims_all_synsets, metric, ic):\n",
    "    \"\"\"Compute the WordNet <metric> similarity for all the pairs of tokens needed in the MSRPCorpus\n",
    "    using the lemmas and POS tags of the given tokens to extract the wordnet synsets\"\"\"\n",
    "    count_first = 0\n",
    "    count_max = 0\n",
    "\n",
    "    # For each pair of phrase in training dataset\n",
    "    for id1, id2 in pairs:\n",
    "        # For each token in first phrase\n",
    "        for token1, lemma1, stem1, tag1 in parsed_texts[id1]:\n",
    "            # Only look at WordNet POS tags\n",
    "            wn_tag1 = tags_mapping.get(tag1[:2], None)\n",
    "            if wn_tag1:\n",
    "                # index of token1 in the vocabulary\n",
    "                token1_id = lemmapos2index.get((lemma1, wn_tag1), None)\n",
    "                if token1_id == None:\n",
    "                    print(\"Error, lemma1 not in vocabulary\", lemma1)\n",
    "                    raise\n",
    "                # Extract the synsets from first phrase tokens\n",
    "                syns_tk1 = wn.synsets(lemma=lemma1, pos=wn_tag1)\n",
    "                # Compare to each token in phrase 2\n",
    "                for token2, lemma2, stem2, tag2 in parsed_texts[id2]:\n",
    "                    # Only look at WordNet POS tags\n",
    "                    wn_tag2 = tags_mapping.get(tag2[:2], None)\n",
    "                    # Compute similarity of those tokens with the same POS tag\n",
    "                    if wn_tag2 and wn_tag1 == wn_tag2:\n",
    "                        # index of token2 in the vocabulary\n",
    "                        token2_id = lemmapos2index.get((lemma2, wn_tag2), None)\n",
    "                        if token2_id == None:\n",
    "                            print(\"Error, lemma2 not in vocabulary\")\n",
    "                            raise\n",
    "                        # Extract the synsets from second prhase tokens\n",
    "                        syns_tk2 = wn.synsets(lemma=lemma2, pos=wn_tag2)\n",
    "\n",
    "                        # Validating we have not computed this similarity before\n",
    "                        # *****************\n",
    "                        # *** IMPORTANT *** We need to apply the same method when retrieving similarity scores***\n",
    "                        # *****************\n",
    "                        sim_key = (token1_id, token2_id) if token1_id > token2_id else (token2_id, token1_id)\n",
    "                        # Must be different tokens\n",
    "                        if sim_key[0] != sim_key[1] and len(syns_tk1)*len(syns_tk2) != 0 and sim_key not in sims_all_synsets:\n",
    "                            all_sims = []\n",
    "                            #sims_first_synset[sim_key] = 0.0\n",
    "                            #sims_all_synsets[sim_key] = 0.0\n",
    "                            for s1 in syns_tk1:\n",
    "                                for s2 in syns_tk2:\n",
    "                                    # The similarity must be different of None and Zero\n",
    "                                    res = wordnet_similarity(s1, s2, metric=metric, ic=ic)\n",
    "                                    if res != 0.0:\n",
    "                                        all_sims.append(res)\n",
    "                            if len(all_sims) > 0:\n",
    "                                sims_first_synset[sim_key] = all_sims[0]\n",
    "                                sims_all_synsets[sim_key] = max(all_sims)\n",
    "                                count_max += len(syns_tk1)*len(syns_tk2)\n",
    "                                count_first += 0 if len(syns_tk1)*len(syns_tk2) == 0 else 1\n",
    "        # For debugging. Stopping early.\n",
    "        #if count_first > 1:\n",
    "        #    break\n",
    "        # Finish debugging\n",
    "    print(count_first, \"new token pairs computed.\")\n",
    "    print(count_max, \"Wordnet similarity computed.\")\n",
    "    return None"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### WordNet similarity\n",
    "path, lch, wup, res, jcn, lin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "METRIC: lin\n",
      "Using only tokens\n",
      "Training dataset\n",
      "177413 new token pairs computed.\n",
      "13452394 Wordnet similarity computed.\n",
      "Test dataset\n",
      "55911 new token pairs computed.\n",
      "4225212 Wordnet similarity computed.\n",
      "Total pairs: 233324\n",
      "Using only lemmas\n",
      "Training dataset\n",
      "174284 new token pairs computed.\n",
      "12462095 Wordnet similarity computed.\n",
      "Test dataset\n",
      "52244 new token pairs computed.\n",
      "3635155 Wordnet similarity computed.\n",
      "Total pairs: 226528\n",
      "Using lemmas+pos\n",
      "Training dataset\n",
      "64372 new token pairs computed.\n",
      "2129263 Wordnet similarity computed.\n",
      "Test dataset\n",
      "21304 new token pairs computed.\n",
      "640119 Wordnet similarity computed.\n",
      "Total pairs: 85676\n",
      "\n",
      "\n",
      "METRIC: jcn\n",
      "Using only tokens\n",
      "Training dataset\n",
      "227892 new token pairs computed.\n",
      "15621875 Wordnet similarity computed.\n",
      "Test dataset\n",
      "73320 new token pairs computed.\n",
      "4928741 Wordnet similarity computed.\n",
      "Total pairs: 301212\n",
      "Using only lemmas\n",
      "Training dataset\n",
      "215651 new token pairs computed.\n",
      "13761833 Wordnet similarity computed.\n",
      "Test dataset\n",
      "65973 new token pairs computed.\n",
      "4044083 Wordnet similarity computed.\n",
      "Total pairs: 281624\n",
      "Using lemmas+pos\n",
      "Training dataset\n",
      "80897 new token pairs computed.\n",
      "2406770 Wordnet similarity computed.\n",
      "Test dataset\n",
      "26956 new token pairs computed.\n",
      "723447 Wordnet similarity computed.\n",
      "Total pairs: 107853\n",
      "\n",
      "\n",
      "METRIC: res\n",
      "Using only tokens\n",
      "Training dataset\n",
      "177316 new token pairs computed.\n",
      "13446893 Wordnet similarity computed.\n",
      "Test dataset\n",
      "55892 new token pairs computed.\n",
      "4224538 Wordnet similarity computed.\n",
      "Total pairs: 233208\n",
      "Using only lemmas\n",
      "Training dataset\n",
      "174190 new token pairs computed.\n",
      "12457161 Wordnet similarity computed.\n",
      "Test dataset\n",
      "52227 new token pairs computed.\n",
      "3634720 Wordnet similarity computed.\n",
      "Total pairs: 226417\n",
      "Using lemmas+pos\n",
      "Training dataset\n",
      "64327 new token pairs computed.\n",
      "2127980 Wordnet similarity computed.\n",
      "Test dataset\n",
      "21293 new token pairs computed.\n",
      "639580 Wordnet similarity computed.\n",
      "Total pairs: 85620\n",
      "\n",
      "\n",
      "METRIC: lin\n",
      "Using only tokens\n",
      "Training dataset\n",
      "177413 new token pairs computed.\n",
      "13452394 Wordnet similarity computed.\n",
      "Test dataset\n",
      "55911 new token pairs computed.\n",
      "4225212 Wordnet similarity computed.\n",
      "Total pairs: 233324\n",
      "Using only lemmas\n",
      "Training dataset\n",
      "174284 new token pairs computed.\n",
      "12462095 Wordnet similarity computed.\n",
      "Test dataset\n",
      "52244 new token pairs computed.\n",
      "3635155 Wordnet similarity computed.\n",
      "Total pairs: 226528\n",
      "Using lemmas+pos\n",
      "Training dataset\n",
      "64372 new token pairs computed.\n",
      "2129263 Wordnet similarity computed.\n",
      "Test dataset\n",
      "21304 new token pairs computed.\n",
      "640119 Wordnet similarity computed.\n",
      "Total pairs: 85676\n",
      "\n",
      "\n",
      "METRIC: jcn\n",
      "Using only tokens\n",
      "Training dataset\n",
      "227892 new token pairs computed.\n",
      "15621875 Wordnet similarity computed.\n",
      "Test dataset\n",
      "73320 new token pairs computed.\n",
      "4928741 Wordnet similarity computed.\n",
      "Total pairs: 301212\n",
      "Using only lemmas\n",
      "Training dataset\n",
      "215651 new token pairs computed.\n",
      "13761833 Wordnet similarity computed.\n",
      "Test dataset\n",
      "65973 new token pairs computed.\n",
      "4044083 Wordnet similarity computed.\n",
      "Total pairs: 281624\n",
      "Using lemmas+pos\n",
      "Training dataset\n",
      "80897 new token pairs computed.\n",
      "2406770 Wordnet similarity computed.\n",
      "Test dataset\n",
      "26956 new token pairs computed.\n",
      "723447 Wordnet similarity computed.\n",
      "Total pairs: 107853\n",
      "\n",
      "\n",
      "METRIC: res\n",
      "Using only tokens\n",
      "Training dataset\n",
      "177316 new token pairs computed.\n",
      "13446893 Wordnet similarity computed.\n",
      "Test dataset\n",
      "55892 new token pairs computed.\n",
      "4224538 Wordnet similarity computed.\n",
      "Total pairs: 233208\n",
      "Using only lemmas\n",
      "Training dataset\n",
      "174190 new token pairs computed.\n",
      "12457161 Wordnet similarity computed.\n",
      "Test dataset\n",
      "52227 new token pairs computed.\n",
      "3634720 Wordnet similarity computed.\n",
      "Total pairs: 226417\n",
      "Using lemmas+pos\n",
      "Training dataset\n",
      "64327 new token pairs computed.\n",
      "2127980 Wordnet similarity computed.\n",
      "Test dataset\n",
      "21293 new token pairs computed.\n",
      "639580 Wordnet similarity computed.\n",
      "Total pairs: 85620\n",
      "\n",
      "\n",
      "METRIC: lin\n",
      "Using only tokens\n",
      "Training dataset\n",
      "177413 new token pairs computed.\n",
      "13452394 Wordnet similarity computed.\n",
      "Test dataset\n",
      "55911 new token pairs computed.\n",
      "4225212 Wordnet similarity computed.\n",
      "Total pairs: 233324\n",
      "Using only lemmas\n",
      "Training dataset\n",
      "174284 new token pairs computed.\n",
      "12462095 Wordnet similarity computed.\n",
      "Test dataset\n",
      "52244 new token pairs computed.\n",
      "3635155 Wordnet similarity computed.\n",
      "Total pairs: 226528\n",
      "Using lemmas+pos\n",
      "Training dataset\n",
      "64372 new token pairs computed.\n",
      "2129263 Wordnet similarity computed.\n",
      "Test dataset\n",
      "21304 new token pairs computed.\n",
      "640119 Wordnet similarity computed.\n",
      "Total pairs: 85676\n",
      "\n",
      "\n",
      "METRIC: jcn\n",
      "Using only tokens\n",
      "Training dataset\n",
      "227892 new token pairs computed.\n",
      "15621875 Wordnet similarity computed.\n",
      "Test dataset\n",
      "73320 new token pairs computed.\n",
      "4928741 Wordnet similarity computed.\n",
      "Total pairs: 301212\n",
      "Using only lemmas\n",
      "Training dataset\n",
      "215651 new token pairs computed.\n",
      "13761833 Wordnet similarity computed.\n",
      "Test dataset\n",
      "65973 new token pairs computed.\n",
      "4044083 Wordnet similarity computed.\n",
      "Total pairs: 281624\n",
      "Using lemmas+pos\n",
      "Training dataset\n",
      "80897 new token pairs computed.\n",
      "2406770 Wordnet similarity computed.\n",
      "Test dataset\n",
      "26956 new token pairs computed.\n",
      "723447 Wordnet similarity computed.\n",
      "Total pairs: 107853\n",
      "\n",
      "\n",
      "METRIC: res\n",
      "Using only tokens\n",
      "Training dataset\n",
      "177316 new token pairs computed.\n",
      "13446893 Wordnet similarity computed.\n",
      "Test dataset\n",
      "55892 new token pairs computed.\n",
      "4224538 Wordnet similarity computed.\n",
      "Total pairs: 233208\n",
      "Using only lemmas\n",
      "Training dataset\n",
      "174190 new token pairs computed.\n",
      "12457161 Wordnet similarity computed.\n",
      "Test dataset\n",
      "52227 new token pairs computed.\n",
      "3634720 Wordnet similarity computed.\n",
      "Total pairs: 226417\n",
      "Using lemmas+pos\n",
      "Training dataset\n",
      "64327 new token pairs computed.\n",
      "2127980 Wordnet similarity computed.\n",
      "Test dataset\n",
      "21293 new token pairs computed.\n",
      "639580 Wordnet similarity computed.\n",
      "Total pairs: 85620\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for ic_data, ic_name in zip([bnc_ic_2000, semcor_ic, brown_ic], ['bnc_ic_2000', 'semcor_ic', 'brown_ic']):\n",
    "    for metric in ['lin', 'jcn', 'res']:#['lin', 'jcn', 'res', 'path', 'lch', 'wup']:\n",
    "        tags_mapping = {\"NN\":wn.NOUN, \"VB\":wn.VERB, \"JJ\":wn.ADJ, \"RB\":wn.ADV}\n",
    "        date_obj = datetime.date.today()\n",
    "        date_str = \"{:04d}\".format(date_obj.year) + \"{:02d}\".format(date_obj.month) + \"{:02d}\".format(date_obj.day)\n",
    "\n",
    "        print(\"METRIC:\", metric)\n",
    "        #***********************************************\n",
    "        print(\"Using only tokens\")\n",
    "        print(\"Training dataset\")\n",
    "        sims_first_synset = {}\n",
    "        sims_all_synsets = {}\n",
    "        compute_msrpc_wordnet_sims_tokens(train_pairs, vocab_tokens, token2index, tags_mapping,\n",
    "                                           sims_first_synset, sims_all_synsets, metric, ic_data)#bnc_ic_2007)\n",
    "        print(\"Test dataset\")\n",
    "        compute_msrpc_wordnet_sims_tokens(test_pairs, vocab_tokens, token2index, tags_mapping,\n",
    "                                           sims_first_synset, sims_all_synsets, metric, ic_data)#bnc_ic_2007)\n",
    "        print(\"Total pairs:\", len(sims_first_synset))\n",
    "        pickle.dump(sims_first_synset, open(metric+\"_first_tokens_\"+ic_name+\"_\"+date_str+\".pickle\", \"wb\"), protocol=pickle.HIGHEST_PROTOCOL)\n",
    "        pickle.dump(sims_all_synsets, open(metric+\"_all_tokens_\"+ic_name+\"_\"+date_str+\".pickle\", \"wb\"), protocol=pickle.HIGHEST_PROTOCOL)\n",
    "        #***********************************************\n",
    "\n",
    "        #***********************************************\n",
    "        print(\"Using only lemmas\")\n",
    "        print(\"Training dataset\")\n",
    "        sims_first_synset = {}\n",
    "        sims_all_synsets = {}\n",
    "        compute_msrpc_wordnet_sims_lemmas(train_pairs, vocab_lemmas, lemma2index, tags_mapping,\n",
    "                                           sims_first_synset, sims_all_synsets, metric, ic_data)#bnc_ic_2007)\n",
    "        print(\"Test dataset\")\n",
    "        compute_msrpc_wordnet_sims_lemmas(test_pairs, vocab_lemmas, lemma2index, tags_mapping,\n",
    "                                           sims_first_synset, sims_all_synsets, metric, ic_data)#bnc_ic_2007)\n",
    "        print(\"Total pairs:\", len(sims_first_synset))\n",
    "        pickle.dump(sims_first_synset, open(metric+\"_first_lemmas_\"+ic_name+\"_\"+date_str+\".pickle\", \"wb\"), protocol=pickle.HIGHEST_PROTOCOL)\n",
    "        pickle.dump(sims_all_synsets, open(metric+\"_all_lemmas_\"+ic_name+\"_\"+date_str+\".pickle\", \"wb\"), protocol=pickle.HIGHEST_PROTOCOL)\n",
    "        #***********************************************\n",
    "\n",
    "        #***********************************************\n",
    "        print(\"Using lemmas+pos\")\n",
    "        print(\"Training dataset\")\n",
    "        sims_first_synset = {}\n",
    "        sims_all_synsets = {}\n",
    "        compute_msrpc_wordnet_sims_lemmaspos(train_pairs, vocab_lemmapos, lemmapos2index, tags_mapping,\n",
    "                                           sims_first_synset, sims_all_synsets, metric, ic_data)#bnc_ic_2007)\n",
    "        print(\"Test dataset\")\n",
    "        compute_msrpc_wordnet_sims_lemmaspos(test_pairs, vocab_lemmapos, lemmapos2index, tags_mapping,\n",
    "                                           sims_first_synset, sims_all_synsets, metric, ic_data)#bnc_ic_2007)\n",
    "        print(\"Total pairs:\", len(sims_first_synset))\n",
    "        pickle.dump(sims_first_synset, open(metric+\"_first_lemmapos_\"+ic_name+\"_\"+date_str+\".pickle\", \"wb\"), protocol=pickle.HIGHEST_PROTOCOL)\n",
    "        pickle.dump(sims_all_synsets, open(metric+\"_all_lemmapos_\"+ic_name+\"_\"+date_str+\".pickle\", \"wb\"), protocol=pickle.HIGHEST_PROTOCOL)\n",
    "        #***********************************************\n",
    "        print(\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "s = wn.synsets(\"hit\", \"n\")[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import nltk.corpus.reader.wordnet as wn2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(8.907087158260376, 8.907087158260376, 8.907087158260376)"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wn2._lcs_ic(s, s,bnc_ic_2007)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1e+300"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wn.jcn_similarity(s, s, bnc_ic_2007)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8.907087158260376"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wn.res_similarity(s, s, bnc_ic_2007)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wn.lin_similarity(s, s, bnc_ic_2007)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1e+300"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wn2._INF"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Serializing the WordNet similarity scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pickle.dump(sims_first_synset, open(metric+\"_first_\"+date+\".pickle\", \"wb\"), protocol=pickle.HIGHEST_PROTOCOL)\n",
    "pickle.dump(sims_all_synsets, open(metric+\"_all_\"+date+\".pickle\", \"wb\"), protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Computing the WordNet similarity scores of the vocabulary terms\n",
    "We compute these scores in order to normalize the similarity measures that do not have a range between 0 and 1.\n",
    "\n",
    "path, lch, wup, res, jcn, lin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "for metric in ['path', 'lch', 'wup', 'res', 'jcn', 'lin']:\n",
    "    print(metric)\n",
    "    wn_tags = set(['v','n','r','a'])\n",
    "    vocab_sim = {}\n",
    "    #metric = \"lch\"\n",
    "\n",
    "    # The tag is already a wordnet tag where it is one of the wn_tags\n",
    "    for lemma, tag in vocab:\n",
    "        idx = token_to_index[(lemma, tag)]\n",
    "        if tag in wn_tags:\n",
    "            syns_tk = wn.synsets(lemma=lemma, pos=tag)\n",
    "            all_sims = []\n",
    "            for s in syns_tk:\n",
    "                try:\n",
    "                    res = wordnet_similarity(s, s, metric=metric, ic=bnc_ic_2000)\n",
    "                    if res > 0.0:\n",
    "                        all_sims.append(res)\n",
    "                except ZeroDivisionError:\n",
    "                    pass\n",
    "            if len(all_sims) > 0:\n",
    "                vocab_sim[idx] = max(all_sims)\n",
    "\n",
    "    print(\"Vocab size:\", len(vocab))\n",
    "    \n",
    "    for pos in ['n','v','r','a']:\n",
    "        print(pos)\n",
    "        non_zero = [x for idx, x in vocab_sim.items() if x > 0.0 and index_to_token[idx][1]==pos]\n",
    "    \n",
    "        print(\"Sims computed:\", len(vocab_sim))\n",
    "\n",
    "        print(\"Max:\", max(non_zero, default=-1))\n",
    "        print(\"Min\", min(non_zero, default=-1))\n",
    "        print(\"Mean:\", -1 if len(non_zero) == 0 else sum(non_zero)/len(non_zero))\n",
    "\n",
    "    pickle.dump(vocab_sim, open(\"vocab_\"+metric+\"_\"+date+\".pickle\", \"wb\"), protocol=pickle.HIGHEST_PROTOCOL)\n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "WordNet similarity ranges:\n",
    "- path: 0.0 - 1.0\n",
    "- lch:\n",
    "    - nouns: 0.0 - 3.6375861597263857\n",
    "    - verbs: 0.0 - 3.258096538021482\n",
    "- wup: 0.0 - 1.0\n",
    "- res: 0.0 - 1e+300    **This number (inf) is too big. Need to check the maximum value in the similarity file**\n",
    "- jcn: 0.0 - 1e+300    **This number (inf) is too big. Need to check the maximum value in the similarity file**\n",
    "- lin: 0.0 - 1.0\n",
    "\n",
    "**Need to check that all the similarities computed fall within these ranges**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Questions**\n",
    "\n",
    "- Shall we generate an index for the words or not? **Yes. Done!**\n",
    "- Shall we remove the tokens composed by only punctuation symbols? **Yes. Done!**\n",
    "- Shall we replace the numbers by a unique identifier?\n",
    "- What is the range of each WordNet similarity measure? **Done for our specific corpus!**\n",
    "- How to serialize the Information Count from British National Corpus (BNC)? **I implemented it manually. Done!**\n",
    "\n",
    "**Tasks**\n",
    "\n",
    "- Define the similarities needed in both datasets. **Done!**\n",
    "- Run example for all WordNet similarity measures. **Done!**\n",
    "- Serialize all the similarities needed in both, training and test dataset for each of the WordNet similarities. Try with first synset and with max of all synsets. **Done!**\n",
    "    - Save the values without normalization. We need to know the ranges of the WordNet metrics or the maximum value to normalize. **Done for MSRPC!**\n",
    "- When serializing the WordNet similarity values, we also need to serialize the vocabulary, token_to_index, and index_to_token data structures. **Done**\n",
    "\n",
    "**Notes**\n",
    "- The WordNet metrics that need normalization, because their values are not in the 0-1 range, won't be easily normalized because there is not a score for the similarity between the same lemmas other than Inf.\n",
    "    - An idea to normalize is compute the similarity of a large amount of words between themself and pick the highest value."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Computing IDF over the BNC Corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def compute_idf(corpus):\n",
    "    df = {}\n",
    "    idf = {}\n",
    "    for fid in corpus.fileids():\n",
    "        words = set([token.lower() for token in corpus.words(fid)])\n",
    "        for word in words:\n",
    "            df[word] = df.get(word, 0) + 1\n",
    "    N = len(corpus.fileids())\n",
    "    for word in df:\n",
    "        idf[word] = np.log(N/df[word])\n",
    "    return idf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#corpus = nltk.corpus.brown\n",
    "\n",
    "# BNC 2007 XML version\n",
    "corpus = nltk.corpus.reader.BNCCorpusReader(root='D:/Corpus/2554/2554/download/Texts/',\n",
    "                                            fileids=r'[A-K]/\\w*/\\w*.xml') \n",
    "idf = compute_idf(corpus)\n",
    "len(idf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pickle.dump(idf, open(\"idf_20170810.pickle\", \"wb\"), protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}