SoftCosine WordNet Experiments.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "import math\n",
    "import nltk\n",
    "from nltk.corpus import wordnet as wn\n",
    "import numpy as np\n",
    "import string\n",
    "import itertools\n",
    "import datetime\n",
    "import sys\n",
    "import re\n",
    "import os\n",
    "import Stemmer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Functions"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Similarity Metrics"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Softcosine similarity function\n",
    "$softcosine(a,b)=\\frac{ \\sum_{i}^{N}{ \\sum_{j}^{N}{s_{ij} a_i b_j} } }{ \\sqrt{ \\sum{ \\sum_{i,j}^{N}{s_{ij} a_i a_j} } } \\sqrt{ \\sum{ \\sum_{i,j}^{N}{s_{ij} b_i b_j} } }} $"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def soft_euclidean_norm(vect, ft_id2wn_id, term_sims):\n",
    "    \"\"\"Compute the soft euclidean norm given a vector <v> in dict format and a \"\"\"\n",
    "    norm = 0.0\n",
    "    for token1_id in vect:\n",
    "        for token2_id in vect:\n",
    "            # Here me need to check if (a_i, a_j) is the same as (a_j, a_i)******************************\n",
    "            # The similarity between the same terms is 1\n",
    "            if token1_id == token2_id:\n",
    "                norm += vect[token1_id] * vect[token2_id]\n",
    "            else:\n",
    "                sims = []\n",
    "                for wn_id1 in ft_id2wn_id[token1_id]:\n",
    "                    for wn_id2 in ft_id2wn_id[token2_id]:\n",
    "                        sim_key = (wn_id1, wn_id2) if wn_id1 > wn_id2 else (wn_id2, wn_id1)\n",
    "                        if sim_key in term_sims:\n",
    "                            sims.append(term_sims[sim_key])\n",
    "                if len(sims) > 0:\n",
    "                    norm += max(sims) * vect[token1_id] * vect[token2_id]\n",
    "    return math.sqrt(norm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def softcosine(vect1, vect2, ft_id2wn_id, term_sims):\n",
    "    \"\"\"Compute the softcosine similarity given two vectors\n",
    "    <a> and <b> in a dictionary format, and the .\"\"\"\n",
    "    dot_prod = 0.0\n",
    "    det = soft_euclidean_norm(vect1, ft_id2wn_id, term_sims) * soft_euclidean_norm(vect2, ft_id2wn_id, term_sims)\n",
    "    if det == 0.0:\n",
    "        return 0.0\n",
    "    for token1_id in vect1:\n",
    "        for token2_id in vect2:\n",
    "            if token1_id == token2_id:\n",
    "                dot_prod += vect1[token1_id] * vect2[token2_id]\n",
    "            else:\n",
    "                sims = []\n",
    "                for wn_id1 in ft_id2wn_id[token1_id]:\n",
    "                    for wn_id2 in ft_id2wn_id[token2_id]:\n",
    "                        sim_key = (wn_id1, wn_id2) if wn_id1 > wn_id2 else (wn_id2, wn_id1)\n",
    "                        if sim_key in term_sims:\n",
    "                            sims.append(term_sims[sim_key])\n",
    "                if len(sims) > 0:\n",
    "                    dot_prod += max(sims) * vect1[token1_id] * vect2[token2_id]\n",
    "    return dot_prod / det"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Cosine similarity function\n",
    "$cosine(a,b)=\\frac{a \\cdot b}{|a| |b|}=\\frac{\\sum_{i}^{N}{a_i b_i}}{\\sqrt{ \\sum_{i}^{N}{a_i^2} } \\sqrt{ \\sum_{i}^{N}{b_i^2} }}$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def euclidean_norm(vect):\n",
    "    norm = 0.0\n",
    "    for token_id in vect:\n",
    "        norm += vect[token_id]**2\n",
    "    return math.sqrt(norm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def cosine(vect1, vect2):\n",
    "    dot_prod = 0.0\n",
    "    det = euclidean_norm(vect1) * euclidean_norm(vect2)\n",
    "    if det == 0.0:\n",
    "        return 0.0\n",
    "    for token_id in vect1:\n",
    "        if token_id in vect2:\n",
    "            dot_prod += vect1[token_id] * vect2[token_id]\n",
    "    return dot_prod / det"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Stevenson similarity metric\n",
    "Fernando, S. and Stevenson, M. (2008). **A semantic similarity approach to paraphrase detection**, *Computational Linguistics UK (CLUK 2008) 11th Annual Research Colloquium*\n",
    "\n",
    "$stevenson\\_sim(a,b)=\\frac{a \\cdot W \\cdot b}{|a| |b|}=\\frac{ \\sum_{i}^{N}{ \\sum_{j}^{N}{W_{ij} a_i b_j} } }{\\sqrt{ \\sum_{i}^{N}{a_i^2} } \\sqrt{ \\sum_{i}^{N}{b_i^2} }}$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def stevenson_sim(vect1, vect2, ft_id2wn_id, term_sims):\n",
    "    dot_prod = 0.0\n",
    "    det = euclidean_norm(vect1) * euclidean_norm(vect2)\n",
    "    if det == 0.0:\n",
    "        return 0.0\n",
    "    for token1_id in vect1:\n",
    "        for token2_id in vect2:\n",
    "            if token1_id == token2_id:\n",
    "                dot_prod += vect1[token1_id] * vect2[token2_id]\n",
    "            else:\n",
    "                sims = []\n",
    "                for wn_id1 in ft_id2wn_id[token1_id]:\n",
    "                    for wn_id2 in ft_id2wn_id[token2_id]:\n",
    "                        sim_key = (wn_id1, wn_id2) if wn_id1 > wn_id2 else (wn_id2, wn_id1)\n",
    "                        if sim_key in term_sims:\n",
    "                            sims.append(term_sims[sim_key])\n",
    "                if len(sims) > 0:\n",
    "                    dot_prod += max(sims) * vect1[token1_id] * vect2[token2_id]\n",
    "    return dot_prod / det"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Mihalcea's similarity method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def mihalcea_sim(vect1, vect2, ft_id2wn_id, ft_id2token_id, term_sims, idf, idf_median_value):\n",
    "    sum1 = 0.0\n",
    "    sum_idf1 = 0.0\n",
    "    for token1_id in vect1:\n",
    "        max_sim = 0.0\n",
    "        for token2_id in vect2:\n",
    "            if token1_id == token2_id:\n",
    "                max_sim = 1.0\n",
    "                break\n",
    "            for wn_id1 in ft_id2wn_id[token1_id]:\n",
    "                for wn_id2 in ft_id2wn_id[token2_id]:\n",
    "                    sim_key = (wn_id1, wn_id2) if wn_id1 > wn_id2 else (wn_id2, wn_id1)\n",
    "                    if sim_key in term_sims:\n",
    "                        sval = term_sims[sim_key]\n",
    "                        max_sim = sval if sval > max_sim else max_sim\n",
    "        #idf_val = idf.get(list(ft_id2token_id[token1_id])[0], idf_median_value)\n",
    "        idf_val = max([idf.get(x, idf_median_value) for x in ft_id2token_id[token1_id]])\n",
    "        sum1 += max_sim * (idf_val)\n",
    "        sum_idf1 += idf_val\n",
    "        \n",
    "    sum2 = 0.0\n",
    "    sum_idf2 = 0.0\n",
    "    for token2_id in vect2:\n",
    "        max_sim = 0.0\n",
    "        for token1_id in vect1:\n",
    "            if token2_id == token1_id:\n",
    "                max_sim = 1.0\n",
    "                break\n",
    "            for wn_id2 in ft_id2wn_id[token2_id]:\n",
    "                for wn_id1 in ft_id2wn_id[token1_id]:\n",
    "                    sim_key = (wn_id1, wn_id2) if wn_id1 > wn_id2 else (wn_id2, wn_id1)\n",
    "                    if sim_key in term_sims:\n",
    "                        sval = term_sims[sim_key]\n",
    "                        max_sim = sval if sval > max_sim else max_sim\n",
    "        #idf_val = idf.get(list(ft_id2token_id[token2_id])[0], idf_median_value)\n",
    "        idf_val = max([idf.get(x, idf_median_value) for x in ft_id2token_id[token2_id]])\n",
    "        sum2 += max_sim * (idf_val)\n",
    "        sum_idf2 += idf_val\n",
    "        \n",
    "    return 1/2 * (sum1/sum_idf1 + sum2/sum_idf2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Function to compute similarity metrics for all pairs in MSRPCorpus\n",
    "Training instances"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**TODO**\n",
    "- Implement the tf-idf and log enthropy measures.\n",
    "    - Compute the the idf and enthropy over some of the existing corpora\n",
    "- Implement Mihalcea's method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def compute_sims(pairs, vectors, feature, ft_id2wn_id, ft_id2token_id, metric=\"cosine\",\n",
    "                 weight_scheme=\"tf\", term_sims={}, idf={}, idf_median_value=0.0):\n",
    "    res = []\n",
    "    for text_id1, text_id2 in pairs:\n",
    "        # Vectorization\n",
    "        if weight_scheme == \"tf\":\n",
    "            v1 = nltk.FreqDist([x[feature] for x in vectors[text_id1]])\n",
    "            v2 = nltk.FreqDist([x[feature] for x in vectors[text_id2]])\n",
    "            #v1 = nltk.FreqDist([eval(feature+\"_id\") for token_id, lemma_id, stem_id, lemmapos_id in vectors[text_id1]])\n",
    "            #v2 = nltk.FreqDist([eval(feature+\"_id\") for token_id, lemma_id, stem_id, lemmapos_id in vectors[text_id2]])\n",
    "        elif weight_scheme == \"binary\":\n",
    "            v1 = {}.fromkeys(set([x[feature] for x in vectors[text_id1]]), 1)\n",
    "            v2 = {}.fromkeys(set([x[feature] for x in vectors[text_id2]]), 1)\n",
    "            #v1 = {}.fromkeys(set([eval(feature+\"_id\") for token_id, lemma_id, stem_id, lemmapos_id in vectors[text_id1]]), 1)\n",
    "            #v2 = {}.fromkeys(set([eval(feature+\"_id\") for token_id, lemma_id, stem_id, lemmapos_id in vectors[text_id2]]), 1)\n",
    "        else:\n",
    "            print(\"Unrecognized weighting scheme\")\n",
    "            raise\n",
    "        \n",
    "        # Similarity computation\n",
    "        if metric == \"softcosine\":\n",
    "            res.append(softcosine(v1, v2, ft_id2wn_id, term_sims))\n",
    "        elif metric == \"cosine\":\n",
    "            res.append(cosine(v1, v2))\n",
    "        elif metric == \"stevenson\":\n",
    "            res.append(stevenson_sim(v1, v2, ft_id2wn_id, term_sims))\n",
    "        elif metric == \"mihalcea\":\n",
    "            res.append(mihalcea_sim(v1, v2, ft_id2wn_id, ft_id2token_id, term_sims, idf, idf_median_value))\n",
    "        else:\n",
    "            print(\"Unimplemented similarity metric\")\n",
    "            raise\n",
    "        #break\n",
    "    return res"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Optimizing the similarity threshold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def compute_scores(pair_sims, y, thresholds):\n",
    "    \"\"\"Receives a float between 0 and 1, or an iterable that returns such floats.\n",
    "    The function returns the <tuple>/<list of tuples> of scores\n",
    "    (threshold, accuracy, f_measure, precision, recall)\"\"\"\n",
    "    # (accuracy, f_measure, precision, recall)\n",
    "    if hasattr(thresholds, '__iter__'):\n",
    "        measures = []\n",
    "        for th in thresholds: #range(5, 101, 5):\n",
    "            results = [1 if val >= th else 0 for val in pair_sims]\n",
    "            measures.append((th,)+evaluate(results, y))\n",
    "        return measures\n",
    "    else:\n",
    "        results = [1 if val >= thresholds else 0 for val in pair_sims]\n",
    "        return (thresholds,)+evaluate(results, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Preprocessing"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Check if a token is made purely of punctuation symbols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def is_punct_token(token, puncts):\n",
    "    if sum([1 for c in token if c in puncts]) == len(token):\n",
    "        return True\n",
    "    return False"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluation metrics\n",
    "accuracy, f_measure, precision, recall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def evaluate(result, truth):\n",
    "    \"\"\"Receive the gold_standard and values returned by our system.\n",
    "    Return a tuple as (accuracy, f_measure, precision, recall)\"\"\"\n",
    "    tp, fp, tn, fn = 0, 0, 0, 0\n",
    "    for r, t in zip(result, truth):\n",
    "        if t == 1:\n",
    "            if r == 1:\n",
    "                tp += 1\n",
    "            else:\n",
    "                fn += 1\n",
    "        else:\n",
    "            if r == 1:\n",
    "                fp += 1\n",
    "            else:\n",
    "                tn += 1\n",
    "    accuracy = 0.0 if tp+tn+fp+fn == 0 else (tp+tn) / (tp+tn+fp+fn)\n",
    "    precision = 0.0 if tp+fp == 0 else tp / (tp+fp)\n",
    "    recall = 0.0 if (tp+fn) == 0 else tp / (tp+fn) \n",
    "    f_measure = (0 if precision + recall == 0 else \n",
    "                 2*precision*recall/(precision+recall))\n",
    "    return accuracy, f_measure, precision, recall"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Normalization strategy for each WordNet metric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def normalize(term_sims, metric):\n",
    "    if metric == \"path\":\n",
    "        # Not need for normalization.\n",
    "        # Creating a copy of the similarity matrix\n",
    "        return dict(term_sims.items())\n",
    "    \n",
    "    elif metric == \"lch\":\n",
    "        # MinMax normalization\n",
    "        data = np.array([val for (tkid1, tkid2), val in term_sims.items()])\n",
    "        _max = np.max(data)\n",
    "        _min = np.min(data)\n",
    "        res = {}\n",
    "        for key, val in term_sims.items():\n",
    "            res[key] = (val - _min)/(_max - _min)\n",
    "        return res\n",
    "    \n",
    "    elif metric == \"wup\":\n",
    "        # Creating a copy of the similarity matrix\n",
    "        return dict(term_sims.items())\n",
    "    \n",
    "    elif metric == \"res\":\n",
    "        # MinMax normalization except for extremly big values (WordNet Inf=1e300)\n",
    "        data = np.array([val for (tkid1, tkid2), val in term_sims.items() if val < 10000])\n",
    "        _max = np.max(data)\n",
    "        _min = np.min(data)\n",
    "        res = {}\n",
    "        for key, val in term_sims.items():\n",
    "            # Exception\n",
    "            if val > 10000:\n",
    "                res[key] = 1.0\n",
    "            else:\n",
    "                res[key] = (val - _min)/(_max - _min)\n",
    "        return res\n",
    "    \n",
    "    elif metric == \"jcn\":\n",
    "        # MinMax normalization except for extremly big values (WordNet Inf=1e300)\n",
    "        data = np.array([val for (tkid1, tkid2), val in term_sims.items() if val < 10000])\n",
    "        _max = np.max(data)\n",
    "        _min = np.min(data)\n",
    "        res = {}\n",
    "        for key, val in term_sims.items():\n",
    "            # Exception\n",
    "            if val > 10000:\n",
    "                res[key] = 1.0\n",
    "            else:\n",
    "                res[key] = (val - _min)/(_max - _min)\n",
    "        return res\n",
    "    \n",
    "    elif metric == \"lin\":\n",
    "        # Not need for normalization.\n",
    "        # Creating a copy of the similarity matrix\n",
    "        return dict(term_sims.items())\n",
    "    print(\"Unrecognize metric\")\n",
    "    raise"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MAIN"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Loading the parsed MSRPCorpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "[parsed_texts,\n",
    " train_pairs,\n",
    " train_y,\n",
    " test_pairs,\n",
    " test_y] = pickle.load(open(\"msrpc_parsed_20170821.pickle\", 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "[vocab_tokens, token2index, index2token] = pickle.load(open(\"tokens_data_20170821.pickle\", 'rb'))\n",
    "[vocab_lemmas, lemma2index, index2lemma] = pickle.load(open(\"lemmas_data_20170821.pickle\", 'rb'))\n",
    "[vocab_stems, stem2index, index2stem] = pickle.load(open(\"stems_data_20170821.pickle\", 'rb'))\n",
    "[vocab_lemmapos, lemmapos2index, index2lemmapos] = pickle.load(open(\"lemmapos_data_20170821.pickle\", 'rb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Loading WordNet similarity \"matrix\"\n",
    "It is a dictionary (token1_id, token2_id): sim_value, where token1_id > token2_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# PATH Similarity\n",
    "path_all_tokens   = pickle.load(open(\"path_all_tokens_20170821.pickle\", 'rb'))\n",
    "path_all_lemmas   = pickle.load(open(\"path_all_lemmas_20170821.pickle\", 'rb'))\n",
    "path_all_lemmapos = pickle.load(open(\"path_all_lemmapos_20170821.pickle\", 'rb'))\n",
    "\n",
    "path_first_tokens   = pickle.load(open(\"path_first_tokens_20170821.pickle\", 'rb'))\n",
    "path_first_lemmas   = pickle.load(open(\"path_first_lemmas_20170821.pickle\", 'rb'))\n",
    "path_first_lemmapos = pickle.load(open(\"path_first_lemmapos_20170821.pickle\", 'rb'))\n",
    "\n",
    "\n",
    "# LCH Similarity\n",
    "lch_all_tokens   = pickle.load(open(\"lch_all_tokens_20170821.pickle\", 'rb'))\n",
    "lch_all_lemmas   = pickle.load(open(\"lch_all_lemmas_20170821.pickle\", 'rb'))\n",
    "lch_all_lemmapos = pickle.load(open(\"lch_all_lemmapos_20170821.pickle\", 'rb'))\n",
    "\n",
    "lch_first_tokens   = pickle.load(open(\"lch_first_tokens_20170821.pickle\", 'rb'))\n",
    "lch_first_lemmas   = pickle.load(open(\"lch_first_lemmas_20170821.pickle\", 'rb'))\n",
    "lch_first_lemmapos = pickle.load(open(\"lch_first_lemmapos_20170821.pickle\", 'rb'))\n",
    "\n",
    "\n",
    "# WUP Similarity\n",
    "wup_all_tokens   = pickle.load(open(\"wup_all_tokens_20170821.pickle\", 'rb'))\n",
    "wup_all_lemmas   = pickle.load(open(\"wup_all_lemmas_20170821.pickle\", 'rb'))\n",
    "wup_all_lemmapos = pickle.load(open(\"wup_all_lemmapos_20170821.pickle\", 'rb'))\n",
    "\n",
    "wup_first_tokens   = pickle.load(open(\"wup_first_tokens_20170821.pickle\", 'rb'))\n",
    "wup_first_lemmas   = pickle.load(open(\"wup_first_lemmas_20170821.pickle\", 'rb'))\n",
    "wup_first_lemmapos = pickle.load(open(\"wup_first_lemmapos_20170821.pickle\", 'rb'))\n",
    "\n",
    "\n",
    "# RES Similarity\n",
    "res_all_tokens_bnc_ic_2007   = pickle.load(open(\"res_all_tokens_bnc_ic_2007_20170821.pickle\", 'rb'))\n",
    "res_all_tokens_bnc_ic_2000   = pickle.load(open(\"res_all_tokens_bnc_ic_2000_20170821.pickle\", 'rb'))\n",
    "res_all_tokens_semcor_ic     = pickle.load(open(\"res_all_tokens_semcor_ic_20170821.pickle\", 'rb'))\n",
    "res_all_tokens_brown_ic      = pickle.load(open(\"res_all_tokens_brown_ic_20170821.pickle\", 'rb'))\n",
    "\n",
    "res_all_lemmas_bnc_ic_2007   = pickle.load(open(\"res_all_lemmas_bnc_ic_2007_20170821.pickle\", 'rb'))\n",
    "res_all_lemmas_bnc_ic_2000   = pickle.load(open(\"res_all_lemmas_bnc_ic_2000_20170821.pickle\", 'rb'))\n",
    "res_all_lemmas_semcor_ic     = pickle.load(open(\"res_all_lemmas_semcor_ic_20170821.pickle\", 'rb'))\n",
    "res_all_lemmas_brown_ic      = pickle.load(open(\"res_all_lemmas_brown_ic_20170821.pickle\", 'rb'))\n",
    "\n",
    "res_all_lemmapos_bnc_ic_2007   = pickle.load(open(\"res_all_lemmapos_bnc_ic_2007_20170821.pickle\", 'rb'))\n",
    "res_all_lemmapos_bnc_ic_2000   = pickle.load(open(\"res_all_lemmapos_bnc_ic_2000_20170821.pickle\", 'rb'))\n",
    "res_all_lemmapos_semcor_ic     = pickle.load(open(\"res_all_lemmapos_semcor_ic_20170821.pickle\", 'rb'))\n",
    "res_all_lemmapos_brown_ic      = pickle.load(open(\"res_all_lemmapos_brown_ic_20170821.pickle\", 'rb'))\n",
    "\n",
    "res_first_tokens_bnc_ic_2007   = pickle.load(open(\"res_first_tokens_bnc_ic_2007_20170821.pickle\", 'rb'))\n",
    "res_first_tokens_bnc_ic_2000   = pickle.load(open(\"res_first_tokens_bnc_ic_2000_20170821.pickle\", 'rb'))\n",
    "res_first_tokens_semcor_ic     = pickle.load(open(\"res_first_tokens_semcor_ic_20170821.pickle\", 'rb'))\n",
    "res_first_tokens_brown_ic      = pickle.load(open(\"res_first_tokens_brown_ic_20170821.pickle\", 'rb'))\n",
    "\n",
    "res_first_lemmas_bnc_ic_2007   = pickle.load(open(\"res_first_lemmas_bnc_ic_2007_20170821.pickle\", 'rb'))\n",
    "res_first_lemmas_bnc_ic_2000   = pickle.load(open(\"res_first_lemmas_bnc_ic_2000_20170821.pickle\", 'rb'))\n",
    "res_first_lemmas_semcor_ic     = pickle.load(open(\"res_first_lemmas_semcor_ic_20170821.pickle\", 'rb'))\n",
    "res_first_lemmas_brown_ic      = pickle.load(open(\"res_first_lemmas_brown_ic_20170821.pickle\", 'rb'))\n",
    "\n",
    "res_first_lemmapos_bnc_ic_2007   = pickle.load(open(\"res_first_lemmapos_bnc_ic_2007_20170821.pickle\", 'rb'))\n",
    "res_first_lemmapos_bnc_ic_2000   = pickle.load(open(\"res_first_lemmapos_bnc_ic_2000_20170821.pickle\", 'rb'))\n",
    "res_first_lemmapos_semcor_ic     = pickle.load(open(\"res_first_lemmapos_semcor_ic_20170821.pickle\", 'rb'))\n",
    "res_first_lemmapos_brown_ic      = pickle.load(open(\"res_first_lemmapos_brown_ic_20170821.pickle\", 'rb'))\n",
    "\n",
    "\n",
    "# JCN Similarity\n",
    "jcn_all_tokens_bnc_ic_2007   = pickle.load(open(\"jcn_all_tokens_bnc_ic_2007_20170821.pickle\", 'rb'))\n",
    "jcn_all_tokens_bnc_ic_2000   = pickle.load(open(\"jcn_all_tokens_bnc_ic_2000_20170821.pickle\", 'rb'))\n",
    "jcn_all_tokens_semcor_ic     = pickle.load(open(\"jcn_all_tokens_semcor_ic_20170821.pickle\", 'rb'))\n",
    "jcn_all_tokens_brown_ic      = pickle.load(open(\"jcn_all_tokens_brown_ic_20170821.pickle\", 'rb'))\n",
    "\n",
    "jcn_all_lemmas_bnc_ic_2007   = pickle.load(open(\"jcn_all_lemmas_bnc_ic_2007_20170821.pickle\", 'rb'))\n",
    "jcn_all_lemmas_bnc_ic_2000   = pickle.load(open(\"jcn_all_lemmas_bnc_ic_2000_20170821.pickle\", 'rb'))\n",
    "jcn_all_lemmas_semcor_ic     = pickle.load(open(\"jcn_all_lemmas_semcor_ic_20170821.pickle\", 'rb'))\n",
    "jcn_all_lemmas_brown_ic      = pickle.load(open(\"jcn_all_lemmas_brown_ic_20170821.pickle\", 'rb'))\n",
    "\n",
    "jcn_all_lemmapos_bnc_ic_2007   = pickle.load(open(\"jcn_all_lemmapos_bnc_ic_2007_20170821.pickle\", 'rb'))\n",
    "jcn_all_lemmapos_bnc_ic_2000   = pickle.load(open(\"jcn_all_lemmapos_bnc_ic_2000_20170821.pickle\", 'rb'))\n",
    "jcn_all_lemmapos_semcor_ic     = pickle.load(open(\"jcn_all_lemmapos_semcor_ic_20170821.pickle\", 'rb'))\n",
    "jcn_all_lemmapos_brown_ic      = pickle.load(open(\"jcn_all_lemmapos_brown_ic_20170821.pickle\", 'rb'))\n",
    "\n",
    "jcn_first_tokens_bnc_ic_2007   = pickle.load(open(\"jcn_first_tokens_bnc_ic_2007_20170821.pickle\", 'rb'))\n",
    "jcn_first_tokens_bnc_ic_2000   = pickle.load(open(\"jcn_first_tokens_bnc_ic_2000_20170821.pickle\", 'rb'))\n",
    "jcn_first_tokens_semcor_ic     = pickle.load(open(\"jcn_first_tokens_semcor_ic_20170821.pickle\", 'rb'))\n",
    "jcn_first_tokens_brown_ic      = pickle.load(open(\"jcn_first_tokens_brown_ic_20170821.pickle\", 'rb'))\n",
    "\n",
    "jcn_first_lemmas_bnc_ic_2007   = pickle.load(open(\"jcn_first_lemmas_bnc_ic_2007_20170821.pickle\", 'rb'))\n",
    "jcn_first_lemmas_bnc_ic_2000   = pickle.load(open(\"jcn_first_lemmas_bnc_ic_2000_20170821.pickle\", 'rb'))\n",
    "jcn_first_lemmas_semcor_ic     = pickle.load(open(\"jcn_first_lemmas_semcor_ic_20170821.pickle\", 'rb'))\n",
    "jcn_first_lemmas_brown_ic      = pickle.load(open(\"jcn_first_lemmas_brown_ic_20170821.pickle\", 'rb'))\n",
    "\n",
    "jcn_first_lemmapos_bnc_ic_2007   = pickle.load(open(\"jcn_first_lemmapos_bnc_ic_2007_20170821.pickle\", 'rb'))\n",
    "jcn_first_lemmapos_bnc_ic_2000   = pickle.load(open(\"jcn_first_lemmapos_bnc_ic_2000_20170821.pickle\", 'rb'))\n",
    "jcn_first_lemmapos_semcor_ic     = pickle.load(open(\"jcn_first_lemmapos_semcor_ic_20170821.pickle\", 'rb'))\n",
    "jcn_first_lemmapos_brown_ic      = pickle.load(open(\"jcn_first_lemmapos_brown_ic_20170821.pickle\", 'rb'))\n",
    "\n",
    "\n",
    "# LIN Similarity\n",
    "lin_all_tokens_bnc_ic_2007   = pickle.load(open(\"lin_all_tokens_bnc_ic_2007_20170821.pickle\", 'rb'))\n",
    "lin_all_tokens_bnc_ic_2000   = pickle.load(open(\"lin_all_tokens_bnc_ic_2000_20170821.pickle\", 'rb'))\n",
    "lin_all_tokens_semcor_ic     = pickle.load(open(\"lin_all_tokens_semcor_ic_20170821.pickle\", 'rb'))\n",
    "lin_all_tokens_brown_ic      = pickle.load(open(\"lin_all_tokens_brown_ic_20170821.pickle\", 'rb'))\n",
    "\n",
    "lin_all_lemmas_bnc_ic_2007   = pickle.load(open(\"lin_all_lemmas_bnc_ic_2007_20170821.pickle\", 'rb'))\n",
    "lin_all_lemmas_bnc_ic_2000   = pickle.load(open(\"lin_all_lemmas_bnc_ic_2000_20170821.pickle\", 'rb'))\n",
    "lin_all_lemmas_semcor_ic     = pickle.load(open(\"lin_all_lemmas_semcor_ic_20170821.pickle\", 'rb'))\n",
    "lin_all_lemmas_brown_ic      = pickle.load(open(\"lin_all_lemmas_brown_ic_20170821.pickle\", 'rb'))\n",
    "\n",
    "lin_all_lemmapos_bnc_ic_2007   = pickle.load(open(\"lin_all_lemmapos_bnc_ic_2007_20170821.pickle\", 'rb'))\n",
    "lin_all_lemmapos_bnc_ic_2000   = pickle.load(open(\"lin_all_lemmapos_bnc_ic_2000_20170821.pickle\", 'rb'))\n",
    "lin_all_lemmapos_semcor_ic     = pickle.load(open(\"lin_all_lemmapos_semcor_ic_20170821.pickle\", 'rb'))\n",
    "lin_all_lemmapos_brown_ic      = pickle.load(open(\"lin_all_lemmapos_brown_ic_20170821.pickle\", 'rb'))\n",
    "\n",
    "lin_first_tokens_bnc_ic_2007   = pickle.load(open(\"lin_first_tokens_bnc_ic_2007_20170821.pickle\", 'rb'))\n",
    "lin_first_tokens_bnc_ic_2000   = pickle.load(open(\"lin_first_tokens_bnc_ic_2000_20170821.pickle\", 'rb'))\n",
    "lin_first_tokens_semcor_ic     = pickle.load(open(\"lin_first_tokens_semcor_ic_20170821.pickle\", 'rb'))\n",
    "lin_first_tokens_brown_ic      = pickle.load(open(\"lin_first_tokens_brown_ic_20170821.pickle\", 'rb'))\n",
    "\n",
    "lin_first_lemmas_bnc_ic_2007   = pickle.load(open(\"lin_first_lemmas_bnc_ic_2007_20170821.pickle\", 'rb'))\n",
    "lin_first_lemmas_bnc_ic_2000   = pickle.load(open(\"lin_first_lemmas_bnc_ic_2000_20170821.pickle\", 'rb'))\n",
    "lin_first_lemmas_semcor_ic     = pickle.load(open(\"lin_first_lemmas_semcor_ic_20170821.pickle\", 'rb'))\n",
    "lin_first_lemmas_brown_ic      = pickle.load(open(\"lin_first_lemmas_brown_ic_20170821.pickle\", 'rb'))\n",
    "\n",
    "lin_first_lemmapos_bnc_ic_2007   = pickle.load(open(\"lin_first_lemmapos_bnc_ic_2007_20170821.pickle\", 'rb'))\n",
    "lin_first_lemmapos_bnc_ic_2000   = pickle.load(open(\"lin_first_lemmapos_bnc_ic_2000_20170821.pickle\", 'rb'))\n",
    "lin_first_lemmapos_semcor_ic     = pickle.load(open(\"lin_first_lemmapos_semcor_ic_20170821.pickle\", 'rb'))\n",
    "lin_first_lemmapos_brown_ic      = pickle.load(open(\"lin_first_lemmapos_brown_ic_20170821.pickle\", 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "idf = pickle.load(open(\"idf_bnc2007_20170810.pickle\", 'rb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Possible Configurations TODO list\n",
    "- Remove stopwords **Implemented!**\n",
    "- Remove punctuation **Implemented!**\n",
    "- Use binary or term frequency **Implemented!**\n",
    "- Different kind of term_sims normalizations **Implemented!**\n",
    "- Different WordNet term_sims **Implemented!**\n",
    "- Softcosine or stevenson metric **Implemented!**\n",
    "- Set as 0 all term similarity values below a given threshold **Implemented!**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "remove_stopwords_opt = [False, True]\n",
    "remove_punctuation_opt = [False, True]\n",
    "termsim_metric_opt = [\"path\", \"lch\", \"wup\", \"res\", \"jcn\", \"lin\"]\n",
    "normalization_opt = [True, False]\n",
    "termsim_threshold_opt = [0.0, 0.25, 0.5, 0.75]#np.linspace(0.0, 1.0, 21) # Every 5% # np.linspace(0.5, 1.0, 10)\n",
    "synsets_taken_opt = [\"all\", \"first\"]\n",
    "wscheme_opt = [\"tf\", \"binary\"]\n",
    "sim_metric_opt = [\"mihalcea\", \"softcosine\", \"stevenson\"]\n",
    "\n",
    "# This order should be kept given that it is used later\n",
    "features_opt = [\"token\", \"lemma\", \"stem\", \"lemmapos\"]\n",
    "synset_getter_opt = [\"token\", \"lemma\", \"lemmapos\"]\n",
    "ic_data_opt = [\"bnc_ic_2007\", \"bnc_ic_2000\", \"semcor_ic\", \"brown_ic\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab_tokens\n",
      "vocab_lemmas\n",
      "vocab_lemmapos\n"
     ]
    }
   ],
   "source": [
    "for x in synset_getter_opt:\n",
    "    print(\"vocab_\"+x+ ('' if x[-1]=='s' else 's'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Running experiments with different combinations of parameters"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Generating configurations\n",
    "Generating the configurations trying to avoid any unnecessary repetition"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "configurations = list(itertools.product(remove_stopwords_opt,\n",
    "                                        remove_punctuation_opt,\n",
    "                                        termsim_metric_opt,\n",
    "                                        normalization_opt,\n",
    "                                        #termsim_threshold_opt,\n",
    "                                        synsets_taken_opt,\n",
    "                                        wscheme_opt,\n",
    "                                        sim_metric_opt))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# No normalization, No ic_data\n",
    "configurations = list(itertools.product(remove_stopwords_opt,\n",
    "                                        [True], #remove_punctuation_opt,\n",
    "                                        [\"path\", \"wup\"], # metrics\n",
    "                                        features_opt,\n",
    "                                        synset_getter_opt,\n",
    "                                        [None], # ic_data not needed\n",
    "                                        [False], # normalization not needed\n",
    "                                        termsim_threshold_opt,\n",
    "                                        synsets_taken_opt,\n",
    "                                        wscheme_opt,\n",
    "                                        sim_metric_opt))\n",
    "\n",
    "# No normalization, Yes ic_data\n",
    "configurations += list(itertools.product(remove_stopwords_opt,\n",
    "                                         [True], # remove_punctuation_opt,\n",
    "                                         [\"lin\"], # metric\n",
    "                                         features_opt,\n",
    "                                         synset_getter_opt,\n",
    "                                         ic_data_opt, # ic_data needed\n",
    "                                         [False], # normalization not needed\n",
    "                                         termsim_threshold_opt,\n",
    "                                         synsets_taken_opt,\n",
    "                                         wscheme_opt,\n",
    "                                         sim_metric_opt))\n",
    "\n",
    "# Yes normalization, No ic_data\n",
    "configurations += list(itertools.product(remove_stopwords_opt,\n",
    "                                         [True], # remove_punctuation_opt,\n",
    "                                         [\"lch\"], # metric\n",
    "                                         features_opt,\n",
    "                                         synset_getter_opt,\n",
    "                                         [None], # ic_data not needed\n",
    "                                         [True], # normalization needed\n",
    "                                         termsim_threshold_opt,\n",
    "                                         synsets_taken_opt,\n",
    "                                         wscheme_opt,\n",
    "                                         sim_metric_opt))\n",
    "\n",
    "# Yes normalization, Yes ic_data\n",
    "configurations += list(itertools.product(remove_stopwords_opt, \n",
    "                                         [True], # remove_punctuation_opt,\n",
    "                                         [\"jcn\", \"res\"], # metrics\n",
    "                                         features_opt,\n",
    "                                         synset_getter_opt,\n",
    "                                         ic_data_opt, # ic_data needed\n",
    "                                         [True], # normalization needed\n",
    "                                         termsim_threshold_opt,\n",
    "                                         synsets_taken_opt,\n",
    "                                         wscheme_opt,\n",
    "                                         sim_metric_opt))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Converting from string to indexes given certain pre-processing options\n",
    "Preprocessing options:\n",
    "- Removing stopwords\n",
    "- Removing punctuation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tags_mapping = {\"NN\":wn.NOUN, \"VB\":wn.VERB, \"JJ\":wn.ADJ, \"RB\":wn.ADV}\n",
    "stopwords = set(nltk.corpus.stopwords.words(\"english\"))\n",
    "puncts = set(string.punctuation)\n",
    "vectors_no_prep = {}\n",
    "vectors_no_sw = {}\n",
    "vectors_no_punct = {}\n",
    "vectors_prep = {}\n",
    "for idx in parsed_texts:   \n",
    "    vectors_no_prep[idx] = [(token2index[token],\n",
    "                             lemma2index[lemma],\n",
    "                             stem2index[stem],\n",
    "                             lemmapos2index[(lemma, tags_mapping.get(tag[:2], tag))])\n",
    "                            for token, lemma, stem, tag in parsed_texts[idx]\n",
    "                           ]\n",
    "    \n",
    "    vectors_no_sw[idx] = [(token2index[token],\n",
    "                           lemma2index[lemma],\n",
    "                           stem2index[stem],\n",
    "                           lemmapos2index[(lemma, tags_mapping.get(tag[:2], tag))])\n",
    "                          for token, lemma, stem, tag in parsed_texts[idx]\n",
    "                          if token not in stopwords # Removing stopwords\n",
    "                         ]\n",
    "\n",
    "    vectors_no_punct[idx] = [(token2index[token],\n",
    "                              lemma2index[lemma],\n",
    "                              stem2index[stem],\n",
    "                              lemmapos2index[(lemma, tags_mapping.get(tag[:2], tag))])\n",
    "                             for token, lemma, stem, tag in parsed_texts[idx]\n",
    "                             if not is_punct_token(token, puncts) # Removing punctuation\n",
    "                            ]\n",
    "    \n",
    "    vectors_prep[idx] = [(token2index[token],\n",
    "                          lemma2index[lemma],\n",
    "                          stem2index[stem],\n",
    "                          lemmapos2index[(lemma, tags_mapping.get(tag[:2], tag))])\n",
    "                         for token, lemma, stem, tag in parsed_texts[idx]\n",
    "                         if token not in stopwords # Removing stopwords\n",
    "                         and not is_punct_token(token, puncts) # Removing punctuation\n",
    "                        ]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Mapping feature indexes to wordnet getter indexes\n",
    "For the particular case of the MSRP Corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mappings = {}\n",
    "for feature in features_opt:\n",
    "    for syngetter in synset_getter_opt:\n",
    "        comb_dict = {}\n",
    "        for idx in vectors_no_prep:\n",
    "            for token_id, lemma_id, stem_id, lemmapos_id in vectors_no_prep[idx]:\n",
    "                ft_idx = eval(feature+\"_id\")\n",
    "                wn_idx = eval(syngetter+\"_id\")       \n",
    "                if ft_idx in comb_dict:\n",
    "                    comb_dict[ft_idx].add(wn_idx)\n",
    "                else:\n",
    "                    comb_dict[ft_idx] = set([wn_idx])\n",
    "        mappings[(feature, syngetter)] = comb_dict\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Experiments with given configurations in the Training Corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8.30622521603216"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idf_median_value = np.median(list(idf.values()))\n",
    "idf_median_value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Progress ...\n",
      "[==                                      ] 5%   964/17280"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-24-7d5f5ab91b81>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     61\u001b[0m                         \u001b[0mterm_sims\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mterm_sims\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m                         \u001b[0midf\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0midf\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m---> 63\u001b[0;31m                         \u001b[0midf_median_value\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0midf_median_value\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     64\u001b[0m                        )\n\u001b[1;32m     65\u001b[0m     \u001b[1;31m#sims = compute_sims(test_pairs, vectors, weight_scheme=wscheme, metric=sim_metric, term_sims=term_sims)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-9-62501e14e4ae>\u001b[0m in \u001b[0;36mcompute_sims\u001b[0;34m(pairs, vectors, feature, ft_id2wn_id, ft_id2token_id, metric, weight_scheme, term_sims, idf, idf_median_value)\u001b[0m\n\u001b[1;32m     20\u001b[0m         \u001b[1;31m# Similarity computation\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m     21\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mmetric\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m\"softcosine\"\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m---> 22\u001b[0;31m             \u001b[0mres\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msoftcosine\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mv1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mv2\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mft_id2wn_id\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mterm_sims\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     23\u001b[0m         \u001b[1;32melif\u001b[0m \u001b[0mmetric\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m\"cosine\"\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m             \u001b[0mres\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcosine\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mv1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mv2\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-4-1f4ca0558e91>\u001b[0m in \u001b[0;36msoftcosine\u001b[0;34m(vect1, vect2, ft_id2wn_id, term_sims)\u001b[0m\n\u001b[1;32m      3\u001b[0m     <a> and <b> in a dictionary format, and the .\"\"\"\n\u001b[1;32m      4\u001b[0m     \u001b[0mdot_prod\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;36m0.0\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m     \u001b[0mdet\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0msoft_euclidean_norm\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvect1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mft_id2wn_id\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mterm_sims\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m*\u001b[0m \u001b[0msoft_euclidean_norm\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mvect2\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mft_id2wn_id\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mterm_sims\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mdet\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m0.0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[1;36m0.0\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-3-89186ca31c4f>\u001b[0m in \u001b[0;36msoft_euclidean_norm\u001b[0;34m(vect, ft_id2wn_id, term_sims)\u001b[0m\n\u001b[1;32m     16\u001b[0m                             \u001b[0msims\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mterm_sims\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0msim_key\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msims\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m>\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m---> 18\u001b[0;31m                     \u001b[0mnorm\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[0mmax\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msims\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m*\u001b[0m \u001b[0mvect\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mtoken1_id\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m*\u001b[0m \u001b[0mvect\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0mtoken2_id\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     19\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mmath\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msqrt\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnorm\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "all_scores = {}\n",
    "count = 0\n",
    "total_configurations = len(configurations)\n",
    "print(\"Progress ...\")\n",
    "for configuration in configurations:\n",
    "    (remove_stopwords,\n",
    "     remove_punctuation, \n",
    "     termsim_metric,\n",
    "     feature,\n",
    "     syngetter,\n",
    "     ic_data,\n",
    "     normalization,\n",
    "     termsim_th,\n",
    "     synsets_taken,\n",
    "     wscheme,\n",
    "     sim_metric) = configuration\n",
    "    \n",
    "    # Converting from word vectors to index vectors from the vocabulary\n",
    "    vocab_syngetter = eval(\"vocab_\"+syngetter+ ('' if syngetter[-1]=='s' else 's'))\n",
    "    syngetter2index = eval(syngetter+\"2index\")\n",
    "    index2syngetter = eval(\"index2\"+syngetter)\n",
    "    \n",
    "    vocab_feature = eval(\"vocab_\"+feature+ ('' if feature[-1]=='s' else 's'))\n",
    "    feature2index = eval(feature+\"2index\")\n",
    "    index2feature = eval(\"index2\"+feature)\n",
    "    \n",
    "    # Choosing the appropiated preprocessed vecotrs\n",
    "    if remove_stopwords:\n",
    "        if remove_punctuation:\n",
    "            vectors = vectors_prep\n",
    "        else:\n",
    "            vectors = vectors_no_sw\n",
    "    else:\n",
    "        if remove_punctuation:\n",
    "            vectors = vectors_no_punct\n",
    "        else:\n",
    "            vectors = vectors_no_prep\n",
    "        \n",
    "    \n",
    "    # Choosing the term similarity matrix\n",
    "    if ic_data:\n",
    "        term_sims = eval(termsim_metric+\"_\"+synsets_taken+\"_\"+syngetter+('' if syngetter[-1]=='s' else 's')+\"_\"+ic_data)\n",
    "    else:\n",
    "        term_sims = eval(termsim_metric+\"_\"+synsets_taken+\"_\"+syngetter+('' if syngetter[-1]=='s' else 's'))\n",
    "    \n",
    "    # Normalizing the term similarity matrix\n",
    "    if normalization:\n",
    "        term_sims = normalize(term_sims, termsim_metric)\n",
    "        \n",
    "    # Setting to zero term similarities below termsim_th\n",
    "    term_sims = dict((key, value) for key, value in term_sims.items() if value >= termsim_th)\n",
    "    \n",
    "    # Computing pair of texts similarities\n",
    "    sims = compute_sims(train_pairs,\n",
    "                        vectors,\n",
    "                        features_opt.index(feature),\n",
    "                        mappings[(feature, syngetter)],\n",
    "                        mappings[(feature, 'token')],\n",
    "                        weight_scheme=wscheme,\n",
    "                        metric=sim_metric,\n",
    "                        term_sims=term_sims,\n",
    "                        idf=idf,\n",
    "                        idf_median_value=idf_median_value\n",
    "                       )\n",
    "    #sims = compute_sims(test_pairs, vectors, weight_scheme=wscheme, metric=sim_metric, term_sims=term_sims)\n",
    "    \n",
    "    # Computing scores\n",
    "    scores = np.array(compute_scores(sims, train_y, np.linspace(0.05, 1.0, 20))) # [x/100.0 for x in range(5, 101, 5)]))\n",
    "    #scores = np.array(compute_scores(sims, test_y, np.linspace(0.05, 1.0, 20))) # [x/100.0 for x in range(5, 101, 5)]))\n",
    "    \n",
    "    all_scores[configuration] = scores[scores[:,1].argsort()[::-1]]\n",
    "    \n",
    "    #print(configuration)\n",
    "    #print(all_scores[configuration][:3], \"\\n\\n\")\n",
    "    #break\n",
    "    sys.stdout.write('\\r')\n",
    "    # the exact output you're looking for:\n",
    "    count += 1\n",
    "    i = count * 40.0 / total_configurations\n",
    "    sys.stdout.write(\"[{:40}] {:.0f}%   {}/{}\".format('='*int(i), int(2.5*i), count, total_configurations))\n",
    "    sys.stdout.flush()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Saving the results in HD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'20180504'"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "date_obj = datetime.date.today()\n",
    "date_str = \"{:04d}\".format(date_obj.year) + \"{:02d}\".format(date_obj.month) + \"{:02d}\".format(date_obj.day) \n",
    "date_str"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pickle.dump(all_scores, open(\"results_\"+date_str+\"_simple_normalization.pickle\", \"wb\"), protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Extracting best results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Configuration format\n",
    "1. Removing stopwords flag  :  [True, False]\n",
    "\n",
    "2. Removing punctuation flag  :  [True, False]\n",
    "\n",
    "3. Wordnet similarity metrics  :  [\"path\", \"lch\", \"wup\", \"res\", \"jcn\", \"lin\"]\n",
    "\n",
    "4. Features extracted to compute similarity  :  [\"token\", \"lemma\", \"stem\", \"lemmapos\"]\n",
    "\n",
    "5. Features used to extract synsets  :  [\"token\", \"lemma\", \"stem\", \"lemmapos\"]\n",
    "\n",
    "6. Information Content used in some WordNet metrics  :  [\"bnc_ic_2007\", \"bnc_ic_2000\", \"semcor_ic\", \"brown_ic\"]\n",
    "\n",
    "7. Normalization flag : [True, False]\n",
    "\n",
    "8. Term-Term similarity minimum threslhold  :  [0.0, 0.25, 0.5, 0.75]            \n",
    "\n",
    "9. Synsets selection strategy (all-vs-all, first)  :  [\"all\", \"first\"]\n",
    "\n",
    "10. Features weighting scheme  :  [\"tf\", \"binary\"]\n",
    "\n",
    "11. Text similarity method  :  [\"mihalcea\", \"softcosine\", \"stevenson\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((False,\n",
       "  True,\n",
       "  'res',\n",
       "  'stem',\n",
       "  'lemmapos',\n",
       "  'semcor_ic',\n",
       "  True,\n",
       "  0.0,\n",
       "  'first',\n",
       "  'tf',\n",
       "  'mihalcea'),\n",
       " 0.54999999999999993,\n",
       " 0.73307163886162907,\n",
       " 0.82779360557138326)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# res = (threshold, accuracy, f_measure, precision, recall)\n",
    "max([(conf, res[0][0], res[0][1], res[0][2])\n",
    "     for conf, res in all_scores.items()\n",
    "     #if conf[2] == \"path\"\n",
    "    ]\n",
    "    , key=lambda x:x[3])\n",
    "#[(conf, res[0][0], res[0][1]) for conf, res in all_scores.items()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "17280"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(all_scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.55      ,  0.72988224,  0.82227603,  0.73997676,  0.92517254],\n",
       "       [ 0.6       ,  0.72669284,  0.80963773,  0.76444014,  0.8605158 ],\n",
       "       [ 0.5       ,  0.72105005,  0.82434729,  0.7172043 ,  0.96912459],\n",
       "       [ 0.65      ,  0.71393523,  0.7869152 ,  0.79183523,  0.78205594],\n",
       "       [ 0.45      ,  0.70387635,  0.81781132,  0.69963843,  0.98401744],\n",
       "       [ 0.7       ,  0.69357213,  0.75418225,  0.82302405,  0.69596803],\n",
       "       [ 0.4       ,  0.68719333,  0.81085892,  0.68530592,  0.9927352 ],\n",
       "       [ 0.35      ,  0.68204122,  0.80924345,  0.68027716,  0.99854704],\n",
       "       [ 0.3       ,  0.67590775,  0.80644689,  0.67583497,  0.99963676],\n",
       "       [ 0.15      ,  0.67566241,  0.80638547,  0.67558282,  1.        ],\n",
       "       [ 0.2       ,  0.67566241,  0.80638547,  0.67558282,  1.        ],\n",
       "       [ 0.25      ,  0.67566241,  0.80638547,  0.67558282,  1.        ],\n",
       "       [ 0.1       ,  0.67541708,  0.80626739,  0.67541708,  1.        ],\n",
       "       [ 0.05      ,  0.67541708,  0.80626739,  0.67541708,  1.        ],\n",
       "       [ 0.75      ,  0.64842983,  0.69016216,  0.8525641 ,  0.5797312 ],\n",
       "       [ 0.8       ,  0.58709519,  0.59494585,  0.88159772,  0.44896477],\n",
       "       [ 0.85      ,  0.5107949 ,  0.45785753,  0.91027027,  0.30584817],\n",
       "       [ 0.9       ,  0.42836114,  0.2808642 ,  0.93429158,  0.16527425],\n",
       "       [ 0.95      ,  0.37708538,  0.15112003,  0.94957983,  0.08209226],\n",
       "       [ 1.        ,  0.34887144,  0.07202797,  0.96261682,  0.03741373]])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_scores[(False, True, 'jcn', 'lemma', 'lemma', 'semcor_ic', True, 0.0, 'all', 'binary', 'stevenson')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.5       ,  0.71344455,  0.80946166,  0.73467575,  0.90119869],\n",
       "       [ 0.45      ,  0.70878312,  0.8132767 ,  0.7172586 ,  0.93897566],\n",
       "       [ 0.55      ,  0.70632974,  0.79222357,  0.75864362,  0.82891391],\n",
       "       [ 0.4       ,  0.70019627,  0.8136627 ,  0.70118265,  0.96912459],\n",
       "       [ 0.35      ,  0.69381747,  0.81261261,  0.69260302,  0.98292772],\n",
       "       [ 0.6       ,  0.69332679,  0.76851852,  0.78390631,  0.75372321],\n",
       "       [ 0.3       ,  0.6844946 ,  0.80919881,  0.68397291,  0.99055576],\n",
       "       [ 0.1       ,  0.67566241,  0.80638547,  0.67558282,  1.        ],\n",
       "       [ 0.15      ,  0.67541708,  0.80615385,  0.67558939,  0.99927352],\n",
       "       [ 0.25      ,  0.67541708,  0.80558413,  0.67645607,  0.99564112],\n",
       "       [ 0.05      ,  0.67541708,  0.80626739,  0.67541708,  1.        ],\n",
       "       [ 0.2       ,  0.67443572,  0.80539669,  0.67535662,  0.99745732],\n",
       "       [ 0.65      ,  0.67100098,  0.72826748,  0.82355637,  0.65274246],\n",
       "       [ 0.7       ,  0.63076546,  0.66828301,  0.84977578,  0.55067199],\n",
       "       [ 0.75      ,  0.57801766,  0.58373669,  0.87454677,  0.43806756],\n",
       "       [ 0.8       ,  0.50245339,  0.44590164,  0.89966924,  0.29640392],\n",
       "       [ 0.85      ,  0.41781158,  0.25913206,  0.92222222,  0.15074464],\n",
       "       [ 0.9       ,  0.36432777,  0.11358194,  0.97647059,  0.06029786],\n",
       "       [ 0.95      ,  0.33881256,  0.04126645,  1.        ,  0.02106793],\n",
       "       [ 1.        ,  0.33071639,  0.01799856,  1.        ,  0.009081  ]])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_scores[(True, True, 'jcn', 'lemmapos', 'lemmapos', 'bnc_ic_2000', True, 0.0, 'all', 'tf', 'mihalcea')]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Serializing all results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluating best configuration for each WordNet metric on the test set"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Best-k configurations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for metric in [\"path\", \"lch\", \"wup\", \"res\", \"jcn\", \"lin\"]:\n",
    "    print(metric)\n",
    "    # conf = (0-remove_stopwords, 1-remove_punctuation, 2-termsim_metric, 3-normalization,\n",
    "    #         4-termsim_th, 5-synsets_taken, 6-wscheme, 7-sim_metric)\n",
    "    #\n",
    "    # scores = (0-threshold, 1-accuracy, 2-f_measure, 3-precision, 4-recall)\n",
    "    #\n",
    "    for conf, score in sorted([(conf, scores[0,:])\n",
    "                               for conf, scores in all_scores.items()\n",
    "                               if conf[2] == metric],\n",
    "                              key=lambda tup:tup[1][1], # tup=(conf, scores)\n",
    "                              reverse=True)[:10]:\n",
    "        print(conf)\n",
    "        print(score, \"\\n\")\n",
    "    print(\"\\n\\n\")\n",
    "    #break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "list(all_scores.items())[0][1][0,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "options = [\n",
    "           (False, True, 'jcn', True, 0.0, 'all', 'binary', 'softcosine'),\n",
    "           (False, True, 'jcn', True, 0.0, 'all', 'binary', 'stevenson'),\n",
    "           (False, True, 'jcn', True, 0.0, 'all', 'binary', 'stevenson')\n",
    "          ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Loading best configurations from training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_configurations = pickle.load(open(\"test_configurations.pickle\", \"rb\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluating best configurations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Progress ...\n",
      "[========================================] 100%   18/18"
     ]
    }
   ],
   "source": [
    "test_scores = {}\n",
    "count = 0\n",
    "total_configurations = len(test_configurations)\n",
    "print(\"Progress ...\")\n",
    "for configuration in test_configurations:\n",
    "    (remove_stopwords,\n",
    "     remove_punctuation, \n",
    "     termsim_metric,\n",
    "     feature,\n",
    "     syngetter,\n",
    "     ic_data,\n",
    "     normalization,\n",
    "     termsim_th,\n",
    "     synsets_taken,\n",
    "     wscheme,\n",
    "     sim_metric) = configuration\n",
    "    \n",
    "    # Converting from word vectors to index vectors from the vocabulary\n",
    "    vocab_syngetter = eval(\"vocab_\"+syngetter+ ('' if syngetter[-1]=='s' else 's'))\n",
    "    syngetter2index = eval(syngetter+\"2index\")\n",
    "    index2syngetter = eval(\"index2\"+syngetter)\n",
    "    \n",
    "    vocab_feature = eval(\"vocab_\"+feature+ ('' if feature[-1]=='s' else 's'))\n",
    "    feature2index = eval(feature+\"2index\")\n",
    "    index2feature = eval(\"index2\"+feature)\n",
    "    \n",
    "    # Choosing the appropiated preprocessed vecotrs\n",
    "    if remove_stopwords:\n",
    "        if remove_punctuation:\n",
    "            vectors = vectors_prep\n",
    "        else:\n",
    "            vectors = vectors_no_sw\n",
    "    else:\n",
    "        if remove_punctuation:\n",
    "            vectors = vectors_no_punct\n",
    "        else:\n",
    "            vectors = vectors_no_prep\n",
    "        \n",
    "    \n",
    "    # Choosing the term similarity matrix\n",
    "    if ic_data:\n",
    "        term_sims = eval(termsim_metric+\"_\"+synsets_taken+\"_\"+syngetter+('' if syngetter[-1]=='s' else 's')+\"_\"+ic_data)\n",
    "    else:\n",
    "        term_sims = eval(termsim_metric+\"_\"+synsets_taken+\"_\"+syngetter+('' if syngetter[-1]=='s' else 's'))\n",
    "    \n",
    "    # Normalizing the term similarity matrix\n",
    "    if normalization:\n",
    "        term_sims = normalize(term_sims, termsim_metric)\n",
    "        \n",
    "    # Setting to zero term similarities below termsim_th\n",
    "    term_sims = dict((key, value) for key, value in term_sims.items() if value >= termsim_th)\n",
    "    \n",
    "    # Computing pair of texts similarities\n",
    "    sims = compute_sims(test_pairs,\n",
    "                        vectors,\n",
    "                        features_opt.index(feature),\n",
    "                        mappings[(feature, syngetter)],\n",
    "                        mappings[(feature, 'token')],\n",
    "                        weight_scheme=wscheme,\n",
    "                        metric=sim_metric,\n",
    "                        term_sims=term_sims,\n",
    "                        idf=idf,\n",
    "                        idf_median_value=idf_median_value\n",
    "                       )\n",
    "    #sims = compute_sims(test_pairs, vectors, weight_scheme=wscheme, metric=sim_metric, term_sims=term_sims)\n",
    "    \n",
    "    # Computing scores\n",
    "    scores = np.array(compute_scores(sims, test_y, np.linspace(0.05, 1.0, 20))) # [x/100.0 for x in range(5, 101, 5)]))\n",
    "    \n",
    "    test_scores[configuration] = scores[scores[:,1].argsort()[::-1]]\n",
    "    \n",
    "    sys.stdout.write('\\r')\n",
    "    # the exact output you're looking for:\n",
    "    count += 1\n",
    "    i = count * 40.0 / total_configurations\n",
    "    sys.stdout.write(\"[{:40}] {:.0f}%   {}/{}\".format('='*int(i), int(2.5*i), count, total_configurations))\n",
    "    sys.stdout.flush()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Storing test results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open(\"./Results/test_results_20180504.pickle\", \"wb\") as fid:\n",
    "    pickle.dump(test_scores, fid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}