tatoeba_ds.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from collections import defaultdict\n",
    "import random\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_to_data = '/home/hamza/3A_projets/data_all/glose/sentences.csv'\n",
    "df_sents = pd.read_csv(path_to_data,sep='\\t',header=None)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_sent_en = df_sents[df_sents[1]=='eng'] ## select english sentences\n",
    "df_sent_en = df_sent_en[2].to_frame()\n",
    "df_sent_en.rename(columns={2:'sentence'},inplace=True)\n",
    "df_sent_en = df_sent_en.sample(200000)  ## we take a subset of 100 000 sentences to have a trainable model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Many examples of tatoeba are in fact composed by more than one sentence. so we will use spacy sentence segmentation to parse them : this is relevant since spacy is good especially when the separators are clear (dots , two points,etc...). The model that I will implement will be especially relevant to separate sentences that do not contain these separators. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 200000/200000 [25:27<00:00, 130.90it/s]\n"
     ]
    }
   ],
   "source": [
    "import spacy\n",
    "from tqdm import tqdm\n",
    "#!python3 -m spacy download en_core_web_sm\n",
    "nlp = spacy.load('en_core_web_sm')\n",
    "k=0\n",
    "all_sentences = df_sent_en.sentence.values\n",
    "sentences = []\n",
    "for text in tqdm(all_sentences,total = len(all_sentences)) :\n",
    "    doc = nlp(text)\n",
    "    if 'and' in text.split(): \n",
    "        p = random.random()\n",
    "        if p<0.15 : \n",
    "            sentences.append(text) ## we take 15% of examples that contain and \n",
    "    if len(list(doc.sents))<=1 :\n",
    "        sentences.append(text)\n",
    "    else : \n",
    "        for sentence in doc.sents :\n",
    "            sentences.append(sentence.text)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "#import pickle\n",
    "#pickle.dump(sentences,open('data/sentences.pkl','wb'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tokenize "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "df_sentences = pd.DataFrame(data={'sentence':sentences})\n",
    "df_sentences['tokenized'] = df_sentences.sentence.apply(lambda sent : nltk.word_tokenize(sent))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The dot ends the majority of the sentences. We will change this by randomly removing dots , interrogation points. We also add randomly commas , two points , 'and' connector ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "## constitue new sentences \n",
    "change_sents = []\n",
    "tokenized_sents = list(df_sentences['tokenized'].values)\n",
    "probs = {'.':0.85, '?':0.7 , '!' : 0.35}\n",
    "conjunction = {'and':0.3 , ',' :0.3,';':0.2,':':0.2}\n",
    "n = len(tokenized_sents)\n",
    "for j,sent in enumerate(tokenized_sents)  :\n",
    "    if sent[-1] in ['.','?','!'] : \n",
    "        k = np.random.binomial(size=1, n=1, p= probs[sent[-1]])\n",
    "        if k == 0 : \n",
    "            change_sents.append(sent)\n",
    "        else : \n",
    "            if len(sent[:-1])==0 : \n",
    "                continue\n",
    "            else :\n",
    "                q = np.random.binomial(size=1, n=1, p=0.4)\n",
    "                if q == 0 :\n",
    "                    change_sents.append(sent[:-1])\n",
    "                elif q==1 :\n",
    "                    p = random.random()\n",
    "                    if p < conjunction['and'] : \n",
    "                        if j != n-1 : \n",
    "                            tokenized_sents[j+1].insert(0,'and')\n",
    "                            change_sents.append(sent[:-1])\n",
    "                    elif conjunction['and']<=p < conjunction['and'] + conjunction[','] :\n",
    "                        sent=sent[:-1]+[',']\n",
    "                        change_sents.append(sent)\n",
    "                    elif conjunction['and'] + conjunction[',']<=p < conjunction['and'] + conjunction[','] + conjunction[':']:\n",
    "                        sent=sent[:-1]+[':']\n",
    "                        change_sents.append(sent)\n",
    "                    elif conjunction['and'] + conjunction[','] + conjunction[':'] <=p :\n",
    "                        sent=sent[:-1]+[';']\n",
    "                        change_sents.append(sent)\n",
    "                        \n",
    "                    \n",
    "    else :\n",
    "        change_sents.append(sent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{',': 20368,\n",
       " 'lawyer': 50,\n",
       " 'thinking': 29,\n",
       " ':': 13908,\n",
       " 'anymore': 433,\n",
       " 'U.S': 3,\n",
       " ';': 13559,\n",
       " 'dramatically': 3,\n",
       " '.': 27062,\n",
       " 'well': 420,\n",
       " 'Islamophobic': 3,\n",
       " 'today': 739,\n",
       " 'attended': 3,\n",
       " 'delivered': 3,\n",
       " 'Australia': 554,\n",
       " 'taxi': 18,\n",
       " 'that': 5960,\n",
       " 'them': 732,\n",
       " 'bad': 74,\n",
       " 'stay': 63,\n",
       " 'required': 5,\n",
       " 'boss': 33,\n",
       " 'company': 67,\n",
       " 'suit': 16,\n",
       " 'floor': 66,\n",
       " 'register': 8,\n",
       " 'up': 431,\n",
       " 'diner': 1,\n",
       " 'mad': 59,\n",
       " 'weather': 34,\n",
       " 'coins': 10,\n",
       " 'adopt': 1,\n",
       " 'sentence': 38,\n",
       " 'home': 522,\n",
       " 'blaze': 1,\n",
       " 'skate': 3,\n",
       " 'border': 7,\n",
       " 'you': 2123,\n",
       " 'now': 853,\n",
       " 'fainted': 3,\n",
       " 'Tatoeba': 28,\n",
       " '?': 7016,\n",
       " 'choice': 74,\n",
       " 'future': 64,\n",
       " 'wine': 77,\n",
       " 'me': 1841,\n",
       " 'mouth': 32,\n",
       " 'married': 148,\n",
       " 'teachers': 26,\n",
       " 'meeting': 119,\n",
       " 'disappointed': 37,\n",
       " 'Yoshkar-Ola': 2,\n",
       " 'wrong': 224,\n",
       " 'Tom': 1666,\n",
       " 'great': 39,\n",
       " 'fast': 64,\n",
       " 'secrets': 10,\n",
       " 'morning': 277,\n",
       " 'computer': 61,\n",
       " 'hometown': 5,\n",
       " 'alike': 9,\n",
       " 'mosque': 58,\n",
       " 'assignment': 3,\n",
       " '2': 3,\n",
       " 'says': 43,\n",
       " 'failure': 26,\n",
       " 'try': 44,\n",
       " 'ones': 23,\n",
       " 'mirror': 23,\n",
       " 'side': 37,\n",
       " 'all': 225,\n",
       " 'day': 484,\n",
       " 'this': 716,\n",
       " 'jacket': 16,\n",
       " 'couples': 1,\n",
       " 'block': 5,\n",
       " 'tongue': 18,\n",
       " 'Pona': 1,\n",
       " 'happen': 214,\n",
       " 'early': 194,\n",
       " 'fraud': 3,\n",
       " 'her': 1164,\n",
       " 'neither': 3,\n",
       " 'avoided': 3,\n",
       " 'children': 182,\n",
       " 'too': 219,\n",
       " 'life': 330,\n",
       " 'Tuesday': 5,\n",
       " 'structure': 7,\n",
       " 'ill': 26,\n",
       " 'yet': 330,\n",
       " 'house': 396,\n",
       " 'long': 151,\n",
       " 'matter': 52,\n",
       " 'soldiers': 6,\n",
       " 'overflows': 1,\n",
       " 'us': 783,\n",
       " 'anything': 299,\n",
       " 'pregnant': 28,\n",
       " 'sister': 76,\n",
       " 'parents': 94,\n",
       " 'examination': 23,\n",
       " 'skating': 4,\n",
       " 'herself': 295,\n",
       " 'work': 351,\n",
       " 'time': 848,\n",
       " 'afternoon': 149,\n",
       " 'together': 302,\n",
       " 'things': 134,\n",
       " 'soon': 231,\n",
       " 'feet': 27,\n",
       " 'thief': 10,\n",
       " 'jobs': 32,\n",
       " 'stye': 1,\n",
       " 'understand': 65,\n",
       " 'camera': 30,\n",
       " 'leave': 190,\n",
       " 'concerned': 38,\n",
       " 'eating': 42,\n",
       " 'plans': 35,\n",
       " 'tremble': 3,\n",
       " 'juggler': 2,\n",
       " 'singer': 33,\n",
       " 'true': 147,\n",
       " 'else': 124,\n",
       " 'planet': 18,\n",
       " 'beautiful': 75,\n",
       " 'persuasive': 1,\n",
       " 'shelf': 19,\n",
       " 'myself': 184,\n",
       " 'digits': 1,\n",
       " 'red': 34,\n",
       " 'ideas': 29,\n",
       " 'towel': 18,\n",
       " 'Mary': 1070,\n",
       " 'one': 164,\n",
       " 'jockey': 2,\n",
       " 'ride': 24,\n",
       " 'quit': 25,\n",
       " 'available': 14,\n",
       " '!': 1592,\n",
       " 'longer': 100,\n",
       " 'sand': 11,\n",
       " 'Quran': 31,\n",
       " 'tired': 166,\n",
       " 'kid': 74,\n",
       " 'hour': 66,\n",
       " 'tonight': 178,\n",
       " 'it': 1665,\n",
       " 'woman': 74,\n",
       " 'way': 218,\n",
       " 'song': 40,\n",
       " 'French': 588,\n",
       " 'tradition': 7,\n",
       " 'fired': 58,\n",
       " 'arthropods': 1,\n",
       " 'assertive': 5,\n",
       " 'again': 704,\n",
       " 'English': 136,\n",
       " 'argue': 12,\n",
       " \"''\": 1295,\n",
       " 'boyfriend': 40,\n",
       " 'nature': 21,\n",
       " 'park': 63,\n",
       " 'piano': 37,\n",
       " 'hands': 66,\n",
       " 'library': 47,\n",
       " 'German': 25,\n",
       " 'fussy': 2,\n",
       " 'qualifications': 1,\n",
       " 'forbidden': 3,\n",
       " 'expert': 13,\n",
       " 'ago': 197,\n",
       " 'religious': 10,\n",
       " 'interest': 9,\n",
       " 'disoriented': 4,\n",
       " 'yesterday': 323,\n",
       " 'composure': 4,\n",
       " 'confusion': 4,\n",
       " 'thirsty': 37,\n",
       " 'band': 22,\n",
       " 'area': 38,\n",
       " 'birthday': 47,\n",
       " 'old': 108,\n",
       " 'pizza': 22,\n",
       " 'room': 406,\n",
       " 'him': 1638,\n",
       " 'key': 40,\n",
       " 'man': 196,\n",
       " 'coat': 61,\n",
       " 'lunch': 97,\n",
       " 'sing': 71,\n",
       " 'year': 254,\n",
       " 'help': 433,\n",
       " 'dryer': 3,\n",
       " 'dangerous': 43,\n",
       " 'did': 198,\n",
       " 'more': 123,\n",
       " 'God': 49,\n",
       " 'god': 8,\n",
       " 'count': 7,\n",
       " 'grief': 6,\n",
       " 'surfing': 3,\n",
       " 'meanings': 2,\n",
       " 'problems': 82,\n",
       " 'promise': 33,\n",
       " 'father': 132,\n",
       " 'die': 90,\n",
       " 'region': 18,\n",
       " 'think': 91,\n",
       " 'place': 106,\n",
       " 'baby': 51,\n",
       " '``': 392,\n",
       " 'not': 200,\n",
       " 'Boston': 752,\n",
       " 'funeral': 12,\n",
       " 'in': 145,\n",
       " 'privately': 1,\n",
       " 'souvenir': 2,\n",
       " 'idea': 126,\n",
       " 'mule': 3,\n",
       " 'books': 91,\n",
       " 'to': 464,\n",
       " 'orphans': 2,\n",
       " 'mistake': 122,\n",
       " 'is': 299,\n",
       " 'enough': 106,\n",
       " 'wood': 17,\n",
       " 'charge': 19,\n",
       " 'say': 112,\n",
       " 'sandstorm': 1,\n",
       " 'relatives': 6,\n",
       " 'died': 65,\n",
       " 'character': 16,\n",
       " 'hear': 18,\n",
       " 'here': 1065,\n",
       " 'tomorrow': 442,\n",
       " 'apartment': 39,\n",
       " 'class': 121,\n",
       " \"o'clock\": 47,\n",
       " 'chopper': 1,\n",
       " 'other': 187,\n",
       " 'secret': 53,\n",
       " 'on': 266,\n",
       " 'point': 43,\n",
       " 'done': 212,\n",
       " 'predators': 3,\n",
       " 'alright': 5,\n",
       " 'learners': 4,\n",
       " 'painted': 6,\n",
       " 'world': 173,\n",
       " 'hospital': 86,\n",
       " 'gas': 24,\n",
       " 'horse': 39,\n",
       " 'money': 389,\n",
       " 'week': 357,\n",
       " 'interested': 55,\n",
       " 'disrupted': 1,\n",
       " 'paid': 16,\n",
       " 'fingers': 16,\n",
       " 'Filby': 1,\n",
       " 'predictable': 2,\n",
       " 'filming': 4,\n",
       " 'Layla': 275,\n",
       " 'news': 77,\n",
       " \"'s\": 87,\n",
       " 'LARP': 1,\n",
       " 'sleep': 97,\n",
       " 'ashamed': 7,\n",
       " 'they': 141,\n",
       " 'published': 2,\n",
       " 'rest': 36,\n",
       " 'punctual': 12,\n",
       " 'ghosts': 14,\n",
       " 'actor': 18,\n",
       " 'classmates': 15,\n",
       " 'plate': 9,\n",
       " 'hair': 109,\n",
       " 'sleeping': 47,\n",
       " 'country': 132,\n",
       " 'file': 8,\n",
       " 'serious': 42,\n",
       " 'she': 367,\n",
       " 'anyplace': 2,\n",
       " 'soccer': 15,\n",
       " 'safety': 15,\n",
       " 'live': 51,\n",
       " 'worried': 51,\n",
       " 'do': 1065,\n",
       " 'stormy': 1,\n",
       " 'America': 34,\n",
       " 'water': 162,\n",
       " 'eats': 6,\n",
       " 'number': 72,\n",
       " 'reference': 3,\n",
       " 'Algeria': 250,\n",
       " 'train': 88,\n",
       " 'beach': 59,\n",
       " 'hand': 103,\n",
       " 'invitation': 16,\n",
       " 'triumphantly': 1,\n",
       " 'quitter': 1,\n",
       " 'he': 457,\n",
       " 'girlfriend': 62,\n",
       " 'inside': 41,\n",
       " 'couscous': 12,\n",
       " 'furious': 15,\n",
       " 'nosy': 4,\n",
       " 'department': 7,\n",
       " 'gone': 49,\n",
       " '2013': 52,\n",
       " 'retire': 20,\n",
       " 'view': 18,\n",
       " 'punches': 2,\n",
       " 'fire': 60,\n",
       " 'scarecrow': 1,\n",
       " 'story': 88,\n",
       " 'supper': 18,\n",
       " 'article': 10,\n",
       " 'diet': 15,\n",
       " 'Brazil': 16,\n",
       " 'tense': 19,\n",
       " 'furnished': 1,\n",
       " 'possible': 85,\n",
       " 'people': 190,\n",
       " 'payment': 4,\n",
       " 'fictitious': 1,\n",
       " 'coincidental': 1,\n",
       " 'yours': 60,\n",
       " 'indoors': 5,\n",
       " 'spirits': 8,\n",
       " 'bus': 108,\n",
       " 'right': 263,\n",
       " 'defibrillator': 1,\n",
       " 'days': 162,\n",
       " 'sock': 2,\n",
       " 'busy': 165,\n",
       " 'shoulder': 21,\n",
       " 'mansion': 6,\n",
       " 'smile': 40,\n",
       " 'hate': 5,\n",
       " 'jokes': 11,\n",
       " 'for': 152,\n",
       " 'chief': 2,\n",
       " 'Japan': 49,\n",
       " 'Islam': 105,\n",
       " 'meat': 48,\n",
       " 'tools': 8,\n",
       " 'sandwiches': 12,\n",
       " 'go': 304,\n",
       " 'before': 201,\n",
       " 'already': 81,\n",
       " 'bicycle': 64,\n",
       " 'wonderful': 10,\n",
       " 'learn': 23,\n",
       " 'information': 35,\n",
       " \"'\": 53,\n",
       " 'dress': 85,\n",
       " 'Algerians': 4,\n",
       " 'statements': 2,\n",
       " 'photographer': 8,\n",
       " 'pity': 4,\n",
       " 'prediction': 1,\n",
       " 'truth': 180,\n",
       " 'fault': 30,\n",
       " 'risk': 20,\n",
       " 'good': 177,\n",
       " 'ring': 20,\n",
       " 'street': 78,\n",
       " 'late': 208,\n",
       " 'ankle': 6,\n",
       " 'arrive': 21,\n",
       " 'can': 106,\n",
       " 'knee': 4,\n",
       " 'problem': 207,\n",
       " 'cake': 47,\n",
       " 'yes': 27,\n",
       " 'uncle': 14,\n",
       " 'simple': 22,\n",
       " 'equitably': 1,\n",
       " 'sheep': 17,\n",
       " 'brothers': 24,\n",
       " 'ICU': 1,\n",
       " 'humor': 18,\n",
       " 'are': 158,\n",
       " 'wall': 55,\n",
       " 'correctly': 18,\n",
       " 'worries': 4,\n",
       " '2:30': 193,\n",
       " 'screaming': 19,\n",
       " 'language': 132,\n",
       " 'oil': 18,\n",
       " 'sociology': 1,\n",
       " 'history': 34,\n",
       " 'bed': 136,\n",
       " 'law': 30,\n",
       " 'affairs': 11,\n",
       " 'rent': 24,\n",
       " '2016': 1,\n",
       " 'thanks': 10,\n",
       " 'out': 399,\n",
       " 'instinctively': 1,\n",
       " 'something': 192,\n",
       " 'talk': 60,\n",
       " 'met': 20,\n",
       " 'absorbent': 1,\n",
       " 'evangelist': 1,\n",
       " 'silk': 3,\n",
       " 'questions': 90,\n",
       " 'improvement': 4,\n",
       " 'car': 444,\n",
       " 'air': 37,\n",
       " 'dying': 22,\n",
       " 'trace': 4,\n",
       " 'decision': 60,\n",
       " 'suicides': 1,\n",
       " 'contest': 16,\n",
       " 'alive': 52,\n",
       " 'buying': 3,\n",
       " 'got': 19,\n",
       " 'restaurant': 43,\n",
       " 'happens': 29,\n",
       " 'station': 100,\n",
       " 'drive': 59,\n",
       " 'while': 72,\n",
       " 'casting': 1,\n",
       " 'food': 153,\n",
       " 'obedient': 6,\n",
       " 'principles': 7,\n",
       " 'yourself': 145,\n",
       " 'be': 223,\n",
       " 'slowly': 30,\n",
       " 'stop': 55,\n",
       " 'alone': 261,\n",
       " 'Monday': 263,\n",
       " 'Iraq': 5,\n",
       " 'Jane': 4,\n",
       " 'know': 176,\n",
       " 'tear': 1,\n",
       " 'satisfactory': 11,\n",
       " 'guys': 16,\n",
       " 'afraid': 55,\n",
       " 'unmarried': 8,\n",
       " 'bread': 27,\n",
       " 'gods': 6,\n",
       " 'basement': 17,\n",
       " 'office': 113,\n",
       " 'chest': 18,\n",
       " 'coronavirus': 10,\n",
       " 'portrait': 3,\n",
       " 'need': 77,\n",
       " 'back': 340,\n",
       " 'acquitted': 1,\n",
       " 'residences': 1,\n",
       " 'YouTube': 8,\n",
       " 'Toyota': 3,\n",
       " 'engine': 11,\n",
       " 'come': 141,\n",
       " 'months': 85,\n",
       " 'corner': 11,\n",
       " 'fuming': 1,\n",
       " 'specified': 1,\n",
       " 'backpack': 7,\n",
       " 'translation': 11,\n",
       " 'native': 6,\n",
       " 'doing': 154,\n",
       " 'seventeen': 2,\n",
       " 'recently': 40,\n",
       " 'rocks': 11,\n",
       " 'so': 66,\n",
       " 'cry': 110,\n",
       " 'easy': 65,\n",
       " 'times': 114,\n",
       " 'cancer': 18,\n",
       " 'imaginary': 2,\n",
       " 'CIA': 2,\n",
       " 'much': 210,\n",
       " 'anger': 28,\n",
       " 'wolves': 2,\n",
       " 'does': 65,\n",
       " 'eyes': 84,\n",
       " 'later': 137,\n",
       " 'friends': 236,\n",
       " 'GAAP': 1,\n",
       " 'night': 380,\n",
       " 'river': 55,\n",
       " 'hurry': 43,\n",
       " 'chips': 14,\n",
       " 'Algiers': 67,\n",
       " 'headline': 1,\n",
       " 'treasurer': 2,\n",
       " 'town': 109,\n",
       " 'Skura': 41,\n",
       " 'explain': 6,\n",
       " 'himself': 411,\n",
       " 'favor': 22,\n",
       " 'tears': 34,\n",
       " 'question': 117,\n",
       " 'meal': 22,\n",
       " 'evil': 18,\n",
       " 'sleepy': 63,\n",
       " 'grandfather': 23,\n",
       " 'girl': 96,\n",
       " 'school': 378,\n",
       " 'polite': 12,\n",
       " 'argument': 13,\n",
       " 'last': 32,\n",
       " 'study': 57,\n",
       " 'stronger': 10,\n",
       " 'cold': 155,\n",
       " 'implications': 1,\n",
       " 'tea': 51,\n",
       " 'painter': 5,\n",
       " 'balloon': 6,\n",
       " 'party': 269,\n",
       " 'Hokkaido': 3,\n",
       " 'win': 360,\n",
       " 'Conquest': 1,\n",
       " '...': 81,\n",
       " 'Fuck': 1,\n",
       " 'heart': 66,\n",
       " 'was': 142,\n",
       " 'oboe': 3,\n",
       " 'case': 31,\n",
       " 'there': 554,\n",
       " 'table': 105,\n",
       " 'through': 25,\n",
       " 'mind': 118,\n",
       " 'field': 22,\n",
       " 'terrain': 4,\n",
       " 'either': 61,\n",
       " 'patient': 27,\n",
       " 'testify': 7,\n",
       " 'precinct': 1,\n",
       " 'job': 306,\n",
       " 'potatoes': 14,\n",
       " 'seen': 33,\n",
       " 'won': 27,\n",
       " 'themselves': 222,\n",
       " 'success': 37,\n",
       " 'politics': 19,\n",
       " 'lovers': 2,\n",
       " 'percent': 6,\n",
       " 'worse': 37,\n",
       " 'fool': 12,\n",
       " 'ready': 92,\n",
       " 'peace': 31,\n",
       " 'eternal': 3,\n",
       " 'I': 39,\n",
       " 'programming': 2,\n",
       " 'embarrassed': 34,\n",
       " 'correct': 41,\n",
       " 'doctor': 100,\n",
       " 'murder': 35,\n",
       " 'supermarket': 18,\n",
       " 'joke': 31,\n",
       " 'beers': 3,\n",
       " 'fly': 17,\n",
       " 'escaped': 9,\n",
       " 'arrived': 42,\n",
       " 'shaken': 7,\n",
       " 'dream': 28,\n",
       " 'TV': 60,\n",
       " 'shoes': 75,\n",
       " 'reading': 27,\n",
       " 'real': 23,\n",
       " 'England': 19,\n",
       " 'steam': 4,\n",
       " 'ends': 5,\n",
       " 'Hypothermia': 1,\n",
       " 'off': 189,\n",
       " 'thousands': 2,\n",
       " 'Paris': 25,\n",
       " 'healthy': 26,\n",
       " 'takfirism': 2,\n",
       " 'cheeks': 10,\n",
       " 'setting': 2,\n",
       " 'trust': 22,\n",
       " 'translate': 5,\n",
       " 'boy': 69,\n",
       " 'chocolate': 20,\n",
       " 'regards': 1,\n",
       " 'windshield': 4,\n",
       " 'Thanks': 7,\n",
       " 'sun': 37,\n",
       " 'death': 91,\n",
       " 'Anna': 2,\n",
       " 'complex': 12,\n",
       " 'failed': 15,\n",
       " 'land': 25,\n",
       " 'better': 154,\n",
       " 'node': 1,\n",
       " 'jail': 38,\n",
       " 'arrangement': 4,\n",
       " 'Africa': 31,\n",
       " 'tasty': 6,\n",
       " 'agreed': 9,\n",
       " 'please': 151,\n",
       " 'circumstances': 6,\n",
       " 'stairwell': 4,\n",
       " 'art': 24,\n",
       " 'studied': 3,\n",
       " 'leg': 32,\n",
       " 'away': 251,\n",
       " 'desk': 52,\n",
       " 'angry': 126,\n",
       " 'tied': 1,\n",
       " 'whore': 3,\n",
       " 'teacher': 161,\n",
       " 'five-thirty': 1,\n",
       " 'mourning': 3,\n",
       " 'aching': 1,\n",
       " 'universe': 22,\n",
       " 'slow': 16,\n",
       " 'guitarist': 10,\n",
       " 'arm': 41,\n",
       " 'novel': 21,\n",
       " 'everything': 147,\n",
       " 'health': 67,\n",
       " 'Market': 1,\n",
       " 'mystery': 11,\n",
       " 'fun': 143,\n",
       " 'nation': 10,\n",
       " 'breakfast': 65,\n",
       " 'said': 122,\n",
       " 'network': 5,\n",
       " 'dad': 13,\n",
       " 'laughing': 28,\n",
       " 'daughter': 56,\n",
       " 'skillful': 1,\n",
       " 'driving': 29,\n",
       " 'bananas': 8,\n",
       " 'answer': 75,\n",
       " 'dishes': 32,\n",
       " 'bike': 22,\n",
       " 'understood': 10,\n",
       " 'neighborhood': 25,\n",
       " 'colors': 13,\n",
       " 'family': 163,\n",
       " 'like': 111,\n",
       " 'weed': 14,\n",
       " 'have': 80,\n",
       " 'scholar': 8,\n",
       " 'pond': 11,\n",
       " 'lot': 154,\n",
       " 'pleased': 29,\n",
       " 'happy': 225,\n",
       " 'bluffing': 5,\n",
       " 'daytime': 2,\n",
       " 'quiet': 44,\n",
       " 'thirty': 46,\n",
       " 'low': 11,\n",
       " 'spikes': 1,\n",
       " 'video': 35,\n",
       " 'kids': 75,\n",
       " 'secretary': 8,\n",
       " 'game': 71,\n",
       " 'pain': 47,\n",
       " 'instructor': 6,\n",
       " 'guy': 100,\n",
       " 'address': 32,\n",
       " 'err': 2,\n",
       " 'heard': 20,\n",
       " 'shaking': 7,\n",
       " 'barefooted': 2,\n",
       " 'taxes': 15,\n",
       " 'jeans': 13,\n",
       " 'hello': 2,\n",
       " 'parties': 20,\n",
       " 'minutes': 73,\n",
       " 'beer': 42,\n",
       " 'anybody': 51,\n",
       " 'why': 36,\n",
       " 'perfect': 26,\n",
       " 'SpaceChem': 1,\n",
       " 'mood': 12,\n",
       " 'snow': 55,\n",
       " 'playing': 8,\n",
       " 'friend': 140,\n",
       " 'stocking': 1,\n",
       " 'manners': 7,\n",
       " 'with': 113,\n",
       " 'left': 120,\n",
       " 'broken': 39,\n",
       " 'roof': 25,\n",
       " 'newspapers': 5,\n",
       " 'pushy': 2,\n",
       " 'month': 119,\n",
       " 'beginners': 2,\n",
       " 'thing': 156,\n",
       " 'gambling': 1,\n",
       " 'drunk': 68,\n",
       " 'raining': 31,\n",
       " 'model': 12,\n",
       " 'sugar': 24,\n",
       " 'dropped': 5,\n",
       " 'homework': 78,\n",
       " 'line': 32,\n",
       " 'cousins': 13,\n",
       " 'refund': 4,\n",
       " 'opened': 3,\n",
       " 'trouble': 79,\n",
       " 'risky': 4,\n",
       " 'full': 25,\n",
       " 'dog': 165,\n",
       " 'words': 54,\n",
       " 'dollars': 53,\n",
       " 'generation': 7,\n",
       " 'nothing': 68,\n",
       " 'truck': 25,\n",
       " 'normal': 20,\n",
       " 'Berber': 182,\n",
       " 'cell': 2,\n",
       " 'customers': 10,\n",
       " 'trapped': 5,\n",
       " 'China': 31,\n",
       " 'deserted': 5,\n",
       " 'detect': 1,\n",
       " 'door': 167,\n",
       " 'fishing': 14,\n",
       " 'genuine': 6,\n",
       " 'surprise': 21,\n",
       " 'gift': 24,\n",
       " 'waiting': 30,\n",
       " 'bands': 1,\n",
       " 'but': 4,\n",
       " 'phone': 93,\n",
       " 'agree': 23,\n",
       " 'credible': 1,\n",
       " 'love': 87,\n",
       " 'brother': 108,\n",
       " 'urbanized': 1,\n",
       " 'lawyers': 4,\n",
       " 'instead': 10,\n",
       " 'end': 45,\n",
       " 'pounds': 17,\n",
       " 'violent': 7,\n",
       " 'telephone': 9,\n",
       " 'digested': 1,\n",
       " 'happened': 264,\n",
       " 'enticing': 1,\n",
       " 'develop': 2,\n",
       " 'Spanish': 19,\n",
       " 'hurt': 74,\n",
       " 'supportive': 10,\n",
       " 'notebook': 11,\n",
       " 'swimming': 47,\n",
       " 'Sascha': 1,\n",
       " 'helpful': 21,\n",
       " 'excited': 30,\n",
       " 'milk': 49,\n",
       " 'thigh': 1,\n",
       " 'rock': 13,\n",
       " 'holster': 1,\n",
       " 'any': 11,\n",
       " 'confused': 50,\n",
       " 'charade': 1,\n",
       " 'directly': 4,\n",
       " 'survived': 7,\n",
       " 'asleep': 87,\n",
       " 'countries': 25,\n",
       " 'policeman': 16,\n",
       " 'Jesus': 8,\n",
       " 'wrote': 11,\n",
       " 'faster': 21,\n",
       " 'rich': 73,\n",
       " 'someone': 22,\n",
       " 'mountains': 14,\n",
       " 'lying': 38,\n",
       " 'earnest': 1,\n",
       " 'statement': 10,\n",
       " 'automated': 2,\n",
       " 'killed': 28,\n",
       " 'eat': 133,\n",
       " 'resolutions': 3,\n",
       " 'responsibility': 12,\n",
       " 'chicken-house': 1,\n",
       " 'movies': 38,\n",
       " 'wet': 15,\n",
       " 'around': 82,\n",
       " 'beans': 10,\n",
       " 'audience': 14,\n",
       " 'harder': 36,\n",
       " 'Jerusalem': 3,\n",
       " 'upset': 79,\n",
       " 'sins': 5,\n",
       " 'antagonist': 1,\n",
       " 'example': 12,\n",
       " 'banister': 1,\n",
       " 'offer': 32,\n",
       " 'reputable': 1,\n",
       " 'student': 42,\n",
       " 'wounds': 8,\n",
       " 'imprecise': 1,\n",
       " 'messages': 12,\n",
       " 'downstairs': 24,\n",
       " 'complaint': 7,\n",
       " 'teens': 4,\n",
       " 'forest': 26,\n",
       " 'zombie': 2,\n",
       " 'unison': 1,\n",
       " 'nerves': 4,\n",
       " 'useful': 18,\n",
       " 'club': 27,\n",
       " 'involved': 23,\n",
       " 'frightened': 35,\n",
       " 'haircut': 6,\n",
       " 'particular': 3,\n",
       " 'marionette': 1,\n",
       " 'watch': 36,\n",
       " 'boring': 25,\n",
       " 'unafraid': 9,\n",
       " 'towels': 9,\n",
       " 'cream': 42,\n",
       " 'hungry': 121,\n",
       " 'trunk': 5,\n",
       " 'distaff': 1,\n",
       " 'coming': 78,\n",
       " 'race': 33,\n",
       " 'Kazunari': 1,\n",
       " 'chalk': 3,\n",
       " 'firefighter': 4,\n",
       " 'crying': 68,\n",
       " 'LSD': 3,\n",
       " 'dirty': 25,\n",
       " 'semester': 7,\n",
       " 'knowledge': 14,\n",
       " 'shit': 22,\n",
       " 'moonlight': 3,\n",
       " 'treat': 1,\n",
       " 'kitchen': 49,\n",
       " 'event': 11,\n",
       " 'Egypt': 48,\n",
       " 'doomed': 2,\n",
       " 'homeless': 10,\n",
       " 'cars': 22,\n",
       " 'stairs': 21,\n",
       " 'ensemble': 1,\n",
       " 'wavelength': 3,\n",
       " 'attack': 23,\n",
       " 'spiders': 6,\n",
       " 'dumb': 9,\n",
       " 'Zealand': 9,\n",
       " 'details': 30,\n",
       " 'devils': 2,\n",
       " 'request': 18,\n",
       " 'neighbor': 13,\n",
       " 'collector': 2,\n",
       " 'talking': 32,\n",
       " 'stage': 11,\n",
       " 'guilty': 37,\n",
       " 'displeased': 14,\n",
       " 'duty': 22,\n",
       " 'air-conditioned': 2,\n",
       " 'minute': 45,\n",
       " 'lucky': 38,\n",
       " 'oddly': 1,\n",
       " 'foot': 19,\n",
       " 'single': 27,\n",
       " 'e-mail': 6,\n",
       " 'tribe': 2,\n",
       " 'challenge': 17,\n",
       " 'faucet': 3,\n",
       " 'far': 35,\n",
       " 'fine': 93,\n",
       " 'wait': 58,\n",
       " 'years': 261,\n",
       " 'survive': 14,\n",
       " 'next': 50,\n",
       " 'chance': 83,\n",
       " 'present': 37,\n",
       " 'Cairo': 38,\n",
       " 'lonely': 56,\n",
       " 'awake': 29,\n",
       " 'October': 79,\n",
       " 'candy': 9,\n",
       " 'overnight': 7,\n",
       " 'swim': 92,\n",
       " 'speech': 27,\n",
       " '911': 1,\n",
       " 'California': 5,\n",
       " 'talkative': 5,\n",
       " 'personally': 8,\n",
       " 'spring': 20,\n",
       " 'beverage': 3,\n",
       " 'age': 64,\n",
       " 'person': 150,\n",
       " 'cents': 2,\n",
       " 'share': 7,\n",
       " 'marked': 2,\n",
       " 'Bank': 3,\n",
       " 'rascal': 1,\n",
       " 'Advil': 1,\n",
       " 'visit': 19,\n",
       " 'noisy': 11,\n",
       " 'graduate': 5,\n",
       " 'painting': 16,\n",
       " 'cap': 5,\n",
       " 'head': 74,\n",
       " 'irritated': 24,\n",
       " 'exhausted': 32,\n",
       " 'Sundays': 9,\n",
       " 'instruction': 1,\n",
       " 'deal': 25,\n",
       " 'name': 149,\n",
       " 'joint': 2,\n",
       " 'temper': 9,\n",
       " 'knot': 3,\n",
       " 'poetry': 9,\n",
       " 'down': 175,\n",
       " 'first': 91,\n",
       " 'briefcase': 8,\n",
       " 'resigned': 9,\n",
       " 'needed': 53,\n",
       " 'beginner': 6,\n",
       " 'important': 64,\n",
       " 'Island': 3,\n",
       " 'confess': 10,\n",
       " 'singing': 27,\n",
       " 'lost': 73,\n",
       " 'gold': 16,\n",
       " 'colony': 2,\n",
       " 'Joe': 1,\n",
       " 'ran': 8,\n",
       " 'his': 20,\n",
       " 'Holmes': 2,\n",
       " 'surrender': 8,\n",
       " 'events': 4,\n",
       " 'Peek-a-boo': 1,\n",
       " 'lake': 26,\n",
       " 'renovated': 2,\n",
       " 'expected': 37,\n",
       " 'uptown': 1,\n",
       " 'yourselves': 47,\n",
       " 'bacteria': 4,\n",
       " 'everyone': 56,\n",
       " 'free': 43,\n",
       " 'book': 160,\n",
       " 'living': 20,\n",
       " 'hot': 52,\n",
       " 'Now': 2,\n",
       " 'caught': 20,\n",
       " 'wash': 3,\n",
       " 'best': 78,\n",
       " 'equally': 3,\n",
       " 'total': 3,\n",
       " 'sick': 110,\n",
       " 'shy': 32,\n",
       " 'efforts': 6,\n",
       " 'dark': 53,\n",
       " 'check': 14,\n",
       " 'nasty': 3,\n",
       " 'hard': 107,\n",
       " 'nudes': 2,\n",
       " 'advice': 54,\n",
       " 'hits': 2,\n",
       " 'instruments': 4,\n",
       " 'relax': 21,\n",
       " 'partner': 5,\n",
       " 'ex-wife': 2,\n",
       " 'want': 113,\n",
       " 'shell': 2,\n",
       " 'marriage': 24,\n",
       " 'robot': 4,\n",
       " 'T-shirt': 5,\n",
       " 'over': 118,\n",
       " 'successful': 17,\n",
       " 'predecessor': 1,\n",
       " 'ability': 15,\n",
       " 'Hebrew': 3,\n",
       " 'rights': 14,\n",
       " 'rescheduled': 1,\n",
       " 'weeks': 65,\n",
       " 'diary': 9,\n",
       " 'interesting': 63,\n",
       " 'relaxing': 4,\n",
       " 'plump': 1,\n",
       " 'stranded': 3,\n",
       " 'map': 13,\n",
       " 'fat': 25,\n",
       " 'matters': 9,\n",
       " 'beliefs': 6,\n",
       " 'open': 74,\n",
       " 'perfection': 4,\n",
       " 'Netherlands': 6,\n",
       " 'run': 24,\n",
       " 'diarrhea': 2,\n",
       " 'city': 79,\n",
       " '2006': 10,\n",
       " 'mist': 3,\n",
       " 'cool': 29,\n",
       " 'household': 9,\n",
       " 'start': 52,\n",
       " 'grant': 1,\n",
       " 'child': 78,\n",
       " 'read': 53,\n",
       " 'impeached': 1,\n",
       " 'wins': 12,\n",
       " 'could': 41,\n",
       " 'forever': 31,\n",
       " ...}"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## see distribution of last elements\n",
    "final_chars = defaultdict(int)\n",
    "for sent in change_sents:\n",
    "    final_chars[sent[-1]]+=1\n",
    "dict(final_chars) "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create labels "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "tab_labels = []\n",
    "for sent in change_sents :\n",
    "    labels = [0] * len(sent)\n",
    "    labels[-1] = 1 \n",
    "    tab_labels.append(labels)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## concatenante randomly sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "K = random.randrange(1,6) ## we choose uptil 6 sentences to concatenante\n",
    "\n",
    "grouped_sent , grouped_labels = []  , [] \n",
    "p=0\n",
    "for i, (sent,label) in enumerate(zip(change_sents,tab_labels),1) :\n",
    "    if len(sent)> 400 : \n",
    "        if i%K ==0 :\n",
    "            p+=1\n",
    "            K = random.randrange(1,6)\n",
    "        continue\n",
    "    try : \n",
    "        grouped_sent[p]+=sent\n",
    "        grouped_labels[p]+=label\n",
    "    except : \n",
    "        grouped_sent.append(sent)\n",
    "        grouped_labels.append(label)\n",
    "    if i%K ==0 :\n",
    "        p+=1\n",
    "        K = random.randrange(1,6)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "94381"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(grouped_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle \n",
    "pickle.dump(grouped_sent, open('/home/hamza/3A_projets/data_all/glose/tatoeba_sentences.pkl','wb'))\n",
    "pickle.dump(grouped_labels, open('/home/hamza/3A_projets/data_all/glose/tatoeba_grouped_labels.pkl','wb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Is', 'she', 'the', 'lady', 'who', 'was', 'dressed', 'in', 'blue', '?', 'At', 'nightfall', ',', 'when', 'the', 'last', 'rays', 'of', 'the', 'sun', 'color', 'the', 'prairie', 'sky', 'red', ',', 'the', 'howls', 'of', 'coyotes', 'resonate', 'through', 'the', 'hills', '.', 'I', 'just', 'want', 'to', 'see', 'if', 'I', 'can', 'do', 'that']\n"
     ]
    }
   ],
   "source": [
    "max_ = 0\n",
    "for k,sent in enumerate(grouped_sent) : \n",
    "    if k==3644 : \n",
    "        print(sent)\n",
    "    if len(sent)>max_ :\n",
    "        max_=len(sent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "89102"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(grouped_sent)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}