12. NMT With Attention.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import, division, print_function, unicode_literals\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.ticker as ticker\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import unicodedata\n",
    "import re\n",
    "import numpy as np\n",
    "import os\n",
    "import io\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Setup the file\n",
    "# path_to_dataset = \"./datasets\"\n",
    "# path_to_file = os.path.dirname(path_to_dataset)+\"/hin-eng/hin.txt\"\n",
    "path_to_file = \"./datasets/hin-eng/hin.txt\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "path_to_zip = tf.keras.utils.get_file('spa-eng.zip', origin='http://storage.googleapis.com/download.tensorflow.org/data/spa-eng.zip', extract=True)\n",
    "path_to_file = os.path.dirname(path_to_zip)+\"/spa-eng/spa.txt\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert unicode to ascii. Unicode normalization methods are NFC, NFKC, NFD, NFKD.\n",
    "def unicode_to_ascii(s):\n",
    "    return ''.join(c for c in unicodedata.normalize('NFD', s) if unicodedata.category(c) != 'Mn')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess_sentence(w):\n",
    "    # w = unicode_to_ascii(w.lower().strip())\n",
    "    w = w.lower().strip()\n",
    "    \n",
    "    # creating a space between a word and the punctuation\n",
    "    w = re.sub(r\"([?.!,¿।])\", r\" \\1 \", w)\n",
    "    w = re.sub(r'[\" \"]+', \" \", w)\n",
    "    \n",
    "    # replacing everything with space except (a-z, A-Z, \".\", \"?\", \"!\", \",\")\n",
    "    # w = re.sub(r\"[^a-zA-Z?.!,¿]+\", \" \", w)\n",
    "    \n",
    "    w = w.rstrip().strip()\n",
    "    \n",
    "    # adding a start and an end token to the sentence\n",
    "    # so that the model know when to start and stop predicting.\n",
    "    w = '<start> ' + w + ' <end>'\n",
    "    return w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<start> got it ? <end>\n",
      "<start> समझे कि नहीं ? <end>\n"
     ]
    }
   ],
   "source": [
    "print(preprocess_sentence(\"Got it?\"))\n",
    "print(preprocess_sentence(\"समझे कि नहीं?\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove the accents, clean the sentences, and return word pairs in the format: [ENGLISH, HINDI]\n",
    "def create_dataset(path, num_examples):\n",
    "    lines = io.open(path, encoding='UTF-8').read().strip().split('\\n')\n",
    "    cleaned_lines = []\n",
    "    for l in lines[:num_examples]:\n",
    "        cleaned_line = re.sub(r\"CC-BY 2.0 \\(France\\).*\", \"\", l)\n",
    "        cleaned_lines.append(cleaned_line.strip())\n",
    "    word_pairs = [[preprocess_sentence(w) for w in l.split('\\t')]  for l in cleaned_lines[:num_examples]]\n",
    "    return zip(*word_pairs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<start> when i was a kid , touching bugs didn't bother me a bit . now i can hardly stand looking at pictures of them . <end>\n",
      "<start> जब मैं बच्चा था , मुझे कीड़ों को छूने से कोई परेशानी नहीं होती थी , पर अब मैं उनकी तस्वीरें देखना भी बर्दाश्त नहीं कर सकता । <end>\n"
     ]
    }
   ],
   "source": [
    "en, hin = create_dataset(path_to_file, None)\n",
    "print(en[-1])\n",
    "print(hin[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def max_length(tensor):\n",
    "    return max(len(t) for t in tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize(lang):\n",
    "    lang_tokenizer = tf.keras.preprocessing.text.Tokenizer(filters='')\n",
    "    lang_tokenizer.fit_on_texts(lang)\n",
    "    \n",
    "    tensor = lang_tokenizer.texts_to_sequences(lang)\n",
    "    \n",
    "    tensor = tf.keras.preprocessing.sequence.pad_sequences(tensor, padding='post')\n",
    "    \n",
    "    return tensor, lang_tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_dataset(path, num_examples=None):\n",
    "    # creating cleaned input, output pairs\n",
    "    targ_lang, inp_lang = create_dataset(path, num_examples)\n",
    "    \n",
    "    input_tensor, inp_lang_tokenizer = tokenize(inp_lang)\n",
    "    target_tensor, targ_lang_tokenizer = tokenize(targ_lang)\n",
    "    \n",
    "    return input_tensor, target_tensor, inp_lang_tokenizer, targ_lang_tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Limit the size of the dataset to experiment faster\n",
    "num_examples = 30000\n",
    "input_tensor, target_tensor, inp_lang, targ_lang = load_dataset(path_to_file, num_examples)\n",
    "\n",
    "# Calculate max_length of the target tensors\n",
    "max_length_targ, max_length_inp = max_length(target_tensor), max_length(input_tensor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2223 2223 556 556\n"
     ]
    }
   ],
   "source": [
    "# Creating training and validation sets using an 80-20 split\n",
    "input_tensor_train, input_tensor_val, target_tensor_train, target_tensor_val = train_test_split(input_tensor, target_tensor, test_size=0.2)\n",
    "\n",
    "# Show length\n",
    "print(len(input_tensor_train), len(target_tensor_train), len(input_tensor_val), len(target_tensor_val))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert(lang, tensor):\n",
    "    for t in tensor:\n",
    "        if t!=0:\n",
    "            print (\"%d ----> %s\" % (t, lang.index_word[t]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input Language; index to word mapping\n",
      "1 ----> <start>\n",
      "13 ----> मुझे\n",
      "38 ----> अपने\n",
      "1243 ----> आलसी\n",
      "189 ----> होने\n",
      "23 ----> पर\n",
      "902 ----> शर्म\n",
      "437 ----> आती\n",
      "4 ----> है\n",
      "3 ----> ।\n",
      "2 ----> <end>\n",
      "\n",
      "Target Language; index to word mapping\n",
      "1 ----> <start>\n",
      "5 ----> i\n",
      "65 ----> am\n",
      "850 ----> ashamed\n",
      "12 ----> of\n",
      "657 ----> having\n",
      "68 ----> been\n",
      "1116 ----> lazy\n",
      "3 ----> .\n",
      "2 ----> <end>\n"
     ]
    }
   ],
   "source": [
    "print (\"Input Language; index to word mapping\")\n",
    "convert(inp_lang, input_tensor_train[0])\n",
    "print ()\n",
    "print (\"Target Language; index to word mapping\")\n",
    "convert(targ_lang, target_tensor_train[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "BUFFER_SIZE = len(input_tensor_train)\n",
    "BATCH_SIZE = 64\n",
    "steps_per_epoch = len(input_tensor_train)//BATCH_SIZE\n",
    "embedding_dim = 256\n",
    "units = 1024\n",
    "vocab_inp_size = len(inp_lang.word_index)+1\n",
    "vocab_tar_size = len(targ_lang.word_index)+1\n",
    "\n",
    "dataset = tf.data.Dataset.from_tensor_slices((input_tensor_train, target_tensor_train)).shuffle(BUFFER_SIZE)\n",
    "dataset = dataset.batch(BATCH_SIZE, drop_remainder=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(TensorShape([64, 30]), TensorShape([64, 27]))"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "example_input_batch, example_target_batch = next(iter(dataset))\n",
    "example_input_batch.shape, example_target_batch.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Encoder(tf.keras.Model):\n",
    "    def __init__(self, vocab_size, embedding_dim, enc_units, batch_sz):\n",
    "        super(Encoder, self).__init__()\n",
    "        self.batch_sz = batch_sz\n",
    "        self.enc_units = enc_units\n",
    "        self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
    "        self.gru = tf.keras.layers.GRU(self.enc_units,\n",
    "                                       return_sequences=True,\n",
    "                                       return_state=True,\n",
    "                                       recurrent_initializer='glorot_uniform')\n",
    "\n",
    "    def call(self, x, hidden):\n",
    "        x = self.embedding(x)\n",
    "        output, state = self.gru(x, initial_state = hidden)\n",
    "        return output, state\n",
    "\n",
    "    def initialize_hidden_state(self):\n",
    "        return tf.zeros((self.batch_sz, self.enc_units))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder = Encoder(vocab_inp_size, embedding_dim, units, BATCH_SIZE)\n",
    "\n",
    "# sample input\n",
    "sample_hidden = encoder.initialize_hidden_state()\n",
    "sample_output, sample_hidden = encoder(example_input_batch, sample_hidden)\n",
    "# print ('Encoder output shape: (batch size, sequence length, units) {}'.format(sample_output.shape))\n",
    "# print ('Encoder Hidden state shape: (batch size, units) {}'.format(sample_hidden.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "class BahdanauAttention(tf.keras.layers.Layer):\n",
    "    def __init__(self, units):\n",
    "        super(BahdanauAttention, self).__init__()\n",
    "        self.W1 = tf.keras.layers.Dense(units)\n",
    "        self.W2 = tf.keras.layers.Dense(units)\n",
    "        self.V = tf.keras.layers.Dense(1)\n",
    "\n",
    "    def call(self, query, values):\n",
    "        # hidden shape == (batch_size, hidden size)\n",
    "        # hidden_with_time_axis shape == (batch_size, 1, hidden size)\n",
    "        # we are doing this to perform addition to calculate the score\n",
    "        hidden_with_time_axis = tf.expand_dims(query, 1)\n",
    "        \n",
    "        # score shape == (batch_size, max_length, 1)\n",
    "        # we get 1 at the last axis because we are applying score to self.V\n",
    "        # the shape of the tensor before applying self.V is (batch_size, max_length, units)\n",
    "        score = self.V(tf.nn.tanh(self.W1(values) + self.W2(hidden_with_time_axis)))\n",
    "        \n",
    "        # attention_weights shape == (batch_size, max_length, 1)\n",
    "        attention_weights = tf.nn.softmax(score, axis=1)\n",
    "\n",
    "        # context_vector shape after sum == (batch_size, hidden_size)\n",
    "        context_vector = attention_weights * values\n",
    "        context_vector = tf.reduce_sum(context_vector, axis=1)\n",
    "\n",
    "        return context_vector, attention_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Attention result shape: (batch size, units) (64, 1024)\n",
      "Attention weights shape: (batch_size, sequence_length, 1) (64, 30, 1)\n"
     ]
    }
   ],
   "source": [
    "attention_layer = BahdanauAttention(10)\n",
    "attention_result, attention_weights = attention_layer(sample_hidden, sample_output)\n",
    "\n",
    "print(\"Attention result shape: (batch size, units) {}\".format(attention_result.shape))\n",
    "print(\"Attention weights shape: (batch_size, sequence_length, 1) {}\".format(attention_weights.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Decoder(tf.keras.Model):\n",
    "    def __init__(self, vocab_size, embedding_dim, dec_units, batch_sz):\n",
    "        super(Decoder, self).__init__()\n",
    "        self.batch_sz = batch_sz\n",
    "        self.dec_units = dec_units\n",
    "        self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
    "        self.gru = tf.keras.layers.GRU(self.dec_units,\n",
    "                                       return_sequences=True,\n",
    "                                       return_state=True,\n",
    "                                       recurrent_initializer='glorot_uniform')\n",
    "        self.fc = tf.keras.layers.Dense(vocab_size)\n",
    "\n",
    "        # used for attention\n",
    "        self.attention = BahdanauAttention(self.dec_units)\n",
    "\n",
    "    def call(self, x, hidden, enc_output):\n",
    "        # enc_output shape == (batch_size, max_length, hidden_size)\n",
    "        context_vector, attention_weights = self.attention(hidden, enc_output)\n",
    "\n",
    "        # x shape after passing through embedding == (batch_size, 1, embedding_dim)\n",
    "        x = self.embedding(x)\n",
    "\n",
    "        # x shape after concatenation == (batch_size, 1, embedding_dim + hidden_size)\n",
    "        x = tf.concat([tf.expand_dims(context_vector, 1), x], axis=-1)\n",
    "\n",
    "        # passing the concatenated vector to the GRU\n",
    "        output, state = self.gru(x)\n",
    "\n",
    "        # output shape == (batch_size * 1, hidden_size)\n",
    "        output = tf.reshape(output, (-1, output.shape[2]))\n",
    "\n",
    "        # output shape == (batch_size, vocab)\n",
    "        x = self.fc(output)\n",
    "\n",
    "        return x, state, attention_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Decoder output shape: (batch_size, vocab size) (64, 2389)\n"
     ]
    }
   ],
   "source": [
    "decoder = Decoder(vocab_tar_size, embedding_dim, units, BATCH_SIZE)\n",
    "\n",
    "sample_decoder_output, _, _ = decoder(tf.random.uniform((BATCH_SIZE, 1)), sample_hidden, sample_output)\n",
    "\n",
    "print ('Decoder output shape: (batch_size, vocab size) {}'.format(sample_decoder_output.shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Optimizer and the loss function\n",
    "optimizer = tf.keras.optimizers.Adam()\n",
    "loss_object = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True, reduction='none')\n",
    "\n",
    "def loss_function(real, pred):\n",
    "    mask = tf.math.logical_not(tf.math.equal(real, 0))\n",
    "    loss_ = loss_object(real, pred)\n",
    "\n",
    "    mask = tf.cast(mask, dtype=loss_.dtype)\n",
    "    loss_ *= mask\n",
    "\n",
    "    return tf.reduce_mean(loss_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Checkpoints\n",
    "checkpoint_dir = './NMT_attention_training_checkpoints'\n",
    "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt\")\n",
    "checkpoint = tf.train.Checkpoint(optimizer=optimizer, encoder=encoder, decoder=decoder)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "@tf.function\n",
    "def train_step(inp, targ, enc_hidden):\n",
    "    loss = 0\n",
    "\n",
    "    with tf.GradientTape() as tape:\n",
    "        enc_output, enc_hidden = encoder(inp, enc_hidden)\n",
    "\n",
    "        dec_hidden = enc_hidden\n",
    "\n",
    "        dec_input = tf.expand_dims([targ_lang.word_index['<start>']] * BATCH_SIZE, 1)\n",
    "\n",
    "        # Teacher forcing - feeding the target as the next input\n",
    "        for t in range(1, targ.shape[1]):\n",
    "            # passing enc_output to the decoder\n",
    "            predictions, dec_hidden, _ = decoder(dec_input, dec_hidden, enc_output)\n",
    "\n",
    "            loss += loss_function(targ[:, t], predictions)\n",
    "\n",
    "            # using teacher forcing\n",
    "            dec_input = tf.expand_dims(targ[:, t], 1)\n",
    "\n",
    "    batch_loss = (loss / int(targ.shape[1]))\n",
    "\n",
    "    variables = encoder.trainable_variables + decoder.trainable_variables\n",
    "\n",
    "    gradients = tape.gradient(loss, variables)\n",
    "\n",
    "    optimizer.apply_gradients(zip(gradients, variables))\n",
    "\n",
    "    return batch_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1 Loss 1.4187\n",
      "Time taken for 1 epoch 3.6038808822631836 sec\n",
      "\n",
      "Epoch 2 Loss 1.3398\n",
      "Time taken for 1 epoch 4.044708728790283 sec\n",
      "\n",
      "Epoch 3 Loss 1.2688\n",
      "Time taken for 1 epoch 3.616734504699707 sec\n",
      "\n",
      "Epoch 4 Loss 1.1852\n",
      "Time taken for 1 epoch 4.015035152435303 sec\n",
      "\n",
      "Epoch 5 Loss 1.1040\n",
      "Time taken for 1 epoch 3.6132664680480957 sec\n",
      "\n",
      "Epoch 6 Loss 1.0315\n",
      "Time taken for 1 epoch 4.020668029785156 sec\n",
      "\n",
      "Epoch 7 Loss 0.9740\n",
      "Time taken for 1 epoch 3.589599132537842 sec\n",
      "\n",
      "Epoch 8 Loss 0.9062\n",
      "Time taken for 1 epoch 3.9772090911865234 sec\n",
      "\n",
      "Epoch 9 Loss 0.8479\n",
      "Time taken for 1 epoch 3.560030460357666 sec\n",
      "\n",
      "Epoch 10 Loss 0.8001\n",
      "Time taken for 1 epoch 4.050275087356567 sec\n",
      "\n",
      "Epoch 11 Loss 0.7428\n",
      "Time taken for 1 epoch 3.5467545986175537 sec\n",
      "\n",
      "Epoch 12 Loss 0.6917\n",
      "Time taken for 1 epoch 4.0569908618927 sec\n",
      "\n",
      "Epoch 13 Loss 0.6401\n",
      "Time taken for 1 epoch 3.6547935009002686 sec\n",
      "\n",
      "Epoch 14 Loss 0.5869\n",
      "Time taken for 1 epoch 4.006633281707764 sec\n",
      "\n",
      "Epoch 15 Loss 0.5399\n",
      "Time taken for 1 epoch 3.661085367202759 sec\n",
      "\n",
      "Epoch 16 Loss 0.4872\n",
      "Time taken for 1 epoch 4.1071295738220215 sec\n",
      "\n",
      "Epoch 17 Loss 0.4439\n",
      "Time taken for 1 epoch 3.6677956581115723 sec\n",
      "\n",
      "Epoch 18 Loss 0.3930\n",
      "Time taken for 1 epoch 4.065694332122803 sec\n",
      "\n",
      "Epoch 19 Loss 0.3421\n",
      "Time taken for 1 epoch 3.705244541168213 sec\n",
      "\n",
      "Epoch 20 Loss 0.2974\n",
      "Time taken for 1 epoch 4.146350622177124 sec\n",
      "\n"
     ]
    }
   ],
   "source": [
    "EPOCHS = 20\n",
    "\n",
    "for epoch in range(EPOCHS):\n",
    "    start = time.time()\n",
    "\n",
    "    enc_hidden = encoder.initialize_hidden_state()\n",
    "    total_loss = 0\n",
    "\n",
    "    for (batch, (inp, targ)) in enumerate(dataset.take(steps_per_epoch)):\n",
    "        batch_loss = train_step(inp, targ, enc_hidden)\n",
    "        total_loss += batch_loss\n",
    "\n",
    "    if batch % 100 == 0:\n",
    "        print('Epoch {} Batch {} Loss {:.4f}'.format(epoch + 1, batch, batch_loss.numpy()))\n",
    "    \n",
    "    # saving (checkpoint) the model every 2 epochs\n",
    "    if (epoch + 1) % 2 == 0:\n",
    "        checkpoint.save(file_prefix = checkpoint_prefix)\n",
    "\n",
    "    print('Epoch {} Loss {:.4f}'.format(epoch + 1, total_loss / steps_per_epoch))\n",
    "    print('Time taken for 1 epoch {} sec\\n'.format(time.time() - start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(sentence):\n",
    "    attention_plot = np.zeros((max_length_targ, max_length_inp))\n",
    "\n",
    "    sentence = preprocess_sentence(sentence)\n",
    "\n",
    "    inputs = [inp_lang.word_index[i] for i in sentence.split(' ')]\n",
    "    inputs = tf.keras.preprocessing.sequence.pad_sequences([inputs],\n",
    "                                                         maxlen=max_length_inp,\n",
    "                                                         padding='post')\n",
    "    inputs = tf.convert_to_tensor(inputs)\n",
    "\n",
    "    result = ''\n",
    "\n",
    "    hidden = [tf.zeros((1, units))]\n",
    "    enc_out, enc_hidden = encoder(inputs, hidden)\n",
    "\n",
    "    dec_hidden = enc_hidden\n",
    "    dec_input = tf.expand_dims([targ_lang.word_index['<start>']], 0)\n",
    "\n",
    "    for t in range(max_length_targ):\n",
    "        predictions, dec_hidden, attention_weights = decoder(dec_input,\n",
    "                                                             dec_hidden,\n",
    "                                                             enc_out)\n",
    "\n",
    "        # storing the attention weights to plot later on\n",
    "        attention_weights = tf.reshape(attention_weights, (-1, ))\n",
    "        attention_plot[t] = attention_weights.numpy()\n",
    "\n",
    "        predicted_id = tf.argmax(predictions[0]).numpy()\n",
    "\n",
    "        result += targ_lang.index_word[predicted_id] + ' '\n",
    "\n",
    "        if targ_lang.index_word[predicted_id] == '<end>':\n",
    "            return result, sentence, attention_plot\n",
    "\n",
    "        # the predicted ID is fed back into the model\n",
    "        dec_input = tf.expand_dims([predicted_id], 0)\n",
    "\n",
    "    return result, sentence, attention_plot\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function for plotting the attention weights\n",
    "def plot_attention(attention, sentence, predicted_sentence):\n",
    "    fig = plt.figure(figsize=(10,10))\n",
    "    ax = fig.add_subplot(1, 1, 1)\n",
    "    ax.matshow(attention, cmap='viridis')\n",
    "\n",
    "    fontdict = {'fontsize': 14}\n",
    "\n",
    "    ax.set_xticklabels([''] + sentence, fontdict=fontdict, rotation=90)\n",
    "    ax.set_yticklabels([''] + predicted_sentence, fontdict=fontdict)\n",
    "\n",
    "    ax.xaxis.set_major_locator(ticker.MultipleLocator(1))\n",
    "    ax.yaxis.set_major_locator(ticker.MultipleLocator(1))\n",
    "\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def translate(sentence):\n",
    "    result, sentence, attention_plot = evaluate(sentence)\n",
    "\n",
    "    print('Input: %s' % (sentence))\n",
    "    print('Predicted translation: {}'.format(result))\n",
    "\n",
    "    attention_plot = attention_plot[:len(result.split(' ')), :len(sentence.split(' '))]\n",
    "    plot_attention(attention_plot, sentence.split(' '), result.split(' '))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.training.tracking.util.CheckpointLoadStatus at 0x7f2f89f32e10>"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# restoring the latest checkpoint in checkpoint_dir and then test\n",
    "checkpoint.restore(tf.train.latest_checkpoint(checkpoint_dir))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input: <start> आप कहाँ हैं ? <end>\n",
      "Predicted translation: where were you ? <end> \n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAmYAAAJwCAYAAAAjo60MAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO3de7Std13f+883F4IBA4IIEctFKIoil7CRm2I4sYJIzxkgw1bkVnpMDxb1nBbx0qK9KA6sl3IOngPBGwhHubQWFAHBsEEQjCQgBhBMuVRMIWBVCIEkJN/+MWdksdiQvbPXns93rvV6jbHHXvOZl/Vdv7FH1jvP88xnVncHAIDlnbT0AAAArAgzAIAhhBkAwBDCDABgCGEGADCEMAMAGEKYAQAMIcwAAIYQZgAAQwgzAIAhhNmWqaq/X1XnV9U3LD0LALC3hNn2eXySs5M8ceE5AIA9Vj7EfHtUVSX5QJLXJPmHSb6yu69ZdCgAYM/YY7Zdzk7ypUl+IMlnkjxs0WkAgD0lzLbL45O8tLuvSPKb69sAwD7hUOaWqKqbJPnvSb6ju/+gqu6Z5M1Jzuzuv1l2OgBgL9hjtj2+M8nHuvsPkqS7357kz5P840WnAoCFVNVNqupxVXWzpWfZK8Jsezw2yQt2bXtBkidsfhQAGOG7kvxqVr8j9wWHMrdAVf29JO9Pctfu/vMd278qq3dpfl13v3eh8QBgEVX1uiS3TnJFdx9aep69IMwAgK1TVXdI8t4k35jkLUnO6u53LTnTXnAoc0tU1e3W1zE74n2bngcAFvbYJH+wPuf6d7NPrlQgzLbH+5PcavfGqrrl+j4AOEgel+TX11+/MMn3fKEdGNtEmG2PSnKk4843TfLpDc8CAIupqgckOTPJS9ebfjvJ6Um+dbGh9sgpSw/AF1dV//f6y07y01V1xY67T87q2PrbNz4YACzn8Ule1t2XJ0l3X1VVL87qSgWvWXKw4yXM5vuG9d+V5K5Jrtpx31VJLkrys5seCgCWUFWnZXWZjO/eddcLkry6qm56XbBtI+/K3ALrY+YvTvLE7v7E0vMAwFKq6suz+qzoF3T3tbvue0yS13b3hxcZbg8Isy1QVSdndR7ZPfbDW4EBgCNz8v8W6O5rknwwyY2WngUAOHHsMdsSVfX4rI6nP6a7P7b0PACwSVX1/hz56gSfp7u/+gSPc8I4+X97PCXJHZP8ZVV9KMknd97Z3XdfZCoA2Ixn7fj6pkn+RZILkrx5ve3+WV2p4Oc2PNeeEmbb46XX/xAA2J+6+++Cq6p+LckzuvvpOx9TVT+a5Os3PNqecigT2JiqOpyjP1eykny4ux9x4iYCtlFVfTyrz8a8ZNf2Oye5qLvPWGay42ePGbBJN+vuex3tg6vqj0/kMMDW+mSSs5Ncsmv72Umu2P3gbSLMtkRV3SjJv8rqDQC3S3Lqzvu7++Ql5oJjdKy76O3SB47kF5L8YlUdSvKW9bb7ZfWJAP9mqaH2gjDbHv8+yT9K8tNZ/YP8oSR3SPKPkzxtubEAYLO6+2eq6gNJfjCrTwFIkncneXx3v3ixwfaAc8y2xPptwk/q7ldV1SeS3LO7/2tVPSnJOd39qIVHhOtVVRd191nH8PgLuvsbT+RMAJPYY7Y9bp3kuqv+X57k5uuvX5XkGYtMBAALq6qbZ9cF87v7fyw0znETZtvjvyX5yvXflyR5SJILs7puy6cWnAuOxU2q6leO8rG1/gPwOarq9kmendXJ/jvf6V1ZnZu6teddC7Pt8VtJzsnqJMdnJvmNqvreJLdN8h+WHAyOwbdn1xtXrof/6QCO5FezOnL0T5Ncmn30RiHnmG2pqrpvkgcmeW93/87S88DRqKofyGcPwx+NS7v7l07UPPuZtWY/q6rLk9yvuy9eepa9Jsy2RFU9KMkfdvdndm0/JckDuvsNy0wGR6+q3pHVx4sd7SHKf+/k/xvGWrOfVdWfJnlCd1+49Cx7TZhtiaq6JsmZ3X3Zru23THKZ65ixDarqbcd6gdnuvs+JnGm/stbsZ1X1vyT5kSTft/vq/9vOOWbb47oTGne7ZXZ9oDkM5gKzm2Ot2c9eluS0JO+pqiuTfM7RJB/JxAlTVS9ff9lJXrD+B3idk5PcLckfbnwwAFjOk5ce4EQRZvP91frvSvLX+dx3qV2V5I1JnrvpoQBgKd39vKVnOFGE2XDd/U+SZP3REz/b3Q5bss1OXb+R5Wi4jtnxsdbsa1V16ySPTXKnJE/r7o9V1QOzeofx+5ed7oZz8v+WqKqTkqS7r13fvk2Shyd5V3c7lMlWqKqnJvmyY3jKh7r7F0/UPPuZtWY/q6p7J/n9JO9P8vVJvra731dV/ybJXbr70UvOdzyE2ZaoqlcmeVV3P7Oqbprkz5LcJMlNk/zT7n7+ogNusar6yhzb3uMru/sjJ2qe/cxab461Zj+rqtcleUN3/8T686PvsQ6z+yf5ze6+/cIj3mAOZW6PQ0meuv76kUk+nuSOSb4nq2sVCbMb7vwkF+XoD+XcKYnrPd0wO9f6+v6vsGKtj4e1Zj+7d1ZX/d/tv2f12dJbS5htj5sm+Zv119+W5Le6++qqOj+Jww/H51PHstu7qv74RA6zz1nrzbHW7GefypEP1X9tksuOsH1rnHT9D2GI/5bkgVV1k6w+wPw16+23SHLFYlPtD673tDnWenOsNfvZy5L8RFWdtr7dVXWHJM9I8p+WGmovCLPt8fNJfj3Jh5L8ZZLrPoLpQUn+dKmhAGABT8lqx8RHk5ye1aWjLknyt0n+9YJzHTeHMrdEdz+nqt6a5HZJXnPduzOT/NckT1tuMgDYrO7+eJJvWn8001lZ7Wi6qLtfu+xkx0+YbYGqulmSu3f3HyTZ/YGtf5PkXZuf6kBzvafNsdabY63ZCjt/J3b3+Vm90eW6+x6Y1WWk/nqxAY+TMNsO1yZ5ZVU9pLvfdN3GqrpHVv8gb7vYZPvDVVV1LNeC++gJm2T/s9abY63Zr/b170RhtgW6+xNV9bIkj0vyph13PTbJq7v7Y8tMtm+8P8ltjuHxHzxRgxwA1npzrDX70n7/nSjMtsfzk/xGVX1/d1+1/iSAR2cff5DrBn1Nkvvl6A7lVD77xguOnbXeHGvNfrZvfycKs+3xmqyu2/LwJP85yTlJbpTkt5ccap+o7r7qqB9c5VycG85ab4613qCqeniSO2d11fkPLz3PAbBvfye6XMaWWL8L8wVZ7bpNVrtsX9TdVy831b7hek+bY603x1pvSFX9SJLfSvJDSf6kqr5h4ZH2vf38O1GYbZfnJ3loVd0uySOSPG/heQBIvi+rzyy+bZJnJnlNVX1bVd2uqk6pqjPX/91mb+3L34kOZW6R7n5nVV2c5IVJPtTdFyw9EwC5Rdbn6HX309fnO71yfd99svpv9l2SnLzMePvTfv2dKMy2z/OT/Mck/2rpQfaRL6mqHz/KxzoP5/hY682x1pvz3iRfl+QDSdLdP1lVv5zkzCTvzupw2+mLTbe/7bvfidXttIJtUlW3SPL9SZ7jBNO9UVUPSvIlx/CUv+3ut5yoefYza7051npzqurJSR7c3d+59CwHzX78nSjMAACGcPI/AMAQwgwAYAhhtoWq6tylZzgorPXmWOvNsM6bY603Y7+tszDbTvvqH+Fw1npzrPVmWOfNsdabsa/WWZgBAAxx4N+VeaM6rW+cmyw9xjG5Olfm1Jy29BgHgrXeHGu9Gdu6zne5+xVLj3DMPvpX1+RWt9yua8q+9x3bd7m1bf03/Yn89ce6+1a7tx/4C8zeODfJfeucpccA4It49avfvvQIB8JDbnuvpUc4MF577Us+eKTtDmUCAAwhzAAAhhBmAABDCDMAgCGEGQDAEMIMAGAIYQYAMIQwAwAYQpgBAAwhzAAAhhBmAABDCDMAgCGEGQDAEMIMAGAIYQYAMIQwAwAYQpgBAAwhzAAAhhBmAABDCDMAgCGEGQDAEMIMAGAIYQYAMIQwAwAYQpgBAAwhzAAAhhBmAABDCDMAgCGEGQDAEMIMAGAIYQYAMIQwAwAYQpgBAAwhzAAAhhBmAABDCDMAgCGEGQDAEMIMAGCIjYRZVZ1dVV1VX76J7wcAsI3sMQMAGGKrw6yqbrT0DAAAe+UGh1lVPbSqPlFVp6xv33l9uPLZOx7zk1X12h1Pu0dV/VFVXVFVb62qs3a95gOq6vXr+/+yqv6/qjpjx/2H19t+tqo+muRN6+03q6rzquqy9Uyvr6pDN/RnAwBYwvHsMXtjkhsnuS6Azk7ysfXf2bHt8I7bP53kR5KcleSvkrywqipJquobkvxekpcnuUeSRya5Z5Jf2fV9H5Okknxzksetn/+KJLdN8vAk90ryhiTnV9WZx/HzAQBs1A0Os+6+PMmFSR683nR2kmcluX1VnVlVpye5Tz43zJ7W3a/r7j9L8u+SfG1WQZUkP5TkRd39c9395939R0melOQ7q+ordrzG+7v7X3b3n3X3u9ff/55JHtXdF3T3Jd39tCTvS/LYI81eVeeu99i99epceUOXAABgTx3vOWaH89k9ZN+S5JVJ/mi97QFJPpPkgh2Pf8eOry9d/31ddN07yWOq6vLr/mR9qDLJnXY878JdM9w7yelJPrrruXfb9by/093ndfeh7j50ak47mp8TAOCEO+U4n384yZOr6q5Jzsgqmg5ntRfrsiRv7u6r1kcrk+TqHc/t9d8n7fj7l5L8whG+z1/u+PqTu+47KclHsjq0udvHj+aHAACY4HjD7I1JTkvy1CRv7O5rqupwkudmFUuvOobXuijJ13f3Jcc4w0VJbp3k2u5+3zE+FwBgjOM6lLnjPLPHJHndevNbknxVkvvlc88vuz7PSPKNVfXsqrrX+l2eD6+q51zP816b1SHPl1XVt1fVHavq/lX1b6vqSHvRAABG2ovrmB3Oas/b4STp7k9ndZ7Zlfnc88u+qO5+R5IHJblDktcn+ZOs3sX5ket5Xid5WJLzs9pT954kL07yNfnseWwAAOPVqmsOrjPqFn3fOmfpMQD4Il596duXHuFAeMht77X0CAfGa699yYXd/XnXXN3qK/8DAOwnwgwAYAhhBgAwhDADABhCmAEADCHMAACGEGYAAEMIMwCAIYQZAMAQwgwAYAhhBgAwhDADABhCmAEADCHMAACGEGYAAEMIMwCAIYQZAMAQwgwAYAhhBgAwhDADABhCmAEADCHMAACGEGYAAEMIMwCAIYQZAMAQwgwAYAhhBgAwhDADABhCmAEADCHMAACGEGYAAEMIMwCAIYQZAMAQwgwAYAhhBgAwhDADABhCmAEADHHK0gMAbK2TTl56ggPjYXc/Z+kRDoRTbnf60iMcHB848mZ7zAAAhhBmAABDCDMAgCGEGQDAEMIMAGAIYQYAMIQwAwAYQpgBAAwhzAAAhhBmAABDCDMAgCGEGQDAEMIMAGAIYQYAMIQwAwAYQpgBAAwhzAAAhhBmAABDCDMAgCGEGQDAEMIMAGAIYQYAMIQwAwAYQpgBAAwhzAAAhhBmAABDCDMAgCGEGQDAEMIMAGAIYQYAMIQwAwAYQpgBAAwhzAAAhhBmAABDCDMAgCGEGQDAEMIMAGAIYQYAMIQwAwAYQpgBAAwhzAAAhhBmAABDCDMAgCG2Osyq6kZLzwAAsFdOaJhV1UOr6hNVdcr69p2rqqvq2Tse85NV9dr1119XVa9YP+eyqvqNqrrNjsf+WlX9TlX9cFV9KMmH1ttvVFXPqKoPVdUVVfXHVfWQE/mzAQDstRO9x+yNSW6c5ND69tlJPrb+Ozu2Ha6qM5O8IcnFSb4xybcmuWmSl1XVzjm/Jcndkzw0yTnrbb+63v7oJHdL8rwkv11V99jrHwgA4EQ5oWHW3ZcnuTDJg9ebzk7yrCS3r6ozq+r0JPdJcjjJk5L8SXf/cHe/u7vfkeRxWUXaoR0v++kkT+zui7v7T6vqTkm+O8l3dfcbuvt93f2sJL+b5J8daa6qOreq3lpVb706V+71jw0AcINs4hyzw/nsHrJvSfLKJH+03vaAJJ9JckGSeyd5UFVdft2fJH+xft6ddrzexd29s6bOSlJJ3rXrud+x63l/p7vP6+5D3X3o1Jy2Bz8iAMDxO2UD3+NwkidX1V2TnJHVHrTDWe1FuyzJm7v7qvXhylckecoRXuMjO77+5K77TkrSWe15u3rXfZ863uEBADZlE2H2xiSnJXlqkjd29zVVdTjJc7MKrletH3dRku9K8sHu3h1YX8zbstpjdpvuft2eTQ0AsGEn/FDmjvPMHpPkunB6S5KvSnK/rPaeJckvJrlZkhdV1X2r6qur6lur6ryq+tIv8vrvTfLCJL9WVY9aP+9QVT2lqh55gn4sAIA9t6nrmB3Oau/c4STp7k9ndZ7ZlVmdX5buvjTJA5Ncm9VetHdmFWtXrv98Mf8kq3dm/kySP0vyO0kelOSDe/pTAACcQNXdS8+wqDPqFn3fOuf6Hwiw20knLz3BgXHyLW6+9AgHQt3k9KVHODBe9YFfuLC7D+3evtVX/gcA2E+EGQDAEMIMAGAIYQYAMIQwAwAYQpgBAAwhzAAAhhBmAABDCDMAgCGEGQDAEMIMAGAIYQYAMIQwAwAYQpgBAAwhzAAAhhBmAABDCDMAgCGEGQDAEMIMAGAIYQYAMIQwAwAYQpgBAAwhzAAAhhBmAABDCDMAgCGEGQDAEMIMAGAIYQYAMIQwAwAYQpgBAAwhzAAAhhBmAABDCDMAgCGEGQDAEMIMAGAIYQYAMIQwAwAY4pSlBwDYWtdes/QEB8a1f/O3S49wIJx0zbVLj3Dg2WMGADCEMAMAGEKYAQAMIcwAAIYQZgAAQwgzAIAhhBkAwBDCDABgCGEGADCEMAMAGEKYAQAMIcwAAIYQZgAAQwgzAIAhhBkAwBDCDABgCGEGADCEMAMAGEKYAQAMIcwAAIYQZgAAQwgzAIAhhBkAwBDCDABgCGEGADCEMAMAGEKYAQAMIcwAAIYQZgAAQwgzAIAhhBkAwBDCDABgCGEGADCEMAMAGEKYAQAMIcwAAIYQZgAAQwgzAIAhhBkAwBDCDABgCGEGADCEMAMAGEKYAQAMsXiYVdXjquqvquq0XdtfWFUvX3/9z6rqkqq6av339+56bFfVo3Zt+0BVPeXE/wQAAHtj8TBL8pKs5vjfrttQVTdL8ogkv1xVj0jyrCT/Mcndkjwzyf9bVf9wgVkBAE6YU5YeoLs/VVUvTPLEJC9eb350ko8neUWS1yf59e5+1vq+91bVvZP8cJLfviHfs6rOTXJuktw4px/H9AAAe2fCHrMkeW6Sf1BVX7W+/cQkz+vuzyS5a5I37Xr8G5N83Q39Zt19Xncf6u5Dp+a0638CAMAGjAiz7v6TJBcleUJV3S3JoSS/cn1P2/V17br/1L2bEADgxBsRZmvPTfKEJP97kjd193vW29+d5IG7HvtNSd614/ZHk5x53Y2quvXO2wAA22Dxc8x2+I0kP5/kSUn+jx3b/0OSl1TVhUl+L8lDk3xPkkfueMz5Sf55Vf1hkmuSPD3JpzcxNADAXhmzx6y7P5HVyf9X5rNvAkh3/5ck35/k/8pqL9kPJvm+7t554v+/TPK+JIeTvDTJLyW5bCODAwDskUl7zJLV4ccXdfcnd27s7mcnefYXelJ3X5rk23dt/k97Px4AwIkzIsyq6suSfHOSb0tyj4XHAQBYxIgwS/K2JLdI8mPdffHSwwAALGFEmHX3HZaeAQBgaWNO/gcAOOiEGQDAEMIMAGAIYQYAMIQwAwAYQpgBAAwhzAAAhhBmAABDCDMAgCGEGQDAEMIMAGAIYQYAMIQwAwAYQpgBAAwhzAAAhhBmAABDCDMAgCGEGQDAEMIMAGAIYQYAMIQwAwAYQpgBAAwhzAAAhhBmAABDCDMAgCGEGQDAEMIMAGAIYQYAMIQwAwAYQpgBAAwhzAAAhhBmAABDCDMAgCGEGQDAEMIMAGCIU5YeAACuzxUPP2vpEQ6EDz3yM0uPcHA89sib7TEDABhCmAEADCHMAACGEGYAAEMIMwCAIYQZAMAQwgwAYAhhBgAwhDADABhCmAEADCHMAACGEGYAAEMIMwCAIYQZAMAQwgwAYAhhBgAwhDADABhCmAEADCHMAACGEGYAAEMIMwCAIYQZAMAQwgwAYAhhBgAwhDADABhCmAEADCHMAACGEGYAAEMIMwCAIYQZAMAQwgwAYAhhBgAwhDADABhCmAEADCHMAACGEGYAAEMIMwCAIYQZAMAQwgwAYAhhBgAwhDADABhiX4VZVT25qt5WVZ+sqr+oqh9deiYAgKN1ytID7LFzkvx4kncmeVCSX6qqd3b3y5cdCwDg+u2rMOvuR+y4+b6qenqSOy81DwDAsdhXhzJ3qqofS3Jqkt9cehYAgKOxr/aYXaeq/nWSH0jyD7r70iPcf26Sc5Pkxjl9w9MBABzZvguzqvrKJP8uyXd099uP9JjuPi/JeUlyRt2iNzgeAMAXtB8PZZ6ZpJK8e+lBAACOxX4Ms3cnuU+SzzuECQAw2X4Ms7sleUGSWy09CADAsdiPYXZ6kq/J6h2ZAABbY9+d/N/dh7M6xwwAYKvsxz1mAABbSZgBAAwhzAAAhhBmAABDCDMAgCGEGQDAEMIMAGAIYQYAMIQwAwAYQpgBAAwhzAAAhhBmAABDCDMAgCGEGQDAEMIMAGAIYQYAMIQwAwAYQpgBAAwhzAAAhhBmAABDCDMAgCGEGQDAEMIMAGAIYQYAMIQwAwAYQpgBAAwhzAAAhhBmAABDCDMAgCGEGQDAEMIMAGAIYQYAMIQwAwAYQpgBAAwhzAAAhhBmAABDCDMAgCFOWXoAALg+X/JfLlh6hAPhq//2rKVHODA++AW222MGADCEMAMAGEKYAQAMIcwAAIYQZgAAQwgzAIAhhBkAwBDCDABgCGEGADCEMAMAGEKYAQAMIcwAAIYQZgAAQwgzAIAhhBkAwBDCDABgCGEGADCEMAMAGEKYAQAMIcwAAIYQZgAAQwgzAIAhhBkAwBDCDABgCGEGADCEMAMAGEKYAQAMIcwAAIYQZgAAQwgzAIAhhBkAwBDCDABgCGEGADCEMAMAGEKYAQAMIcwAAIYQZgAAQwgzAIAhhBkAwBDCDABgCGEGADDE1oRZVT2lqj6w9BwAACfK1oQZAMB+tydhVlVnVNXN9+K1juF73qqqbrzJ7wkAcCLd4DCrqpOr6iFV9f8n+XCSe6y336yqzquqy6rqE1X1+qo6tON5T6iqy6vqnKq6uKo+WVWvq6o77nr9p1bVh9ePfX6Sm+4a4WFJPrz+Xg+8oT8HAMAUxxxmVfX1VfUzSf4iyYuSfDLJQ5O8oaoqySuS3DbJw5PcK8kbkpxfVWfueJnTkvxokicmuX+Smyd59o7v8V1JfjLJTyQ5K8l7kvyLXaO8MMmjk3xpktdU1SVV9eO7A+8L/AznVtVbq+qtV+fKY10CAIAT4qjCrKpuWVU/UFUXJnlbkq9N8oNJbtPd39vdb+juTvLgJPdM8qjuvqC7L+nupyV5X5LH7njJU5L88/Vj3pHkZ5OcvQ67JPk/kzyvu5/T3e/t7p9KcsHOmbr7M939u9393Uluk+Tp6+//51V1uKqeWFW797Jd99zzuvtQdx86NacdzRIAAJxwR7vH7PuTPDPJp5Pcpbv/1+5+SXd/etfj7p3k9CQfXR+CvLyqLk9ytyR32vG4K7v7PTtuX5rkRkm+bH37rknevOu1d9/+O9398e7+le5+cJL7JLl1kl9O8qij/PkAABZ3ylE+7rwkVyd5XJKLq+q3kvx6kt/v7mt2PO6kJB9J8s1HeI2P7/j6M7vu6x3PP2ZVdVpWh04fk9W5Z+/Maq/by27I6wEALOGoQqi7L+3un+rur0nyrUkuT/KbST5UVT9XVfdcP/SirPZWXbs+jLnzz2XHMNe7k9xv17bPuV0r31RVz8nqzQf/T5JLkty7u8/q7md2918fw/cEAFjUMe+h6u63dPeTkpyZ1SHOuyT546r65iSvTfKmJC+rqm+vqjtW1f2r6t+u7z9az0zy+Kr63qr6+1X1o0nuu+sxj0nye0nOSPLdSf5ed/9Qd198rD8TAMAER3so8/N095VJXprkpVX1FUmu6e6uqodl9Y7K5yb5iqwObb4pyfOP4bVfVFVfneSnsjpn7eVJfj7JE3Y87PezevPBxz//FQAAtk+t3kx5cJ1Rt+j71jlLjwEAi7vmwWctPcKB8brzf+zC7j60e7uPZAIAGEKYAQAMIcwAAIYQZgAAQwgzAIAhhBkAwBDCDABgCGEGADCEMAMAGEKYAQAMIcwAAIYQZgAAQwgzAIAhhBkAwBDCDABgCGEGADCEMAMAGEKYAQAMIcwAAIYQZgAAQwgzAIAhhBkAwBDCDABgCGEGADCEMAMAGEKYAQAMIcwAAIYQZgAAQwgzAIAhhBkAwBDCDABgCGEGADCEMAMAGEKYAQAMIcwAAIYQZgAAQwgzAIAhTll6AABghpNfd9HSIxx49pgBAAwhzAAAhhBmAABDCDMAgCGEGQDAEMIMAGAIYQYAMIQwAwAYQpgBAAwhzAAAhhBmAABDCDMAgCGEGQDAEMIMAGAIYQYAMIQwAwAYQpgBAAwhzAAAhhBmAABDCDMAgCGEGQDAEMIMAGAIYQYAMIQwAwAYQpgBAAwhzAAAhhBmAABDCDMAgCGEGQDAEMIMAGAIYQYAMIQwAwAYQpgBAAwhzAAAhhBmAABDCDMAgCGEGQDAEMIMAGAIYQYAMIQwAwAYQpgBAAwhzAAAhhBmAABDCDMAgCGEGQDAEKcsPcASqurcJOcmyY1z+sLTAACsHMg9Zt19Xncf6u5Dp+a0pccBAEhyQMMMAGAiYQYAMIQwAwAYQpgBAAwhzAAAhhBmAABDCDMAgCGEGQDAEMIMAGAIYQYAMIQwAwAYQpgBAAwhzAAAhhBmAABDCDMAgCGEGQDAEMIMAGAIYQYAMIQwAwAYQpgBAAwhzAAAhhBmAABDCDMAgCGEGQDAEMIMAGAIYQYAMIQwAwAYQpgBAAwhzAAAhhBmAABDCDMAgCGEGQDAEMIMAGAIYQYAMIQwAwAYQpgBAAwhzAAAhhBmAABDCDMAgCGEGen7ugcAAAClSURBVADAEMIMAGAIYQYAMIQwAwAYorp76RkWVVUfTfLBpec4Rl+e5GNLD3FAWOvNsdabYZ03x1pvxrau8+27+1a7Nx74MNtGVfXW7j609BwHgbXeHGu9GdZ5c6z1Zuy3dXYoEwBgCGEGADCEMNtO5y09wAFirTfHWm+Gdd4ca70Z+2qdnWMGADCEPWYAAEMIMwCAIYQZAMAQwgwAYAhhBgAwxP8EWpiyF8+JSsUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 720x720 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "translate(u'आप कहाँ हैं?')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tftest",
   "language": "python",
   "name": "tftest"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}