train_model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " <h2 style=\"color:black; direction:rtl\">ترجمه ماشین با <span style=\"color:#cc0066\">Attention</span></h2>\n",
    " <h6 style=\"color:#666699\" >Copyright 2018 The TensorFlow Authors.</h6>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<p style=\"direction:rtl;text-align:right\">این نوت بوک از  یک مدل seq2seq برای ترجمه فارسی به انگلیسی با استفاده از tf.keras وtf.enable_eager_execution است.</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "NMPCER-kCydU",
    "outputId": "6b1a070c-5bee-479d-8473-c4ac62175cbc"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.12.0\n"
     ]
    }
   ],
   "source": [
    "from __future__ import absolute_import, division, print_function\n",
    "\n",
    "# Import TensorFlow >= 1.10 and enable eager execution\n",
    "import tensorflow as tf\n",
    "\n",
    "tf.enable_eager_execution()\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "import unicodedata\n",
    "import re\n",
    "import numpy as np\n",
    "import os\n",
    "import time\n",
    "\n",
    "print(tf.__version__)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "YO4bmdL-7qB0"
   },
   "source": [
    "<h3 style=\"direction:rtl;text-align:right\">دانلود و تهیه مجموعه داده</h3>\n",
    "<h6 style=\"direction:rtl;text-align:right\">در اینجا مراحل مورد نیاز برای تهیه داده ها را انجام می دهیم:</h6>\n",
    "<ol style=\"direction:rtl;text-align:right\">\n",
    "    <li>یک علامت شروع و پایان را به هر جمله اضافه کنید.</li>\n",
    "    <li>جملات را با حذف کاراکتر های خاص پاک کنید.</li>\n",
    "    <li>یک  word index  ایجاد کنید و آنرا برعکس کنید (dictionaries mapping from word → id and id → word).</li>\n",
    "    <li>هر جمله را به حداکثر طول برسانید.</li>\n",
    "</ol>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "PSuqkeVeC_n7"
   },
   "outputs": [],
   "source": [
    "# Download the file\n",
    "#add file name\n",
    "path_to_file = \"fra.txt\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "rbKGvOVkDCLx"
   },
   "outputs": [],
   "source": [
    "# Converts the unicode file to ascii\n",
    "def unicode_to_ascii(s):\n",
    "    return ''.join(c for c in unicodedata.normalize('NFD', s)\n",
    "        if unicodedata.category(c) != 'Mn')\n",
    "\n",
    "\n",
    "def preprocess_sentence(w):\n",
    "    w = unicode_to_ascii(w.lower().strip())\n",
    "    \n",
    "    # creating a space between a word and the punctuation following it\n",
    "    # eg: \"he is a boy.\" => \"he is a boy .\" \n",
    "    # Reference:- https://stackoverflow.com/questions/3645931/python-padding-punctuation-with-white-spaces-keeping-punctuation\n",
    "    w = re.sub(r\"([?.!,¿])\", r\" \\1 \", w)\n",
    "    w = re.sub(r'[\" \"]+', \" \", w)\n",
    "    \n",
    "    # replacing everything with space except (a-z, A-Z, \".\", \"?\", \"!\", \",\")\n",
    "    w = re.sub(r\"[^a-zA-Z?.!,¿]+\", \" \", w)\n",
    "    \n",
    "    w = w.rstrip().strip()\n",
    "    \n",
    "    # adding a start and an end token to the sentence\n",
    "    # so that the model know when to start and stop predicting.\n",
    "    w = '<start> ' + w + ' <end>'\n",
    "    return w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "U_9OntDsDESl"
   },
   "outputs": [],
   "source": [
    "# 1. Remove the accents\n",
    "# 2. Clean the sentences\n",
    "# 3. Return word pairs in the format: [ENGLISH, SPANISH]\n",
    "def create_dataset(path, num_examples):\n",
    "    lines = open(path, encoding='UTF-8').read().strip().split('\\n')\n",
    "    \n",
    "    word_pairs = [[preprocess_sentence(w) for w in l.split('\\t')]  for l in lines[:num_examples]]\n",
    "    \n",
    "    return word_pairs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "2V0X0Yk-DcIE"
   },
   "outputs": [],
   "source": [
    "\n",
    "# This class creates a word -> index mapping (e.g,. \"dad\" -> 5) and vice-versa \n",
    "# (e.g., 5 -> \"dad\") for each language,\n",
    "class LanguageIndex():\n",
    "    \n",
    "    def __init__(self, lang):\n",
    "        \n",
    "        self.lang = lang\n",
    "        self.word2idx = {}\n",
    "        self.idx2word = {}\n",
    "        self.vocab = set()\n",
    "\n",
    "        self.create_index()\n",
    "    \n",
    "    def create_index(self):\n",
    "        \n",
    "        for phrase in self.lang:\n",
    "          self.vocab.update(phrase.split(' '))\n",
    "\n",
    "        self.vocab = sorted(self.vocab)\n",
    "\n",
    "        self.word2idx['<pad>'] = 0\n",
    "        for index, word in enumerate(self.vocab):\n",
    "          self.word2idx[word] = index + 1\n",
    "\n",
    "        for word, index in self.word2idx.items():\n",
    "          self.idx2word[index] = word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "bv7yawOlDeBB"
   },
   "outputs": [],
   "source": [
    "def max_length(tensor):\n",
    "    return max(len(t) for t in tensor)\n",
    "\n",
    "\n",
    "def load_dataset(path, num_examples):\n",
    "    # creating cleaned input, output pairs\n",
    "    pairs = create_dataset(path, num_examples)\n",
    "\n",
    "    # index language using the class defined above    \n",
    "    inp_lang = LanguageIndex(sp for en, sp in pairs)\n",
    "    targ_lang = LanguageIndex(en for en, sp in pairs)\n",
    "    \n",
    "    # Vectorize the input and target languages\n",
    "    \n",
    "    # Spanish sentences\n",
    "    input_tensor = [[inp_lang.word2idx[s] for s in sp.split(' ')] for en, sp in pairs]\n",
    "    \n",
    "    # English sentences\n",
    "    target_tensor = [[targ_lang.word2idx[s] for s in en.split(' ')] for en, sp in pairs]\n",
    "    \n",
    "    # Calculate max_length of input and output tensor\n",
    "    # Here, we'll set those to the longest sentence in the dataset\n",
    "    max_length_inp, max_length_tar = max_length(input_tensor), max_length(target_tensor)\n",
    "    \n",
    "    # Padding the input and output tensor to the maximum length\n",
    "    input_tensor = tf.keras.preprocessing.sequence.pad_sequences(input_tensor, \n",
    "                                                                 maxlen=max_length_inp,\n",
    "                                                                 padding='post')\n",
    "    \n",
    "    target_tensor = tf.keras.preprocessing.sequence.pad_sequences(target_tensor, \n",
    "                                                                  maxlen=max_length_tar, \n",
    "                                                                  padding='post')\n",
    "    \n",
    "    return input_tensor, target_tensor, inp_lang, targ_lang, max_length_inp, max_length_tar"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3 style=\"direction:rtl;text-align:right\">محدود کردن اندازه مجموعه داده ها برای آزمایش سریع تر (اختیاری)</h3>\n",
    "<p style=\"direction:rtl;text-align:right\">آموزش  مجموعه داده بالای 100،000 زمان بر خواهد بود. برای آموزش سریع تر می توانیم اندازه مجموعه داده ها را مثلا به 30،000 جمله محدود کنیم.البته در این حالت میزان دقت کاش میابد!</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zkbDJUQ0Df7v"
   },
   "outputs": [],
   "source": [
    "# Try experimenting with the size of that dataset\n",
    "num_examples =31000\n",
    "input_tensor, target_tensor, inp_lang, targ_lang, max_length_inp, max_length_targ = load_dataset(path_to_file, num_examples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "vxh1Be0sDh9T",
    "outputId": "b7c020ac-150a-460f-9c0f-5c05293cf035"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(24800, 24800, 6200, 6200)"
      ]
     },
     "execution_count": 10,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Creating training and validation sets using an 80-20 split\n",
    "input_tensor_train, input_tensor_val, target_tensor_train, target_tensor_val = train_test_split(input_tensor, target_tensor, test_size=0.2)\n",
    "\n",
    "# Show length\n",
    "len(input_tensor_train), len(target_tensor_train), len(input_tensor_val), len(target_tensor_val)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3 style=\"direction:rtl;text-align:right\">یک مجموعه داده tf.data ایجاد کنید</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "WypHR7i3DkOL"
   },
   "outputs": [],
   "source": [
    "BUFFER_SIZE = len(input_tensor_train)\n",
    "BATCH_SIZE = 64\n",
    "N_BATCH = BUFFER_SIZE//BATCH_SIZE\n",
    "embedding_dim = 256\n",
    "units = 1024\n",
    "vocab_inp_size = len(inp_lang.word2idx)\n",
    "vocab_tar_size = len(targ_lang.word2idx)\n",
    "\n",
    "dataset = tf.data.Dataset.from_tensor_slices((input_tensor_train, target_tensor_train)).shuffle(BUFFER_SIZE)\n",
    "dataset = dataset.batch(BATCH_SIZE, drop_remainder=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3 style=\"direction:rtl;text-align:right\">مدل رمزگذار و رمزگشایی را بنویسید</h3>\n",
    "<p style=\"direction:rtl;text-align:right\">این نوت بوک مکانیزم Attention را از آموزش seq2seq پیاده سازی می کند. نمودار زیر نشان می دهد که هر کلمه ورودی یک وزن با توجه به مکانیزم attention دارد که سپس توسط رمزگذار برای پیش بینی کلمه بعدی در جمله استفاده می شود.</p>\n",
    "<img src=\"img/dig1.png\">\n",
    "\n",
    "<p style=\"direction:rtl;text-align:right\">ورودی از طریق یک مدل رمزگذار قرار می گیرد که به ما خروجی رمزگذار اندازه (batch_size، max_length، hidden_size) \n",
    "    و حالت پنهان رمزگذار \n",
    "    (batch_size، hidden_size) را می دهد.</p>\n",
    "    \n",
    "<h5 style=\"direction:rtl;text-align:right\">در اینجا معادلات اجرا می شوند:</h5>   \n",
    "<img src=\"img/e1.png\">\n",
    "<br><br>\n",
    "<p style=\"direction:rtl;text-align:right\">ما ازBahdanau attention  استفاده می کنیم. قبل از نوشتن فرم ساده، تصمیم گیری در مورد نماد انجام می شود:</p>\n",
    "\n",
    "<ul style=\"direction:rtl;text-align:right\">\n",
    "    <li>FC = Fully connected (dense) layer</li>\n",
    "    <li>EO = Encoder output</li>\n",
    "    <li>H = hidden state</li>\n",
    "    <li>X = input to the decoder</li>\n",
    "    </ul>\n",
    "<br>  \n",
    "<p style=\"direction:rtl;text-align:right\">و شبه کد:</p>\n",
    "\n",
    "<ul style=\"direction:rtl;text-align:right\">\n",
    "    <li style=\"direction:ltr;text-align:left\">score = FC(tanh(FC(EO) + FC(H)))</li>\n",
    "    <li style=\"direction:ltr;text-align:left\">attention weights = softmax(score, axis = 1)</li>\n",
    "    <span style=\"direction:rtl;text-align:right\">Softmax به طور پیش فرض بر روی محور آخر اعمال می شود، اما در اینجا ما می خواهیم آن را در محور اول اعمال کنیم، زیرا شکل نمره (batch_size، max_length، 1) است. Max_length طول ورودی ما است. از آنجا که ما سعی می کنیم وزن را برای هر ورودی اختصاص دهیم، softmax باید بر آن محور اعمال شود.</span>\n",
    "    \n",
    "<li style=\"direction:ltr;text-align:left\">context vector = sum(attention weights * EO, axis = 1).</li>\n",
    "<li style=\"direction:ltr;text-align:left\">embedding output = The input to the decoder X is passed through an embedding layer.</li>\n",
    "<li style=\"direction:ltr;text-align:left\">merged vector = concat(embedding output, context vector)</li>\n",
    "<li style=\"direction:ltr;text-align:left\">This merged vector is then given to the GRU</li>\n",
    "</ul>\n",
    "<br>\n",
    "<p style=\"direction:rtl;text-align:right\">شکل همه بردارها در هر مرحله در comment در کد مشخص شده است:</p>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "SyahVlADDmLj"
   },
   "outputs": [],
   "source": [
    "def gru(units):\n",
    "  # If you have a GPU, we recommend using CuDNNGRU(provides a 3x speedup than GRU)\n",
    "  # the code automatically does that.\n",
    "  if tf.test.is_gpu_available():\n",
    "    return tf.keras.layers.CuDNNGRU(units, \n",
    "                                    return_sequences=True, \n",
    "                                    return_state=True, \n",
    "                                    recurrent_initializer='glorot_uniform')\n",
    "  else:\n",
    "    return tf.keras.layers.GRU(units, \n",
    "                               return_sequences=True, \n",
    "                               return_state=True, \n",
    "                               recurrent_activation='sigmoid', \n",
    "                               recurrent_initializer='glorot_uniform')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-V6UbMMzDoUr"
   },
   "outputs": [],
   "source": [
    "class Encoder(tf.keras.Model):\n",
    "    def __init__(self, vocab_size, embedding_dim, enc_units, batch_sz):\n",
    "        super(Encoder, self).__init__()\n",
    "        self.batch_sz = batch_sz\n",
    "        self.enc_units = enc_units\n",
    "        self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
    "        self.gru = gru(self.enc_units)\n",
    "        \n",
    "    def call(self, x, hidden):\n",
    "        x = self.embedding(x)\n",
    "        output, state = self.gru(x, initial_state = hidden)        \n",
    "        return output, state\n",
    "    \n",
    "    def initialize_hidden_state(self):\n",
    "        return tf.zeros((self.batch_sz, self.enc_units))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "QNFpHnKSDqpi"
   },
   "outputs": [],
   "source": [
    "class Decoder(tf.keras.Model):\n",
    "    def __init__(self, vocab_size, embedding_dim, dec_units, batch_sz):\n",
    "        super(Decoder, self).__init__()\n",
    "        self.batch_sz = batch_sz\n",
    "        self.dec_units = dec_units\n",
    "        self.embedding = tf.keras.layers.Embedding(vocab_size, embedding_dim)\n",
    "        self.gru = gru(self.dec_units)\n",
    "        self.fc = tf.keras.layers.Dense(vocab_size)\n",
    "        \n",
    "        # used for attention\n",
    "        self.W1 = tf.keras.layers.Dense(self.dec_units)\n",
    "        self.W2 = tf.keras.layers.Dense(self.dec_units)\n",
    "        self.V = tf.keras.layers.Dense(1)\n",
    "        \n",
    "    def call(self, x, hidden, enc_output):\n",
    "        # enc_output shape == (batch_size, max_length, hidden_size)\n",
    "        \n",
    "        # hidden shape == (batch_size, hidden size)\n",
    "        # hidden_with_time_axis shape == (batch_size, 1, hidden size)\n",
    "        # we are doing this to perform addition to calculate the score\n",
    "        hidden_with_time_axis = tf.expand_dims(hidden, 1)\n",
    "        \n",
    "        # score shape == (batch_size, max_length, 1)\n",
    "        # we get 1 at the last axis because we are applying tanh(FC(EO) + FC(H)) to self.V\n",
    "        score = self.V(tf.nn.tanh(self.W1(enc_output) + self.W2(hidden_with_time_axis)))\n",
    "        \n",
    "        # attention_weights shape == (batch_size, max_length, 1)\n",
    "        attention_weights = tf.nn.softmax(score, axis=1)\n",
    "        \n",
    "        # context_vector shape after sum == (batch_size, hidden_size)\n",
    "        context_vector = attention_weights * enc_output\n",
    "        context_vector = tf.reduce_sum(context_vector, axis=1)\n",
    "        \n",
    "        # x shape after passing through embedding == (batch_size, 1, embedding_dim)\n",
    "        x = self.embedding(x)\n",
    "        \n",
    "        # x shape after concatenation == (batch_size, 1, embedding_dim + hidden_size)\n",
    "        x = tf.concat([tf.expand_dims(context_vector, 1), x], axis=-1)\n",
    "        \n",
    "        # passing the concatenated vector to the GRU\n",
    "        output, state = self.gru(x)\n",
    "        \n",
    "        # output shape == (batch_size * 1, hidden_size)\n",
    "        output = tf.reshape(output, (-1, output.shape[2]))\n",
    "        \n",
    "        # output shape == (batch_size * 1, vocab)\n",
    "        x = self.fc(output)\n",
    "        \n",
    "        return x, state, attention_weights\n",
    "        \n",
    "    def initialize_hidden_state(self):\n",
    "        return tf.zeros((self.batch_sz, self.dec_units))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "szq9b8p3DsqO"
   },
   "outputs": [],
   "source": [
    "encoder = Encoder(vocab_inp_size, embedding_dim, units, BATCH_SIZE)\n",
    "decoder = Decoder(vocab_tar_size, embedding_dim, units, BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "s_SDs4WkEF5H",
    "outputId": "08ff39c7-2de0-401c-8e02-2fcd0ccbc0f3"
   },
   "source": [
    "<h3 style=\"direction:rtl;text-align:right\">تعریف  optimizer و عملکرد loss function</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "o9RCDPUWDusL"
   },
   "outputs": [],
   "source": [
    "optimizer = tf.train.AdamOptimizer()\n",
    "\n",
    "\n",
    "def loss_function(real, pred):\n",
    "    \n",
    "    mask = 1 - np.equal(real, 0)\n",
    "    loss_ = tf.nn.sparse_softmax_cross_entropy_with_logits(labels=real, logits=pred) * mask\n",
    "    return tf.reduce_mean(loss_)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "Sc2ja4MjD5OF",
    "outputId": "c7ae7d75-b5e9-43c0-d905-4319b7634253"
   },
   "source": [
    "<h3 style=\"text-align:right\"rtl;>Checkpoints (Object-based saving)</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "jQLMhM9xDwlV"
   },
   "outputs": [],
   "source": [
    "checkpoint_dir = './training_checkpoints'\n",
    "checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt\")\n",
    "checkpoint = tf.train.Checkpoint(optimizer=optimizer,\n",
    "                                 encoder=encoder,\n",
    "                                 decoder=decoder)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3 style=\"direction:rtl;text-align:right\">Training</h3>\n",
    "\n",
    "<ol style=\"direction:rtl;text-align:right\">\n",
    "    <li >ورودی از طریق رمزگذار عبور می کند که خروجی رمزگذار  و hidden state اش را برمیگرداند.</li>\n",
    "    <li >خروجی رمزگذار،  hidden state رمزگذار و ورودی رمزگذار (که علامت شروع است) به رمزگذار منتقل می شود.</li>\n",
    "    <li >رمزگشا پیش بینی ها و  hidden state رمزگشایی را برمیگرداند.</li>\n",
    "    <li >و سپس hidden state رمزگذار  به مدل منتقل می شود و پیش بینی ها برای محاسبه loss استفاده می شود.</li>\n",
    "    <li >استفاده ازteacher forcing برای تصمیم گیری ورودی بعدی به رمزگشای.</li>\n",
    "    <li >اما Teacher forcing تکنیکی است که کلمه مورد نظر به عنوان ورودی بعدی به رمزگشایی منتقل می شود.</li>\n",
    "    <li >مرحله نهایی محاسبه گرادیان ها و اعمال آن به optimizer  و backpropagate است.</li>\n",
    "</ol>    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 255
    },
    "colab_type": "code",
    "id": "9PswP7dPDyWc",
    "outputId": "1d6b1535-a455-46d8-e674-3deecfe54524"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1 Batch 0 Loss 1.0012\n",
      "Epoch 1 Batch 100 Loss 1.0194\n",
      "Epoch 1 Batch 200 Loss 0.8746\n",
      "Epoch 1 Batch 300 Loss 0.9165\n",
      "Epoch 1 Loss 0.9968\n",
      "Time taken for 1 epoch 103.46005201339722 sec\n",
      "\n",
      "Epoch 2 Batch 0 Loss 0.8353\n",
      "Epoch 2 Batch 100 Loss 0.5950\n",
      "Epoch 2 Batch 200 Loss 0.7256\n",
      "Epoch 2 Batch 300 Loss 0.7323\n",
      "Epoch 2 Loss 0.6700\n",
      "Time taken for 1 epoch 103.70070195198059 sec\n",
      "\n"
     ]
    }
   ],
   "source": [
    "EPOCHS = 2\n",
    "\n",
    "for epoch in range(EPOCHS):\n",
    "    start = time.time()\n",
    "    \n",
    "    hidden = encoder.initialize_hidden_state()\n",
    "    total_loss = 0\n",
    "    \n",
    "    for (batch, (inp, targ)) in enumerate(dataset):\n",
    "        loss = 0\n",
    "        \n",
    "        with tf.GradientTape() as tape:\n",
    "            enc_output, enc_hidden = encoder(inp, hidden)\n",
    "            \n",
    "            dec_hidden = enc_hidden\n",
    "            \n",
    "            dec_input = tf.expand_dims([targ_lang.word2idx['<start>']] * BATCH_SIZE, 1)       \n",
    "            \n",
    "            # Teacher forcing - feeding the target as the next input\n",
    "            for t in range(1, targ.shape[1]):\n",
    "                # passing enc_output to the decoder\n",
    "                predictions, dec_hidden, _ = decoder(dec_input, dec_hidden, enc_output)\n",
    "                \n",
    "                loss += loss_function(targ[:, t], predictions)\n",
    "                \n",
    "                # using teacher forcing\n",
    "                dec_input = tf.expand_dims(targ[:, t], 1)\n",
    "        \n",
    "        batch_loss = (loss / int(targ.shape[1]))\n",
    "        \n",
    "        total_loss += batch_loss\n",
    "        \n",
    "        variables = encoder.variables + decoder.variables\n",
    "        \n",
    "        gradients = tape.gradient(loss, variables)\n",
    "        \n",
    "        optimizer.apply_gradients(zip(gradients, variables))\n",
    "        \n",
    "        if batch % 100 == 0:\n",
    "            print('Epoch {} Batch {} Loss {:.4f}'.format(epoch + 1,\n",
    "                                                         batch,\n",
    "                                                         batch_loss.numpy()))\n",
    "    # saving (checkpoint) the model every 2 epochs\n",
    "    if (epoch + 1) % 2 == 0:\n",
    "      checkpoint.save(file_prefix = checkpoint_prefix)\n",
    "    \n",
    "    print('Epoch {} Loss {:.4f}'.format(epoch + 1,\n",
    "                                        total_loss / N_BATCH))\n",
    "    print('Time taken for 1 epoch {} sec\\n'.format(time.time() - start))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3 style=\"direction:rtl;text-align:right\">Translate</h3>\n",
    "\n",
    "<ul style=\"direction:rtl;text-align:right\">\n",
    "    \n",
    "<li>تابع ارزیابی شبیه به حلقه training است، به جز اینکه از  teacher forcing استفاده نمی کنیم.ورودی به رمزگشای در هر گام زمانی، پیش بینی های قبلی خود را همراه با hidden state و خروجی رمزگذار است.</li>\n",
    "<li>توقف پیش بینی زمانی که مدل end token  را پیش بینی میکند</li>\n",
    "<li>و وزن attention را برای هر گام زمانی ذخیره کنید.</li>\n",
    "<li></li>\n",
    "</ul>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "y3Dhe8mmD0OI"
   },
   "outputs": [],
   "source": [
    "def evaluate(sentence, encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ):\n",
    "    attention_plot = np.zeros((max_length_targ, max_length_inp))\n",
    "    \n",
    "    sentence = preprocess_sentence(sentence)\n",
    "\n",
    "    inputs = [inp_lang.word2idx[i] for i in sentence.split(' ')]\n",
    "    inputs = tf.keras.preprocessing.sequence.pad_sequences([inputs], maxlen=max_length_inp, padding='post')\n",
    "    inputs = tf.convert_to_tensor(inputs)\n",
    "    \n",
    "    result = ''\n",
    "\n",
    "    hidden = [tf.zeros((1, units))]\n",
    "    enc_out, enc_hidden = encoder(inputs, hidden)\n",
    "\n",
    "    dec_hidden = enc_hidden\n",
    "    dec_input = tf.expand_dims([targ_lang.word2idx['<start>']], 0)\n",
    "\n",
    "    for t in range(max_length_targ):\n",
    "        predictions, dec_hidden, attention_weights = decoder(dec_input, dec_hidden, enc_out)\n",
    "        \n",
    "        # storing the attention weigths to plot later on\n",
    "        attention_weights = tf.reshape(attention_weights, (-1, ))\n",
    "        attention_plot[t] = attention_weights.numpy()\n",
    "\n",
    "        predicted_id = tf.argmax(predictions[0]).numpy()\n",
    "\n",
    "        result += targ_lang.idx2word[predicted_id] + ' '\n",
    "\n",
    "        if targ_lang.idx2word[predicted_id] == '<end>':\n",
    "            return result, sentence, attention_plot\n",
    "        \n",
    "        # the predicted ID is fed back into the model\n",
    "        dec_input = tf.expand_dims([predicted_id], 0)\n",
    "\n",
    "    return result, sentence, attention_plot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "kcw7YwAyD2r4"
   },
   "outputs": [],
   "source": [
    "# function for plotting the attention weights\n",
    "def plot_attention(attention, sentence, predicted_sentence):\n",
    "    fig = plt.figure(figsize=(10,10))\n",
    "    ax = fig.add_subplot(1, 1, 1)\n",
    "    ax.matshow(attention, cmap='viridis')\n",
    "    \n",
    "    fontdict = {'fontsize': 14}\n",
    "    \n",
    "    ax.set_xticklabels([''] + sentence, fontdict=fontdict, rotation=90)\n",
    "    ax.set_yticklabels([''] + predicted_sentence, fontdict=fontdict)\n",
    "\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "n7E-D_pWD4jo"
   },
   "outputs": [],
   "source": [
    "def translate(sentence, encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ):\n",
    "    result, sentence, attention_plot = evaluate(sentence, encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ)\n",
    "        \n",
    "    print('Input: {}'.format(sentence))\n",
    "    print('Predicted translation: {}'.format(result))\n",
    "    \n",
    "    attention_plot = attention_plot[:len(result.split(' ')), :len(sentence.split(' '))]\n",
    "    plot_attention(attention_plot, sentence.split(' '), result.split(' '))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "aBQzPssmD7xI",
    "outputId": "771d8aa9-6dab-42cc-e924-5a7d5a3674ed"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.training.checkpointable.util.CheckpointLoadStatus at 0x7f262903ccc0>"
      ]
     },
     "execution_count": 32,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# restoring the latest checkpoint in checkpoint_dir\n",
    "checkpoint.restore(tf.train.latest_checkpoint(checkpoint_dir))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "HqgNgI7SD-Jb",
    "outputId": "39361c33-3f28-42b6-8680-b2a29651cb07"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'./training_checkpoints/ckpt-1'"
      ]
     },
     "execution_count": 33,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.train.latest_checkpoint(checkpoint_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<h3 style=\"text=align:right;direction:rtl\">Restore the latest checkpoint and test</h3>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 662
    },
    "colab_type": "code",
    "id": "3ERExpKXEmjA",
    "outputId": "2e6ff5ad-916c-4c4a-8023-0bd68c670dca"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input: <start> bonjour <end>\n",
      "Predicted translation: shut up here . <end> \n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAU0AAAJjCAYAAACIvkMfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAH9JJREFUeJzt3Xlw1PX9x/HXhggBBIQRqEKBoBhE\nTSCAVECEAHIYhMigTSsoghweqNQyttSjhVKtoGWCiFAUSxH5BVKpyJUgKCDBRjElHhEEJNxCRK6Q\nJcn394fjtimXb9jNJ9k8HzPOhP1mwys78cl3d7OJz/M8TwCAHyXC9QAAqEiIJgAYEE0AMCCaAGBA\nNAHAgGgCgAHRBAADogkABkQTlcL48eNdT0CYIJqoFLKysrRz507XMxAGfLyMEpXB9OnT9c477+jm\nm2/WlVdeqSpVqpQ6/stf/tLRMlQ0RLMcyMjI0DfffKPk5GTXU8JWQkLCWY/5fD6tWrWqDNegIiOa\njuXn56t79+6qVauWnn32WXXs2NH1JADnQDQdmzZtmr766iu1bt1a69at06xZs1xPCktbt2495/Gr\nr766jJagoiOaDvn9fiUkJGjGjBlq3ry5unXrpvnz56t58+aup4Wdli1byufz6b+/3H0+X+Dtzz//\n3MUsVECRrgdUZosXL1azZs10/fXXS5LuvPNOzZkzR3/4wx8cLws///uYZUlJib7++mvNnz9f99xz\nj6NV4WvPnj0qKCjQVVdd5XpK0HGm6VC/fv00duxYdevWTZK0f/9+JSYmKiMjQ3Xq1HG8rnLIz8/X\nPffco7ffftv1lLDh9/vVrVs3HT9+XIsWLQq7cPJ9mo6sXbtWRUVFgWBKUsOGDdW9e3e98cYbDpdV\nLhEREdq1a5frGWHln//8p376058qOTlZr7/+uus5QceZpiPZ2dmKjIzUddddV+ryvXv3Kjc3V127\ndnUzLEz9+c9/Pu2ykydPKjMzU5dddhn/UAVRv3799Mgjj+j6669XYmKi0tPTVbduXdezgoYzTUfi\n4uI0d+7c0y6/4oor1K5dO40aNcrBqvC1efPm0/7buXOnOnbsqClTprieFzbWrVsnv9+vHj166Cc/\n+Ym6du2qN9980/WsoOKJIAe+/vpr7dixQ++884769Olz2vEdO3Zow4YNDpaFrzP9A4XgmzNnTqkn\n1oYOHaqRI0dq+PDhuuSSSxwuCx6i6cCWLVs0depUnTp1SiNHjjzteLVq1Xh1UAhs3LhRS5cu1e7d\nu+Xz+dS0aVMNGDAg8N0LuDhbtmzRp59+qpdeeilw2XXXXaerrrpKS5YsUVJSksN1wcNjmg4lJiZq\nyZIlrmdUCgsWLNCECRPUsWNHNWnSRJK0fft2bdy4UdOnT1eXLl0cL6z4li9frpMnT2rAgAGlLv/o\no4+0adMmDR8+3NGy4OJM06G4uDjXEyqNV199VdOmTTvtCbb09HT95S9/IZpB0Lt37zNe3rZtW7Vt\n27aM14QO0XTohx9X9sOZD0Jn3759ZwxjQkKCnnjiCQeLwscjjzzyo9936tSpIVxSNoimQ/3799fo\n0aP5cWVloHHjxvroo4/Uvn37UpdnZ2erfv36jlaFhxo1agTeLi4uVnp6upo3b67o6Gh5nqetW7cq\nLy9P/fv3d7gyeHhM0yF+XFnZSU1N1aRJk5SYmBh4hcq2bdu0ZMkSjRkzRvfee6/bgWHi6aefVnx8\n/GmBXLhwobKzszVhwgRHy4KHaJZThw8f1mWXXeZ6RlhZs2aNFi5cqLy8PPn9fjVp0kRJSUlnfSwO\ndu3atVNmZqYiI0vfiT116pRuuukmZWVlOVoWPNw9L4cOHDigxMREffjhh66nhJWuXbvySqsQq1On\njlavXq2ePXuWuvz9999XrVq1HK0KLqLp0LZt2zR+/Hh9+umnOnXqVKlj1157raNV4ePFF1/UY489\nJunML6P8bz6fT3Xq1FFiYqKuvPLKspgXlkaNGqUxY8YoJiZGjRs3VlFRkfbt26fc3Fz97ne/cz0v\nKLh77tC9996ryy+/XLfeeqvGjh2rqVOnKicnR1lZWUpJSeHu+UUaNmyYZs+eLUkaPHjwed9///79\nioyM1NKlS0M9Laxt375dGRkZ2r9/v/x+vxo0aKAuXbooNjbW9bSgIJoOtW/fXuvXr1fVqlUVGxur\nf//735KklStXKiMj47xnRwguv9+v+Ph45eTkuJ6Ccoy75w5VrVpVJSUlkqTq1asrPz9f9erVU9eu\nXfXb3/7W8brw4nme1qxZo6+++konT54sdczn8+nBBx9U1apVCeZF2rJli1JSUs54O0un/zDoioho\nOnTjjTdq1KhRmjFjhm644QZNmjRJgwcP1qZNm0p97xsu3rhx47Rs2TI1bdpUUVFRpY79EE1cvMcf\nf1y1a9dWUlKSqlev7npOSHD33KHDhw/r+eef1zPPPKMdO3Zo5MiR2rNnj2rWrKkJEyaob9++rieG\njbZt22revHlq2bKl6ylhrU2bNtqwYcNp/zCFE840HapZs6b++Mc/SpJatGihVatW6eDBg6pXr572\n7dvneF14qVOnjpo1a+Z6Rti79tprdeDAgbB+aTBnmg7FxcUpOzv7tMuPHj2qbt26hcU3ApcXixcv\n1ubNm/Xoo4/q0ksvdT0nbK1cuVKzZ89Wv3791KhRI0VElP4557fccoujZcFDNB1YtmyZli1bpoyM\njNO+CVj6/lde5OXl8YOIg+j222/Xnj17dPz4cdWuXfu0/5m5rYPjXA9/+Hy+sPhVydw9dyA2NlZ7\n9+5VRkbGGZ/wadWqlcaNG+dgWfgaOnSo6wmVwhdffOF6QshxpunQzJkzNWLECNczgKAqLi5WZmam\n9u3bp4EDB0qSjh07FjYPixBNh/x+vxYtWhT41RarVq3SwoUL1axZMz388MN821EQFRUV6eWXXy71\n6y6aNGmigQMH8hOOguiLL77Q6NGjdfz4cZ04cUI5OTnavXu3BgwYoFmzZql169auJ140fhulQ5Mm\nTdKiRYskff869EceeUSXXXaZPvnkE/3pT39yvC68PPfcc3rrrbc0cOBATZkyRZMnT9Ztt92m1157\nLfBSS1y8CRMmKCkpSZmZmYHHjRs1aqTHH39czz33nON1QeLBmY4dO3oHDhzwPM/zXnjhBW/o0KGe\n53neoUOHvM6dO7ucFnY6derkbd269bTLP//8c+/WW291sCg8tW7d2issLPQ8z/NiY2MDlxcVFXlt\n2rRxNSuoONN06MSJE4GfGr5u3Tr16NFDklSvXj0dPXrU5bSwU1BQcMbvHbz66qt16NAhB4vCU926\ndXX48OHTLt+2bZuqVavmYFHwEU2HmjZtqrS0NC1dulS5ubmBaGZlZalBgwaO14WXFi1aaP78+add\n/uabbyo6OtrBovCUkJCgMWPG6L333pPnedq8ebNSU1M1atQoJSYmup4XFDwR5NB7772nRx99VH6/\nX6NHj9ZDDz2kb7/9Vj169NDYsWP5HUFBtGnTJt13331q0KBB4NddfPXVV9q/f79eeuklderUyfHC\n8OD3+/X8888rLS1Nx48fl/T92ecvfvELjRw5UlWrVnW88OIRTUfeffddFRcXq1u3biosLFTNmjUl\nSTk5OXr//ff1wAMPOF4YfvLz87VkyRKtW7dO9evXV3R0tG677TZdccUVrqeFhR++pnv27CnP83To\n0CFFRUVpx44dys7ODpuTAL653ZGGDRtq1KhR6tq1ayCYkpSSknLOX7iGC7N3716NGzdOWVlZ+uE8\nITIyUp988omeeuopHg4Jgv/+mr7kkkt0+eWXSwq/r2ke03TkuuuuU7NmzfT2228HLtu6dauys7M1\nYMAAh8vC07hx43TJJZdo1qxZWrlypVasWKHp06eroKCA33seJJXla5ozTYfuueceTZ06VXfccYck\n6bXXXtNdd90VNs8ylic5OTlau3ZtqVelNG3aVPHx8erSpYvDZeGlMnxNc6bpUPfu3eX3+7V27Vod\nPHhQy5Yt09133+16Vlhq0qSJjh07dtrlBQUFatSokYNF4akyfE1XeeaZZ55xPaKy8vl8ioiI0MKF\nC3XgwAHVr18/rO7GuLZ161bl5+crPz9fjRo10gsvvKBq1aqpuLhYhw4d0r/+9S9NmTJF9913n1q0\naOF6blioDF/TPHvuWEFBgbp27apTp07pjTfe4CeLB1HLli3l8/l0vi/xcPmRZeVFuH9NE81yYMuW\nLTp27JjatGnjekpY2b17949+X+6iB1c4f00TTQAw4IkgADAgmgBgQDQBwIBoAoBB2L4iqGfEINcT\nzGb+e4pGxP7K9YxKgdu67FTE2zq9JPWsxzjTLEeirz/9h+QiNLity0643dZEEwAMiCYAGBBNADAg\nmgBgQDQBwIBoAoAB0QQAA6IJAAZEEwAMiCYAGBBNADAgmgBgQDQBwIBoAoAB0QQAA6IJAAZEEwAM\niCYAGBBNADAgmgBgQDQBwIBoAoAB0QQAA6IJAAZEEwAMiCYAGBBNADAgmgBgQDQBwIBoAoAB0QQA\nA6IJAAZEEwAMiCYAGBBNADAgmgBgQDQBwIBoAoAB0QQAgzKPZkxMjFavXl3Wfy0ABEWFO9OcM2eO\n/H6/6xkAKqkKFc38/Hw9++yzOnXqlOspACqpkEVz1qxZSkhIUFxcnLp37665c+cGjh06dEjDhg1T\nbGysevXqpc8++0yStHHjRsXExOj48eOB933iiSc0ZswY7d+/X126dJHnefrZz36m1NTUUE0HgLMK\nSTQ//vhjpaSk6OWXX1Z2drZefPFFpaSkKDc3V5K0YMECjR8/Xhs2bFCjRo00efLk837Mhg0bavbs\n2ZKkzMxMDRo0KBTTAeCcIkPxQY8ePSpJqlGjhiQpNjZWmZmZioj4vtH9+vVT8+bNJUk9evTQjBkz\ngr5h5r+nKPr6JkH/uKGWXsIZdFnhti474XRbhySaN910kzp27Kg+ffroxhtvVOfOnZWUlKS6detK\nkho3bhx436ioKBUWFgZ9w4jYXwX9Y4ZaekmqekZwBl0WuK3LTkW8rc8V+ZDcPa9atapmzJihhQsX\nqm3btkpLS1Pfvn2Vl5cnSfL5fD/6YxUXF4diIgBckJBEs6ioSEeOHFHLli314IMP6q233lKtWrWU\nnp5+zutVq1ZNknTy5MnAZT+EFgDKg5BEc/bs2Ro8eLB27dolSdq+fbsOHz6sJk3O/Rhj48aNVaVK\nFS1fvlxFRUV655139PXXXweOR0VFBT7eiRMnQjEdAM4pJNEcOnSo4uPjdeeddyouLk6jR4/W8OHD\n1aNHj3Ne7/LLL9fjjz+uadOmqUOHDvr444/Vv3//wPFrr71W8fHx+vnPf66///3voZgOAOfk8zzP\ncz0iFCraA89SxXzAvKLiti47FfG2LvMnggAgXBFNADAgmgBgQDQBwIBoAoAB0QQAA6IJAAZEEwAM\niCYAGBBNADAgmgBgQDQBwIBoAoAB0QQAA6IJAAZEEwAMiCYAGBBNADAgmgBgQDQBwIBoAoAB0QQA\nA6IJAAZEEwAMiCYAGBBNADAgmgBgQDQBwIBoAoAB0QQAA6IJAAZEEwAMiCYAGBBNADAgmgBgQDQB\nwIBoAoBBpOsBoVLQ/0bXEy5IRdwddaDQ9YQL4t0U53qCycpFr7uecMFW7PnE9YSg4UwTAAyIJgAY\nEE0AMCCaAGBANAHAgGgCgAHRBAADogkABkQTAAyIJgAYEE0AMCCaAGBANAHAgGgCgAHRBAADogkA\nBkQTAAyIJgAYEE0AMCCaAGBANAHAgGgCgAHRBAADogkABkQTAAyIJgAYEE0AMCCaAGBANAHAgGgC\ngAHRBAADogkABkQTAAyIJgAYEE0AMCCaAGBANAHAgGgCgAHRBAADogkABkQTAAyIJgAYEE0AMHAe\nzV27dikmJkZffvll4LKUlBTdcccdSktLU0JCgtLS0tSlSxfFxcXpN7/5jfx+v8PFACoz59E8n4MH\nD+rjjz/W8uXL9dZbb2n9+vWaM2eO61kAKqlI1wPOp7CwUGPGjFGNGjUUHR2tAQMGaNWqVRoxYsQ5\nr/e3qfeqeZP6ZbQyeNb949euJ1QaGet/53qCUUXb+x8RP9niekLQlPto1qxZUw0aNAj8+corr9SB\nAwfOe70hj8wJ4arQWPePX6tz0vOuZ5hFHSh0PcEsY/3v1KPTRNczTFYuet31hAsS8ZMtKtnXwvUM\nk3NFvlzePS8uLg68XVJSUuqY53ny+XxlPQkAJJWDaFarVk2SdPLkycBleXl5gbcLCgr0zTffBP68\nZ88eNWzYsOwGAsB/cR7NevXqqVatWlqxYoWKi4uVmZmprKyswPGqVatq+vTpKigo0LZt27R48WL1\n6NHD4WIAlZnzaFapUkVPP/203n77bbVr104LFizQkCFDAsdr1qypVq1aqVevXkpKSlKnTp1KHQeA\nslQungjq16+f+vXrV+qyYcOGKS0tTZ7nadCgQRo0aJCjdQDwH87PNAGgIiGaAGBQrqN5xx13aOPG\nja5nAEBAuY4mAJQ3RBMADIgmABgQTQAwIJoAYEA0AcCAaAKAAdEEAAOiCQAGRBMADIgmABgQTQAw\nIJoAYEA0AcCAaAKAAdEEAAOiCQAGRBMADIgmABgQTQAwIJoAYEA0AcCAaAKAAdEEAAOiCQAGRBMA\nDIgmABgQTQAwIJoAYEA0AcCAaAKAAdEEAAOiCQAGRBMADIgmABhEuh4QKjWWZ7uecEEq4m6fz+d6\nwgWpsinX9QSTXncMcT3hgqR/UPG2p39w9mOcaQKAAdEEAAOiCQAGRBMADIgmABgQTQAwIJoAYEA0\nAcCAaAKAAdEEAAOiCQAGRBMADIgmABgQTQAwIJoAYEA0AcCAaAKAAdEEAAOiCQAGRBMADIgmABgQ\nTQAwIJoAYEA0AcCAaAKAAdEEAAOiCQAGRBMADIgmABgQTQAwIJoAYEA0AcCAaAKAAdEEAAOiCQAG\nRBMADIgmABgQTQAwIJoAYEA0AcAg6NGMiYnR6tWrg/1hAaBc4EwTAAyIJgAYhCSahw4d0rBhwxQb\nG6tevXrps88+CxzLzc3Vvffeq/bt26tDhw566qmnVFhYKElKS0tT7969NXnyZLVp00Z5eXkqKSnR\ntGnT1LNnT8XFxWnAgAHasGFDKGYDwHmFJJoLFizQ+PHjtWHDBjVq1EiTJ0+WJBUUFGj48OFq3769\n1q9fr3/84x/KycnRtGnTAtc9ePCgfD6fPvzwQzVu3Fh/+9vftHjxYr3yyivKyspScnKyHnjgAR0+\nfDgU0wHg3Lwgu+aaa7zXX3898Od58+Z5N998s+d5nrd06VKvQ4cOpd5/xYoVXqdOnTzP87xFixZ5\nMTEx3nfffRc43rdvX2/u3LmlrpOYmOjNnz//nDu25+y8qM8DAM4kMhQhbty4ceDtqKiowN3vvLw8\nHT58WDfccEOp9y8pKZHf75ckXXrppapdu3bg2M6dO/Xss8/queee++/Qa+/evefcMKLdby768yhr\nKwv+rlur3+16hpnP53M9wWzFibnqVWOw6xkmJa2vcT3hgqR/8KR6dpzgeoZJ+gdPnvVYSKJ5tv+J\nqlWrpujoaC1btuys161SpUqpP0dFRen3v/+9+vbtG9SNAHAhyvTZ86ZNm2r37t06duxY4LLvvvtO\nR48ePet1mjRpotzc3FKX7dq1K2QbAeBcyjSanTt3Vv369TVp0iQdPXpU+fn5+vWvf60JE85+6p6c\nnKz58+crKytLxcXFWrVqlRITE7Vt27YyXA4A3wvJ3fOz/mWRkZo+fbomTpyozp07q2bNmrrllls0\nfvz4s15n4MCB2rdvnx577DEdOXJEzZo105QpU9S8efMyXA4A3/N5nue5HhEKFfEJFZ4IKjs8EVR2\nwu2JIF4RBAAGRBMADIgmABgQTQAwIJoAYEA0AcCAaAKAAdEEAAOiCQAGRBMADIgmABgQTQAwIJoA\nYEA0AcCAaAKAAdEEAAOiCQAGRBMADIgmABgQTQAwIJoAYEA0AcCAaAKAAdEEAAOiCQAGRBMADIgm\nABgQTQAwIJoAYEA0AcCAaAKAAdEEAAOiCQAGRBMADIgmABgQTQAwIJoAYEA0AcAg0vWAUPEKC11P\nuCAVcbfnesAFKjl50vUEm42bXS+4cBV5+//gTBMADIgmABgQTQAwIJoAYEA0AcCAaAKAAdEEAAOi\nCQAGRBMADIgmABgQTQAwIJoAYEA0AcCAaAKAAdEEAAOiCQAGRBMADIgmABgQTQAwIJoAYEA0AcCA\naAKAAdEEAAOiCQAGRBMADIgmABgQTQAwIJoAYEA0AcCAaAKAAdEEAAOiCQAGRBMADIgmABgQTQAw\nIJoAYEA0AcCAaAKAAdEEAAOiCQAGRBMADIgmABgQTQAwIJoAYODzPM9zPSIUtufsVPT1TVzPABBm\nwjaaPSMGuZ5gll6SWiF3V0QV8rb2+VwvuCDpxf+nnlXudD3DJL34/856jLvnAGBANAHAgGgCgEGF\ni+a4ceM0a9Ys1zMAVFIVLpp79+5Vfn6+6xkAKqlI1wOs5s6d63oCgEqswp1pAoBLRBMADIgmABgQ\nTQAwIJoAYEA0AcCAaAKAAdEEAAOiCQAGRBMADIgmABgQTQAwIJoAYEA0AcCAaAKAAdEEAAOiCQAG\nRBMADIgmABgQTQAwIJoAYEA0AcCAaAKAAdEEAAOiCQAGRBMADIgmABgQTQAwIJoAYEA0AcCAaAKA\nAdEEAAOiCQAGRBMADIgmABgQTQAwIJoAYBDpegCAH8nzXC+4cBV5+//gTBMADIgmABgQTQAwIJoA\nYEA0AcCAaAKAAdEEAAOiCQAGRBMADIgmABgQTQAwIJoAYEA0AcCAaAKAAdEEAAOiCQAGRBMADIgm\nABgQTQAwIJoAYEA0AcCAaAKAAdEEAAOiCQAGRBMADIgmABgQTQAwIJoAYEA0AcCAaAKAAdEEAAOi\nCQAGRBMADIgmABgQTQAwIJoAYEA0AcCAaAKAAdEEAAOiCQAG5S6aTzzxhMaMGeN6BgCcUbmLJgCU\nZxcczZycHK1atSqYW06zbNkyffnllyH9OwDAwhTNkpISZWRk6O6779aoUaNUVFSkwsJCTZw4Ud26\ndVPr1q2VnJyszz//PHCdmJgYrVixQsnJyWrdurVuv/125ebmBo6npqYqISFB8fHxeuqpp1RcXBw4\ndvLkSQ0ZMkTDhg3T2rVrg/DpAsDF+VHRPHHihObNm6c+ffroxRdfVP/+/fXuu++qV69emjx5sjZv\n3qz58+dr48aN6tChg0aPHq1Tp04Frv/Xv/5VkyZN0gcffKA6deooJSVFkrR9+3Y9+eSTGjdunDIz\nMxUfH6+MjIzA9ZKSkrR69WolJCRowoQJSkxMVGpqqgoLC4N8MwDAj+SdR25urnfjjTd6gwcP9tas\nWeOVlJQEjhUXF3tt2rTx3n333VKXtWvXznv//fc9z/O8a665xpszZ07g+MyZM73evXt7nud5r7zy\nitevX79Sf19SUpL38MMPn7ajuLjYW7FihXfXXXd5N910k5eXl3fO3ds2f32+Tw0AzCLPF9Vjx47J\n7/erTZs2atWqlXw+X+DYoUOHdPz4cT388MOlLi8pKdG+ffsCf27cuHHg7erVqwfOFPfv31/qmCRF\nR0eXOkv9QUREhG644Qa1bt1an332mQoKCs65e0Tsr873qZU76SWp6hkxyPWMSoHbuuxUxNs6vST1\nrMfOG834+HjNmzdPr732mm699Vb17t1b9913n1q0aKGoqChJ0rx58xQXF3fWjxERceZHAfx+f6nH\nMCXJ87zT3u/TTz/Vq6++qtWrV6tv375KS0vT1Vdffb7pABB0P+oxzVatWun555/X8uXLVb9+fd19\n9926//77tWPHDtWtW7fUEzuStGvXrh/1lzdo0EB79+4tddnWrVsDb2dlZWnIkCEaMWKEoqOjlZGR\noYkTJxJMAM6Ynj1v2LChxo4dqzVr1ighIUHZ2dlKTk7WjBkz9OWXX6qoqEgLFixQ//79deTIkfN+\nvC5duig3N1cZGRny+/1auHCh8vLyAsc3bdqk/v37a/Xq1XrooYdUr149+2cIAEF03rvnZ1K9enUl\nJydL+v4u9tGjRzVkyBAVFhYqJiZGM2fOVO3atc/7ceLi4vTkk09q4sSJOnLkiPr06aPbb79d3377\nrSTp/vvvv5B5ABAyPu9MDyKGgYr2wLNUMR8wr6i4rctORbytz/VEEC+jBAADogkABkQTAAyIJgAY\nEE0AMCCaAGBANAHAgGgCgAHRBAADogkABkQTAAyIJgAYEE0AMCCaAGBANAHAgGgCgAHRBAADogkA\nBkQTAAyIJgAYEE0AMCCaAGBANAHAgGgCgAHRBAADogkABkQTAAyIJgAYEE0AMCCaAGBANAHAgGgC\ngAHRBAADogkABkQTAAyIJgAYEE0AMCCaAGBANAHAgGgCgAHRBAADogkABkQTAAyIJgAYEE0AMCCa\nAGBANAHAgGgCgAHRBAADogkABkQTAAyIJgAYEE0AMCCaAGBANAHAgGgCgAHRBAADogkABkQTAAyI\nJgAYEE0AMCCaAGBANAHAgGgCgAHRBAADogkABkQTAAyIJgAYEE0AMCCaAGBANAHAgGgCgAHRBAAD\nogkABkQTAAyIJgAYEE0AMCCaAGBANAHAgGgCgAHRBAADogkABkQTAAyIJgAY+DzP81yPCIXtOTsV\nfX0T1zMAhJmwjWbPiEGuJ5ill6RWyN0VEbd12amIt3V6SepZj3H3HAAMiCYAGBBNADAgmgBgQDQB\nwIBoAoAB0QQAA6IJAAZEEwAMiCYAGBBNADAgmgBgQDQBwIBoAoAB0QQAA6IJAAZEEwAMiCYAGBBN\nADAgmgBgQDQBwIBoAoAB0QQAA6IJAAZEEwAMiCYAGBBNADAgmgBgQDQBwIBoAoAB0QQAA6IJAAZE\nEwAMiCYAGBBNADAgmgBgQDQBwIBoAoAB0QQAA6IJAAZEEwAMiCYAGBBNADDweZ7nuR4BABUFZ5oA\nYEA0AcCAaAKAAdEEAAOiCQAGRBMADP4fQJs274qb6fMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f262a2310f0>"
      ]
     },
     "metadata": {
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "translate(u'', encoder, decoder, inp_lang, targ_lang, max_length_inp, max_length_targ)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "pH39TvFuECqK"
   },
   "outputs": [],
   "source": [
    " "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "l9B1Q1-zEHDX"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "train_new_lang_de.ipynb",
   "provenance": [],
   "version": "0.3.2"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}