machine-translation-model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "386e007d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-25T08:15:24.450742Z",
     "iopub.status.busy": "2022-07-25T08:15:24.449513Z",
     "iopub.status.idle": "2022-07-25T08:15:31.215921Z",
     "shell.execute_reply": "2022-07-25T08:15:31.214606Z"
    },
    "papermill": {
     "duration": 6.781814,
     "end_time": "2022-07-25T08:15:31.218992",
     "exception": false,
     "start_time": "2022-07-25T08:15:24.437178",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import pandas as pd\n",
    "from collections import defaultdict\n",
    "import string \n",
    "import tensorflow as tf\n",
    "import re\n",
    "import os\n",
    "import time\n",
    "from tensorflow import keras \n",
    "from tensorflow.keras.layers import Dense, Input\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.models import Model\n",
    "from tensorflow.keras.callbacks import ModelCheckpoint\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.model_selection import train_test_split\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a57f5480",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-25T08:15:31.239684Z",
     "iopub.status.busy": "2022-07-25T08:15:31.239033Z",
     "iopub.status.idle": "2022-07-25T08:15:31.244176Z",
     "shell.execute_reply": "2022-07-25T08:15:31.243157Z"
    },
    "papermill": {
     "duration": 0.01736,
     "end_time": "2022-07-25T08:15:31.246309",
     "exception": false,
     "start_time": "2022-07-25T08:15:31.228949",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "Encoder_len = 100\n",
    "Decoder_len = 100\n",
    "Batch_size = 128\n",
    "Buffer_size = Batch_size*4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "25c7f2eb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-25T08:15:31.265852Z",
     "iopub.status.busy": "2022-07-25T08:15:31.265206Z",
     "iopub.status.idle": "2022-07-25T08:15:32.881571Z",
     "shell.execute_reply": "2022-07-25T08:15:32.880516Z"
    },
    "papermill": {
     "duration": 1.629244,
     "end_time": "2022-07-25T08:15:32.884295",
     "exception": false,
     "start_time": "2022-07-25T08:15:31.255051",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>english_sentence</th>\n",
       "      <th>hindi_sentence</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>63241</th>\n",
       "      <td>Indian News Service - National News Agency</td>\n",
       "      <td>इण्डियन न्यूज सर्विस - राष्ट्रीय समाचार एजेंसी</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81404</th>\n",
       "      <td>In West Bengal , it seems set to eat humble pi...</td>\n",
       "      <td>पश्चिम बंगाल में तो वह अपमान का घूंट पीने को भ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8803</th>\n",
       "      <td>One american dollar is equal to 60 pakistani r...</td>\n",
       "      <td>एक अमरीकी डालर की कीमत लगभग ६० पाकिस्तानी रुपय...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73434</th>\n",
       "      <td>but between those high highs,</td>\n",
       "      <td>लेकिन इन बेहतरीन लम्हों के बीच</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65711</th>\n",
       "      <td>Every other politician went along because when...</td>\n",
       "      <td>और वजह यह थी कि आर्थिक मामलं पर हमेशा विफल विच...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        english_sentence  \\\n",
       "63241         Indian News Service - National News Agency   \n",
       "81404  In West Bengal , it seems set to eat humble pi...   \n",
       "8803   One american dollar is equal to 60 pakistani r...   \n",
       "73434                      but between those high highs,   \n",
       "65711  Every other politician went along because when...   \n",
       "\n",
       "                                          hindi_sentence  \n",
       "63241     इण्डियन न्यूज सर्विस - राष्ट्रीय समाचार एजेंसी  \n",
       "81404  पश्चिम बंगाल में तो वह अपमान का घूंट पीने को भ...  \n",
       "8803   एक अमरीकी डालर की कीमत लगभग ६० पाकिस्तानी रुपय...  \n",
       "73434                     लेकिन इन बेहतरीन लम्हों के बीच  \n",
       "65711  और वजह यह थी कि आर्थिक मामलं पर हमेशा विफल विच...  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df = pd.read_csv(\"../input/hindienglish-corpora/Hindi_English_Truncated_Corpus.csv\")\n",
    "train_df.drop(['source'],axis=1,inplace=True)\n",
    "mask = (train_df['english_sentence'].str.len()>20) & (train_df['english_sentence'].str.len()<200)\n",
    "train_df = train_df.loc[mask]\n",
    "train_df = train_df.sample(64000, random_state=1)\n",
    "train_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7c53fd30",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-25T08:15:32.904340Z",
     "iopub.status.busy": "2022-07-25T08:15:32.903974Z",
     "iopub.status.idle": "2022-07-25T08:15:32.909444Z",
     "shell.execute_reply": "2022-07-25T08:15:32.908311Z"
    },
    "papermill": {
     "duration": 0.017987,
     "end_time": "2022-07-25T08:15:32.911806",
     "exception": false,
     "start_time": "2022-07-25T08:15:32.893819",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "pd.set_option('display.max_colwidth', None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "16c1e956",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-25T08:15:32.931652Z",
     "iopub.status.busy": "2022-07-25T08:15:32.931289Z",
     "iopub.status.idle": "2022-07-25T08:15:33.047942Z",
     "shell.execute_reply": "2022-07-25T08:15:33.046816Z"
    },
    "papermill": {
     "duration": 0.129774,
     "end_time": "2022-07-25T08:15:33.050758",
     "exception": false,
     "start_time": "2022-07-25T08:15:32.920984",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "\n",
    "eng = train_df['english_sentence']\n",
    "hind = train_df['hindi_sentence']\n",
    "eng = eng.apply(lambda x: \"<SOS> \" + str(x) + \" <EOS>\")\n",
    "hind = hind.apply(lambda x: \"<SOS> \"+ x + \" <EOS>\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "d680d4db",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-25T08:15:33.070595Z",
     "iopub.status.busy": "2022-07-25T08:15:33.070218Z",
     "iopub.status.idle": "2022-07-25T08:15:40.200987Z",
     "shell.execute_reply": "2022-07-25T08:15:40.199879Z"
    },
    "papermill": {
     "duration": 7.144119,
     "end_time": "2022-07-25T08:15:40.204044",
     "exception": false,
     "start_time": "2022-07-25T08:15:33.059925",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "filters = '!\"#$%&()*+,-./:;=?@[\\\\]^_`{|}~\\t\\n'\n",
    "oov_token = '<unk>'\n",
    "eng_tokenizer = tf.keras.preprocessing.text.Tokenizer(filters = filters, oov_token=oov_token)\n",
    "hind_tokenizer = tf.keras.preprocessing.text.Tokenizer(filters = filters, oov_token=oov_token)\n",
    "eng_tokenizer.fit_on_texts(eng)\n",
    "hind_tokenizer.fit_on_texts(hind)\n",
    "inputs = eng_tokenizer.texts_to_sequences(eng)\n",
    "targets = hind_tokenizer.texts_to_sequences(hind)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "64a07fdb",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-25T08:15:40.224645Z",
     "iopub.status.busy": "2022-07-25T08:15:40.224231Z",
     "iopub.status.idle": "2022-07-25T08:15:40.230468Z",
     "shell.execute_reply": "2022-07-25T08:15:40.229390Z"
    },
    "papermill": {
     "duration": 0.019993,
     "end_time": "2022-07-25T08:15:40.233557",
     "exception": false,
     "start_time": "2022-07-25T08:15:40.213564",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "44305 51960\n"
     ]
    }
   ],
   "source": [
    "Encoder_Vocab = len(eng_tokenizer.word_index) + 1\n",
    "Decoder_Vocab = len(hind_tokenizer.word_index) + 1\n",
    "print(Encoder_Vocab, Decoder_Vocab)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2c0b8314",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-25T08:15:40.254834Z",
     "iopub.status.busy": "2022-07-25T08:15:40.253703Z",
     "iopub.status.idle": "2022-07-25T08:15:43.591889Z",
     "shell.execute_reply": "2022-07-25T08:15:43.590747Z"
    },
    "papermill": {
     "duration": 3.352511,
     "end_time": "2022-07-25T08:15:43.595442",
     "exception": false,
     "start_time": "2022-07-25T08:15:40.242931",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-07-25 08:15:41.014605: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-07-25 08:15:41.143276: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-07-25 08:15:41.144225: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-07-25 08:15:41.147466: I tensorflow/core/platform/cpu_feature_guard.cc:142] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 AVX512F FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2022-07-25 08:15:41.147961: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-07-25 08:15:41.149111: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-07-25 08:15:41.150096: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-07-25 08:15:43.161240: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-07-25 08:15:43.162237: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-07-25 08:15:43.163008: I tensorflow/stream_executor/cuda/cuda_gpu_executor.cc:937] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero\n",
      "2022-07-25 08:15:43.163698: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1510] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 15403 MB memory:  -> device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:04.0, compute capability: 6.0\n"
     ]
    }
   ],
   "source": [
    "inputs = tf.keras.preprocessing.sequence.pad_sequences(inputs, maxlen=Encoder_len, padding='post', truncating='post')\n",
    "targets = tf.keras.preprocessing.sequence.pad_sequences(targets, maxlen=Decoder_len, padding='post', truncating='post')\n",
    "inputs = tf.cast(inputs, dtype=tf.int64)\n",
    "targets = tf.cast(targets, dtype=tf.int64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "d078f43f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-25T08:15:43.617603Z",
     "iopub.status.busy": "2022-07-25T08:15:43.616889Z",
     "iopub.status.idle": "2022-07-25T08:15:43.718727Z",
     "shell.execute_reply": "2022-07-25T08:15:43.717594Z"
    },
    "papermill": {
     "duration": 0.115456,
     "end_time": "2022-07-25T08:15:43.721345",
     "exception": false,
     "start_time": "2022-07-25T08:15:43.605889",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "dataset = tf.data.Dataset.from_tensor_slices((inputs, targets)).shuffle(Buffer_size).batch(Batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "cefe5d33",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-25T08:15:43.742690Z",
     "iopub.status.busy": "2022-07-25T08:15:43.742279Z",
     "iopub.status.idle": "2022-07-25T08:15:43.756672Z",
     "shell.execute_reply": "2022-07-25T08:15:43.755433Z"
    },
    "papermill": {
     "duration": 0.028097,
     "end_time": "2022-07-25T08:15:43.759564",
     "exception": false,
     "start_time": "2022-07-25T08:15:43.731467",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def get_angles(position, i, d_model):\n",
    "    angle_rates = 1 / np.power(10000, (2 *(i // 2)) / np.float32(d_model))\n",
    "    return position * angle_rates\n",
    "\n",
    "def positional_encoding(position, d_model):\n",
    "    angle_rads = get_angles(\n",
    "        np.arange(position)[:,np.newaxis],\n",
    "         np.arange(d_model)[np.newaxis, :],\n",
    "        d_model\n",
    "    )\n",
    "    \n",
    "    angle_rads[:, 0::2] = np.sin(angle_rads[:, 0::2])\n",
    "\n",
    "    angle_rads[:, 1::2] = np.cos(angle_rads[:, 1::2])\n",
    "\n",
    "    pos_encoding = angle_rads[np.newaxis, ...]\n",
    "\n",
    "    return tf.cast(pos_encoding, dtype=tf.float32)\n",
    "def create_padding_mask(seq):\n",
    "     seq = tf.cast(tf.math.equal(seq, 0), tf.float32)\n",
    "     return seq[:, tf.newaxis, tf.newaxis, :]\n",
    "\n",
    "def create_look_ahead_mask(size):\n",
    "    mask = 1 - tf.linalg.band_part(tf.ones((size, size)), -1, 0)\n",
    "    return mask\n",
    "def scaled_dot_product_attention(q,k,v,mask):\n",
    "    matmul_qk = tf.matmul(q, k, transpose_b=True)\n",
    "\n",
    "    dk = tf.cast(tf.shape(k)[-1], tf.float32)\n",
    "    scaled_attention_logits = matmul_qk / tf.math.sqrt(dk)\n",
    "    if mask is not None:\n",
    "        scaled_attention_logits += (mask * -1e9)\n",
    "        \n",
    "    attention_weights = tf.nn.softmax(scaled_attention_logits, axis=-1)\n",
    "\n",
    "    output = tf.matmul(attention_weights, v)\n",
    "    return output, attention_weights \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "1550ce82",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-25T08:15:43.780908Z",
     "iopub.status.busy": "2022-07-25T08:15:43.780540Z",
     "iopub.status.idle": "2022-07-25T08:15:43.795952Z",
     "shell.execute_reply": "2022-07-25T08:15:43.794520Z"
    },
    "papermill": {
     "duration": 0.028454,
     "end_time": "2022-07-25T08:15:43.798246",
     "exception": false,
     "start_time": "2022-07-25T08:15:43.769792",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class MultiHeadAttention(tf.keras.layers.Layer):\n",
    "    def __init__(self, d_model, num_heads):\n",
    "        super(MultiHeadAttention, self).__init__()\n",
    "        self.num_heads = num_heads\n",
    "        self.d_model = d_model\n",
    "        assert d_model % self.num_heads==0\n",
    "        self.depth = d_model // self.num_heads\n",
    "        \n",
    "        self.wq = tf.keras.layers.Dense(d_model)\n",
    "        self.wk = tf.keras.layers.Dense(d_model)\n",
    "        self.wv = tf.keras.layers.Dense(d_model)\n",
    "        \n",
    "        self.dense = tf.keras.layers.Dense(d_model) \n",
    "        \n",
    "    def split_heads(self, x, batch_size):\n",
    "        x = tf.reshape(x, (batch_size, -1, self.num_heads, self.depth))\n",
    "        return tf.transpose(x, perm=[0, 2, 1, 3])\n",
    "    \n",
    "    def call(self, v, k, q, mask):\n",
    "        batch_size = tf.shape(q)[0]\n",
    "\n",
    "        q = self.wq(q)\n",
    "        k = self.wk(k)\n",
    "        v = self.wv(v)\n",
    "        \n",
    "        \n",
    "        q = self.split_heads(q, batch_size)\n",
    "        k = self.split_heads(k, batch_size)\n",
    "        v = self.split_heads(v, batch_size)\n",
    "        \n",
    "        scaled_attention, attention_weights = scaled_dot_product_attention(\n",
    "            q, k, v, mask)\n",
    "        \n",
    "        scaled_attention = tf.transpose(scaled_attention, perm=[0, 2, 1, 3])\n",
    "        \n",
    "        concat_attention = tf.reshape(scaled_attention, (batch_size, -1, self.d_model))\n",
    "        output = self.dense(concat_attention)\n",
    "            \n",
    "        return output, attention_weights\n",
    "def point_wise_feed_forward_network(d_model, dff):\n",
    "    return tf.keras.Sequential([\n",
    "        tf.keras.layers.Dense(dff, activation='relu'),\n",
    "        tf.keras.layers.Dense(d_model)\n",
    "    ])  \n",
    "\n",
    "        \n",
    "        \n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "6cc4dae3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-25T08:15:43.819249Z",
     "iopub.status.busy": "2022-07-25T08:15:43.818878Z",
     "iopub.status.idle": "2022-07-25T08:15:43.829570Z",
     "shell.execute_reply": "2022-07-25T08:15:43.828489Z"
    },
    "papermill": {
     "duration": 0.024079,
     "end_time": "2022-07-25T08:15:43.831944",
     "exception": false,
     "start_time": "2022-07-25T08:15:43.807865",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class EncoderLayer(tf.keras.layers.Layer):\n",
    "    def __init__(self, d_model, num_heads, dff, rate=0.1):\n",
    "        super(EncoderLayer, self).__init__()\n",
    "        \n",
    "        self.mha  = MultiHeadAttention(d_model, num_heads)\n",
    "        self.ffn = point_wise_feed_forward_network(d_model, dff)\n",
    "        \n",
    "        self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
    "        self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
    "\n",
    "\n",
    "        self.dropout1 = tf.keras.layers.Dropout(rate)\n",
    "        self.dropout2 = tf.keras.layers.Dropout(rate)\n",
    "        \n",
    "    def call(self, x, training, mask):\n",
    "        attn_output, _ = self.mha(x, x, x ,mask)\n",
    "        attn_output = self.dropout1(attn_output, training=training)\n",
    "        out1 = self.layernorm1(x + attn_output)\n",
    "        \n",
    "        ffn_output = self.ffn(out1)\n",
    "        ffn_output = self.dropout2(ffn_output, training= training)\n",
    "        out2 = self.layernorm2(out1 + ffn_output)\n",
    "        \n",
    "        return out2\n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "8ed9ff74",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-25T08:15:43.853088Z",
     "iopub.status.busy": "2022-07-25T08:15:43.852393Z",
     "iopub.status.idle": "2022-07-25T08:15:43.865658Z",
     "shell.execute_reply": "2022-07-25T08:15:43.864475Z"
    },
    "papermill": {
     "duration": 0.026546,
     "end_time": "2022-07-25T08:15:43.868105",
     "exception": false,
     "start_time": "2022-07-25T08:15:43.841559",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class DecoderLayer(tf.keras.layers.Layer):\n",
    "    def __init__(self, d_model, num_heads, dff, rate=0.1):\n",
    "        super(DecoderLayer, self).__init__()\n",
    "\n",
    "        self.mha1 = MultiHeadAttention(d_model, num_heads)\n",
    "        self.mha2 = MultiHeadAttention(d_model, num_heads)\n",
    "\n",
    "        self.ffn = point_wise_feed_forward_network(d_model, dff)\n",
    "\n",
    "        self.layernorm1 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
    "        self.layernorm2 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
    "        self.layernorm3 = tf.keras.layers.LayerNormalization(epsilon=1e-6)\n",
    "\n",
    "        self.dropout1 = tf.keras.layers.Dropout(rate)\n",
    "        self.dropout2 = tf.keras.layers.Dropout(rate)\n",
    "        self.dropout3 = tf.keras.layers.Dropout(rate)\n",
    "    \n",
    "    \n",
    "    def call(self, x, enc_output, training, look_ahead_mask, padding_mask):\n",
    "        attn1, attn_weights_block1 = self.mha1(x, x, x, look_ahead_mask)\n",
    "        attn1 = self.dropout1(attn1, training=training)\n",
    "        out1 = self.layernorm1(attn1 + x)\n",
    "\n",
    "        attn2, attn_weights_block2 = self.mha2(enc_output, enc_output, out1, padding_mask)\n",
    "        attn2 = self.dropout2(attn2, training=training)\n",
    "        out2 = self.layernorm2(attn2 + out1)\n",
    "\n",
    "        ffn_output = self.ffn(out2)\n",
    "        ffn_output = self.dropout3(ffn_output, training=training)\n",
    "        out3 = self.layernorm3(ffn_output + out2)\n",
    "\n",
    "        return out3, attn_weights_block1, attn_weights_block2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "b683a6dd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-25T08:15:43.888920Z",
     "iopub.status.busy": "2022-07-25T08:15:43.888534Z",
     "iopub.status.idle": "2022-07-25T08:15:43.899680Z",
     "shell.execute_reply": "2022-07-25T08:15:43.898518Z"
    },
    "papermill": {
     "duration": 0.023997,
     "end_time": "2022-07-25T08:15:43.901830",
     "exception": false,
     "start_time": "2022-07-25T08:15:43.877833",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class Encoder(tf.keras.layers.Layer):\n",
    "    def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size, maximum_position_encoding, rate=0.1):\n",
    "        super(Encoder, self).__init__()\n",
    "\n",
    "        self.d_model = d_model\n",
    "        self.num_layers = num_layers\n",
    "\n",
    "        self.embedding = tf.keras.layers.Embedding(input_vocab_size, d_model)\n",
    "        self.pos_encoding = positional_encoding(maximum_position_encoding, self.d_model)\n",
    "\n",
    "        self.enc_layers = [EncoderLayer(d_model, num_heads, dff, rate) for _ in range(num_layers)]\n",
    "\n",
    "        self.dropout = tf.keras.layers.Dropout(rate)\n",
    "        \n",
    "    def call(self, x, training, mask):\n",
    "        seq_len = tf.shape(x)[1]\n",
    "\n",
    "        x = self.embedding(x)\n",
    "        x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n",
    "        x += self.pos_encoding[:, :seq_len, :]\n",
    "\n",
    "        x = self.dropout(x, training=training)\n",
    "    \n",
    "        for i in range(self.num_layers):\n",
    "            x = self.enc_layers[i](x, training, mask)\n",
    "    \n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "1d62539d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-25T08:15:43.923189Z",
     "iopub.status.busy": "2022-07-25T08:15:43.922513Z",
     "iopub.status.idle": "2022-07-25T08:15:43.935372Z",
     "shell.execute_reply": "2022-07-25T08:15:43.934193Z"
    },
    "papermill": {
     "duration": 0.026026,
     "end_time": "2022-07-25T08:15:43.937746",
     "exception": false,
     "start_time": "2022-07-25T08:15:43.911720",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class Decoder(tf.keras.layers.Layer):\n",
    "        \n",
    "    def __init__(self, num_layers, d_model, num_heads, dff, target_vocab_size, maximum_position_encoding, rate=0.1):\n",
    "        super(Decoder, self).__init__()\n",
    "\n",
    "        self.d_model = d_model\n",
    "        self.num_layers = num_layers\n",
    "\n",
    "        self.embedding = tf.keras.layers.Embedding(target_vocab_size, d_model)\n",
    "        self.pos_encoding = positional_encoding(maximum_position_encoding, d_model)\n",
    "\n",
    "        self.dec_layers = [DecoderLayer(d_model, num_heads, dff, rate) for _ in range(num_layers)]\n",
    "        self.dropout = tf.keras.layers.Dropout(rate)\n",
    "    \n",
    "    def call(self, x, enc_output, training, look_ahead_mask, padding_mask):\n",
    "        seq_len = tf.shape(x)[1]\n",
    "        attention_weights = {}\n",
    "\n",
    "        x = self.embedding(x)\n",
    "        x *= tf.math.sqrt(tf.cast(self.d_model, tf.float32))\n",
    "        x += self.pos_encoding[:, :seq_len, :]\n",
    "\n",
    "        x = self.dropout(x, training=training)\n",
    "\n",
    "        for i in range(self.num_layers):\n",
    "            x, block1, block2 = self.dec_layers[i](x, enc_output, training, look_ahead_mask, padding_mask)\n",
    "\n",
    "            attention_weights['decoder_layer{}_block1'.format(i+1)] = block1\n",
    "            attention_weights['decoder_layer{}_block2'.format(i+1)] = block2\n",
    "    \n",
    "        return x, attention_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "3f5ce579",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-25T08:15:43.958439Z",
     "iopub.status.busy": "2022-07-25T08:15:43.958071Z",
     "iopub.status.idle": "2022-07-25T08:15:43.968827Z",
     "shell.execute_reply": "2022-07-25T08:15:43.967759Z"
    },
    "papermill": {
     "duration": 0.02403,
     "end_time": "2022-07-25T08:15:43.971160",
     "exception": false,
     "start_time": "2022-07-25T08:15:43.947130",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class Transformer(tf.keras.Model):\n",
    "    def __init__(self, num_layers, d_model, num_heads, dff, input_vocab_size, target_vocab_size, pe_input, pe_target, rate=0.1):\n",
    "        super(Transformer, self).__init__()\n",
    "\n",
    "        self.encoder = Encoder(num_layers, d_model, num_heads, dff, input_vocab_size, pe_input, rate)\n",
    "\n",
    "        self.decoder = Decoder(num_layers, d_model, num_heads, dff, target_vocab_size, pe_target, rate)\n",
    "\n",
    "        self.final_layer = tf.keras.layers.Dense(target_vocab_size)\n",
    "    \n",
    "    def call(self, inp, tar, training, enc_padding_mask, look_ahead_mask, dec_padding_mask):\n",
    "        enc_output = self.encoder(inp, training, enc_padding_mask)\n",
    "\n",
    "        dec_output, attention_weights = self.decoder(tar, enc_output, training, look_ahead_mask, dec_padding_mask)\n",
    "\n",
    "        final_output = self.final_layer(dec_output)\n",
    "\n",
    "        return final_output, attention_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "aadb61c0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-25T08:15:43.992265Z",
     "iopub.status.busy": "2022-07-25T08:15:43.991894Z",
     "iopub.status.idle": "2022-07-25T08:15:43.997314Z",
     "shell.execute_reply": "2022-07-25T08:15:43.996131Z"
    },
    "papermill": {
     "duration": 0.019276,
     "end_time": "2022-07-25T08:15:44.000306",
     "exception": false,
     "start_time": "2022-07-25T08:15:43.981030",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "num_layers = 4\n",
    "d_model = 128\n",
    "dff = 512\n",
    "num_heads = 8\n",
    "dropout_rate = 0.1\n",
    "EPOCHS = 20"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "17096d7e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-25T08:15:44.020864Z",
     "iopub.status.busy": "2022-07-25T08:15:44.020519Z",
     "iopub.status.idle": "2022-07-25T08:15:44.028387Z",
     "shell.execute_reply": "2022-07-25T08:15:44.027355Z"
    },
    "papermill": {
     "duration": 0.020651,
     "end_time": "2022-07-25T08:15:44.030582",
     "exception": false,
     "start_time": "2022-07-25T08:15:44.009931",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class CustomSchedule(tf.keras.optimizers.schedules.LearningRateSchedule):\n",
    "    def __init__(self, d_model, warmup_steps=4000):\n",
    "        super(CustomSchedule, self).__init__()\n",
    "\n",
    "        self.d_model = d_model\n",
    "        self.d_model = tf.cast(self.d_model, tf.float32)\n",
    "\n",
    "        self.warmup_steps = warmup_steps\n",
    "    \n",
    "    def __call__(self, step):\n",
    "        arg1 = tf.math.rsqrt(step)\n",
    "        arg2 = step * (self.warmup_steps ** -1.5)\n",
    "\n",
    "        return tf.math.rsqrt(self.d_model) * tf.math.minimum(arg1, arg2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "eced9ee8",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-25T08:15:44.051817Z",
     "iopub.status.busy": "2022-07-25T08:15:44.050740Z",
     "iopub.status.idle": "2022-07-25T08:15:44.058062Z",
     "shell.execute_reply": "2022-07-25T08:15:44.057083Z"
    },
    "papermill": {
     "duration": 0.020445,
     "end_time": "2022-07-25T08:15:44.060425",
     "exception": false,
     "start_time": "2022-07-25T08:15:44.039980",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "learning_rate = CustomSchedule(d_model)\n",
    "\n",
    "optimizer = tf.keras.optimizers.Adam(learning_rate, beta_1=0.9, beta_2=0.98, epsilon=1e-9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "6910ff2b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-25T08:15:44.081356Z",
     "iopub.status.busy": "2022-07-25T08:15:44.080409Z",
     "iopub.status.idle": "2022-07-25T08:15:44.355581Z",
     "shell.execute_reply": "2022-07-25T08:15:44.354536Z"
    },
    "papermill": {
     "duration": 0.288041,
     "end_time": "2022-07-25T08:15:44.357927",
     "exception": false,
     "start_time": "2022-07-25T08:15:44.069886",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 0, 'Train Step')"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZUAAAEGCAYAAACtqQjWAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8qNh9FAAAACXBIWXMAAAsTAAALEwEAmpwYAAA0FUlEQVR4nO3deXxV9Zn48c+Tfd9DWAKEJSxBKWpEca+4oO2UaYsj6m9qW6vTVttOl7H66/wcf/7qTO2mtdV23JdRgVJbsXWjWreqQFxQQJDkghC23ASIJBBCkuf3x/kGLuEmuUnuzb3Jfd6vV14593vO+Z7n3kCenPP9nueIqmKMMcaEQ0K0AzDGGDN8WFIxxhgTNpZUjDHGhI0lFWOMMWFjScUYY0zYJEU7gGgqKirSsrKyaIdhjDFDyttvv12vqsXB1sV1UikrK6OqqiraYRhjzJAiIh93t84ufxljjAkbSyrGGGPCxpKKMcaYsLGkYowxJmwsqRhjjAmbiCYVEZknIhtEpFpEbgiyPlVEFrv1K0SkLGDdja59g4hcGND+gIjUiciabo75fRFRESmKyJsyxhjTrYglFRFJBO4CLgIqgMtEpKLLZlcBe1R1MnA7cJvbtwJYCMwA5gF3u/4AHnJtwY45FrgA2BLWN2OMMSYkkTxTmQ1Uq6pPVVuBRcD8LtvMBx52y0uBuSIirn2Rqh5U1U1AtesPVX0V2N3NMW8HrgeGZT1/VWXJqq00HWyLdijGGBNUJJPKGGBrwOta1xZ0G1VtAxqBwhD3PYqIzAe2qerqXra7RkSqRKTK7/eH8j5ixntb93L9H97nh0vfj3YoxhgT1LAYqBeRDOB/Azf1tq2q3qOqlapaWVwctMpAzNqyez8Ayz/cFeVIjDEmuEgmlW3A2IDXpa4t6DYikgTkAg0h7htoEjABWC0im93274jIyAHEH3Nq/M0AtLZ1sNUlGGOMiSWRTCqrgHIRmSAiKXgD78u6bLMMuNItLwBeUu/5xsuAhW522ASgHFjZ3YFU9QNVHaGqZapahne57ERV3RnetxRdNf4mRLzlZ9fsiG4wxhgTRMSSihsjuQ54HvgQWKKqa0XkFhH5nNvsfqBQRKqB7wE3uH3XAkuAdcBzwLWq2g4gIk8AbwJTRaRWRK6K1HuINT5/M2dPKWbG6ByeXTOs8qUxZpiIaJViVX0GeKZL200Byy3AJd3seytwa5D2y0I4bllfY411HR3KpvomTptUyMllBfzs+Q3saDzAqNz0aIdmjDGHDYuB+niwvfEALYc6mFicybzjvKGi5+xsxRgTYyypDBE+N0g/qTiLScVZTBuZzdOrt0c5KmOMOZollSGixt8EwMTiTADmzxrDO1v28nFDczTDMsaYo1hSGSJ8/may05IozkoFYP6s0YjAn961sxVjTOywpDJE1PibmFichbg5xaPz0jl1QiF/fLcWbxa2McZEnyWVIcLnb2ZSUeZRbZ8/cQybG/bz7ta90QnKGGO6sKQyBDQdbGPnJy1MGpF1VPtFx40kNSmBP73bU7EBY4wZPJZUhoBNbubXxC5nKtlpyZxfUcLTq7dzsK09GqEZY8xRLKkMAb56b+ZX1zMVgEsqx7Jn/yFeWGtFJo0x0WdJZQioqWsiQWB8YcYx686cXERpfjqPr7Dnkhljos+SyhBQU99MaX4GqUmJx6xLSBAumz2ON30N+Ny9LMYYEy2WVIaAmromJhVndrv+kspSkhKERau2druNMcYMBksqMa6jQ9nc0MzE4mPHUzqNyE7jvOklLH271gbsjTFRZUklxnUWkpzUQ1IBuPyUcexubrUik8aYqLKkEuM6n/Y4sYfLXwBnTC5iQlEmD/x9s91hb4yJGksqMa5z8L23M5WEBOErp5exeute3tmyZzBCM8aYY1hSiXE1/iay05IoykrpddsFJ5WSm57Mfa9tGoTIjDHmWJZUYpzP33xUIcmeZKQkcdnscTy/didbd+8fhOiMMeZollRinM/f3ON04q6uPG08CSI89MbmyAVljDHdiGhSEZF5IrJBRKpF5IYg61NFZLFbv0JEygLW3ejaN4jIhQHtD4hInYis6dLXz0RkvYi8LyJ/FJG8SL63wXC4kGQv4ymBRuWmc/Hxo1i8aiuN+w9FMDpjjDlWxJKKiCQCdwEXARXAZSJS0WWzq4A9qjoZuB24ze1bASwEZgDzgLtdfwAPubaulgPHqepM4CPgxrC+oSjYdPgRwqGfqQB845xJNB1s48E3bGzFGDO4InmmMhuoVlWfqrYCi4D5XbaZDzzslpcCc8UbPJgPLFLVg6q6Cah2/aGqrwK7ux5MVV9Q1Tb38i2gNNxvaLAdeYRw6GcqANNH5XDe9BIe/Ptm9rXY2YoxZvBEMqmMAQLrhtS6tqDbuITQCBSGuG9Pvgo8G2yFiFwjIlUiUuX3+/vQ5eDz+bsvJNmbb8+dTOOBQzz61scRiMwYY4IbdgP1IvIjoA14LNh6Vb1HVStVtbK4uHhwg+ujGn8zYwuCF5LszczSPM6eUsx9r21if2tb7zsYY0wYRDKpbAPGBrwudW1BtxGRJCAXaAhx32OIyJeBzwJX6DC4rbzG33TMg7n64lvnTmZ3cyuPvWVl8Y0xgyOSSWUVUC4iE0QkBW/gfVmXbZYBV7rlBcBLLhksAxa62WETgHJgZU8HE5F5wPXA51R1yN+k0dGhbKpv7tPMr64qywo4Y3IRv32lxsZWjDGDImJJxY2RXAc8D3wILFHVtSJyi4h8zm12P1AoItXA94Ab3L5rgSXAOuA54FpVbQcQkSeAN4GpIlIrIle5vn4DZAPLReQ9EfldpN7bYNi29wAH2zr6PEjf1Q/nTWN3cyv3vuoLU2TGGNO9pEh2rqrPAM90abspYLkFuKSbfW8Fbg3Sflk3208eULAxxlffv+nEXR1fmstnZo7ivtc38c9zyijOTg1HeMYYE9SwG6gfLmrq+jedOJgfXDCV1rYOfv3SxgH3ZYwxPbGkEqN89aEXkuzNhKJMLj15LI+v2MJmdwZkjDGRYEklRnk1v0IrJBmK78wtJzUpgR//5cOw9GeMMcFYUolRNf6mXh/M1RcjctL41txy/vrhLl7eUBe2fo0xJpAllRjUdLCNXZ8cHNB04mC+cnoZE4oyueXpdbS2dYS1b2OMAUsqMenI0x7Dd6YCkJqUyE3/UIGvvpmHrNikMSYCLKnEIN/h59KH90wF4NNTRzB32gh+9deN7GxsCXv/xpj4ZkklBtUMoJBkKG76hwraVfk/T61hGFSzMcbEEEsqMcg3gEKSoRhfmMl3z5vC8nW7eHbNzogcwxgTnyypxKAaf1PYB+m7uuqMCRw3JoebnlprT4g0xoSNJZUY01lIciDViUORlJjAbV+cyZ79rdz6zLqIHssYEz8sqcSYzkKSk0ZE9kwFYMboXK45ayJLqmr5m927YowJA0sqMebwI4QjfKbS6Ttzy5laks31S9+noengoBzTGDN8WVKJMZGcThxMWnIidyycReP+Q9z45Ac2G8wYMyCWVGKMr76JnDAVkgzV9FE5XD9vKi+s28WSqq2DdlxjzPBjSSXG1NQ1MzGMhSRD9dXTJ3DapEL+79PrDt/Rb4wxfWVJJcb46iM/nTiYhAThF//0KVKTEvjmY+9woLV90GMwxgx9llRiyL6WQ+z65GBYqxP3xajcdG6/dBYbdu3j3/9kd9sbY/rOkkoM2RSmRwgPxDlTR/Ctc8v5wzu1LF5l4yvGmL6JaFIRkXkiskFEqkXkhiDrU0VksVu/QkTKAtbd6No3iMiFAe0PiEidiKzp0leBiCwXkY3ue34k31sk1ByuTjz4l78CfWduOWeWF3HTsrWs2dYY1ViMMUNLxJKKiCQCdwEXARXAZSJS0WWzq4A9qjoZuB24ze1bASwEZgDzgLtdfwAPubaubgBeVNVy4EX3ekjx+ZtJEBgXoUKSoUpMEO64dBZFmSlc/UgVdfusmrExJjSRPFOZDVSrqk9VW4FFwPwu28wHHnbLS4G54k17mg8sUtWDqroJqHb9oaqvAruDHC+wr4eBfwzjexkUPn8z4yJYSLIvCrNSuffKSvbuP8Q1j7xNyyEbuDfG9C6SSWUMEHhRvta1Bd1GVduARqAwxH27KlHVHW55J1ASbCMRuUZEqkSkyu/3h/I+Bo33COHoXvoKNGN0LncsnMV7W/dy/dL3beDeGNOrYTlQr95vv6C/AVX1HlWtVNXK4uLiQY6se+2ukGQ0B+mDuXDGSK6fN5Vlq7fz65eqox2OMSbGRTKpbAPGBrwudW1BtxGRJCAXaAhx3652icgo19coYEhVSNzuCknG0plKp2+cPYkvnDiGXy7/iMWrtkQ7HGNMDItkUlkFlIvIBBFJwRt4X9Zlm2XAlW55AfCSO8tYBix0s8MmAOXAyl6OF9jXlcBTYXgPg2awC0n2hYjwky/M5Kwpxdz45Ae8sNYe7GWMCS5iScWNkVwHPA98CCxR1bUicouIfM5tdj9QKCLVwPdwM7ZUdS2wBFgHPAdcq6rtACLyBPAmMFVEakXkKtfXT4DzRWQjcJ57PWR0FpIcjJL3/ZGSlMBvrziR40vz+NYT77JyU7C5EsaYeCfxPPhaWVmpVVVV0Q4DgB/98QOeXr2d1f9xwaDX/eqL3c2tLPjdG/j3HWTxNXOoGJ0T7ZCMMYNMRN5W1cpg64blQP1Q5PM3M2nE4BeS7KuCzBQeveoUslKTuOK+t/hwxyfRDskYE0MsqcSIGn8TE4ti89JXV2Py0nni6lNJTUrkivtWsGHnvmiHZIyJEZZUYsC+lkPU7YteIcn+KCvK5IlrTiU5Ubj83rf4aJclFmOMJZWYcHiQPganE/dkQlEmT1x9KokJXmKxS2HGGEsqMcBX31lIcuicqXSaWJzFE9ecSlJCApf+95u8/bHNCjMmnvWaVERkioi82FkVWERmisi/Rz60+OHzN5OYIFEvJNlfk4qzWPqNORRmpXLFfSt4ecOQuu/UGBNGoZyp3AvcCBwCUNX38W5kNGFS429ibH56TBSS7K/S/AyW/MscJhZlcfUjVTy9enu0QzLGREEoSSVDVbvezd4WiWDilc/fPOTGU4Ipzk5l0b+cyglj8/n2onf571dqrAilMXEmlKRSLyKTcAUaRWQBsKPnXUyo2jsUX33zkJr51ZOctGQeuWo2Fx83iv96dj3/+48fcKi9I9phGWMGSVII21wL3ANME5FtwCbgiohGFUe27z1Aa4wWkuyvtOREfn3ZCZQVZXDX32rYuvsAd11xIrnpydEOzRgTYaGcqaiqngcUA9NU9YwQ9zMhiJVHCIdbQoLwbxdO42cLZrJiUwNf/O0bbKpvjnZYxpgICyU5/AFAVZtVtfMOt6WRCym+1Lh7VIbL5a+uLqkcyyNfPYX6poN87jev89d1u6IdkjEmgrpNKiIyTUS+COSKyBcCvr4MpA1ahMOcz99EbnoyhZkp0Q4lYuZMKuTp686grDCTrz1SxS9e2EB7hw3gGzMc9TSmMhX4LJAH/ENA+z7g6gjGFFe8RwhnxnwhyYEaW5DB778+h5ueWsOvX6pmdW0jv7p0FvnDOJkaE4+6TSqq+hTwlIjMUdU3BzGmuOLzN3Nmeew81jiS0pITue2LM5k1Np+bl63l4jtf445LZ3HKxMJoh2aMCZNQxlTeFZFrReRuEXmg8yvikcWBzkKSk0YMz/GUYESEy08Zx9JvzCE1KYHL7n2LX76wgTabdmzMsBBKUnkUGAlcCLyC97x4K0kbBp2FJIdKyftwmlmax5+/fSZfOLGUO1+q5tJ73mLr7v3RDssYM0ChJJXJqvp/gGZVfRj4DHBKZMOKD52FJCfH0ZlKoKzUJH5+yae487IT+GjnPi7+1Wssqdpqd+EbM4SFklQOue97ReQ4IBcYEbmQ4kdNnSskWRCfSaXT5z41mme+cyYVo3O4fun7fPnBVexoPBDtsIwx/RBKUrlHRPKBfweWAeuA2yIaVZzw1TcxriCDlCS7l3RsQQZPXH0qt8yfwcpNu7ngl6+yZJWdtRgz1PT620xV71PVPar6qqpOVNURwLOhdC4i80Rkg4hUi8gNQdanishit36FiJQFrLvRtW8QkQt761NE5orIOyLynoi8LiKTQ4kxmmrqmplYFN9nKYESEoQvzSnj+X89ixljcrj+D+/zpQdW8nGD3YlvzFDRY1IRkTkiskBERrjXM0XkceDvvXUsIonAXcBFQAVwmYhUdNnsKmCPqk4GbsedAbntFgIzgHnA3SKS2EufvwWuUNVZwON4Z1Yxq71D2dQwfApJhtO4wgwe/9qp/L/5M3h3y14uuP1V7nxxIwfb2qMdmjGmFz3dUf8z4AHgi8BfROTHwAvACqA8hL5nA9Wq6lPVVmARML/LNvOBh93yUmCueHcBzgcWqepBVd0EVLv+eupTgRy3nAvE9AM9OgtJDreaX+GSkCD885wyXvz+2ZxfUcIvl3/EvDte4/WN9dEOzRjTg57uqP8McIKqtrgxla3Acaq6OcS+x7h9OtVy7Kyxw9uoapuINAKFrv2tLvuOccvd9fk14BkROQB8ApwaLCgRuQa4BmDcuHEhvpXwq3aFJIdTdeJIKMlJ4zeXn8g/Vfq56ak1/K/7V/DZmaO48eLpjMlLj3Z4xpguerr81aKqLQCqugfY2IeEEg3fBS5W1VLgQeCXwTZS1XtUtVJVK4uLo3cne+c9KkPxufTRcNaUYp7717P47nlTWL5uF+f+/GV+/vwGmg7a8+KMiSU9nalMFJFlAa8nBL5W1c/10vc2YGzA61LXFmybWhFJwrts1dDLvse0i0gx8ClVXeHaFwPP9RJfVNW4QpIFVvsqZGnJiXznvHIWVJbys+fW85u/VbNo1VZ+cMEULqkcS2LC8K6fZsxQ0FNS6Tr+8Ys+9r0KKBeRCXgJYSFweZdtlgFXAm8CC4CXVFVd8npcRH4JjMYbw1kJSDd97sGrpjxFVT8Czgc+7GO8g8oXJ4UkI2FMXjp3LDyBL58+gR//eR03PPkBD72xmRsumsbZU4rtMzUminoqKPnKQDp2YyTXAc8DicADqrpWRG4BqlR1GXA/8KiIVAO78ZIEbrslePfEtAHXqmo7QLA+XfvVwB9EpAMvyXx1IPFHWo2/mbOnxEchyUiZNTaP3399Ds+u2cl/PfshX35wFSeX5fODC6ZakUpjokTi+eayyspKraqqGvTj7ms5xPE3v8D186byzXNi/naaIaG1rYPFVVv59Ysbqdt3kDPLi/jBBVP51Ni8aIdmzLAjIm+ramWwdXYrdxQcGaS3mV/hkpKUwD+fOp5Xr/80P7p4Omu2NTL/rr9z9SNVvF+7N9rhGRM3ehpTMRFy5Ln0NvMr3NKSE7n6rIlcdso4Hnh9E/e+5mP5ul2cWV7EtZ+ezCkTCmzMxZgI6jWpiMjTeDcWBmoEqoD/7px2bELn81shyUjLSk3i23PL+crpZfzPW1u4/3UfC+95i5PG53Ptpyfx6akjLLkYEwGhXP7yAU3Ave7rE7znqUxxr00f1fitkORgyU5L5hvnTOL1H57LLfNnsLOxha8+VMVFv3qNP75bS2ubPRzMmHAK5fLXaap6csDrp0VklaqeLCJrIxXYcObzWyHJwZaWnMiX5pRx2exxPPXedn77cjXfXbya/3xmPV86dTyXnzKOwqzUaIdpzJAXyp/KWSJyuJ6JW+4cYW6NSFTDWGchyUkjbJA+GpITE1hwUinLv3s2D33lZKaPyuEXyz9izk9e4odL32f9zk+iHaIxQ1ooZyrfB14XkRq8mw8nAN8UkUyOFIM0Idq2xyskaWcq0ZWQIJwzdQTnTB3Bxl37ePCNzTz5Ti2Lq7Zy2qRC/tep4zm/ooTkRLtEaUxf9JpUVPUZESkHprmmDQGD83dEKrDhqsY9QtjOVGJHeUk2//n54/m3C6byxKot/M+bH/PNx96hKCuVf6os5bLZ4xhbkBHtMI0ZEkKdUnwSUOa2/5SIoKqPRCyqYaymzlUntjOVmJOfmcI3z5nMv5w1iVc+quPxFVv43Ss1/PaVGs4sL+by2eOYO32Enb0Y04NQphQ/CkwC3gM6n5KkgCWVfvDVN5OXYYUkY1lignDutBLOnVbC9r0HWLxqK4tXbeXr//M2xdmp/OOs0XzhxFKmj8rpvTNj4kwoZyqVQIXGcz2XMKqpa2JikRWSHCpG56Xz3fOn8K1zJ/O3DX5+X7WVh97YzL2vbaJiVA5fOHEM82eNoTjbZo4ZA6EllTXASGBHhGOJC756KyQ5FCUlJnB+RQnnV5Swu7mVp1dv58l3avnxXz7kv55dz9lTivnCiWM4b3oJacmJ0Q7XmKgJJakUAetEZCVwsLMxhOepmC4+aTmEf99Bq/k1xBVkpnDlaWVceVoZG3ft48l3t/HHd7bx0vo6MlMSOa+ihM8cP4qzpxaTmmQJxsSXUJLKzZEOIl50FpKcaDW/ho3ykmx+OG8aP7hgKm/5Gvjz+9t5ds1OnnpvO9mpSZw/o4TPzhzFGZOLrYKCiQuhTCke0HNVzBG+w4Uk7UxluElMEE6fXMTpk4u4Zf5xvFHTwJ9Xb+f5tTt58p1t5KQlceGMkVx0/EhOm1Rkl8jMsNVtUhGR11X1DBHZx9EFJQVQVbWpL31U429yhSTtnofhLDkxgbOnFHP2lGJu/fzxvF7t58+rd/Dsmp38/u1aMlISOXtKMedXlHDutBHkZdhMQDN89PTkxzPc9+zBC2d48/mbrZBknElJSjg8PflgWztv1jTwwrpd/HXdLp5ds5PEBGF2WcHhSQB2k6UZ6kJ68qOIJAIlBCQhVd0SwbgGxWA/+fGC219hXEEG9115cu8bm2Gto0N5f1sjy9ft5IW1u9joboqdNjLblY8p5qTx+XajpYlJPT35MZSbH78F/AewC+isE67AzLBFGAfaO5TNDfs5Z+qIaIdiYkBCgjBrbB6zxubxbxdOY3N9M8vX7eKvH+7ivtd8/O6VGrJSkzh9ciHnTB3B2VOKGZ2XHu2wjelVKLO/vgNMVdWGvnYuIvOAXwGJwH2q+pMu61Px7sw/CWgALlXVzW7djcBVeHfxf1tVn++pT/HuJvwxcInb57eqemdfY46UzkKS9rRHE0xZUSZXnzWRq8+ayL6WQ/y9uoFXPvLzyoY6nl+7C4ApJVmcM3UEZ5UXU1mWb4P9JiaFklS24j3psU/cJbO7gPOBWmCViCxT1XUBm10F7FHVySKyELgNuFREKoCFwAxgNPBXEZni9umuzy8DY4FpqtohIjF1StD5COGJNvPL9CI7LZl5x41k3nEjUVU21jXx8oY6XvnIz4N/38Q9r/pISUqgcnw+p00q5LTJRcwck0uSXSozMSCUpOIDXhaRv3D0zY+/7GW/2UC1qvoARGQRMB8ITCrzOXIfzFLgN+6MYz6wSFUPAptEpNr1Rw99fgO4XFU7XHx1Iby3QVNj04lNP4gIU0qymVKSzTVnTaL5YBtv+Rp4o8b7+vkLH8ELH5GVmsQpEwqYM6mQ0ycXMbUkm4QEKwVkBl8oSWWL+0pxX6Eag3eW06kWOKW7bVS1TUQagULX/laXfce45e76nIR3lvN5wI93yWxj16BE5BrgGoBx48Z1XR0xNX4rJGkGLjM1ibnTS5g7vQSA3c2tvFnTwBs19bxR08CL672/pQozUzhlYgEnl3lf00flkGhJxgyCHpOKu4Q1RVWvGKR4BiIVaFHVShH5AvAAcGbXjVT1HuAe8GZ/DVZwPn+Tlbs3YVeQmcJnZo7iMzNHAbB97wHerGng7zX1rPDt5pkPdgKQlZrEiePzmV2Wz8llBXxqbJ6NyZiI6DGpqGq7iIwXkRRV7eujg7fhjXF0KnVtwbapFZEkIBdvwL6nfbtrrwWedMt/BB7sY7wR5atv5hwrJGkibHReOl88qZQvnlQKeElm1ebd3temPd7lMiAlMYGZpblUlhUwe0I+s8bm21m0CYtQx1T+LiLLgObOxhDGVFYB5SIyAe8X/0Lg8i7bLAOuBN4EFgAvqaq6Yz0uIr/EG6gvB1bi3c3fXZ9/Aj4NbALOBj4K4b0Nis5CkjZIbwbb6Lx05s/yyvMD7N3fStXmPazavJuVm3e76cveCXtZYQazxuZxwrh8Zo3NY/qoHLtR1/RZKEmlxn0lACHfXe/GSK4Dnseb/vuAqq4VkVuAKlVdBtwPPOoG4nfjJQncdkvwBuDbgGtVtR0gWJ/ukD8BHhOR7wJNwNdCjTXSOgtJ2nRiE215GSmcV1HCeRXemMyB1nZW1+7lva17eW/LXt6oaeBP720HvGoAx4/JdYnGu6dmTF66PQvI9CikO+qHq8G6o/4Pb9fy/d+v5q/fO5vJ9mx6E8NUlR2NLby3dS/vbtnDu1v28sG2Rg62efc9F2enMnNMLjPG5HK8+yrJSbVEE2cGekd9MXA93j0jaZ3tqnpu2CIc5nz1VkjSDA0iwui8dEbnpXPx8d7g/6H2Dtbv2Me7W/fwnksyf9tQR4f7e7QoK4XjxuRy3OhcjhuTy/GluYzOTbNEE6dCufz1GLAY+CzwdbwxEH8kgxpuauqaGW+FJM0QlZyYwPGlXrL40hyvbX9rGx/u+IQPahtZs/0T1mxr5LWN9bS7TFOQmcKM0TmHk830UdmML8y0ac1xIJSkUqiq94vId9yzVV4RkVWRDmw48dU32YO5zLCSkZLESeMLOGl8weG2lkPtfLjDSzAfbGtkzbZPuPdVH20u0aQlJzC1JJtpI3OYNsp9H5lNvs06G1ZCSSqH3PcdIvIZYDtQ0MP2JkB7h7K5fj+ftkKSZphLS07khHH5nDAu/3Bby6F2Nu5qYv3OT1i/cx/rd37C8g93sbjqyD3MI3PSDieZ6e77xOJMq9A8RIWSVH4sIrnA94FfAznAdyMa1TBSu2c/re0ddqZi4lJacuLhS2edVBV/00HW7/CSzPod+/hw5z7+Xu3jULt3VpOcKEwoyqR8RDaTR2RRXpJF+YhsyooySE2ymzZjWSiPE/6zW2zEuw/E9MGR6cQ268sY8CYDjMhOY0R2GmcF3BB8qL0Dn7+Z9Ts/4cMd+6iua2Lt9kaeWbODzkmqiQnC+IKMoxLN5BFZTCrOIj3Fkk0sCGX21xTgt0CJqh4nIjOBz6nqjyMe3TBg1YmNCU1yYgJTR2YzdWQ282cdaW851I7P38zGOi/RbNzVxMa6fby4vu7wxAARKM1PZ3JxFhOKsphYnMnEokwmFGcyMsdmog2mUC5/3Qv8G/DfAKr6vog8jvfsEtMLKyRpzMCkJSdSMTqHitE5R7W3tnXwcUMzG12i+ahuHz5/M2/6Gmg51HF4u/TkRCa4BDOxKJOJxZlMKMpiQlEmuenJg/12hr1QkkqGqq7skunbIhTPsOPzN9mlL2MiICUpgfKSbMpLsuH4I+0dHcrOT1rYVN+Mr76ZTf5mNtU3sWZbI89+sOPw/TXgVXP2kkwm4wszGV+YwbiCDMYXZJKbYQmnP0JJKvUiMgnvEcKIyAJgR0SjGkZq/M18eqoVkjRmsCQkHLmB8/TJRUeta23rYMvu/Wyq9xKNz+8lnpfW+6lvqj1q29z05MNJZlxBhlv2Es/InDR7Xk03Qkkq1+KVip8mItvwCjYOhVL4Udd44BD1TQeZZKVZjIkJKUkJTB6R5collRy1rvlgG1t27+fjhv1s3b2fj3c383HDfj7Y1shza3Yevt8GvCrPpQXpjC/IYHxhJmNd4hmTl05pQTo5afF7lhPK7C8fcJ6IZAIJqrpPRP4VuCPCsQ15vs5BenuOijExLzM1iemjcpg+KueYdW3tHexobOHjBi/ZbGnwks+W3ftZtXkPTQePHhHITkuiNN8lmfwjX2PyMijNTycvI3nYTh4I5UwFAFVtDnj5PSyp9KpzOrHN/DJmaEtKTGBsQQZjCzI4g6Mvqakqu5tbqd1zgNo9B9i2d7/3fc8Btu7ez1u+hmOSTkZKoksy6V7yOZx00hmTn05RZuqQvbwWclLpYmi+20FW428iKUEYX2iFJI0ZrkSEwqxUCrNS+dTYvGPWqyqNBw4FJJ0D1O7Zzza3/M6WvTQeOHTUPsmJQklOGqNy0xiVm+6+pzEqL/1wW2FmSkwmnv4mlfitl98HPn8z4woyrNyEMXFMRMjLSCEvw6vmHMy+lkNestl9gB2NB9je2MLOxha27z3A6tq9PLe2hda2jqP2SUlMoCQ3lVE56YzKS2Nkbhqjc48knVF5aRRkDH7i6TapiMg+gicPAdIjFtEw4hWStEtfxpieZaclM21kMtNGHjueA0cuse1obHFfB9i+t4WdLgG9u2UvOxtbaG0/OvEkJ3rVC0bmplGSk0pJThojc9IoyUnjtEmFjMhJC3q8geg2qahqyE95NMeyQpLGmHAJvMTW3dlOR4eye38rO/Z6SWdHYws7P2lhl/u+fuc+Xtngp7m1HYBHvjp7cJOKGZjOQpJ246MxZjAkJAhFWakUZaUeVcCzq6aDbexsbGFUbvgTClhSiZgjNb9sOrExJnZkpSZF9LHmER1BFpF5IrJBRKpF5IYg61NFZLFbv0JEygLW3ejaN4jIhX3o804RaYrYmwqRTSc2xsSjiCUVEUkE7gIuAiqAy0SkostmVwF7VHUycDtwm9u3AlgIzADmAXeLSGJvfYpIJZBPDKjxN5NvhSSNMXEmkmcqs4FqVfWpaiuwCJjfZZv5wMNueSkwV7zbTOcDi1T1oKpuAqpdf9326RLOz4DrI/ieQlbjt5lfxpj4E8mkMgbYGvC61rUF3UZV2/AeBFbYw7499XkdsExVeyx2KSLXiEiViFT5/f4+vaG+8PmbmWTjKcaYODMs7soTkdHAJXiPO+6Rqt6jqpWqWllcHJnqwZ2FJO1MxRgTbyKZVLYBYwNel7q2oNuISBKQCzT0sG937ScAk4FqEdkMZIhIdbjeSF9ZIUljTLyKZFJZBZSLyAQRScEbeF/WZZtlwJVueQHwkqqqa1/oZodNAMqBld31qap/UdWRqlqmqmXAfjf4HxU1nc+lt5L3xpg4E7H7VFS1TUSuA54HEoEHVHWtiNwCVKnqMuB+4FF3VrEbL0ngtlsCrMN7yuS1qtoOEKzPSL2H/vK5QpLjCqyQpDEmvkT05kdVfQZ4pkvbTQHLLXhjIcH2vRW4NZQ+g2wT1VMEn7+ZcYVWSNIYE3/st14E1PibmFhkl76MMfHHkkqYtbV38HHDfiaNsEF6Y0z8saQSZrV7DniFJO1MxRgThyyphJmv3gpJGmPilyWVMOssJGkl740x8ciSSpjV+JvIz0gm3wpJGmPikCWVMKvxN9tZijEmbllSCTOfv8nGU4wxccuSShg17j9EfVOrFZI0xsQtSyphVONmftnlL2NMvLKkEkZHHiFsl7+MMfHJkkoYWSFJY0y8s6QSRjX+JiskaYyJa/bbL4x8Np3YGBPnLKmESVt7B5sbmm08xRgT1yyphEntngMcalcrJGmMiWuWVMKks5Cklbw3xsQzSyphUlPnphPbmYoxJo5ZUgkTX30TBZkpVkjSGBPXIppURGSeiGwQkWoRuSHI+lQRWezWrxCRsoB1N7r2DSJyYW99ishjrn2NiDwgIsmRfG9d1dQ1M7HILn0ZY+JbxJKKiCQCdwEXARXAZSJS0WWzq4A9qjoZuB24ze1bASwEZgDzgLtFJLGXPh8DpgHHA+nA1yL13oLx1VshSWOMieSZymygWlV9qtoKLALmd9lmPvCwW14KzBURce2LVPWgqm4Cql1/3fapqs+oA6wESiP43o7SWUjS7lExxsS7SCaVMcDWgNe1ri3oNqraBjQChT3s22uf7rLXPwPPDfgdhKjm8COELakYY+LbcByovxt4VVVfC7ZSRK4RkSoRqfL7/WE54JFHCNvlL2NMfItkUtkGjA14Xeragm4jIklALtDQw7499iki/wEUA9/rLihVvUdVK1W1sri4uI9vKbgaV0hyrBWSNMbEuUgmlVVAuYhMEJEUvIH3ZV22WQZc6ZYXAC+5MZFlwEI3O2wCUI43TtJtnyLyNeBC4DJV7Yjg+zqGz9/EeCskaYwxJEWqY1VtE5HrgOeBROABVV0rIrcAVaq6DLgfeFREqoHdeEkCt90SYB3QBlyrqu0Awfp0h/wd8DHwpjfWz5Oqekuk3l+gGn+zjacYYwwRTCrgzcgCnunSdlPAcgtwSTf73grcGkqfrj2i76U7be0dfNzQzNzpI6JxeGOMiSl2vWaADheStDMVY4yxpDJQNf7O59LbzC9jjLGkMkCHn0tvhSSNMcaSykDV+K2QpDHGdLKkMkA+vxWSNMaYTpZUBqjG32SD9MYY41hSGYDG/YdoaG616sTGGONYUhmAzkKSdqZijDEeSyoDUFPXWZ3YzlSMMQYsqQyIr76Z5EQrJGmMMZ0sqQxATV0T4wqskKQxxnSy34YD4Ku3QpLGGBPIkko/dRaStEF6Y4w5wpJKP211hSRtkN4YY46wpNJPPr9NJzbGmK4sqfSTVSc2xphjWVLpJ5+/mcLMFPIyrJCkMcZ0sqTSTzX+JhtPMcaYLiyp9JNXndjGU4wxJpAllX7Yu7+VhuZWJo2wMxVjjAkU0aQiIvNEZIOIVIvIDUHWp4rIYrd+hYiUBay70bVvEJELe+tTRCa4PqpdnxEb7Kixpz0aY0xQEUsqIpII3AVcBFQAl4lIRZfNrgL2qOpk4HbgNrdvBbAQmAHMA+4WkcRe+rwNuN31tcf1HRGHpxOPsKRijDGBInmmMhuoVlWfqrYCi4D5XbaZDzzslpcCc0VEXPsiVT2oqpuAatdf0D7dPue6PnB9/mOk3liN3xWSzE+P1CGMMWZIimRSGQNsDXhd69qCbqOqbUAjUNjDvt21FwJ7XR/dHQsAEblGRKpEpMrv9/fjbUFZYQafP2EMSVZI0hhjjhJ3vxVV9R5VrVTVyuLi4n71sXD2OH664FNhjswYY4a+SCaVbcDYgNelri3oNiKSBOQCDT3s2117A5Dn+ujuWMYYYyIskkllFVDuZmWl4A28L+uyzTLgSre8AHhJVdW1L3SzwyYA5cDK7vp0+/zN9YHr86kIvjdjjDFBJPW+Sf+oapuIXAc8DyQCD6jqWhG5BahS1WXA/cCjIlIN7MZLErjtlgDrgDbgWlVtBwjWpzvkD4FFIvJj4F3XtzHGmEEk3h/58amyslKrqqqiHYYxxgwpIvK2qlYGWxd3A/XGGGMix5KKMcaYsLGkYowxJmwsqRhjjAmbuB6oFxE/8HE/dy8C6sMYTrhYXH1jcfWNxdU3sRoXDCy28aoa9O7xuE4qAyEiVd3Nfogmi6tvLK6+sbj6JlbjgsjFZpe/jDHGhI0lFWOMMWFjSaX/7ol2AN2wuPrG4uobi6tvYjUuiFBsNqZijDEmbOxMxRhjTNhYUjHGGBM2llT6QUTmicgGEakWkRsG4XibReQDEXlPRKpcW4GILBeRje57vmsXEbnTxfa+iJwY0M+VbvuNInJld8frJZYHRKRORNYEtIUtFhE5yb3XarevDCCum0Vkm/vc3hORiwPW3eiOsUFELgxoD/qzdY9bWOHaF7tHL/QW01gR+ZuIrBORtSLynVj4vHqIK6qfl9svTURWishqF9v/7ak/8R6Psdi1rxCRsv7G3M+4HhKRTQGf2SzXPpj/9hNF5F0R+XMsfFaoqn314Quv5H4NMBFIAVYDFRE+5magqEvbT4Eb3PINwG1u+WLgWUCAU4EVrr0A8Lnv+W45vx+xnAWcCKyJRCx4z8051e3zLHDRAOK6GfhBkG0r3M8tFZjgfp6JPf1sgSXAQrf8O+AbIcQ0CjjRLWcDH7ljR/Xz6iGuqH5eblsBstxyMrDCvb+g/QHfBH7nlhcCi/sbcz/jeghYEGT7wfy3/z3gceDPPX32g/VZ2ZlK380GqlXVp6qtwCJgfhTimA887JYfBv4xoP0R9byF90TMUcCFwHJV3a2qe4DlwLy+HlRVX8V79k3YY3HrclT1LfX+tT8S0Fd/4urOfGCRqh5U1U1ANd7PNejP1v3FeC6wNMh77CmmHar6jlveB3wIjCHKn1cPcXVnUD4vF4+qapN7mey+tIf+Aj/LpcBcd/w+xTyAuLozKD9LESkFPgPc51739NkPymdlSaXvxgBbA17X0vN/yHBQ4AUReVtErnFtJaq6wy3vBEp6iS+ScYcrljFuOZwxXucuPzwg7jJTP+IqBPaqalt/43KXGk7A+ws3Zj6vLnFBDHxe7nLOe0Ad3i/dmh76OxyDW9/ojh/2/wdd41LVzs/sVveZ3S4iqV3jCvH4/f1Z3gFcD3S41z199oPyWVlSGRrOUNUTgYuAa0XkrMCV7i+bmJgbHkuxAL8FJgGzgB3AL6IRhIhkAX8A/lVVPwlcF83PK0hcMfF5qWq7qs4CSvH+Wp4WjTi66hqXiBwH3IgX38l4l7R+OFjxiMhngTpVfXuwjhkKSyp9tw0YG/C61LVFjKpuc9/rgD/i/Ufb5U6Zcd/reokvknGHK5ZtbjksMarqLveLoAO4F+9z609cDXiXL5K6tPdKRJLxfnE/pqpPuuaof17B4oqFzyuQqu4F/gbM6aG/wzG49bnu+BH7fxAQ1zx3KVFV9SDwIP3/zPrzszwd+JyIbMa7NHUu8Cui/Vn1NuhiX8cMiiXhDa5N4Mjg1YwIHi8TyA5YfgNvLORnHD3Y+1O3/BmOHiBc6doLgE14g4P5brmgnzGVcfSAeNhi4djByosHENeogOXv4l03BpjB0QOTPrxByW5/tsDvOXrw85shxCN418bv6NIe1c+rh7ii+nm5bYuBPLecDrwGfLa7/oBrOXrweUl/Y+5nXKMCPtM7gJ9E6d/+ORwZqI/uZ9WfXyrx/oU3s+MjvGu9P4rwsSa6H+ZqYG3n8fCuhb4IbAT+GvAPU4C7XGwfAJUBfX0VbxCuGvhKP+N5Au/SyCG8a6xXhTMWoBJY4/b5Da7qQz/jetQd931gGUf/0vyRO8YGAmbZdPezdT+HlS7e3wOpIcR0Bt6lrfeB99zXxdH+vHqIK6qfl9tvJvCui2ENcFNP/QFp7nW1Wz+xvzH3M66X3Ge2BvgfjswQG7R/+27fcziSVKL6WVmZFmOMMWFjYyrGGGPCxpKKMcaYsLGkYowxJmwsqRhjjAkbSyrGGGPCxpKKMX0kIoUBVWl3ytGVfXusxisilSJyZx+P91VXvfZ9EVkjIvNd+5dFZPRA3osx4WZTio0ZABG5GWhS1Z8HtCXpkdpLA+2/FHgFr6pwoyutUqyqm0TkZbyqwlXhOJYx4WBnKsaEgXuuxu9EZAXwUxGZLSJvuudcvCEiU9125wQ89+JmV7jxZRHxici3g3Q9AtgHNAGoapNLKAvwbpZ7zJ0hpbvncbziCo8+H1AK5mUR+ZXbbo2IzA5yHGPCwpKKMeFTCpymqt8D1gNnquoJwE3Af3azzzS8cuizgf9wNbkCrQZ2AZtE5EER+QcAVV0KVAFXqFfksA34Nd6zPU4CHgBuDegnw233TbfOmIhI6n0TY0yIfq+q7W45F3hYRMrxSqJ0TRad/qJeMcKDIlKHVwb/cAl0VW0XkXl4VXDnAreLyEmqenOXfqYCxwHLvUdkkIhXtqbTE66/V0UkR0Ty1CuMaExYWVIxJnyaA5b/H/A3Vf28e2bJy93sczBguZ0g/yfVG/hcCawUkeV41XBv7rKZAGtVdU43x+k6eGqDqSYi7PKXMZGRy5Ey4V/ubyciMloCnm+O96yTj93yPrzHAYNXCLBYROa4/ZJFZEbAfpe69jOARlVt7G9MxvTEzlSMiYyf4l3++nfgLwPoJxn4uZs63AL4ga+7dQ8BvxORA3jPHFkA3CkiuXj/t+/Aq2wN0CIi77r+vjqAeIzpkU0pNmaYs6nHZjDZ5S9jjDFhY2cqxhhjwsbOVIwxxoSNJRVjjDFhY0nFGGNM2FhSMcYYEzaWVIwxxoTN/wf8cK+ZiXorMQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "temp_learning_rate_schedule = CustomSchedule(d_model)\n",
    "\n",
    "plt.plot(temp_learning_rate_schedule(tf.range(40000, dtype=tf.float32)))\n",
    "plt.ylabel(\"Learning Rate\")\n",
    "plt.xlabel(\"Train Step\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "50f85159",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-25T08:15:44.388560Z",
     "iopub.status.busy": "2022-07-25T08:15:44.388033Z",
     "iopub.status.idle": "2022-07-25T08:15:44.404491Z",
     "shell.execute_reply": "2022-07-25T08:15:44.403221Z"
    },
    "papermill": {
     "duration": 0.039651,
     "end_time": "2022-07-25T08:15:44.407751",
     "exception": false,
     "start_time": "2022-07-25T08:15:44.368100",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "loss_object = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True, reduction='none')\n",
    "def loss_function(real, pred):\n",
    "    mask = tf.math.logical_not(tf.math.equal(real, 0))\n",
    "    loss_ = loss_object(real, pred)\n",
    "\n",
    "    mask = tf.cast(mask, dtype=loss_.dtype)\n",
    "    loss_ *= mask\n",
    "\n",
    "    return tf.reduce_sum(loss_)/tf.reduce_sum(mask)\n",
    "\n",
    "\n",
    "def accuracy_function(real, pred):\n",
    "    accuracies = tf.equal(real, tf.argmax(pred, axis=2))\n",
    "\n",
    "    mask = tf.math.logical_not(tf.math.equal(real, 0))\n",
    "    accuracies = tf.math.logical_and(mask, accuracies)\n",
    "\n",
    "    accuracies = tf.cast(accuracies, dtype=tf.float32)\n",
    "    mask = tf.cast(mask, dtype=tf.float32)\n",
    "    return tf.reduce_sum(accuracies)/tf.reduce_sum(mask)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "f8a57794",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-25T08:15:44.445338Z",
     "iopub.status.busy": "2022-07-25T08:15:44.444850Z",
     "iopub.status.idle": "2022-07-25T08:15:44.503973Z",
     "shell.execute_reply": "2022-07-25T08:15:44.502538Z"
    },
    "papermill": {
     "duration": 0.082118,
     "end_time": "2022-07-25T08:15:44.507263",
     "exception": false,
     "start_time": "2022-07-25T08:15:44.425145",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_loss = tf.keras.metrics.Mean(name='train_loss')\n",
    "train_accuracy = tf.keras.metrics.Mean(name='train_accuracy')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "0ee5a383",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-25T08:15:44.544903Z",
     "iopub.status.busy": "2022-07-25T08:15:44.544314Z",
     "iopub.status.idle": "2022-07-25T08:15:44.741981Z",
     "shell.execute_reply": "2022-07-25T08:15:44.740650Z"
    },
    "papermill": {
     "duration": 0.218692,
     "end_time": "2022-07-25T08:15:44.745569",
     "exception": false,
     "start_time": "2022-07-25T08:15:44.526877",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "transformer = Transformer(\n",
    "    num_layers=num_layers,\n",
    "    d_model=d_model,\n",
    "    num_heads=num_heads,\n",
    "    dff=dff,\n",
    "    input_vocab_size=Encoder_Vocab,\n",
    "    target_vocab_size=Decoder_Vocab,\n",
    "    pe_input=1000,\n",
    "    pe_target=1000,\n",
    "    rate = dropout_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "53d85164",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-25T08:15:44.780133Z",
     "iopub.status.busy": "2022-07-25T08:15:44.779536Z",
     "iopub.status.idle": "2022-07-25T08:15:44.789595Z",
     "shell.execute_reply": "2022-07-25T08:15:44.788328Z"
    },
    "papermill": {
     "duration": 0.030479,
     "end_time": "2022-07-25T08:15:44.792733",
     "exception": false,
     "start_time": "2022-07-25T08:15:44.762254",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def create_masks(inp, tar):\n",
    "    enc_padding_mask = create_padding_mask(inp)\n",
    "    dec_padding_mask = create_padding_mask(inp)\n",
    "\n",
    "    look_ahead_mask = create_look_ahead_mask(tf.shape(tar)[1])\n",
    "    dec_target_padding_mask = create_padding_mask(tar)\n",
    "    combined_mask = tf.maximum(dec_target_padding_mask, look_ahead_mask)\n",
    "  \n",
    "    return enc_padding_mask, combined_mask, dec_padding_mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "a1bd1d6d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-25T08:15:44.826795Z",
     "iopub.status.busy": "2022-07-25T08:15:44.826159Z",
     "iopub.status.idle": "2022-07-25T08:15:44.841915Z",
     "shell.execute_reply": "2022-07-25T08:15:44.840823Z"
    },
    "papermill": {
     "duration": 0.035948,
     "end_time": "2022-07-25T08:15:44.844905",
     "exception": false,
     "start_time": "2022-07-25T08:15:44.808957",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "@tf.function\n",
    "def train_step(inp, tar):\n",
    "    tar_inp = tar[:, :-1]\n",
    "    tar_real = tar[:, 1:]\n",
    "\n",
    "    enc_padding_mask, combined_mask, dec_padding_mask = create_masks(inp, tar_inp)\n",
    "\n",
    "    with tf.GradientTape() as tape:\n",
    "        predictions, _ = transformer(\n",
    "            inp, tar_inp, \n",
    "            True, \n",
    "            enc_padding_mask, \n",
    "            combined_mask, \n",
    "            dec_padding_mask\n",
    "        )\n",
    "        loss = loss_function(tar_real, predictions)\n",
    "\n",
    "    gradients = tape.gradient(loss, transformer.trainable_variables)    \n",
    "    optimizer.apply_gradients(zip(gradients, transformer.trainable_variables))\n",
    "\n",
    "    train_loss(loss)\n",
    "    train_accuracy(accuracy_function(tar_real, predictions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "42496db5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-25T08:15:44.878320Z",
     "iopub.status.busy": "2022-07-25T08:15:44.877861Z",
     "iopub.status.idle": "2022-07-25T09:16:53.058182Z",
     "shell.execute_reply": "2022-07-25T09:16:53.056393Z"
    },
    "papermill": {
     "duration": 3668.212797,
     "end_time": "2022-07-25T09:16:53.073850",
     "exception": false,
     "start_time": "2022-07-25T08:15:44.861053",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2022-07-25 08:15:53.024973: I tensorflow/compiler/mlir/mlir_graph_optimization_pass.cc:185] None of the MLIR Optimization Passes are enabled (registered 2)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1 Batch 0 Loss 10.8644 Accuracy 0.0000\n",
      "Epoch 1 Batch 200 Loss 10.5008 Accuracy 0.0467\n",
      "Epoch 1 Batch 400 Loss 9.6606 Accuracy 0.0537\n",
      "Epoch 1 Loss 9.2475 Accuracy 0.0550\n",
      "Time taken for 1 epoch: 193.6248254776001 secs\n",
      "\n",
      "Epoch 2 Batch 0 Loss 7.3445 Accuracy 0.0550\n",
      "Epoch 2 Batch 200 Loss 7.2368 Accuracy 0.0604\n",
      "Epoch 2 Batch 400 Loss 7.0211 Accuracy 0.0720\n",
      "Epoch 2 Loss 6.9293 Accuracy 0.0777\n",
      "Time taken for 1 epoch: 182.79051160812378 secs\n",
      "\n",
      "Epoch 3 Batch 0 Loss 6.3721 Accuracy 0.0778\n",
      "Epoch 3 Batch 200 Loss 6.3645 Accuracy 0.0887\n",
      "Epoch 3 Batch 400 Loss 6.2476 Accuracy 0.0986\n",
      "Epoch 3 Loss 6.1956 Accuracy 0.1030\n",
      "Time taken for 1 epoch: 182.84664058685303 secs\n",
      "\n",
      "Epoch 4 Batch 0 Loss 5.9185 Accuracy 0.1030\n",
      "Epoch 4 Batch 200 Loss 5.8401 Accuracy 0.1113\n",
      "Epoch 4 Batch 400 Loss 5.7467 Accuracy 0.1194\n",
      "Epoch 4 Loss 5.7033 Accuracy 0.1232\n",
      "Time taken for 1 epoch: 182.86119437217712 secs\n",
      "\n",
      "Epoch 5 Batch 0 Loss 5.3721 Accuracy 0.1233\n",
      "Epoch 5 Batch 200 Loss 5.3911 Accuracy 0.1307\n",
      "Epoch 5 Batch 400 Loss 5.3009 Accuracy 0.1381\n",
      "Epoch 5 Loss 5.2592 Accuracy 0.1416\n",
      "Time taken for 1 epoch: 182.87284684181213 secs\n",
      "\n",
      "Epoch 6 Batch 0 Loss 5.0025 Accuracy 0.1417\n",
      "Epoch 6 Batch 200 Loss 4.9572 Accuracy 0.1487\n",
      "Epoch 6 Batch 400 Loss 4.8722 Accuracy 0.1557\n",
      "Epoch 6 Loss 4.8341 Accuracy 0.1592\n",
      "Time taken for 1 epoch: 182.837096452713 secs\n",
      "\n",
      "Epoch 7 Batch 0 Loss 4.6386 Accuracy 0.1592\n",
      "Epoch 7 Batch 200 Loss 4.5486 Accuracy 0.1662\n",
      "Epoch 7 Batch 400 Loss 4.4649 Accuracy 0.1733\n",
      "Epoch 7 Loss 4.4296 Accuracy 0.1768\n",
      "Time taken for 1 epoch: 182.85953903198242 secs\n",
      "\n",
      "Epoch 8 Batch 0 Loss 4.2232 Accuracy 0.1769\n",
      "Epoch 8 Batch 200 Loss 4.1609 Accuracy 0.1840\n",
      "Epoch 8 Batch 400 Loss 4.0846 Accuracy 0.1911\n",
      "Epoch 8 Loss 4.0538 Accuracy 0.1947\n",
      "Time taken for 1 epoch: 182.9496147632599 secs\n",
      "\n",
      "Epoch 9 Batch 0 Loss 3.8630 Accuracy 0.1947\n",
      "Epoch 9 Batch 200 Loss 3.8033 Accuracy 0.2019\n",
      "Epoch 9 Batch 400 Loss 3.7211 Accuracy 0.2092\n",
      "Epoch 9 Loss 3.6864 Accuracy 0.2129\n",
      "Time taken for 1 epoch: 182.88272142410278 secs\n",
      "\n",
      "Epoch 10 Batch 0 Loss 3.5498 Accuracy 0.2129\n",
      "Epoch 10 Batch 200 Loss 3.4262 Accuracy 0.2204\n",
      "Epoch 10 Batch 400 Loss 3.3573 Accuracy 0.2279\n",
      "Epoch 10 Loss 3.3288 Accuracy 0.2316\n",
      "Time taken for 1 epoch: 182.91844749450684 secs\n",
      "\n",
      "Epoch 11 Batch 0 Loss 3.3196 Accuracy 0.2317\n",
      "Epoch 11 Batch 200 Loss 3.1131 Accuracy 0.2391\n",
      "Epoch 11 Batch 400 Loss 3.0588 Accuracy 0.2466\n",
      "Epoch 11 Loss 3.0354 Accuracy 0.2502\n",
      "Time taken for 1 epoch: 182.8800709247589 secs\n",
      "\n",
      "Epoch 12 Batch 0 Loss 3.1119 Accuracy 0.2502\n",
      "Epoch 12 Batch 200 Loss 2.8603 Accuracy 0.2575\n",
      "Epoch 12 Batch 400 Loss 2.8158 Accuracy 0.2648\n",
      "Epoch 12 Loss 2.7974 Accuracy 0.2683\n",
      "Time taken for 1 epoch: 182.919593334198 secs\n",
      "\n",
      "Epoch 13 Batch 0 Loss 2.6912 Accuracy 0.2683\n",
      "Epoch 13 Batch 200 Loss 2.6498 Accuracy 0.2754\n",
      "Epoch 13 Batch 400 Loss 2.6140 Accuracy 0.2823\n",
      "Epoch 13 Loss 2.5996 Accuracy 0.2857\n",
      "Time taken for 1 epoch: 182.8799283504486 secs\n",
      "\n",
      "Epoch 14 Batch 0 Loss 2.5984 Accuracy 0.2857\n",
      "Epoch 14 Batch 200 Loss 2.4797 Accuracy 0.2924\n",
      "Epoch 14 Batch 400 Loss 2.4500 Accuracy 0.2991\n",
      "Epoch 14 Loss 2.4369 Accuracy 0.3023\n",
      "Time taken for 1 epoch: 182.8470742702484 secs\n",
      "\n",
      "Epoch 15 Batch 0 Loss 2.4711 Accuracy 0.3023\n",
      "Epoch 15 Batch 200 Loss 2.3327 Accuracy 0.3087\n",
      "Epoch 15 Batch 400 Loss 2.3056 Accuracy 0.3150\n",
      "Epoch 15 Loss 2.2967 Accuracy 0.3180\n",
      "Time taken for 1 epoch: 182.82899522781372 secs\n",
      "\n",
      "Epoch 16 Batch 0 Loss 2.0565 Accuracy 0.3180\n",
      "Epoch 16 Batch 200 Loss 2.2064 Accuracy 0.3241\n",
      "Epoch 16 Batch 400 Loss 2.1852 Accuracy 0.3300\n",
      "Epoch 16 Loss 2.1774 Accuracy 0.3329\n",
      "Time taken for 1 epoch: 182.93216395378113 secs\n",
      "\n",
      "Epoch 17 Batch 0 Loss 2.1521 Accuracy 0.3329\n",
      "Epoch 17 Batch 200 Loss 2.0971 Accuracy 0.3386\n",
      "Epoch 17 Batch 400 Loss 2.0794 Accuracy 0.3443\n",
      "Epoch 17 Loss 2.0724 Accuracy 0.3470\n",
      "Time taken for 1 epoch: 182.88296270370483 secs\n",
      "\n",
      "Epoch 18 Batch 0 Loss 1.9907 Accuracy 0.3470\n",
      "Epoch 18 Batch 200 Loss 2.0070 Accuracy 0.3524\n",
      "Epoch 18 Batch 400 Loss 1.9868 Accuracy 0.3577\n",
      "Epoch 18 Loss 1.9804 Accuracy 0.3603\n",
      "Time taken for 1 epoch: 182.87624740600586 secs\n",
      "\n",
      "Epoch 19 Batch 0 Loss 2.0871 Accuracy 0.3603\n",
      "Epoch 19 Batch 200 Loss 1.9150 Accuracy 0.3655\n",
      "Epoch 19 Batch 400 Loss 1.8993 Accuracy 0.3705\n",
      "Epoch 19 Loss 1.8938 Accuracy 0.3730\n",
      "Time taken for 1 epoch: 182.85485172271729 secs\n",
      "\n",
      "Epoch 20 Batch 0 Loss 1.7886 Accuracy 0.3730\n",
      "Epoch 20 Batch 200 Loss 1.8362 Accuracy 0.3779\n",
      "Epoch 20 Batch 400 Loss 1.8229 Accuracy 0.3827\n",
      "Epoch 20 Loss 1.8176 Accuracy 0.3850\n",
      "Time taken for 1 epoch: 182.82518553733826 secs\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(EPOCHS):\n",
    "    start = time.time()\n",
    "\n",
    "    train_loss.reset_states()\n",
    "  \n",
    "    for (batch, (inp, tar)) in enumerate(dataset):\n",
    "        train_step(inp, tar)\n",
    "    \n",
    "        if batch % 200 == 0:\n",
    "            print(f'Epoch {epoch + 1} Batch {batch} Loss {train_loss.result():.4f} Accuracy {train_accuracy.result():.4f}')\n",
    "      \n",
    "    \n",
    "   \n",
    "    print(f'Epoch {epoch + 1} Loss {train_loss.result():.4f} Accuracy {train_accuracy.result():.4f}')\n",
    "    print ('Time taken for 1 epoch: {} secs\\n'.format(time.time() - start))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "65a807aa",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-25T09:16:53.102800Z",
     "iopub.status.busy": "2022-07-25T09:16:53.102478Z",
     "iopub.status.idle": "2022-07-25T09:16:53.112975Z",
     "shell.execute_reply": "2022-07-25T09:16:53.112014Z"
    },
    "papermill": {
     "duration": 0.02778,
     "end_time": "2022-07-25T09:16:53.115080",
     "exception": false,
     "start_time": "2022-07-25T09:16:53.087300",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def evaluate(text):\n",
    "    text = eng_tokenizer.texts_to_sequences([text])\n",
    "    text = tf.keras.preprocessing.sequence.pad_sequences(text, maxlen=Encoder_len, \n",
    "                                                                   padding='post', truncating='post')\n",
    "\n",
    "    encoder_input = tf.expand_dims(text[0], 0)\n",
    "\n",
    "    decoder_input = [hind_tokenizer.word_index['<sos>']]\n",
    "    output = tf.expand_dims(decoder_input, 0)\n",
    "    \n",
    "    for i in range(Decoder_len):\n",
    "        enc_padding_mask, combined_mask, dec_padding_mask = create_masks(encoder_input, output)\n",
    "\n",
    "        predictions, attention_weights = transformer(\n",
    "            encoder_input, \n",
    "            output,\n",
    "            False,\n",
    "            enc_padding_mask,\n",
    "            combined_mask,\n",
    "            dec_padding_mask\n",
    "        )\n",
    "\n",
    "        predictions = predictions[: ,-1:, :]\n",
    "        predicted_id = tf.cast(tf.argmax(predictions, axis=-1), tf.int32)\n",
    "\n",
    "        if predicted_id == hind_tokenizer.word_index['<eos>']:\n",
    "            return tf.squeeze(output, axis=0), attention_weights\n",
    "\n",
    "        output = tf.concat([output, predicted_id], axis=-1)\n",
    "\n",
    "    return tf.squeeze(output, axis=0), attention_weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "a38043a3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-25T09:16:53.143430Z",
     "iopub.status.busy": "2022-07-25T09:16:53.143170Z",
     "iopub.status.idle": "2022-07-25T09:16:53.148578Z",
     "shell.execute_reply": "2022-07-25T09:16:53.147751Z"
    },
    "papermill": {
     "duration": 0.022104,
     "end_time": "2022-07-25T09:16:53.150700",
     "exception": false,
     "start_time": "2022-07-25T09:16:53.128596",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def translate(eng_text):\n",
    "    hind_text = evaluate(text=eng_text)[0].numpy()\n",
    "    hind_text = np.expand_dims(hind_text[1:], 0)  \n",
    "    return hind_tokenizer.sequences_to_texts(hind_text)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "c589bb7f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-25T09:16:53.178313Z",
     "iopub.status.busy": "2022-07-25T09:16:53.177579Z",
     "iopub.status.idle": "2022-07-25T09:16:54.715594Z",
     "shell.execute_reply": "2022-07-25T09:16:54.714600Z"
    },
    "papermill": {
     "duration": 1.553992,
     "end_time": "2022-07-25T09:16:54.717753",
     "exception": false,
     "start_time": "2022-07-25T09:16:53.163761",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'मैं एक विद्यार्थी और सीखने के साथ अध्ययन करना चाहता हूं'"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "translate(\"I am a student and I like to study very much.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "01306f55",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-25T09:16:54.747665Z",
     "iopub.status.busy": "2022-07-25T09:16:54.746682Z",
     "iopub.status.idle": "2022-07-25T09:16:56.148596Z",
     "shell.execute_reply": "2022-07-25T09:16:56.147447Z"
    },
    "papermill": {
     "duration": 1.419551,
     "end_time": "2022-07-25T09:16:56.151436",
     "exception": false,
     "start_time": "2022-07-25T09:16:54.731885",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'मेरा नाम ख़ुशी है मेरी अलौकिक शब्द याद बहुत ज्यादा है।'"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "translate(\"My name is Khushi\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "96ed8893",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-25T09:16:56.180851Z",
     "iopub.status.busy": "2022-07-25T09:16:56.180530Z",
     "iopub.status.idle": "2022-07-25T09:16:57.983109Z",
     "shell.execute_reply": "2022-07-25T09:16:57.981903Z"
    },
    "papermill": {
     "duration": 1.820428,
     "end_time": "2022-07-25T09:16:57.985973",
     "exception": false,
     "start_time": "2022-07-25T09:16:56.165545",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'मैं क्रिकेट खेल खेलने के साथ खेल रहा हूँ क्रिकेट खेल खेले जा रहा हूँ'"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "translate(\"I want to play cricket\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "13803792",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-07-25T09:16:58.016821Z",
     "iopub.status.busy": "2022-07-25T09:16:58.016515Z",
     "iopub.status.idle": "2022-07-25T09:16:59.144294Z",
     "shell.execute_reply": "2022-07-25T09:16:59.143354Z"
    },
    "papermill": {
     "duration": 1.145458,
     "end_time": "2022-07-25T09:16:59.146886",
     "exception": false,
     "start_time": "2022-07-25T09:16:58.001428",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'मैं अपने परिवार के बारे में बहुत खुश हूं'"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "translate(\"I am very happy about my family\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 3707.524691,
   "end_time": "2022-07-25T09:17:02.687082",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2022-07-25T08:15:15.162391",
   "version": "2.3.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}